diff --git a/docs/source/conf.py b/docs/source/conf.py index 7909a7acbd..1826dbbd0d 100644 --- a/docs/source/conf.py +++ b/docs/source/conf.py @@ -116,6 +116,7 @@ (r"py:.*", r".*.hit.l0.utils.*"), (r"py:.*", r".*.hit.l0.data_classes.*"), (r"py:.*", r".*.hit.l1a.*"), + (r"py:.*", r".*ExtendedSpinConfig.*"), (r"py:.*", r".*InitVar*"), (r"py:.*", r".*.glows.utils.constants.TimeTuple.*"), (r"py:.*", r".*glows.utils.constants.DirectEvent.*"), diff --git a/imap_processing/cli.py b/imap_processing/cli.py index bfa8aff830..c366386d83 100644 --- a/imap_processing/cli.py +++ b/imap_processing/cli.py @@ -2069,6 +2069,10 @@ def do_processing( ) datasets = ultra_l1a.ultra_l1a(science_files[0], create_derived_l1b=True) elif self.data_level == "l1b": + if self.repointing is None: + raise ValueError( + "Repointing must be provided for ULTRA L1b processing." + ) science_files = dependencies.get_file_paths(source="ultra", data_type="l1a") l1a_dict = { dataset.attrs["Logical_source"]: dataset @@ -2084,7 +2088,7 @@ def do_processing( ancillary_files = {} for path in anc_paths: ancillary_files[path.stem.split("_")[2]] = path - datasets = ultra_l1b.ultra_l1b(combined, ancillary_files) + datasets = ultra_l1b.ultra_l1b(combined, ancillary_files, self.repointing) elif self.data_level == "l1c": science_files = dependencies.get_file_paths(source="ultra", data_type="l1a") l1a_dict = { diff --git a/imap_processing/tests/ultra/data/l1/imap_ultra_l1b-45sensor-de-product-lookup_20251001_v001.csv b/imap_processing/tests/ultra/data/l1/imap_ultra_l1b-45sensor-de-product-lookup_20251001_v001.csv deleted file mode 100644 index f3c31692cf..0000000000 --- a/imap_processing/tests/ultra/data/l1/imap_ultra_l1b-45sensor-de-product-lookup_20251001_v001.csv +++ /dev/null @@ -1,2 +0,0 @@ -repointing_id_start,repointing_id_end,de_product -0,,imap_ultra_l1b_45sensor-de diff --git a/imap_processing/tests/ultra/data/l1/imap_ultra_l1b-45sensor-extendedspin-config_20251001_v001.csv b/imap_processing/tests/ultra/data/l1/imap_ultra_l1b-45sensor-extendedspin-config_20251001_v001.csv new file mode 100644 index 0000000000..0d19e11199 --- /dev/null +++ b/imap_processing/tests/ultra/data/l1/imap_ultra_l1b-45sensor-extendedspin-config_20251001_v001.csv @@ -0,0 +1,6 @@ +pointing,date,pri_config,cal_config,deflector_Vthresh,cullThresh_0,cullThresh_1,cullThresh_2,cullThresh_3,cullThresh_4 +44,11/10/25,p0,c0,3400,200,7.5,4.5,3.5,3.5 +134,1/22/26,p0,c1,3400,96.5,80.5,8.5,10.5,3.5 +233,4/30/26,p1,c2,3400,200,200,35.5,14.5,8.5 +282,6/17/26,p1,c3,3400,200,200,11.5,26.5,2.5 +337,8/11/26,p1,c4,2900,117.5,73.5,2.5,26.5,8.5 \ No newline at end of file diff --git a/imap_processing/tests/ultra/data/l1/imap_ultra_l1b-90sensor-extendedspin-config_20251001_v001.csv b/imap_processing/tests/ultra/data/l1/imap_ultra_l1b-90sensor-extendedspin-config_20251001_v001.csv new file mode 100644 index 0000000000..b824c5c3f0 --- /dev/null +++ b/imap_processing/tests/ultra/data/l1/imap_ultra_l1b-90sensor-extendedspin-config_20251001_v001.csv @@ -0,0 +1,6 @@ +pointing,date,pri_config,cal_config,deflector_Vthresh,cullThresh_0,cullThresh_1,cullThresh_2,cullThresh_3,cullThresh_4 +44,11/10/25,p0,c0,3400,9.5,13.5,4.5,4.5,0.5 +128,1/16/26,p0,c1,3400,200,118.5,78.5,12.5,5.5 +238,5/4/26,p1,c2,3400,200,200,21.5,6.5,5.5 +282,6/17/26,p1,c3,3400,81.5,49.5,15.5,16.5,9.5 +337,8/11/26,p1,c4,3400,200,97.5,82.5,66.5,8.5 \ No newline at end of file diff --git a/imap_processing/tests/ultra/unit/test_lookup_utils.py b/imap_processing/tests/ultra/unit/test_lookup_utils.py index 051edb56b6..bd2a5c636b 100644 --- a/imap_processing/tests/ultra/unit/test_lookup_utils.py +++ b/imap_processing/tests/ultra/unit/test_lookup_utils.py @@ -1,3 +1,4 @@ +import datetime from unittest import mock import numpy as np @@ -7,6 +8,7 @@ from imap_processing import imap_module_directory from imap_processing.quality_flags import ImapDEOutliersUltraFlags from imap_processing.ultra.l1b.lookup_utils import ( + ExtendedSpinConfig, get_angular_profiles, get_back_position, get_de_product_name, @@ -222,8 +224,8 @@ def test_get_scattering_thresholds(ancillary_files): def test_get_de_product_name_no_repoint(): """Tests function get_de_product_name when the lookup is missing the repoint.""" ancillary_files = { - "l1b-45sensor-de-product-lookup": TEST_PATH - / "imap_ultra_l1b-45sensor-de-product-lookup_20251001_v001.csv" + "l1c-45sensor-de-product-lookup": TEST_PATH + / "imap_ultra_l1c-45sensor-culling-config_20251001_v001.csv" } with mock.patch( "imap_processing.ultra.l1b.lookup_utils.pd.read_csv" @@ -239,14 +241,14 @@ def test_get_de_product_name_no_repoint(): } ) with pytest.raises(ValueError, match="No DE product found for repoint ID 0"): - get_de_product_name("repoint00000", 45, "l1b", ancillary_files) + get_de_product_name("repoint00000", 45, ancillary_files) def test_get_de_product_name_multiple_products(): """Tests function get_de_product_name when the lookup is ambiguous.""" ancillary_files = { - "l1b-45sensor-de-product-lookup": TEST_PATH - / "imap_ultra_l1b-45sensor-de-product-lookup_20251001_v001.csv" + "l1c-45sensor-de-product-lookup": TEST_PATH + / "imap_ultra_l1c-45sensor-culling-config_20251001_v001.csv" } with mock.patch( "imap_processing.ultra.l1b.lookup_utils.pd.read_csv" @@ -262,14 +264,14 @@ def test_get_de_product_name_multiple_products(): } ) with pytest.raises(ValueError, match="Multiple DE products found"): - get_de_product_name("repoint00002", 45, "l1b", ancillary_files) + get_de_product_name("repoint00002", 45, ancillary_files) def test_get_de_product_name(): """Tests function get_de_product_name.""" ancillary_files = { - "l1b-45sensor-de-product-lookup": TEST_PATH - / "imap_ultra_l1b-45sensor-de-product-lookup_20251001_v001.csv" + "l1c-45sensor-de-product-lookup": TEST_PATH + / "imap_ultra_l1c-45sensor-culling-config_20251001_v001.csv" } with mock.patch( "imap_processing.ultra.l1b.lookup_utils.pd.read_csv" @@ -288,9 +290,41 @@ def test_get_de_product_name(): # Test with a repoint in the future. Should return the priority 2 de product # since the last repoint range does not have an end and should be assumed to # cover all future repoints. - de_product = get_de_product_name("repoint00100", 45, "l1b", ancillary_files) + de_product = get_de_product_name("repoint00100", 45, ancillary_files) assert de_product == "imap_ultra_l1b_45sensor-priority-2-de" # Test with valid repoint that falls in the second range. - de_product = get_de_product_name("repoint00003", 45, "l1b", ancillary_files) + de_product = get_de_product_name("repoint00003", 45, ancillary_files) assert de_product == "imap_ultra_l1b_45sensor-priority-1-de" + + +def test_extended_config_class(): + """Tests the ExtendedSpinConfig class.""" + config_path = ( + TEST_PATH / "imap_ultra_l1b-45sensor-extendedspin-config_20251001_v001.csv" + ) + config = ExtendedSpinConfig.from_csv(config_path, "repoint00128") + + assert config.priority == "p0" + assert config.calibration == "c0" + np.testing.assert_array_equal( + config.energy_thresholds, np.array([200, 7.5, 4.5, 3.5, 3.5, 3.5]) + ) + assert config.date == datetime.datetime(2025, 11, 10) + assert config.voltage_threshold == 3400 + + +def test_extended_config_class_last_repoint_range(): + """Tests the ExtendedSpinConfig class.""" + config_path = ( + TEST_PATH / "imap_ultra_l1b-45sensor-extendedspin-config_20251001_v001.csv" + ) + config = ExtendedSpinConfig.from_csv(config_path, "repoint00383") + + assert config.priority == "p1" + assert config.calibration == "c4" + np.testing.assert_array_equal( + config.energy_thresholds, np.array([117.5, 73.5, 2.5, 26.5, 8.5, 8.5]) + ) + assert config.date == datetime.datetime(2026, 8, 11) + assert config.voltage_threshold == 2900 diff --git a/imap_processing/tests/ultra/unit/test_ultra_l1b.py b/imap_processing/tests/ultra/unit/test_ultra_l1b.py index 5b59e17e16..6fcbec06a7 100644 --- a/imap_processing/tests/ultra/unit/test_ultra_l1b.py +++ b/imap_processing/tests/ultra/unit/test_ultra_l1b.py @@ -159,7 +159,7 @@ def test_cdf_de( use_fake_spin_data_for_time(511000000, 511000000 + 86400 * 5) use_fake_repoint_data_for_time(np.arange(511000000, 511000000 + 86400 * 5, 86400)) - l1b_de_dataset = ultra_l1b(data_dict, ancillary_files) + l1b_de_dataset = ultra_l1b(data_dict, ancillary_files, "repoint99999") assert ( l1b_de_dataset[0].attrs["Logical_source_description"] @@ -197,7 +197,7 @@ def test_cdf_de_flags( # Use repoint data that will NOT cover the event times to test flag setting use_fake_repoint_data_for_time(np.arange(0, +86400 * 5, 86400)) - l1b_de_dataset = ultra_l1b(data_dict, ancillary_files) + l1b_de_dataset = ultra_l1b(data_dict, ancillary_files, "repoint99999") # All valid events should be flagged as DURINGREPOINT since the repoint data does # not cover any of the event times valid_events = l1b_de_dataset[0]["event_times"] != FILLVAL_FLOAT32 @@ -220,6 +220,7 @@ def test_ultra_l1b_extendedspin( key: l1b_de_dataset for key in [ "imap_ultra_l1b_45sensor-de", + "imap_ultra_l1b_45sensor-priority-1-de", "imap_ultra_l1a_45sensor-params", ] } @@ -228,10 +229,10 @@ def test_ultra_l1b_extendedspin( data_dict["imap_ultra_l1b_45sensor-status"] = status_dataset ancillary_files = { - "l1b-45sensor-de-product-lookup": TEST_PATH - / "imap_ultra_l1b-45sensor-de-product-lookup_20251001_v001.csv" + "l1b-45sensor-extendedspin-config": TEST_PATH + / "imap_ultra_l1b-45sensor-extendedspin-config_20251001_v001.csv" } - l1b_extendedspin_dataset = ultra_l1b(data_dict, ancillary_files) + l1b_extendedspin_dataset = ultra_l1b(data_dict, ancillary_files, "repoint99999") assert len(l1b_extendedspin_dataset) == 1 assert ( @@ -255,6 +256,7 @@ def test_cdf_extendedspin( key: l1b_de_dataset for key in [ "imap_ultra_l1b_45sensor-de", + "imap_ultra_l1b_45sensor-priority-1-de", "imap_ultra_l1a_45sensor-params", ] } @@ -263,10 +265,10 @@ def test_cdf_extendedspin( data_dict["imap_ultra_l1b_45sensor-status"] = status_dataset ancillary_files = { - "l1b-45sensor-de-product-lookup": TEST_PATH - / "imap_ultra_l1b-45sensor-de-product-lookup_20251001_v001.csv" + "l1b-45sensor-extendedspin-config": TEST_PATH + / "imap_ultra_l1b-45sensor-extendedspin-config_20251001_v001.csv" } - l1b_extendedspin_dataset = ultra_l1b(data_dict, ancillary_files) + l1b_extendedspin_dataset = ultra_l1b(data_dict, ancillary_files, "repoint99999") """Tests that CDF file is created and contains same attributes as xarray.""" l1b_extendedspin_dataset[0].attrs["Data_version"] = "999" l1b_extendedspin_dataset[0].attrs["Repointing"] = "repoint99999" @@ -295,6 +297,7 @@ def test_cdf_goodtimes( key: l1b_de_dataset for key in [ "imap_ultra_l1b_45sensor-de", + "imap_ultra_l1b_45sensor-priority-1-de", "imap_ultra_l1a_45sensor-params", ] } @@ -303,14 +306,15 @@ def test_cdf_goodtimes( data_dict["imap_ultra_l1b_45sensor-status"] = status_dataset ancillary_files = { - "l1b-45sensor-de-product-lookup": TEST_PATH - / "imap_ultra_l1b-45sensor-de-product-lookup_20251001_v001.csv" + "l1b-45sensor-extendedspin-config": TEST_PATH + / "imap_ultra_l1b-45sensor-extendedspin-config_20251001_v001.csv" } - l1b_extendedspin_dataset = ultra_l1b(data_dict, ancillary_files) + l1b_extendedspin_dataset = ultra_l1b(data_dict, ancillary_files, "repoint99999") goodtimes_dataset = ultra_l1b( {"imap_ultra_l1b_45sensor-extendedspin": l1b_extendedspin_dataset[0]}, ancillary_files, + "repoint99999", ) goodtimes_dataset[0].attrs["Data_version"] = "999" goodtimes_dataset[0].attrs["Repointing"] = "repoint99999" @@ -339,6 +343,7 @@ def test_cdf_badtimes( key: l1b_de_dataset for key in [ "imap_ultra_l1b_45sensor-de", + "imap_ultra_l1b_45sensor-priority-1-de", "imap_ultra_l1a_45sensor-params", ] } @@ -347,15 +352,16 @@ def test_cdf_badtimes( data_dict["imap_ultra_l1b_45sensor-status"] = status_dataset ancillary_files = { - "l1b-45sensor-de-product-lookup": TEST_PATH - / "imap_ultra_l1b-45sensor-de-product-lookup_20251001_v001.csv" + "l1b-45sensor-extendedspin-config": TEST_PATH + / "imap_ultra_l1b-45sensor-extendedspin-config_20251001_v001.csv" } - l1b_extendedspin_dataset = ultra_l1b(data_dict, ancillary_files) + l1b_extendedspin_dataset = ultra_l1b(data_dict, ancillary_files, "repoint99999") ancillary_files = {} goodtimes_dataset = ultra_l1b( {"imap_ultra_l1b_45sensor-extendedspin": l1b_extendedspin_dataset[0]}, ancillary_files, + "repoint99999", ) l1b_badtimes_dataset = ultra_l1b( @@ -364,6 +370,7 @@ def test_cdf_badtimes( "imap_ultra_l1b_45sensor-goodtimes": goodtimes_dataset[0], }, ancillary_files, + "repoint99999", ) l1b_badtimes_dataset[0].attrs["Data_version"] = "999" l1b_badtimes_dataset[0].attrs["Repointing"] = "repoint99999" @@ -385,7 +392,7 @@ def test_ultra_l1b_error(mock_data_l1a_rates_dict): with pytest.raises( ValueError, match="Data dictionary does not contain the expected keys." ): - ultra_l1b(mock_data_l1a_rates_dict, ancillary_files) + ultra_l1b(mock_data_l1a_rates_dict, ancillary_files, "repoint99999") @pytest.mark.external_test_data @@ -410,7 +417,7 @@ def test_ultra_l1b_priority_de( data_dict["imap_ultra_l1a_45sensor-priority-1-de"] = de_dataset data_dict[aux_dataset.attrs["Logical_source"]] = aux_dataset - l1b_de_dataset = ultra_l1b(data_dict, ancillary_files) + l1b_de_dataset = ultra_l1b(data_dict, ancillary_files, "repoint99999") assert l1b_de_dataset[0] assert ( diff --git a/imap_processing/tests/ultra/unit/test_ultra_l1b_culling.py b/imap_processing/tests/ultra/unit/test_ultra_l1b_culling.py index a9e2702544..56a5bf456b 100644 --- a/imap_processing/tests/ultra/unit/test_ultra_l1b_culling.py +++ b/imap_processing/tests/ultra/unit/test_ultra_l1b_culling.py @@ -16,6 +16,7 @@ ImapRatesUltraFlags, ) from imap_processing.ultra.constants import UltraConstants +from imap_processing.ultra.l1b.lookup_utils import ExtendedSpinConfig from imap_processing.ultra.l1b.ultra_l1b_culling import ( compare_aux_univ_spin_table, count_rejected_events_per_spin, @@ -42,13 +43,17 @@ get_spin_data, get_valid_de_count_summary, get_valid_earth_angle_events, - get_valid_events_per_energy_range, ) from imap_processing.ultra.l1b.ultra_l1b_extended import get_spin_info from imap_processing.ultra.l1c.l1c_lookup_utils import build_energy_bins TEST_PATH = imap_module_directory / "tests" / "ultra" / "data" / "l1" +REPOINT_47_SPIN_CONFIG = ExtendedSpinConfig.from_csv( + TEST_PATH / "imap_ultra_l1b-45sensor-extendedspin-config_20251001_v001.csv", + "repoint00047", +) + @pytest.fixture def setup_repoint_47_data(): @@ -57,6 +62,7 @@ def setup_repoint_47_data(): de_ds = xr.Dataset( { "de_event_met": ("epoch", de_df.event_times.values), + "event_times": ("epoch", de_df.event_times.values), "energy_spacecraft": ("epoch", de_df.energy_spacecraft.values), "quality_outliers": ("epoch", de_df.quality_outliers.values), "quality_scattering": ("epoch", de_df.quality_scattering.values), @@ -71,7 +77,21 @@ def setup_repoint_47_data(): xspin.spin_start_time.values, spin_bin_size, ) - return de_ds, xspin, spin_tbin_edges + + de_datasets = {"p0": de_ds, "p1": de_ds} + + # Get the energy ranges + energy_ranges = get_binned_energy_ranges(build_energy_bins()[0]) + + de_counts_summary = get_valid_de_count_summary( + de_datasets, + energy_ranges, + spin_tbin_edges, + REPOINT_47_SPIN_CONFIG, + 90, + ) + + return de_datasets, xspin, spin_tbin_edges, energy_ranges, de_counts_summary @pytest.fixture @@ -555,8 +575,8 @@ def test_get_valid_earth_angle_events(mock_spkezr): np.testing.assert_array_equal(actual_flags, expected_flags) -def test_get_valid_events_per_energy_range(): - """Tests get_valid_events_per_energy_range function.""" +def test_get_valid_de_count_summary_valid_events(): + """Tests that get_valid_de_count_summary only counts valid events.""" np.random.seed(0) energy_range_edges = np.array([3, 5, 7, 18]) # 3 example energy bins # example energy values that fall into different bins @@ -573,72 +593,53 @@ def test_get_valid_events_per_energy_range(): # mark event 6 as having an invalid ebin ebin[5] = -1 de_dps_velocity = np.random.random((len(energy), 3)) + # Give each event its own time and spin bin so the counts per spin bin show + # exactly which events were counted as valid. + event_times = np.arange(1, len(energy) + 1) + spin_tbin_edges = np.arange(0.5, len(energy) + 1) de_dataset = xr.Dataset( { "de_dps_velocity": (("epoch", "component"), de_dps_velocity), - "event_times": ("epoch", np.arange(len(energy))), + "event_times": ("epoch", event_times), + "de_event_met": ("epoch", event_times), "energy_spacecraft": ("epoch", energy), "quality_outliers": ("epoch", quality_outliers), "quality_scattering": ("epoch", quality_scattering), "ebin": ("epoch", ebin), } ) - keepout_angle = np.radians(180) - valid_events = get_valid_events_per_energy_range( - de_dataset, energy_range_edges, keepout_angle, 90 + counts = get_valid_de_count_summary( + {"p0": de_dataset, "p1": de_dataset}, + energy_range_edges, + spin_tbin_edges, + REPOINT_47_SPIN_CONFIG, + 90, ) - # Assert that for the first energy bin (3-5), all are false - assert np.array_equal(valid_events[0], np.full(len(valid_events[0]), False)) - # Assert that for the second energy bin (5-7), all are false except - # events 3 and 8 (event 1 had an outlier flag, event 2 had a scattering flag) - expected_flags_ebin2 = np.array( - [ - False, - False, - True, - False, - False, - False, - False, - True, - False, - False, - False, - False, - ] - ) - assert np.array_equal(valid_events[1], expected_flags_ebin2) - # Assert that for the third energy bin (7-18), all are false except events 4 and 7 - # (event 5 was marked as an outlier and event 6 has an invalid ebin) - expected_flags_ebin3 = np.array( - [ - False, - False, - False, - True, - False, - False, - True, - False, - False, - False, - False, - False, - ] - ) - assert np.array_equal(valid_events[2], expected_flags_ebin3) + # Assert that for the first energy bin (3-5), nothing is counted + assert np.array_equal(counts[0], np.zeros(len(energy))) + # Assert that for the second energy bin (5-7), only events 3 and 8 are counted + # (event 1 had a scattering flag, event 2 had an outlier flag) + expected_counts_ebin2 = np.array([0, 0, 1, 0, 0, 0, 0, 1, 0, 0, 0, 0]) + assert np.array_equal(counts[1], expected_counts_ebin2) + # Assert that for the third energy bin (7-18), only events 4 and 7 are counted + # (event 5 was marked as an outlier, event 6 has an invalid ebin, and event 9 + # had a scattering flag) + expected_counts_ebin3 = np.array([0, 0, 0, 1, 0, 0, 1, 0, 0, 0, 0, 0]) + assert np.array_equal(counts[2], expected_counts_ebin3) @mock.patch("imap_processing.ultra.l1b.ultra_l1b_culling.sp.spkezr") -def test_get_valid_events_per_energy_range_ultra45(mock_spkezr): - """Tests get_valid_events_per_energy_range function.""" +def test_get_valid_de_count_summary_ultra45(mock_spkezr): + """Tests the Earth angle cut in get_valid_de_count_summary for ULTRA 45.""" np.random.seed(0) mock_imap_state = np.random.random(6) # Mock IMAP state for testing mock_spkezr.return_value = (mock_imap_state, None) energy_range_edges = np.array([3, 5, 7, 18]) # 3 example energy bins energy = np.arange(18) + event_times = np.full(len(energy), 798033671) + spin_tbin_edges = np.array([798033670, 798033672]) # mark all events with valid outlier and scattering flags and valid ebins. de_dps_velocity = np.random.random((len(energy), 3)) @@ -646,23 +647,30 @@ def test_get_valid_events_per_energy_range_ultra45(mock_spkezr): de_dataset = xr.Dataset( { "velocity_dps_sc": (("epoch", "component"), de_dps_velocity), - "event_times": ("epoch", np.full(len(energy), 798033671)), + "event_times": ("epoch", event_times), + "de_event_met": ("epoch", event_times), "energy_spacecraft": ("epoch", energy), "quality_outliers": ("epoch", np.full(len(energy), 0)), "quality_scattering": ("epoch", np.full(len(energy), 0)), "ebin": ("epoch", np.full(len(energy), 10)), } ) + # ensure that all events fail the earth angle check by setting a very large # keepout angle keepout_angle = np.radians(360) - valid_events = get_valid_events_per_energy_range( - de_dataset, energy_range_edges, keepout_angle, 45 + counts = get_valid_de_count_summary( + {"p0": de_dataset, "p1": de_dataset}, + energy_range_edges, + spin_tbin_edges, + REPOINT_47_SPIN_CONFIG, + 45, + keepout_angle, ) # although all events were valid for outliers, scattering, and ebin, all events # failed the earth angle check for ultra45 - assert not np.any(valid_events) + assert not np.any(counts) @mock.patch( @@ -689,6 +697,7 @@ def test_flag_high_energy(): de_dataset = xr.Dataset( { "de_event_met": ("epoch", np.arange(len(energy))), + "event_times": ("epoch", np.arange(len(energy))), "energy_spacecraft": ("epoch", energy), "quality_outliers": ("epoch", np.full(len(energy), 0)), "quality_scattering": ("epoch", np.full(len(energy), 0)), @@ -700,8 +709,13 @@ def test_flag_high_energy(): spin_tbin_edges = np.arange( start=0, stop=len(energy) + 1, step=4 ) # create spin bins of 4 seconds + de_datasets = {"p0": de_dataset, "p1": de_dataset} de_counts_summary = get_valid_de_count_summary( - de_dataset, energy_range_edges, spin_tbin_edges + de_datasets, + energy_range_edges, + spin_tbin_edges, + REPOINT_47_SPIN_CONFIG, + 90, ) quality_flags = flag_high_energy( de_counts_summary, @@ -743,11 +757,12 @@ def test_validate_high_energy_cull(setup_repoint_47_data): expected_qf = pd.read_csv( TEST_PATH / "validate_high_energy_culling_results_repoint00047_v2.csv" ).to_numpy() - de_ds, _, spin_tbin_edges = setup_repoint_47_data + de_datasets, _, spin_tbin_edges, _, _ = setup_repoint_47_data + # Get the energy ranges energy_ranges = np.array([4.2, 9.4425, 21.2116, 47.2388, 105.202, 316.335]) de_counts_summary = get_valid_de_count_summary( - de_ds, energy_ranges, spin_tbin_edges + de_datasets, energy_ranges, spin_tbin_edges, REPOINT_47_SPIN_CONFIG ) high_energy_combined_spin_bin_radius = 3 e_flags = flag_high_energy( @@ -779,6 +794,7 @@ def test_flag_statistical_outliers(): de_dataset = xr.Dataset( { "de_event_met": ("epoch", np.arange(len(energy))), + "event_times": ("epoch", np.arange(len(energy))), "energy_spacecraft": ("epoch", energy), "quality_outliers": ("epoch", np.full(len(energy), 0)), "quality_scattering": ("epoch", np.full(len(energy), 0)), @@ -788,10 +804,12 @@ def test_flag_statistical_outliers(): spin_tbin_edges = np.arange( start=0, stop=len(energy) + 1, step=spin_step ) # create spin bins of 7 seconds + de_datasets = {"p0": de_dataset, "p1": de_dataset} de_counts_summary = get_valid_de_count_summary( - de_dataset, + de_datasets, energy_range_edges, spin_tbin_edges, + REPOINT_47_SPIN_CONFIG, 90, ) quality_flags, convergence, iterations, std_diff = flag_statistical_outliers( @@ -834,6 +852,7 @@ def test_flag_statistical_outliers_invalid_events(): de_dataset = xr.Dataset( { "de_event_met": ("epoch", np.arange(len(energy))), + "event_times": ("epoch", np.arange(len(energy))), "energy_spacecraft": ("epoch", energy), "quality_outliers": ("epoch", np.full(len(energy), 0)), "quality_scattering": ("epoch", np.full(len(energy), 0)), @@ -844,10 +863,13 @@ def test_flag_statistical_outliers_invalid_events(): start=0, stop=len(energy) + 1, step=5 ) # create spin bins of 5 seconds mask = np.ones((len(energy_range_edges) - 1, len(spin_tbin_edges) - 1), dtype=bool) + de_datasets = {"p0": de_dataset, "p1": de_dataset} + de_counts_summary = get_valid_de_count_summary( - de_dataset, + de_datasets, energy_range_edges, spin_tbin_edges, + REPOINT_47_SPIN_CONFIG, 90, ) quality_flags, convergence, iterations, std_diff = flag_statistical_outliers( @@ -891,9 +913,9 @@ def test_validate_stat_cull(setup_repoint_47_data): results_df = pd.read_csv( TEST_PATH / "validate_stat_culling_results_repoint00047_v3.csv" ) - de_ds, _, spin_tbin_edges = setup_repoint_47_data - # Get the energy ranges - energy_ranges = get_binned_energy_ranges(build_energy_bins()[0]) + de_datasets, _, spin_tbin_edges, energy_ranges, de_counts_summary = ( + setup_repoint_47_data + ) # Create a mask of flagged events to test that the stat cull algorithm # properly ignores these. The test data was created using this exact mask as well. @@ -901,12 +923,6 @@ def test_validate_stat_cull(setup_repoint_47_data): mask[0:2, 0:2] = ( True # This will mark the first 2 energy bins and first 2 spin bins as flagged ) - de_counts_summary = get_valid_de_count_summary( - de_ds, - energy_ranges, - spin_tbin_edges, - 90, - ) # ignored in the statistics calculation and flagging. flags, con, it, std = flag_statistical_outliers( de_counts_summary, spin_tbin_edges, energy_ranges, mask @@ -950,19 +966,14 @@ def test_validate_upstream_ion_cull(setup_repoint_47_data): expected_results = pd.read_csv( TEST_PATH / "validate_upstream_ion_1_culling_results_repoint00047_v1.csv" ).to_numpy() - de_ds, _, spin_tbin_edges = setup_repoint_47_data - intervals, _, _ = build_energy_bins() - energy_ranges = get_binned_energy_ranges(intervals) + de_datasets, _, spin_tbin_edges, energy_ranges, de_counts_summary = ( + setup_repoint_47_data + ) mask = np.zeros((len(energy_ranges) - 1, len(spin_tbin_edges) - 1), dtype=bool) mask[0:2, 0:2] = ( True # This will mark the first 2 energy bins and first 2 spin bins as flagged ) - de_counts_summary = get_valid_de_count_summary( - de_ds, - energy_ranges, - spin_tbin_edges, - 90, - ) + flags = flag_upstream_ion( de_counts_summary, energy_ranges, @@ -978,16 +989,10 @@ def test_validate_upstream_ion_cull(setup_repoint_47_data): @pytest.mark.external_test_data def test_upstream_ion_cull_invalid_channels(setup_repoint_47_data): """Validate upstream ion error handling.""" - de_ds, _, spin_tbin_edges = setup_repoint_47_data - intervals, _, _ = build_energy_bins() - energy_ranges = get_binned_energy_ranges(intervals) - mask = np.zeros((len(energy_ranges) - 1, len(spin_tbin_edges) - 1), dtype=bool) - de_counts_summary = get_valid_de_count_summary( - de_ds, - energy_ranges, - spin_tbin_edges, - 90, + de_datasets, _, spin_tbin_edges, energy_ranges, de_counts_summary = ( + setup_repoint_47_data ) + mask = np.zeros((len(energy_ranges) - 1, len(spin_tbin_edges) - 1), dtype=bool) with pytest.raises( ValueError, match="Channels provided for upstream ion flagging" @@ -1008,20 +1013,13 @@ def test_validate_spectral_cull(setup_repoint_47_data): expected_results = pd.read_csv( TEST_PATH / "validate_spectral_culling_results_repoint00047_v1.csv" ).to_numpy() - de_ds, _, spin_tbin_edges = setup_repoint_47_data - intervals, _, _ = build_energy_bins() - energy_ranges = get_binned_energy_ranges(intervals) + de_datasets, xspin, spin_tbin_edges, energy_ranges, de_counts_summary = ( + setup_repoint_47_data + ) mask = np.zeros((len(energy_ranges) - 1, len(spin_tbin_edges) - 1), dtype=bool) mask[0:2, 0:2] = ( True # This will mark the first 2 energy bins and first 2 spin bins as flagged ) - de_counts_summary = get_valid_de_count_summary( - de_ds, - energy_ranges, - spin_tbin_edges, - 90, - ) - flags = flag_spectral_events( de_counts_summary, energy_ranges, diff --git a/imap_processing/ultra/constants.py b/imap_processing/ultra/constants.py index 3ef2173129..af6814e0d5 100644 --- a/imap_processing/ultra/constants.py +++ b/imap_processing/ultra/constants.py @@ -254,3 +254,7 @@ class UltraConstants: # When True, applies the scattering rejection mask based on the FWHM thresholds # to the L1C fine energy bin maps. APPLY_SCATTERING_REJECTION_L1C: bool = False + + # When true, only use raw direct events for culling in extendedspin.py. Otherwise, + # refer to the priority specified in the extendedspin-config ancillary file. + L1B_USE_RAW_DE_ONLY = False diff --git a/imap_processing/ultra/l1b/extendedspin.py b/imap_processing/ultra/l1b/extendedspin.py index db8cfe2117..b69348f422 100644 --- a/imap_processing/ultra/l1b/extendedspin.py +++ b/imap_processing/ultra/l1b/extendedspin.py @@ -5,6 +5,7 @@ from numpy.typing import NDArray from imap_processing.ultra.constants import UltraConstants +from imap_processing.ultra.l1b.lookup_utils import ExtendedSpinConfig from imap_processing.ultra.l1b.ultra_l1b_culling import ( count_rejected_events_per_spin, expand_bin_flags_to_spins, @@ -33,7 +34,8 @@ def calculate_extendedspin( dict_datasets: dict[str, xr.Dataset], - de_dataset: xr.Dataset, + de_datasets: dict[str, xr.Dataset], + extendedspin_conf: ExtendedSpinConfig, name: str, instrument_id: int, ) -> xr.Dataset: @@ -44,8 +46,11 @@ def calculate_extendedspin( ---------- dict_datasets : dict Dictionary containing all the datasets. - de_dataset : xarray.Dataset - Dataset containing the direct event data. + de_datasets : dict[str, xarray.Dataset] + Dictionary of direct event datasets keyed by priority, e.g. "p0" for raw + de and "p1" for priority 1 de. + extendedspin_conf : ExtendedSpinConfig + Configurations for culling spins. name : str Name of the dataset. instrument_id : int @@ -59,6 +64,9 @@ def calculate_extendedspin( aux_dataset = dict_datasets[f"imap_ultra_l1a_{instrument_id}sensor-aux"] rates_dataset = dict_datasets[f"imap_ultra_l1a_{instrument_id}sensor-rates"] status_dataset = dict_datasets[f"imap_ultra_l1b_{instrument_id}sensor-status"] + # Use the raw de for the spin level quantities since it contains all events. + # The energy dependent culling selects its de dataset per energy range. + de_dataset = de_datasets["p0"] extendedspin_dict = {} rates_qf, spin, energy_bin_geometric_mean, n_sigma_per_energy = flag_rates( @@ -94,13 +102,13 @@ def calculate_extendedspin( # Get the energy ranges energy_ranges = get_binned_energy_ranges(intervals) energy_bin_flags = get_energy_range_flags(energy_ranges) - # Get valid event counts per energy range and spin bin, shared across all of # the culling steps below. de_counts_summary = get_valid_de_count_summary( - de_dataset, + de_datasets, energy_ranges, spin_tbin_edges, + extendedspin_conf, instrument_id, ) diff --git a/imap_processing/ultra/l1b/lookup_utils.py b/imap_processing/ultra/l1b/lookup_utils.py index 2181905d79..278e98408a 100644 --- a/imap_processing/ultra/l1b/lookup_utils.py +++ b/imap_processing/ultra/l1b/lookup_utils.py @@ -1,6 +1,8 @@ """Contains tools for lookup tables for l1b.""" +import datetime import logging +from dataclasses import dataclass import numpy as np import numpy.typing as npt @@ -622,9 +624,8 @@ def get_scattering_thresholds(ancillary_files: dict) -> dict: return threshold_dict -def get_de_product_name( - repoint: str, sensor: int, data_level: str, ancillary_files: dict -) -> str: +# TODO move to l1c_lookup_utils.py +def get_de_product_name(repoint: str, sensor: int, ancillary_files: dict) -> str: """ Get the name of the de product to use for processing. @@ -642,8 +643,6 @@ def get_de_product_name( number. sensor : int Sensor number, either 45 or 90. - data_level : str - Data level, either "l1b" or "l1c". ancillary_files : dict Ancillary files containing the lookup tables to determine which DE product to use based on the repointing ID. @@ -653,13 +652,11 @@ def get_de_product_name( de_product_name : str Name of the de product to use for processing. """ - if data_level not in ["l1b", "l1c"]: - raise ValueError(f"Invalid data level: {data_level}. Must be 'l1b' or 'l1c'.") # load the lookup table. # The lookup table will have columns for repointing_id_start, repointing_id_end, # and de_product. If repointing_id_end is NaN that indicates that the de_product # should be used for all repoint IDs greater than or equal to repointing_id_start. - file_name = f"{data_level}-{sensor}sensor-de-product-lookup" + file_name = f"l1c-{sensor}sensor-de-product-lookup" de_lookup = pd.read_csv(ancillary_files[file_name]) repoint_id = int(repoint.replace("repoint", "")) # Filter the dataset to find where the current repoint ID falls within the @@ -688,3 +685,72 @@ def get_de_product_name( f"Using DE product {product} for repoint ID {repoint_id} based on lookup table" ) return product + + +@dataclass +class ExtendedSpinConfig: + """Pointing dependent l1b culling configurations.""" + + energy_thresholds: np.ndarray # energy thresholds for culling + voltage_threshold: float # voltage threshold for culling + date: datetime.datetime # Date when configuration changed + priority: str # Which de product to use priority 1-4 de or raw de. e.g. p0-p4 + calibration: str # Calibration label + + @classmethod + def from_csv(cls, config_file_path: str, repointing: str) -> "ExtendedSpinConfig": + """ + Construct an ExtendedSpinConfig object from inputs. + + Parameters + ---------- + config_file_path : str + Path to the CSV file containing the configuration data. + repointing : str + The current pointing number for which to retrieve the + configuration. + + Returns + ------- + ExtendedSpinConfig + An instance of ExtendedSpinConfig with the thresholds, date, + priority, and calibration values for the specified pointing number. + """ + repoint_int = int(repointing.replace("repoint", "")) + df = pd.read_csv(config_file_path) + # Each row applies from its pointing number up to (but not including) the + # next row's pointing number. The last row applies to all later pointings. + pointings = np.append(df["pointing"].values, np.inf) + filtered_df = df[ + ((repoint_int < pointings[1:]) & (repoint_int >= pointings[:-1])) + ] + if filtered_df.empty: + raise ValueError( + f"The ancillary file: {config_file_path} contains no " + f"configurations for {repointing}" + ) + if filtered_df.shape[0] > 1: + raise ValueError( + f"The ancillary file: {config_file_path} contains multiple " + f"configurations for {repointing}. It should only contain one" + f" per pointing." + ) + config = filtered_df.iloc[0] + # Get the column names that contain "cullThresh" + thresh_colnames = sorted( + [col for col in df.columns if "cullThresh" in col], + key=lambda threshold: int(threshold.split("_")[-1]), + ) + e_thresholds = np.array([config[col] for col in thresh_colnames]) + # The config provides thresholds for bins 0-4; bin 5 (> + # UltraConstands.MAX_ENERGY_THRESHOLD keV) reuses the bin 4 threshold + e_thresholds = np.append(e_thresholds, e_thresholds[-1]) + date = datetime.datetime.strptime(config["date"], "%m/%d/%y") + + return cls( + energy_thresholds=e_thresholds, + voltage_threshold=config["deflector_Vthresh"], + date=date, + priority=config["pri_config"], + calibration=config["cal_config"], + ) diff --git a/imap_processing/ultra/l1b/ultra_l1b.py b/imap_processing/ultra/l1b/ultra_l1b.py index d698193145..f535674f9e 100644 --- a/imap_processing/ultra/l1b/ultra_l1b.py +++ b/imap_processing/ultra/l1b/ultra_l1b.py @@ -9,12 +9,14 @@ from imap_processing.ultra.l1b.de import calculate_de from imap_processing.ultra.l1b.extendedspin import calculate_extendedspin from imap_processing.ultra.l1b.goodtimes import calculate_goodtimes -from imap_processing.ultra.l1b.lookup_utils import get_de_product_name +from imap_processing.ultra.l1b.lookup_utils import ExtendedSpinConfig logger = logging.getLogger(__name__) -def ultra_l1b(data_dict: dict, ancillary_files: dict) -> list[xr.Dataset]: +def ultra_l1b( + data_dict: dict, ancillary_files: dict, repointing: str +) -> list[xr.Dataset]: """ Will process ULTRA L1A data into L1B CDF files at output_filepath. @@ -24,6 +26,9 @@ def ultra_l1b(data_dict: dict, ancillary_files: dict) -> list[xr.Dataset]: The data itself and its dependent data. ancillary_files : dict Ancillary files. + repointing : str + The repointing ID in the format "repointXXXXX" where XXXXX is the repointing + number. Returns ------- @@ -65,47 +70,32 @@ def ultra_l1b(data_dict: dict, ancillary_files: dict) -> list[xr.Dataset]: ) output_datasets.append(de_dataset) # L1b extended data will be created if L1a hk, rates, - # aux, params, and l1b de data are available + # aux, params, and l1b de (raw and priority 1) data are available elif ( f"imap_ultra_l1b_{instrument_id}sensor-de" in data_dict + and f"imap_ultra_l1b_{instrument_id}sensor-priority-1-de" in data_dict and f"imap_ultra_l1a_{instrument_id}sensor-rates" in data_dict and f"imap_ultra_l1a_{instrument_id}sensor-aux" in data_dict and f"imap_ultra_l1a_{instrument_id}sensor-params" in data_dict and f"imap_ultra_l1b_{instrument_id}sensor-status" in data_dict ): - # get repoint number - repoint = data_dict[f"imap_ultra_l1b_{instrument_id}sensor-de"].attrs.get( - "Repointing", None + # Create dictionary of the de datasets + # For now, only pass in priority 1 and raw de datasets. + de_datasets = { + "p0": data_dict[f"imap_ultra_l1b_{instrument_id}sensor-de"], + "p1": data_dict[f"imap_ultra_l1b_{instrument_id}sensor-priority-1-de"], + } + # Get the extended spin config from ancillary files. + extended_spin_config_anc = ancillary_files[ + f"l1b-{instrument_id}sensor-extendedspin-config" + ] + extendedspin_config = ExtendedSpinConfig.from_csv( + extended_spin_config_anc, repointing ) - if repoint is None: - raise ValueError("Repointing ID attribute is missing from the dataset.") - # Determine which l1b de product to use in calculating the goodtimes - # Will be either the raw de product or a priority 1-4 de product. - de_product_desc = get_de_product_name( - repoint, instrument_id, "l1b", ancillary_files - ) - if de_product_desc not in data_dict: - raise ValueError( - f"Selected L1B DE product '{de_product_desc}' for instrument " - f"{instrument_id} is not present in data_dict. Available L1B DE " - f"products: {data_dict.keys()}" - ) extendedspin_dataset = calculate_extendedspin( - { - f"imap_ultra_l1a_{instrument_id}sensor-aux": data_dict[ - f"imap_ultra_l1a_{instrument_id}sensor-aux" - ], - f"imap_ultra_l1a_{instrument_id}sensor-params": data_dict[ - f"imap_ultra_l1a_{instrument_id}sensor-params" - ], - f"imap_ultra_l1a_{instrument_id}sensor-rates": data_dict[ - f"imap_ultra_l1a_{instrument_id}sensor-rates" - ], - f"imap_ultra_l1b_{instrument_id}sensor-status": data_dict[ - f"imap_ultra_l1b_{instrument_id}sensor-status" - ], - }, - data_dict[de_product_desc], + data_dict, + de_datasets, + extendedspin_config, f"imap_ultra_l1b_{instrument_id}sensor-extendedspin", instrument_id, ) diff --git a/imap_processing/ultra/l1b/ultra_l1b_culling.py b/imap_processing/ultra/l1b/ultra_l1b_culling.py index d8a2e61508..e0b2d31d61 100644 --- a/imap_processing/ultra/l1b/ultra_l1b_culling.py +++ b/imap_processing/ultra/l1b/ultra_l1b_culling.py @@ -24,6 +24,7 @@ from imap_processing.spice.spin import get_spin_data from imap_processing.ultra.constants import UltraConstants from imap_processing.ultra.l1b.lookup_utils import ( + ExtendedSpinConfig, get_scattering_coefficients, get_scattering_thresholds, ) @@ -1061,40 +1062,91 @@ def flag_spectral_events( def get_valid_de_count_summary( - de_dataset: xr.Dataset, + de_datasets: dict[str, xr.Dataset], energy_ranges: NDArray, spin_tbin_edges: NDArray, + spin_config: ExtendedSpinConfig, sensor_id: int = 90, + earth_ang_45: float = UltraConstants.EARTH_ANGLE_45_THRESHOLD, ) -> NDArray: """ Get a summary of valid counts per energy range and spin bin. Parameters ---------- - de_dataset : xr.Dataset - Direct event dataset. + de_datasets : dict[str, xr.Dataset] + Dictionary of raw, and priority 1-4 direct event datasets. energy_ranges : numpy.ndarray Array of energy range edges. spin_tbin_edges : numpy.ndarray Array of spin time bin edges. + spin_config : ExtendedSpinConfig + Spin configuration object containing the direct event priority to use. sensor_id : int Sensor ID (e.g., 45 or 90). + earth_ang_45 : float + Earth angle to use for culling in ULTRA 45. Returns ------- counts : numpy.ndarray A 2D array of counts per energy range and spin bin for valid events. """ - valid_events = get_valid_events_per_energy_range( - de_dataset, energy_ranges, UltraConstants.EARTH_ANGLE_45_THRESHOLD, sensor_id - ) + # Get the specified de priority. E.g. "p0" for raw de or "p1" for priority 1 de. + priority_conf = spin_config.priority + n_energy_ranges = len(energy_ranges) - 1 counts: np.ndarray = np.zeros( - (len(energy_ranges) - 1, len(spin_tbin_edges) - 1), dtype=float + (n_energy_ranges, len(spin_tbin_edges) - 1), dtype=float ) + for i in range(n_energy_ranges): + # By default, use priority 1 de unless it's the last energy bin and the + # config is p0 or if the config is set to use raw de only. + de = de_datasets["p1"] + if UltraConstants.L1B_USE_RAW_DE_ONLY or ( + i == n_energy_ranges - 1 and priority_conf == "p0" + ): + de = de_datasets["p0"] + valid_outliers = de["quality_outliers"].values == 0 + valid_scattering = de["quality_scattering"].values == 0 + # TODO what about species non-proton? For those psets dont cull based on + # High energy? + valid_ebin = np.isin( + de["ebin"].values, UltraConstants.TOFXPH_SPECIES_GROUPS["proton"] + ) + # Exclude events with invalid (fill) event times. + valid_event_times = de["event_times"].values > 0 + energy_mask = ( + (de["energy_spacecraft"].values >= energy_ranges[i]) + & (de["energy_spacecraft"].values < energy_ranges[i + 1]) + & valid_event_times + ) + if not np.any(energy_mask): + continue + # subset the dataset to events within the energy range + de_dataset_subset = de.isel(epoch=energy_mask) + valid_earth_angle: np.ndarray = np.full(np.sum(energy_mask), True, dtype=bool) + # For ultra45, also apply an Earth angle cut to remove times when + # the Earth is in the field of view. ULTRA 90 does not require this since + # Earth is always outside the field of view. + if sensor_id == 45: + valid_earth_angle = get_valid_earth_angle_events( + de_dataset_subset, earth_ang_45 + ) - for i in range(len(energy_ranges) - 1): + # Count events at the valid energy ranges if they meet all the criteria for + # valid events: not flagged as outliers, not flagged as scattering, + # in a valid ebin, and (for ultra45) have a valid Earth angle. + valid_events = np.logical_and.reduce( + [ + valid_outliers[energy_mask], + valid_scattering[energy_mask], + valid_ebin[energy_mask], + valid_earth_angle, + ] + ) counts[i, :], _ = np.histogram( - de_dataset["de_event_met"].values[valid_events[i, :]], bins=spin_tbin_edges + de_dataset_subset["de_event_met"].values[valid_events], + bins=spin_tbin_edges, ) return counts @@ -1132,70 +1184,6 @@ def combine_de_counts_summary( return np.mean(windows, axis=-1) -def get_valid_events_per_energy_range( - de_dataset: xr.Dataset, energy_ranges: NDArray, earth_ang_45: float, sensor_id: int -) -> NDArray: - """ - Get valid events per energy range. - - Parameters - ---------- - de_dataset : xr.Dataset - Direct event dataset. - energy_ranges : numpy.ndarray - Array of energy range edges. - earth_ang_45 : float - Earth angle to use for culling in ULTRA 45. - sensor_id : int - Sensor ID (e.g., 45 or 90). - - Returns - ------- - valid_events_per_range : numpy.ndarray - A boolean array of shape (n_energy_ranges, n_events). - """ - event_energies = de_dataset["energy_spacecraft"].values - valid_events: np.ndarray = np.zeros( - (len(energy_ranges) - 1, len(event_energies)), dtype=bool - ) - valid_outliers = de_dataset["quality_outliers"].values == 0 - valid_scattering = de_dataset["quality_scattering"].values == 0 - # TODO what about species non-proton? For those psets dont cull based on - # High energy? - ebin = de_dataset["ebin"].values - valid_ebin = np.isin(ebin, UltraConstants.TOFXPH_SPECIES_GROUPS["proton"]) - for i in range(len(energy_ranges) - 1): - energy_mask = (event_energies >= energy_ranges[i]) & ( - event_energies < energy_ranges[i + 1] - ) - if not np.any(energy_mask): - continue - # subset the dataset to events within the energy range - de_dataset_subset = de_dataset.isel(epoch=energy_mask) - valid_earth_angle: np.ndarray = np.full(np.sum(energy_mask), True, dtype=bool) - # For ultra45, also apply an Earth angle cut to remove times when - # the Earth is in the field of view. ULTRA 90 does not require this since Earth - # is always outside the field of view. - if sensor_id == 45: - valid_earth_angle = get_valid_earth_angle_events( - de_dataset_subset, earth_ang_45 - ) - - # Flag events at the valid energy ranges if they meet all the criteria for - # valid events: not flagged as outliers, not flagged as scattering, - # in a valid ebin, and (for ultra45) have a valid Earth angle. - valid_events[i, energy_mask] = np.logical_and.reduce( - [ - valid_outliers[energy_mask], - valid_scattering[energy_mask], - valid_ebin[energy_mask], - valid_earth_angle, - ] - ) - - return valid_events - - def get_valid_earth_angle_events( de_dataset_subset: xr.Dataset, earth_ang_45: float = UltraConstants.EARTH_ANGLE_45_THRESHOLD, diff --git a/imap_processing/ultra/l1c/ultra_l1c.py b/imap_processing/ultra/l1c/ultra_l1c.py index 7fe259e66e..9fb4c386a9 100644 --- a/imap_processing/ultra/l1c/ultra_l1c.py +++ b/imap_processing/ultra/l1c/ultra_l1c.py @@ -50,7 +50,7 @@ def ultra_l1c( # Determine which l1b de product to use in calculating the l1c products. # Will be either the raw de product or a priority 1-4 de product. de_product_desc = get_de_product_name( - repoint, instrument_id, "l1c", ancillary_files + repoint, instrument_id, ancillary_files ) if de_product_desc not in data_dict: raise ValueError(