From 2d1eb553b7c1eeadcdab908bc56811cfe549fc44 Mon Sep 17 00:00:00 2001 From: Menlo Innovations - CAVA Project Date: Tue, 29 Sep 2026 09:51:26 -0400 Subject: [PATCH 1/7] Harrison - AFAH/SWYN - #209 - WIP: GLOWS L3d parents now includes all input L3bs and L3cs. Question on if pipeline_settings should be included as well, there is a failing test for this. --- .../science/toolkit/l3d_SolarParamHistory.py | 8 ++++---- tests/glows/test_glows_processor.py | 20 +++++++++++++++++++ .../test_glows_processor_integration.py | 4 ++-- 3 files changed, 26 insertions(+), 6 deletions(-) diff --git a/imap_l3_processing/glows/l3d/science/toolkit/l3d_SolarParamHistory.py b/imap_l3_processing/glows/l3d/science/toolkit/l3d_SolarParamHistory.py index 21d3a875..7e7e48f9 100644 --- a/imap_l3_processing/glows/l3d/science/toolkit/l3d_SolarParamHistory.py +++ b/imap_l3_processing/glows/l3d/science/toolkit/l3d_SolarParamHistory.py @@ -108,8 +108,8 @@ def __init__(self,anc_input_from_instr_team,ext_dependencies): 'filename': None, 'ancillary_data_files': None, 'external_dependeciens': ext_dependencies['lya_raw_data'], - 'l3b_input_filename': None, - 'l3c_input_filename': None + 'l3b_input_filename': [], + 'l3c_input_filename': [] } self.ini_data={} @@ -446,8 +446,8 @@ def _update_l3bc_data(self,data_l3b,data_l3c,CR): ''' cr_params, idx_read_b, idx_read_c = self._generate_cr_solar_params(CR, data_l3b, data_l3c) - self.header['l3b_input_filename']=[data_l3b[i]['header']['filename'] for i in idx_read_b] - self.header['l3c_input_filename']=[data_l3c[i]['header']['filename'] for i in idx_read_c] + self.header['l3b_input_filename'].extend(data_l3b[i]['header']['filename'] for i in idx_read_b) + self.header['l3c_input_filename'].extend(data_l3c[i]['header']['filename'] for i in idx_read_c) # add a row with values for current CR for k in self.ini_data['label']: diff --git a/tests/glows/test_glows_processor.py b/tests/glows/test_glows_processor.py index 62aafaa9..76e37556 100644 --- a/tests/glows/test_glows_processor.py +++ b/tests/glows/test_glows_processor.py @@ -995,6 +995,26 @@ def test_process_glows_l3d_drift(self, mock_save_data, _): glows_l3d_output = process_l3d(l3d_dependencies, Version(1, 4)) + expected_parents = { + "imap_glows_l3b_ion-rate-profile_20100422_v013.cdf", + "imap_glows_l3b_ion-rate-profile_20100519_v013.cdf", + "imap_glows_l3c_sw-profile_20100422_v012.cdf", + "imap_glows_l3c_sw-profile_20100519_v012.cdf", + "imap_glows_plasma-speed-2010a_20100101_v003.dat", + "imap_glows_proton-density-2010a_20100101_v003.dat", + "imap_glows_uv-anisotropy-2010a_20100101_v003.dat", + "imap_glows_photoion-2010a_20100101_v003.dat", + "imap_glows_lya-2010a_20100101_v003.dat", + "imap_glows_electron-density-2010a_20100101_v003.dat", + "imap_glows_pipeline-settings-l3bcde_20100101_v006.json", + "lyman_alpha_composite.nc", + } + + self.assertEqual(1, mock_save_data.call_count) + [data_product] = mock_save_data.call_args_list[0].args + + self.assertEqual(expected_parents, set(data_product.parent_file_names)) + expected_txt_filenames = ["imap_glows_e-dens_19470303_20100629_v004.dat", "imap_glows_lya_19470303_20100629_v004.dat", "imap_glows_p-dens_19470303_20100629_v004.dat", diff --git a/tests/integration/test_glows_processor_integration.py b/tests/integration/test_glows_processor_integration.py index 274ae639..adf4f75e 100644 --- a/tests/integration/test_glows_processor_integration.py +++ b/tests/integration/test_glows_processor_integration.py @@ -319,8 +319,8 @@ def test_l3bcde_first_time_processing(self): for file_path in expected_files: self.assertTrue(file_path.construct_path().exists(), msg=str(file_path.construct_path())) - @run_periodically(timedelta(days=14)) - @run_test_in_docker + # @run_periodically(timedelta(days=14)) + # @run_test_in_docker def test_l3bcde_automatic_reprocessing(self): new_l3a_file = GLOWS_TEST_DATA / "imap_glows_l3a_hist_20251118-repoint00052_v001.cdf" l3bcde_input_files = [ From 0ca55c33102206a4e741ac0f5636280d8e031f8e Mon Sep 17 00:00:00 2001 From: Menlo Innovations - CAVA Project Date: Thu, 1 Oct 2026 08:59:53 -0400 Subject: [PATCH 2/7] Harrison - SWYN - 3566 - GH #209 - GLOWS: Include pipeline settings in the parents of L3d, make the L3d dat files input_file header point to the L3d CDF. --- imap_l3_processing/glows/glows_processor.py | 13 ++++++- .../glows/l3d/glows_l3d_dependencies.py | 2 +- .../glows/l3d/science/generate_l3d.py | 4 ++- .../science/toolkit/l3d_SolarParamHistory.py | 10 +++--- imap_l3_processing/glows/l3d/utils.py | 1 - tests/glows/test_glows_processor.py | 35 +++++++++++-------- tests/periodically_run_tests.json | 12 +++---- 7 files changed, 48 insertions(+), 29 deletions(-) diff --git a/imap_l3_processing/glows/glows_processor.py b/imap_l3_processing/glows/glows_processor.py index e72f5293..5160a023 100644 --- a/imap_l3_processing/glows/glows_processor.py +++ b/imap_l3_processing/glows/glows_processor.py @@ -26,7 +26,7 @@ from imap_l3_processing.glows.l3bc.glows_l3bc_dependencies import GlowsL3BCDependencies from imap_l3_processing.glows.l3bc.glows_l3bc_initializer import GlowsL3BCInitializer, GlowsL3BCInitializerData from imap_l3_processing.glows.l3bc.models import GlowsL3BIonizationRate, GlowsL3CSolarWind, GlowsL3BCProcessorOutput, \ - ExternalDependencies, read_pipeline_settings + ExternalDependencies from imap_l3_processing.glows.l3bc.science.filter_out_bad_days import filter_l3a_files from imap_l3_processing.glows.l3bc.science.generate_l3bc import generate_l3bc from imap_l3_processing.glows.l3bc.utils import get_pointing_date_range @@ -222,6 +222,16 @@ def process_l3d( os.makedirs(PATH_TO_L3D_TOOLKIT / "data_l3d", exist_ok=True) os.makedirs(PATH_TO_L3D_TOOLKIT / "data_l3d_txt", exist_ok=True) + output_l3d_cdf_filename = ScienceFilePath.generate_from_inputs( + instrument="glows", + data_level="l3d", + descriptor=GLOWS_L3D_DESCRIPTOR, + start_time="19470303", + major_version=version.major, + minor_version=version.minor, + cr=dependencies.end_cr, + ).construct_path().name + file_manifest = { "external_files": { key: str(val) for key, val in dependencies.external_files.items() @@ -233,6 +243,7 @@ def process_l3d( for key, val in dependencies.ancillary_files["WawHelioIon"].items() }, }, + "l3d_cdf_filename": output_l3d_cdf_filename } last_processed_cr = None diff --git a/imap_l3_processing/glows/l3d/glows_l3d_dependencies.py b/imap_l3_processing/glows/l3d/glows_l3d_dependencies.py index 873f068a..74a5330d 100644 --- a/imap_l3_processing/glows/l3d/glows_l3d_dependencies.py +++ b/imap_l3_processing/glows/l3d/glows_l3d_dependencies.py @@ -17,7 +17,7 @@ class GlowsL3DDependencies: ancillary_files: dict[str, Path | dict[str, Path]] l3b_file_paths: list[Path] l3c_file_paths: list[Path] - end_cr: float + end_cr: int @classmethod def fetch_dependencies(cls, dependencies: ProcessingInputCollection, external_dependencies: ExternalDependencies): diff --git a/imap_l3_processing/glows/l3d/science/generate_l3d.py b/imap_l3_processing/glows/l3d/science/generate_l3d.py index 0aa07b8f..17c53cab 100644 --- a/imap_l3_processing/glows/l3d/science/generate_l3d.py +++ b/imap_l3_processing/glows/l3d/science/generate_l3d.py @@ -22,6 +22,7 @@ ANC_INPUT_FROM_INSTRUMENT_TEAM = dependencies['ancillary_files'] EXT_DEPENDENCIES = dependencies['external_files'] +L3D_CDF_FILENAME = dependencies['l3d_cdf_filename'] # Create list of L3b/c files and read them l3b_fn_list = np.array(sorted(glob.glob('data_l3b/imap_glows_l3b*.json'))) @@ -35,7 +36,8 @@ # Start generating from the initial files (not using the last entry from L3d) -solar_param_hist.header['ancillary_data_files'] = list(fn_initial.values()) +solar_param_hist.header['ancillary_data_files'] = [*fn_initial.values(), ANC_INPUT_FROM_INSTRUMENT_TEAM['pipeline_settings']] +solar_param_hist.header['l3d_cdf_filename'] = L3D_CDF_FILENAME solar_param_hist.generate_initial_history(fn_initial) solar_param_hist.update_solar_params_hist(EXT_DEPENDENCIES, data_l3b, data_l3c, CR_current) diff --git a/imap_l3_processing/glows/l3d/science/toolkit/l3d_SolarParamHistory.py b/imap_l3_processing/glows/l3d/science/toolkit/l3d_SolarParamHistory.py index 7e7e48f9..58d7e2e8 100644 --- a/imap_l3_processing/glows/l3d/science/toolkit/l3d_SolarParamHistory.py +++ b/imap_l3_processing/glows/l3d/science/toolkit/l3d_SolarParamHistory.py @@ -65,10 +65,10 @@ def __init__(self,anc_input_from_instr_team,ext_dependencies): list of ancillary files used in calculations external_dependeciens : str list of external dependeciens files - l3b_input_filename: str - Input L3b ionization rate profiles file - l3c_input_filename: str - Input L3c solar wind speed and density profiles file + l3b_input_filename: list[str] + All the filenames of input L3b ionization rate profiles file + l3c_input_filename: list[str] + All the filenames of input L3c solar wind speed and density profiles file settings : dict pipeline settings ini_data: dict @@ -321,7 +321,7 @@ def _generate_hdr_template(self): hdr_temp[0]='lines in header: '+str(N)+'\n' hdr_temp[4]='ground software version: '+VERSION+'\n' hdr_temp[5]='creation date: '+ Time.now().iso +'\n' - hdr_temp[6]='input files: ' + ', '.join(self.header['l3b_input_filename']) + ', ' + ', '.join(self.header['l3c_input_filename']) +'\n' + hdr_temp[6]='input files: ' + self.header['l3d_cdf_filename'] + "\n" hdr_temp[7]='external dependencies: ' + self.header['external_dependeciens'] + '\n' hdr_temp[8]='last CR: '+ str(self.CR_last) + '\n' hdr_temp[9]='fill value: '+ str(self.settings['WawHelioIonGlows_fill_value']) + '\n' diff --git a/imap_l3_processing/glows/l3d/utils.py b/imap_l3_processing/glows/l3d/utils.py index 80a13812..883f2fe6 100644 --- a/imap_l3_processing/glows/l3d/utils.py +++ b/imap_l3_processing/glows/l3d/utils.py @@ -140,7 +140,6 @@ def rename_l3d_text_outputs(paths: list[Path], version: str) -> list[Path]: out_paths.append(new_path) return out_paths - def query_for_most_recent_l3d(descriptor: str) -> Optional[dict]: query_result = imap_data_access.query(instrument="glows", data_level="l3d", descriptor=descriptor) sorted_query_result = sorted( diff --git a/tests/glows/test_glows_processor.py b/tests/glows/test_glows_processor.py index 76e37556..1d86e97b 100644 --- a/tests/glows/test_glows_processor.py +++ b/tests/glows/test_glows_processor.py @@ -639,13 +639,11 @@ def test_l3bc_uses_all_l3a_file_names_for_l3b_parents(self, mock_save_data, mock @patch('imap_l3_processing.glows.glows_processor.shutil') @patch("imap_l3_processing.glows.glows_processor.os") @patch("imap_l3_processing.glows.glows_processor.GlowsL3DInitializer") - @patch("imap_l3_processing.glows.glows_processor.read_pipeline_settings") - def test_process_l3d(self, mock_read_pipeline_settings, mock_glows_l3d_initializer, mock_os, mock_shutil, mock_run, + def test_process_l3d(self, mock_glows_l3d_initializer, mock_os, mock_shutil, mock_run, mock_convert_json_to_l3d_data_product, mock_get_parent_file_names_from_l3d_json, mock_rename_l3d, mock_save_data, mock_convert_l3c_to_json, mock_convert_l3b_to_json, _): cr_number = 2092 - mock_read_pipeline_settings.return_value = {'start_cr': cr_number} expected_end_cr = cr_number + 1 glows_l3d_dependencies = GlowsL3DDependencies( @@ -671,8 +669,11 @@ def test_process_l3d(self, mock_read_pipeline_settings, mock_glows_l3d_initializ ) old_l3d = Path('imap_glows_l3d_solar-hist_19470303-cr02090_v001.cdf') - input_major_version = 12 - l3d_output_version = Version(input_major_version, 5) + major_version = 12 + l3d_output_version = Version(major_version, 5) + + expected_l3d_cdf_filename = f'imap_glows_l3d_{GLOWS_L3D_DESCRIPTOR}_19470303-cr0{expected_end_cr}_{l3d_output_version}.cdf' + mock_glows_l3d_initializer.should_process_l3d.return_value = ( l3d_output_version, glows_l3d_dependencies, @@ -701,7 +702,7 @@ def test_process_l3d(self, mock_read_pipeline_settings, mock_glows_l3d_initializ { GLOWS_L3B_DESCRIPTOR: Version(2, 1), GLOWS_L3C_DESCRIPTOR: Version(2, 1), - GLOWS_L3D_DESCRIPTOR: Version(input_major_version, 1), + GLOWS_L3D_DESCRIPTOR: Version(major_version, 1), GLOWS_L3E_HI_45_DESCRIPTOR: Version(2, 1), GLOWS_L3E_HI_90_DESCRIPTOR: Version(2, 1), GLOWS_L3E_LO_DESCRIPTOR: Version(2, 1), @@ -718,7 +719,7 @@ def test_process_l3d(self, mock_read_pipeline_settings, mock_glows_l3d_initializ mock_convert_l3b_to_json.assert_has_calls([call(sentinel.l3b_file_1), call(sentinel.l3b_file_2)]) mock_convert_l3c_to_json.assert_has_calls([call(sentinel.l3c_file_1), call(sentinel.l3c_file_2)]) mock_glows_l3d_initializer.should_process_l3d.assert_called_with( - self.mock_external_deps, [], [], self.mock_reprocess_info, input_major_version) + self.mock_external_deps, [], [], self.mock_reprocess_info, major_version) self.mock_fetch_reprocess_info.assert_called_with(processing_input_collection) self.assertEqual([ Path("imap_glows_e-dens_19470303_20100101_v000.dat"), @@ -755,8 +756,9 @@ def test_process_l3d(self, mock_read_pipeline_settings, mock_glows_l3d_initializ 'phion': str(Path('path/to/phion')), 'lya': str(Path('path/to/lya')), 'e-dens': str(Path('path/to/e-dens')) - } - } + }, + }, + "l3d_cdf_filename": expected_l3d_cdf_filename }) expected_working_directory = Path(l3d.__file__).parent / 'science' @@ -892,8 +894,7 @@ def test_process_l3d_adds_parent_file_names_to_output(self, mock_spicepy, mock_r @patch('imap_l3_processing.glows.glows_processor.os') @patch('imap_l3_processing.glows.glows_processor.run') @patch('imap_l3_processing.glows.glows_processor.convert_json_to_l3d_data_product') - @patch('imap_l3_processing.glows.glows_processor.read_pipeline_settings') - def test_process_l3d_handles_unexpected_exception_from_science(self, mock_read_pipeline_settings, + def test_process_l3d_handles_unexpected_exception_from_science(self, mock_convert_json_to_l3d, mock_run, mock_os, _, __): ancillary_files = { @@ -911,7 +912,6 @@ def test_process_l3d_handles_unexpected_exception_from_science(self, mock_read_p external_files = { 'lya_raw_data': Path('path/to/lya'), } - mock_read_pipeline_settings.return_value = {'start_cr': 2091} l3b_file_paths = [] l3c_file_paths = [] expected_cr = 2096 @@ -948,7 +948,7 @@ def test_process_l3d_handles_unexpected_exception_from_science(self, mock_read_p mock_run.side_effect = [CalledProcessError(cmd="", returncode=1, stderr=unexpected_exception)] with self.assertRaises(Exception) as context: - process_l3d(l3d_dependencies, 1) + process_l3d(l3d_dependencies, Version(1, 1)) self.assertEqual(unexpected_exception, context.exception.stderr) mock_convert_json_to_l3d.assert_not_called() @@ -1088,7 +1088,14 @@ def test_process_glows_l3d_drift(self, mock_save_data, _): with CDF(str(tmp_dir / expected_cdf_filename)) as actual_cdf: for filename, cdf_var_name, length_of_data, first_line, last_line in test_cases: with self.subTest(msg=filename): - actual = np.loadtxt(PATH_TO_L3D_TOOLKIT / "data_l3d_txt" / filename) + l3d_text_path = PATH_TO_L3D_TOOLKIT / "data_l3d_txt" / filename + actual = np.loadtxt(l3d_text_path) + + actual_input_file_line = next((l for l in l3d_text_path.read_text().split("\n") if l.startswith("# input files:")), None) + self.assertIsNotNone(actual_input_file_line) + + expected_input_file_line = f"# input files: {expected_cdf_filename}" + self.assertEqual(expected_input_file_line, actual_input_file_line) self.assertEqual(length_of_data, len(actual)) self.assertEqual(length_of_data, len(actual_cdf[cdf_var_name][...])) diff --git a/tests/periodically_run_tests.json b/tests/periodically_run_tests.json index 2eb85e0c..906e9ac0 100644 --- a/tests/periodically_run_tests.json +++ b/tests/periodically_run_tests.json @@ -1,13 +1,13 @@ { - "test_spectral_fit_against_validation_data": "2026-09-25T14:46:51.493678", + "test_spectral_fit_against_validation_data": "2026-10-01T08:50:41.360170", "test_codice_lo_3d_distributions": "2026-05-28T09:43:22.990231", - "test_swe_processor_with_local_data": "2026-09-21T13:49:18.260259", - "test_ultra_all_sp_maps": "2026-09-25T14:47:48.096216", - "test_ultra_combined_nsp_maps": "2026-09-21T13:48:32.412322", - "test_ultra_combined_sp_maps": "2026-09-21T13:48:49.416421", + "test_swe_processor_with_local_data": "2026-10-01T08:53:28.218357", + "test_ultra_all_sp_maps": "2026-10-01T08:52:29.438105", + "test_ultra_combined_nsp_maps": "2026-10-01T08:52:41.585975", + "test_ultra_combined_sp_maps": "2026-10-01T08:52:58.910527", "test_glows_l3abcde_from_l2": "2026-09-20T09:28:16.106080", "test_l3bcde_automatic_reprocessing": "2026-09-20T10:25:40.483342", "test_l3bcde_first_time_processing": "2026-09-20T11:20:19.798340", "test_proton_normalized_errors_are_standard": "2026-09-25T14:48:27.874117", - "test_alpha_normalized_errors_are_standard": "2026-09-21T13:50:04.172539" + "test_alpha_normalized_errors_are_standard": "2026-10-01T08:54:13.637480" } \ No newline at end of file From 3ba6c8c995e6c4e810788a2548dfa7ad8f8b6f1c Mon Sep 17 00:00:00 2001 From: Menlo Innovations - CAVA Project Date: Thu, 1 Oct 2026 09:25:05 -0400 Subject: [PATCH 3/7] Harrison - SWYN - 3566 - GH #209 - GLOWS: Call the same helper to build the output L3d filename and check that L3d produces up to the CR we expect to ensure the text files point to the correct L3d filename --- imap_l3_processing/glows/glows_processor.py | 27 +++++++++------------ imap_l3_processing/models.py | 18 +++++++++++++- imap_l3_processing/utils.py | 15 +----------- tests/glows/test_glows_processor.py | 1 - 4 files changed, 29 insertions(+), 32 deletions(-) diff --git a/imap_l3_processing/glows/glows_processor.py b/imap_l3_processing/glows/glows_processor.py index 5160a023..1f7de1ec 100644 --- a/imap_l3_processing/glows/glows_processor.py +++ b/imap_l3_processing/glows/glows_processor.py @@ -1,3 +1,4 @@ +import collections import json import logging import os @@ -216,21 +217,18 @@ def process_l3d( dependencies: GlowsL3DDependencies, version: Version ) -> Optional[GlowsL3DProcessorOutput]: - [create_glows_l3b_json_file_from_cdf(l3b) for l3b in dependencies.l3b_file_paths] - [create_glows_l3c_json_file_from_cdf(l3c) for l3c in dependencies.l3c_file_paths] + collections.deque((create_glows_l3b_json_file_from_cdf(l3b) for l3b in dependencies.l3b_file_paths), maxlen=0) + collections.deque((create_glows_l3c_json_file_from_cdf(l3c) for l3c in dependencies.l3c_file_paths), maxlen=0) os.makedirs(PATH_TO_L3D_TOOLKIT / "data_l3d", exist_ok=True) os.makedirs(PATH_TO_L3D_TOOLKIT / "data_l3d_txt", exist_ok=True) - output_l3d_cdf_filename = ScienceFilePath.generate_from_inputs( - instrument="glows", - data_level="l3d", - descriptor=GLOWS_L3D_DESCRIPTOR, - start_time="19470303", - major_version=version.major, - minor_version=version.minor, - cr=dependencies.end_cr, - ).construct_path().name + start_date = datetime(1947, 3, 3) + data_product_metadata = InputMetadata(instrument="glows", data_level="l3d", descriptor=GLOWS_L3D_DESCRIPTOR, + start_date=start_date, end_date=start_date, + version=VersionMap({GLOWS_L3D_DESCRIPTOR: version})) + + output_l3d_cdf_filename = data_product_metadata.to_science_file_path(cr_number=dependencies.end_cr).construct_path().name file_manifest = { "external_files": { @@ -267,7 +265,7 @@ def process_l3d( if output.stdout: last_processed_cr = int(output.stdout.split('= ')[-1]) - if last_processed_cr: + if last_processed_cr == dependencies.end_cr: output_text_files = [] for text_file in os.listdir(PATH_TO_L3D_TOOLKIT / 'data_l3d_txt'): if str(last_processed_cr) in text_file: @@ -280,14 +278,11 @@ def process_l3d( file_name = f'imap_glows_l3d_solar-params-history_19470303-cr0{last_processed_cr}_v00.json' - start_date = datetime(1947, 3, 3) - data_product_metadata = InputMetadata(instrument="glows", data_level="l3d", descriptor=GLOWS_L3D_DESCRIPTOR, - start_date=start_date, end_date=start_date, version=VersionMap({GLOWS_L3D_DESCRIPTOR: version})) parent_file_names = get_parent_file_names_from_l3d_json(PATH_TO_L3D_TOOLKIT / 'data_l3d') l3d_data_product = convert_json_to_l3d_data_product(PATH_TO_L3D_TOOLKIT / 'data_l3d' / file_name, data_product_metadata, parent_file_names) - l3d_data_product_path = save_data(l3d_data_product, cr_number=last_processed_cr) + l3d_data_product_path = save_data(l3d_data_product, cr_number=dependencies.end_cr) return GlowsL3DProcessorOutput(l3d_data_product_path, txt_files_with_correct_version, last_processed_cr) return None diff --git a/imap_l3_processing/models.py b/imap_l3_processing/models.py index 6b1b1b47..6b4861da 100644 --- a/imap_l3_processing/models.py +++ b/imap_l3_processing/models.py @@ -7,7 +7,7 @@ from typing import Union, Optional, TypeVar, Generic import numpy as np -from imap_data_access.file_validation import Version +from imap_data_access.file_validation import Version, ScienceFilePath from imap_l3_processing.data_utils import rebin @@ -38,6 +38,22 @@ class InputMetadata: def logical_source(self): return f"imap_{self.instrument}_{self.data_level}_{self.descriptor}" + def to_science_file_path(self, cr_number: Optional[int] = None, extension: str = "cdf") -> ScienceFilePath: + assert self.repointing is None or cr_number is None, "You cannot call save_data with both a repointing in the metadata while passing in a CR number" + formatted_start_date = self.start_date.strftime("%Y%m%d") + version = self.version.lookup(self.descriptor) + + return ScienceFilePath.generate_from_inputs( + instrument=self.instrument, + data_level=self.data_level, + descriptor=self.descriptor, + start_time=formatted_start_date, + major_version=version.major, + minor_version=version.minor, + extension=extension, + repointing=self.repointing, + cr=cr_number, + ) @dataclass class DataProductVariable: diff --git a/imap_l3_processing/utils.py b/imap_l3_processing/utils.py index 0c8c8a89..48016f00 100644 --- a/imap_l3_processing/utils.py +++ b/imap_l3_processing/utils.py @@ -66,20 +66,7 @@ class SpiceKernelTypes(enum.Enum): def save_data(data: DataProduct, delete_if_present: bool = False, folder_path: Path = None, cr_number=None) -> Path: - assert data.input_metadata.repointing is None or cr_number is None, "You cannot call save_data with both a repointing in the metadata while passing in a CR number" - formatted_start_date = data.input_metadata.start_date.strftime("%Y%m%d") - version = data.input_metadata.version.lookup(data.input_metadata.descriptor) - science_file_path = ScienceFilePath.generate_from_inputs( - instrument=data.input_metadata.instrument, - data_level=data.input_metadata.data_level, - descriptor=data.input_metadata.descriptor, - start_time=formatted_start_date, - repointing=data.input_metadata.repointing, - cr=cr_number, - major_version=version.major, - minor_version=version.minor, - ) - + science_file_path = data.input_metadata.to_science_file_path(cr_number=cr_number) file_path = science_file_path.construct_path() if folder_path is not None: file_path = folder_path / file_path.name diff --git a/tests/glows/test_glows_processor.py b/tests/glows/test_glows_processor.py index 1d86e97b..58d10cbf 100644 --- a/tests/glows/test_glows_processor.py +++ b/tests/glows/test_glows_processor.py @@ -1227,7 +1227,6 @@ def test_process_l3e_invoked_from_top_level_process(self, mock_process_l3d, mock mock_process_l3d.return_value = process_l3d_result mock_determine_spacecraft_info.return_value = sentinel.spacecraft_info, GlowsL3Flags.PREDICTIVE_EPHEMERIS, ["spice kernel"] - input_major_version = 5 input_version_map = VersionMap( From a2d3a4c04f4ffeb44dfcdcc8f18a632117d846ba Mon Sep 17 00:00:00 2001 From: Menlo Innovations - CAVA Project Date: Thu, 1 Oct 2026 09:29:44 -0400 Subject: [PATCH 4/7] Harrison - SWYN - 3566 - GH #209 - Add back run_periodically on GLOWS integration test. --- imap_l3_processing/glows/l3d/utils.py | 1 + tests/integration/test_glows_processor_integration.py | 4 ++-- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/imap_l3_processing/glows/l3d/utils.py b/imap_l3_processing/glows/l3d/utils.py index 883f2fe6..80a13812 100644 --- a/imap_l3_processing/glows/l3d/utils.py +++ b/imap_l3_processing/glows/l3d/utils.py @@ -140,6 +140,7 @@ def rename_l3d_text_outputs(paths: list[Path], version: str) -> list[Path]: out_paths.append(new_path) return out_paths + def query_for_most_recent_l3d(descriptor: str) -> Optional[dict]: query_result = imap_data_access.query(instrument="glows", data_level="l3d", descriptor=descriptor) sorted_query_result = sorted( diff --git a/tests/integration/test_glows_processor_integration.py b/tests/integration/test_glows_processor_integration.py index adf4f75e..274ae639 100644 --- a/tests/integration/test_glows_processor_integration.py +++ b/tests/integration/test_glows_processor_integration.py @@ -319,8 +319,8 @@ def test_l3bcde_first_time_processing(self): for file_path in expected_files: self.assertTrue(file_path.construct_path().exists(), msg=str(file_path.construct_path())) - # @run_periodically(timedelta(days=14)) - # @run_test_in_docker + @run_periodically(timedelta(days=14)) + @run_test_in_docker def test_l3bcde_automatic_reprocessing(self): new_l3a_file = GLOWS_TEST_DATA / "imap_glows_l3a_hist_20251118-repoint00052_v001.cdf" l3bcde_input_files = [ From 136cce96cd6658eb25e5db5c17596f4ce39d114f Mon Sep 17 00:00:00 2001 From: Menlo Innovations - CAVA Project Date: Thu, 1 Oct 2026 12:16:25 -0400 Subject: [PATCH 5/7] Harrison - SWYN - 3566 - GH #209 - Move L3e initializer logic to be in the same file. Refactor for clarity to prep for thinking about gating L3d processing on whether to produce L3e. --- .../glows/l3e/glows_l3e_initializer.py | 115 +++++- .../glows/l3e/glows_l3e_utils.py | 94 +---- .../glows/l3e/reprocess_info.py | 6 +- tests/glows/l3e/test_glows_l3e_initializer.py | 382 +++++++++++++++++- tests/glows/l3e/test_glows_l3e_utils.py | 381 +---------------- tests/glows/l3e/test_reprocess_info.py | 6 +- 6 files changed, 499 insertions(+), 485 deletions(-) diff --git a/imap_l3_processing/glows/l3e/glows_l3e_initializer.py b/imap_l3_processing/glows/l3e/glows_l3e_initializer.py index bf8b0436..0706ab50 100644 --- a/imap_l3_processing/glows/l3e/glows_l3e_initializer.py +++ b/imap_l3_processing/glows/l3e/glows_l3e_initializer.py @@ -1,25 +1,30 @@ +from __future__ import annotations + import logging from dataclasses import dataclass from pathlib import Path from typing import Optional import imap_data_access +import numpy as np from imap_data_access import ( ProcessingInputCollection, AncillaryInput, ScienceInput, RepointInput, ) +from imap_data_access.file_validation import Version +from imap_processing.spice.repoint import set_global_repoint_table_paths, get_repoint_data +from spacepy.pycdf import CDF +from imap_l3_processing.glows.descriptors import GLOWS_L3E_DESCRIPTORS, GLOWS_L3E_HI_90_DESCRIPTOR, \ + GLOWS_L3E_HI_45_DESCRIPTOR, GLOWS_L3E_LO_DESCRIPTOR, GLOWS_L3E_ULTRA_SF_DESCRIPTOR, GLOWS_L3E_ULTRA_HF_DESCRIPTOR from imap_l3_processing.glows.l3bc.utils import get_pointing_date_range from imap_l3_processing.glows.l3d.models import GlowsL3DProcessorOutput from imap_l3_processing.glows.l3d.utils import get_most_recently_uploaded_ancillary from imap_l3_processing.glows.l3e.glows_l3e_dependencies import GlowsL3EDependencies from imap_l3_processing.glows.l3e.glows_l3e_utils import ( - find_first_updated_cr, - identify_versions_for_l3e_output_files, - GlowsL3eVersionsForRepointings, -) + GlowsL3eVersionsForRepointings, get_repoint_numbers_within_cr_window, ) from imap_l3_processing.glows.l3e.reprocess_info import ReprocessInfo from imap_l3_processing.models import VersionMap from imap_l3_processing.utils import FurnishMetakernelOutput @@ -38,11 +43,11 @@ class GlowsL3EInitializerOutput: class GlowsL3EInitializer: @staticmethod def get_repointings_to_process( - l3d_output: GlowsL3DProcessorOutput, - previous_l3d: Optional[str], - repointing_file_path: Path, - version_map: VersionMap, - reprocess_info: ReprocessInfo, + l3d_output: GlowsL3DProcessorOutput, + previous_l3d: Optional[str], + repointing_file_path: Path, + version_map: VersionMap, + reprocess_info: ReprocessInfo, ) -> Optional[GlowsL3EInitializerOutput]: pipeline_settings_l3bcde = get_most_recently_uploaded_ancillary(imap_data_access.query(table='ancillary', instrument='glows', descriptor='pipeline-settings-l3bcde')) energy_grid_lo = get_most_recently_uploaded_ancillary(imap_data_access.query(table='ancillary', instrument='glows', descriptor='energy-grid-lo')) @@ -106,3 +111,95 @@ def get_repointings_to_process( metakernel_without_predict_ephem=furnished_metakernels[1], ) +def query_existing_l3es_versions(descriptor: str) -> dict[int, Version]: + existing_l3es_for_descriptor = {} + l3e_files = imap_data_access.query(instrument='glows', data_level='l3e', version="latest", descriptor=descriptor) + for l3e in l3e_files: + if 'major_version' in l3e.keys() and 'minor_version' in l3e.keys(): + existing_l3es_for_descriptor[int(l3e['repointing'])] = Version(l3e["major_version"], l3e["minor_version"]) + elif 'version' in l3e.keys(): + existing_l3es_for_descriptor[int(l3e['repointing'])] = Version.from_version(l3e['version']) + else: + continue + return existing_l3es_for_descriptor + +def identify_versions_for_l3e_output_files(start_cr_of_mission: int, end_cr_of_mission: int, first_updated_cr_from_l3d: Optional[int], + repointing_path: Path, version_map: VersionMap, reprocess_info: ReprocessInfo) -> GlowsL3eVersionsForRepointings: + + set_global_repoint_table_paths([repointing_path]) + repointing_data = get_repoint_data() + + all_pointing_numbers = get_repoint_numbers_within_cr_window(start_cr_of_mission, end_cr_of_mission, repointing_data) + pointing_numbers_updated_by_l3d = get_repoint_numbers_within_cr_window(first_updated_cr_from_l3d, end_cr_of_mission, repointing_data) + + updated_pointings_per_instruments = {} + updated_pointing_numbers = {} + + for descriptor in GLOWS_L3E_DESCRIPTORS: + new_major_version = version_map.lookup(descriptor).major + + repointings_to_force_processing = reprocess_info.get_repoints_for_descriptor( + descriptor, repointing_data + ) + + existing_file_versions = query_existing_l3es_versions(descriptor) + out_of_date_repointings = find_out_of_date_l3es(existing_file_versions, all_pointing_numbers, new_major_version) + + repointings_to_process = pointing_numbers_updated_by_l3d | repointings_to_force_processing | out_of_date_repointings + + new_file_versions = {} + for pointing_number in sorted(repointings_to_process): + if previous_version := existing_file_versions.get(pointing_number): + new_version = Version(new_major_version, previous_version.minor + 1) + else: + new_version = Version(new_major_version, 1) + new_file_versions[pointing_number] = new_version + + updated_pointings_per_instruments[descriptor] = new_file_versions + updated_pointing_numbers = updated_pointing_numbers | new_file_versions.keys() + + + return GlowsL3eVersionsForRepointings(list(updated_pointing_numbers), + updated_pointings_per_instruments[GLOWS_L3E_HI_90_DESCRIPTOR], + updated_pointings_per_instruments[GLOWS_L3E_HI_45_DESCRIPTOR], + updated_pointings_per_instruments[GLOWS_L3E_LO_DESCRIPTOR], + updated_pointings_per_instruments[GLOWS_L3E_ULTRA_SF_DESCRIPTOR], + updated_pointings_per_instruments[GLOWS_L3E_ULTRA_HF_DESCRIPTOR], + ) + +def find_out_of_date_l3es(existing_l3es: dict[int, Version], repointings_to_check: set[int], current_major_version: int) -> set[int]: + out_of_date_l3es = set() + for repointing in repointings_to_check: + if existing_l3e_version := existing_l3es.get(repointing): + if existing_l3e_version.major < current_major_version: + out_of_date_l3es.add(repointing) + else: + out_of_date_l3es.add(repointing) + return out_of_date_l3es + + +def find_first_updated_cr(new_l3d: Path, old_l3d: str) -> Optional[int]: + downloaded_old_l3d = imap_data_access.download(old_l3d) + + old_l3d_cdf = CDF(str(downloaded_old_l3d)) + new_l3d_cdf = CDF(str(new_l3d)) + + for i, cr in enumerate(old_l3d_cdf['cr_grid'][...]): + lya_matches = np.isclose(old_l3d_cdf['lyman_alpha'][i], new_l3d_cdf['lyman_alpha'][i]) + phion_matches = np.isclose(old_l3d_cdf['phion'][i], new_l3d_cdf['phion'][i]) + plasma_speed_flag_matches = np.isclose(old_l3d_cdf['plasma_speed_flag'][i], new_l3d_cdf['plasma_speed_flag'][i]) + proton_density_flag_matches = np.isclose(old_l3d_cdf['proton_density_flag'][i], new_l3d_cdf['proton_density_flag'][i]) + uv_anisotropy_flag_matches = np.isclose(old_l3d_cdf['uv_anisotropy_flag'][i], new_l3d_cdf['uv_anisotropy_flag'][i]) + glows_flags_matches = np.isclose(old_l3d_cdf['glows_flags'][i], new_l3d_cdf['glows_flags'][i]) + + plasma_speed_matches = np.all(np.isclose(old_l3d_cdf['plasma_speed'][i], new_l3d_cdf['plasma_speed'][i])) + proton_density_matches = np.all(np.isclose(old_l3d_cdf['proton_density'][i], new_l3d_cdf['proton_density'][i])) + uv_anisotropy_matches = np.all(np.isclose(old_l3d_cdf['uv_anisotropy'][i], new_l3d_cdf['uv_anisotropy'][i])) + + if np.any(np.logical_not([lya_matches, phion_matches, plasma_speed_matches, plasma_speed_flag_matches, proton_density_matches, proton_density_flag_matches, uv_anisotropy_matches, uv_anisotropy_flag_matches, glows_flags_matches])): + return int(cr) + + if old_l3d_cdf['cr_grid'].shape != new_l3d_cdf['cr_grid'].shape: + return int(old_l3d_cdf['cr_grid'][-1]) + 1 + + return None diff --git a/imap_l3_processing/glows/l3e/glows_l3e_utils.py b/imap_l3_processing/glows/l3e/glows_l3e_utils.py index 480849a2..b4495199 100644 --- a/imap_l3_processing/glows/l3e/glows_l3e_utils.py +++ b/imap_l3_processing/glows/l3e/glows_l3e_utils.py @@ -13,21 +13,17 @@ from imap_l3_processing.glows.quality_flags import GlowsL3Flags from imap_data_access.file_validation import Version -from imap_processing.spice.repoint import set_global_repoint_table_paths, get_repoint_data from spacepy.pycdf import CDF from typing import Optional from imap_l3_processing.constants import ONE_AU_IN_KM, TT2000_EPOCH, ONE_SECOND_IN_NANOSECONDS -from imap_l3_processing.glows.descriptors import GLOWS_L3E_ULTRA_HF_DESCRIPTOR, GLOWS_L3E_ULTRA_SF_DESCRIPTOR, \ - GLOWS_L3E_LO_DESCRIPTOR, GLOWS_L3E_HI_45_DESCRIPTOR, GLOWS_L3E_HI_90_DESCRIPTOR, GLOWS_L3E_DESCRIPTORS from imap_l3_processing.glows.l3bc.l3bc_toolkit.funcs import jd_fm_Carrington from imap_l3_processing.glows.l3e.glows_l3e_call_arguments import GlowsL3eCallArguments, GlowsL3eSpacecraftInfo from imap_l3_processing.utils import FurnishMetakernelOutput -from imap_l3_processing.models import VersionMap if typing.TYPE_CHECKING: - from imap_l3_processing.glows.l3e.reprocess_info import ReprocessInfo + pass def determine_call_args_for_l3e_executable(start_date: datetime, repointing_midpoint: datetime, elongation: float, spacecraft_info: GlowsL3eSpacecraftInfo) -> GlowsL3eCallArguments: @@ -99,59 +95,6 @@ class GlowsL3eVersionsForRepointings: ultra_sf_repointings: dict[int, Version] ultra_hf_repointings: dict[int, Version] -def identify_versions_for_l3e_output_files(start_cr_of_mission: int, end_cr_of_mission: int, first_updated_cr_from_l3d: Optional[int], - repointing_path: Path, version_map: VersionMap, reprocess_info: ReprocessInfo) -> GlowsL3eVersionsForRepointings: - - set_global_repoint_table_paths([repointing_path]) - repointing_data = get_repoint_data() - - all_pointing_numbers = get_repoint_numbers_within_cr_window(start_cr_of_mission, end_cr_of_mission, repointing_data) - pointing_numbers_updated_by_l3d = get_repoint_numbers_within_cr_window(first_updated_cr_from_l3d, end_cr_of_mission, repointing_data) - - updated_pointings_per_instruments = {} - updated_pointing_numbers = {} - for descriptor in GLOWS_L3E_DESCRIPTORS: - repointings_to_force_processing = reprocess_info.get_repoints_for_descriptor( - descriptor, repointing_data - ) - repointings_to_process = set(pointing_numbers_updated_by_l3d + repointings_to_force_processing) - l3e_files = imap_data_access.query(instrument='glows', data_level='l3e', version="latest", descriptor=descriptor) - existing_file_versions = {} - for l3e in l3e_files: - if 'major_version' in l3e.keys() and 'minor_version' in l3e.keys(): - existing_file_versions[int(l3e['repointing'])] = Version(l3e["major_version"], l3e["minor_version"]) - elif 'version' in l3e.keys(): - existing_file_versions[int(l3e['repointing'])] = Version.from_version(l3e['version']) - else: - continue - new_file_versions = {} - for pointing_number in all_pointing_numbers: - new_major_version = version_map.lookup(descriptor).major - if pointing_number in existing_file_versions: - previous_version = existing_file_versions[pointing_number] - higher_major_version_given = new_major_version > previous_version.major - - if ( - higher_major_version_given - or pointing_number in repointings_to_process - ): - new_file_versions[pointing_number] = Version(new_major_version, previous_version.minor + 1) - - else: - new_file_versions[pointing_number] = Version(new_major_version, 1) - - updated_pointings_per_instruments[descriptor] = new_file_versions - updated_pointing_numbers = updated_pointing_numbers | new_file_versions.keys() - - - return GlowsL3eVersionsForRepointings(list(updated_pointing_numbers), - updated_pointings_per_instruments[GLOWS_L3E_HI_90_DESCRIPTOR], - updated_pointings_per_instruments[GLOWS_L3E_HI_45_DESCRIPTOR], - updated_pointings_per_instruments[GLOWS_L3E_LO_DESCRIPTOR], - updated_pointings_per_instruments[GLOWS_L3E_ULTRA_SF_DESCRIPTOR], - updated_pointings_per_instruments[GLOWS_L3E_ULTRA_HF_DESCRIPTOR], - ) - def compute_glows_flags_for_repoint(l3d_cdf_path: Path, repoint_midpoint: datetime) -> int: with CDF(str(l3d_cdf_path)) as cdf: @@ -172,33 +115,6 @@ def compute_glows_flags_for_repoint(l3d_cdf_path: Path, repoint_midpoint: dateti return int(np.bitwise_or.reduce(selected.astype(np.uint16), initial=0)) - -def find_first_updated_cr(new_l3d: Path, old_l3d: str) -> Optional[int]: - downloaded_old_l3d = imap_data_access.download(old_l3d) - - old_l3d_cdf = CDF(str(downloaded_old_l3d)) - new_l3d_cdf = CDF(str(new_l3d)) - - for i, cr in enumerate(old_l3d_cdf['cr_grid'][...]): - lya_matches = np.isclose(old_l3d_cdf['lyman_alpha'][i], new_l3d_cdf['lyman_alpha'][i]) - phion_matches = np.isclose(old_l3d_cdf['phion'][i], new_l3d_cdf['phion'][i]) - plasma_speed_flag_matches = np.isclose(old_l3d_cdf['plasma_speed_flag'][i], new_l3d_cdf['plasma_speed_flag'][i]) - proton_density_flag_matches = np.isclose(old_l3d_cdf['proton_density_flag'][i], new_l3d_cdf['proton_density_flag'][i]) - uv_anisotropy_flag_matches = np.isclose(old_l3d_cdf['uv_anisotropy_flag'][i], new_l3d_cdf['uv_anisotropy_flag'][i]) - glows_flags_matches = np.isclose(old_l3d_cdf['glows_flags'][i], new_l3d_cdf['glows_flags'][i]) - - plasma_speed_matches = np.all(np.isclose(old_l3d_cdf['plasma_speed'][i], new_l3d_cdf['plasma_speed'][i])) - proton_density_matches = np.all(np.isclose(old_l3d_cdf['proton_density'][i], new_l3d_cdf['proton_density'][i])) - uv_anisotropy_matches = np.all(np.isclose(old_l3d_cdf['uv_anisotropy'][i], new_l3d_cdf['uv_anisotropy'][i])) - - if np.any(np.logical_not([lya_matches, phion_matches, plasma_speed_matches, plasma_speed_flag_matches, proton_density_matches, proton_density_flag_matches, uv_anisotropy_matches, uv_anisotropy_flag_matches, glows_flags_matches])): - return int(cr) - - if old_l3d_cdf['cr_grid'].shape != new_l3d_cdf['cr_grid'].shape: - return int(old_l3d_cdf['cr_grid'][-1]) + 1 - - return None - def get_lo_pivot_angle_from_l1b_file(path: Path) -> float: with CDF(str(path)) as cdf: epoch = cdf['epoch'][...] @@ -236,9 +152,9 @@ def get_lo_pivot_angles(repointings: list[int]) -> dict[int, LoPivotAngle]: result[repointing] = LoPivotAngle(parent_filename=None, pivot_angle=90.0) return result -def get_repoint_numbers_within_cr_window(start_cr_number: int | None, end_cr_number: int, repointing_data) -> list[int]: +def get_repoint_numbers_within_cr_window(start_cr_number: int | None, end_cr_number: int, repointing_data) -> set[int]: if start_cr_number is None: - return [] + return set() first_carrington_start_date = Time(jd_fm_Carrington(float(start_cr_number)), format='jd') last_cr_end_date = Time(jd_fm_Carrington(float(end_cr_number + 0.5)), format='jd') @@ -251,10 +167,10 @@ def get_repoint_numbers_within_cr_window(start_cr_number: int | None, end_cr_num repoint_ends = vectorized_date_conv(repointing_data["repoint_end_utc"]) repoint_ids = repointing_data["repoint_id"] - repoint_numbers = [] + repoint_numbers = set() for i in range(len(repoint_ids)): if i + 1 < len(repoint_ids) and start_ns < (repoint_starts[i + 1] + repoint_ends[i])/2 < end_ns: - repoint_numbers.append(int(repoint_ids[i])) + repoint_numbers.add(int(repoint_ids[i])) return repoint_numbers diff --git a/imap_l3_processing/glows/l3e/reprocess_info.py b/imap_l3_processing/glows/l3e/reprocess_info.py index efea8156..4583edd8 100644 --- a/imap_l3_processing/glows/l3e/reprocess_info.py +++ b/imap_l3_processing/glows/l3e/reprocess_info.py @@ -79,17 +79,17 @@ def should_reprocess_l3d(self) -> bool: def get_repoints_for_descriptor( self, descriptor: str, repointing_data - ) -> list[int]: + ) -> set[int]: reprocess_targets = self.products_to_reprocess.get(descriptor, None) if reprocess_targets is None: - return [] + return set() repoints = set(reprocess_targets.repoints) for cr in reprocess_targets.carrington_rotations: repoints.update(get_repoint_numbers_within_cr_window(cr, cr, repointing_data)) - return sorted(repoints) + return repoints def fetch_reprocess_info(input_collection: ProcessingInputCollection) -> ReprocessInfo: [reprocessing_file_path] = input_collection.get_file_paths(source="glows", descriptor=GLOWS_REPROCESSING_DESCRIPTOR) diff --git a/tests/glows/l3e/test_glows_l3e_initializer.py b/tests/glows/l3e/test_glows_l3e_initializer.py index b3e6ca80..b6cd0aac 100644 --- a/tests/glows/l3e/test_glows_l3e_initializer.py +++ b/tests/glows/l3e/test_glows_l3e_initializer.py @@ -1,17 +1,22 @@ import unittest from datetime import datetime from pathlib import Path -from unittest.mock import patch, call, sentinel, Mock +from unittest.mock import patch, call, sentinel, Mock, create_autospec +import numpy as np from imap_data_access import RepointInput from imap_data_access.file_validation import Version -from imap_l3_processing.glows.descriptors import GLOWS_L3BCDE_DESCRIPTORS +from imap_l3_processing.glows.descriptors import GLOWS_L3BCDE_DESCRIPTORS, GLOWS_L3E_HI_45_DESCRIPTOR, \ + GLOWS_L3E_HI_90_DESCRIPTOR, GLOWS_L3E_LO_DESCRIPTOR, GLOWS_L3E_ULTRA_SF_DESCRIPTOR, GLOWS_L3E_ULTRA_HF_DESCRIPTOR, \ + GLOWS_L3E_DESCRIPTORS from imap_l3_processing.glows.l3d.models import GlowsL3DProcessorOutput -from imap_l3_processing.glows.l3e.glows_l3e_initializer import GlowsL3EInitializer, GlowsL3EInitializerOutput +from imap_l3_processing.glows.l3e.glows_l3e_initializer import GlowsL3EInitializer, GlowsL3EInitializerOutput, \ + find_first_updated_cr, identify_versions_for_l3e_output_files from imap_l3_processing.glows.l3e.glows_l3e_utils import GlowsL3eVersionsForRepointings +from imap_l3_processing.glows.l3e.reprocess_info import ReprocessInfo from imap_l3_processing.models import VersionMap -from tests.test_helpers import create_mock_query_results +from tests.test_helpers import create_mock_query_results, get_test_data_path MODULE = 'imap_l3_processing.glows.l3e.glows_l3e_initializer' @@ -265,3 +270,372 @@ def test_get_repointings_to_process_uses_mission_start_cr_when_no_previous_l3d(s sentinel.version_map, sentinel.reprocess_info, ) + + @patch('imap_l3_processing.glows.l3e.glows_l3e_initializer.imap_data_access.download') + @patch('imap_l3_processing.glows.l3e.glows_l3e_initializer.CDF') + def test_find_first_updated_cr(self, mock_CDF, mock_download): + num_crs = 10 + mock_download.return_value = sentinel.download_old_l3d + + old_l3d = { + 'cr_grid': np.arange(num_crs) + 0.5, + 'lyman_alpha': np.arange(num_crs), + 'phion': np.arange(num_crs), + 'plasma_speed': np.arange(0, 20).reshape((num_crs, 2)), + 'plasma_speed_flag': np.arange(num_crs), + 'proton_density': np.arange(0, 20).reshape((num_crs, 2)), + 'proton_density_flag': np.arange(num_crs), + 'uv_anisotropy': np.arange(0, 20).reshape((num_crs, 2)), + 'uv_anisotropy_flag': np.arange(num_crs), + 'glows_flags': np.arange(num_crs), + } + + new_lyman_alpha = np.arange(num_crs) + new_lyman_alpha[1] = 10 + + new_phion = np.arange(num_crs) + new_phion[2] = 10 + + new_plasma_speed_flag = np.arange(num_crs) + new_plasma_speed_flag[3] = 10 + + new_proton_density_flag = np.arange(num_crs) + new_proton_density_flag[4] = 10 + + new_uv_anisotropy_flag = np.arange(num_crs) + new_uv_anisotropy_flag[5] = 10 + + new_plasma_speed = np.arange(0, 20).reshape((num_crs, 2)) + new_plasma_speed[6, :] = 10 + + new_proton_density = np.arange(0, 20).reshape((num_crs, 2)) + new_proton_density[7, :] = 10 + + new_uv_anisotropy = np.arange(0, 20).reshape((num_crs, 2)) + new_uv_anisotropy[8, :] = 10 + + new_glows_flags = np.arange(num_crs) + new_glows_flags[9] = 10 + + cases = [ + ('cr_grid', np.append(old_l3d['cr_grid'], 10.5), 10), + ('lyman_alpha', new_lyman_alpha, 1), + ('phion', new_phion, 2), + ('plasma_speed_flag', new_plasma_speed_flag, 3), + ('proton_density_flag', new_proton_density_flag, 4), + ('uv_anisotropy_flag', new_uv_anisotropy_flag, 5), + + ('plasma_speed', new_plasma_speed, 6), + ('proton_density', new_proton_density, 7), + ('uv_anisotropy', new_uv_anisotropy, 8), + ('glows_flags', new_glows_flags, 9), + ('no_change', None, None) + ] + + for case, change, expected in cases: + mock_download.reset_mock() + mock_CDF.reset_mock() + + with self.subTest(case=case): + new_l3d = {**old_l3d} + if case != "no_change": + new_l3d[case] = change + + mock_CDF.side_effect = [old_l3d, new_l3d] + + actual_cr = find_first_updated_cr( + sentinel.new_l3d_path, sentinel.old_l3d_filename + ) + + mock_download.assert_called_once_with(sentinel.old_l3d_filename) + mock_CDF.assert_has_calls( + [ + call(str(sentinel.download_old_l3d)), + call(str(sentinel.new_l3d_path)), + ] + ) + + self.assertEqual(actual_cr, expected) + + @patch('imap_l3_processing.glows.l3e.glows_l3e_initializer.get_repoint_numbers_within_cr_window') + @patch('imap_l3_processing.glows.l3e.glows_l3e_initializer.imap_data_access') + def test_identify_versions_for_l3e_output_files_gives_minor_version_1_for_non_existing_l3e(self, + mock_imap_data_access, + mock_get_repoint_numbers_within_cr_window): + start_cr_of_mission = 2093 + end_cr_of_mission = 2094 + first_cr_updated_in_l3d = None + repointing_path = get_test_data_path("fake_1_day_repointing_file.csv") + version_map = VersionMap({desc: Version(3 + i, 5) for i, desc in enumerate(GLOWS_L3E_DESCRIPTORS)}) + + mock_imap_data_access.query.side_effect = [ + create_mock_query_results([]), + create_mock_query_results([]), + create_mock_query_results([]), + create_mock_query_results([]), + create_mock_query_results([]), + ] + + all_repointing_numbers = set(range(3682, 3736)) + updated_repointing_numbers = set() + mock_get_repoint_numbers_within_cr_window.side_effect = [ + all_repointing_numbers, + updated_repointing_numbers + ] + reprocess_info = ReprocessInfo({}) + + result = identify_versions_for_l3e_output_files(start_cr_of_mission, end_cr_of_mission, first_cr_updated_in_l3d, + repointing_path, version_map, reprocess_info) + + mock_imap_data_access.query.assert_has_calls([ + call(instrument='glows', data_level='l3e', version="latest", descriptor=GLOWS_L3E_HI_45_DESCRIPTOR), + call(instrument='glows', data_level='l3e', version="latest", descriptor=GLOWS_L3E_HI_90_DESCRIPTOR), + call(instrument='glows', data_level='l3e', version="latest", descriptor=GLOWS_L3E_LO_DESCRIPTOR), + call(instrument='glows', data_level='l3e', version="latest", descriptor=GLOWS_L3E_ULTRA_SF_DESCRIPTOR), + call(instrument='glows', data_level='l3e', version="latest", descriptor=GLOWS_L3E_ULTRA_HF_DESCRIPTOR) + ]) + + expected_versions_for_hi45_repoint_number = {repoint_number: Version(3, 1) for repoint_number in + all_repointing_numbers} + expected_versions_for_hi90_repoint_number = {repoint_number: Version(4, 1) for repoint_number in + all_repointing_numbers} + expected_versions_for_lo_repoint_number = {repoint_number: Version(5, 1) for repoint_number in + all_repointing_numbers} + expected_versions_for_ultra_sf_repoint_number = {repoint_number: Version(6, 1) for repoint_number in + all_repointing_numbers} + expected_versions_for_ultra_hf_repoint_number = {repoint_number: Version(7, 1) for repoint_number in + all_repointing_numbers} + + self.assertCountEqual(all_repointing_numbers, result.repointing_numbers) + self.assertEqual(expected_versions_for_hi90_repoint_number, result.hi_90_repointings) + self.assertEqual(expected_versions_for_hi45_repoint_number, result.hi_45_repointings) + self.assertEqual(expected_versions_for_lo_repoint_number, result.lo_repointings) + self.assertEqual( + expected_versions_for_ultra_sf_repoint_number, result.ultra_sf_repointings + ) + self.assertEqual(expected_versions_for_ultra_hf_repoint_number, result.ultra_hf_repointings) + + @patch('imap_l3_processing.glows.l3e.glows_l3e_initializer.get_repoint_numbers_within_cr_window') + @patch('imap_l3_processing.glows.l3e.glows_l3e_initializer.imap_data_access') + def test_identify_versions_for_l3e_output_files_increments_major_and_minor_when_given_higher_major_version(self, + mock_imap_data_access, + mock_get_repoint_numbers_within_cr_window): + start_cr_of_mission = 2093 + end_cr_of_mission = 2094 + first_cr_updated_in_l3d = None + repointing_path = get_test_data_path("fake_1_day_repointing_file.csv") + version_map = VersionMap({desc: Version(3 + i, 5) for i, desc in enumerate(GLOWS_L3E_DESCRIPTORS)}) + + all_repointing_numbers = set(range(3682, 3736)) + updated_repointing_numbers = set() + old_major_version = 2 + mock_get_repoint_numbers_within_cr_window.reset_mock() + mock_imap_data_access.reset_mock() + + mock_get_repoint_numbers_within_cr_window.side_effect = [ + all_repointing_numbers, + updated_repointing_numbers + ] + + mock_imap_data_access.query.side_effect = [ + create_mock_query_results([ + f'imap_glows_l3e_survival-probability-hi-90_20250101-repoint03682_{Version(old_major_version, 1)}.cdf', + f'imap_glows_l3e_survival-probability-hi-90_20250101-repoint03683_{Version(3, 1)}.cdf', + f'imap_glows_l3e_survival-probability-hi-90_20250101-repoint03735_{Version(3, 1)}.cdf' + ]), + create_mock_query_results([ + f'imap_glows_l3e_survival-probability-hi-45_20250101-repoint03683_{Version(old_major_version, 2)}.cdf', + f'imap_glows_l3e_survival-probability-hi-45_20250101-repoint03684_{Version(4, 2)}.cdf', + f'imap_glows_l3e_survival-probability-hi-45_20250101-repoint03735_{Version(4, 2)}.cdf' + ]), + create_mock_query_results([ + f'imap_glows_l3e_survival-probability-lo_20250101-repoint03684_{Version(old_major_version, 3)}.cdf', + f'imap_glows_l3e_survival-probability-lo_20250101-repoint03685_{Version(5, 3)}.cdf', + f'imap_glows_l3e_survival-probability-lo_20250101-repoint03735_{Version(5, 3)}.cdf' + ]), + create_mock_query_results([ + f'imap_glows_l3e_survival-probability-ul-sf_20250101-repoint03685_{Version(old_major_version, 4)}.cdf', + f'imap_glows_l3e_survival-probability-ul-sf_20250101-repoint03686_{Version(6, 4)}.cdf', + f'imap_glows_l3e_survival-probability-ul-sf_20250101-repoint03735_{Version(6, 4)}.cdf' + ]), + create_mock_query_results([ + f'imap_glows_l3e_survival-probability-ul-hf_20250101-repoint03686_{Version(old_major_version, 5)}.cdf', + f'imap_glows_l3e_survival-probability-ul-hf_20250101-repoint03687_{Version(7, 5)}.cdf', + f'imap_glows_l3e_survival-probability-ul-hf_20250101-repoint03735_{Version(7, 5)}.cdf' + ]) + ] + + result = identify_versions_for_l3e_output_files(start_cr_of_mission, end_cr_of_mission, + first_cr_updated_in_l3d, repointing_path, + version_map, ReprocessInfo({})) + + mock_imap_data_access.query.assert_has_calls([ + call(instrument='glows', data_level='l3e', version="latest", descriptor=GLOWS_L3E_HI_45_DESCRIPTOR), + call(instrument='glows', data_level='l3e', version="latest", descriptor=GLOWS_L3E_HI_90_DESCRIPTOR), + call(instrument='glows', data_level='l3e', version="latest", descriptor=GLOWS_L3E_LO_DESCRIPTOR), + call(instrument='glows', data_level='l3e', version="latest", descriptor=GLOWS_L3E_ULTRA_SF_DESCRIPTOR), + call(instrument='glows', data_level='l3e', version="latest", descriptor=GLOWS_L3E_ULTRA_HF_DESCRIPTOR) + ]) + + self.assertCountEqual(list(range(3682, 3735)), result.repointing_numbers) + + self.assertNotIn(3683, result.hi_45_repointings) + self.assertNotIn(3684, result.hi_90_repointings) + self.assertNotIn(3685, result.lo_repointings) + self.assertNotIn(3686, result.ultra_sf_repointings) + self.assertNotIn(3687, result.ultra_hf_repointings) + + self.assertEqual(Version(3, 2), result.hi_45_repointings[3682]) + self.assertEqual(Version(4, 3), result.hi_90_repointings[3683]) + self.assertEqual(Version(5, 4), result.lo_repointings[3684]) + self.assertEqual(Version(6, 5), result.ultra_sf_repointings[3685]) + self.assertEqual(Version(7, 6), result.ultra_hf_repointings[3686]) + + @patch('imap_l3_processing.glows.l3e.glows_l3e_initializer.get_repoint_numbers_within_cr_window') + @patch('imap_l3_processing.glows.l3e.glows_l3e_initializer.imap_data_access') + def test_identify_versions_for_l3e_output_files_increments_minor_when_same_major_and_updated_l3d_covers_pointing( + self, mock_imap_data_access, mock_get_repoint_numbers_within_cr_window + ): + start_cr_of_mission = 2093 + end_cr_of_mission = 2095 + first_cr_updated_in_l3d = None + repointing_path = get_test_data_path("fake_1_day_repointing_file.csv") + version_map = VersionMap({desc: Version(3 + i, 5) for i, desc in enumerate(GLOWS_L3E_DESCRIPTORS)}) + + all_repointing_numbers = set(range(3682, 3763)) + updated_repointing_numbers = set(range(3709, 3763)) + + mock_get_repoint_numbers_within_cr_window.side_effect = [ + all_repointing_numbers, + updated_repointing_numbers + ] + + mock_imap_data_access.query.side_effect = [ + create_mock_query_results([ + f'imap_glows_l3e_survival-probability-hi-90_20250101-repoint{repoint:05d}_{Version(3, 1)}.cdf' for + repoint in all_repointing_numbers + ]), + create_mock_query_results([ + f'imap_glows_l3e_survival-probability-hi-45_20250101-repoint{repoint:05d}_{Version(4, 2)}.cdf' for + repoint in all_repointing_numbers + ]), + create_mock_query_results([ + f'imap_glows_l3e_survival-probability-lo_20250101-repoint{repoint:05d}_{Version(5, 3)}.cdf' for repoint + in all_repointing_numbers + ]), + create_mock_query_results([ + f'imap_glows_l3e_survival-probability-ul-sf_20250101-repoint{repoint:05d}_{Version(6, 4)}.cdf' for + repoint in all_repointing_numbers + ]), + create_mock_query_results([ + f'imap_glows_l3e_survival-probability-ul-hf_20250101-repoint{repoint:05d}_{Version(7, 5)}.cdf' for + repoint in all_repointing_numbers + ]) + ] + + result = identify_versions_for_l3e_output_files(start_cr_of_mission, end_cr_of_mission, + first_cr_updated_in_l3d, repointing_path, + version_map, ReprocessInfo({})) + + mock_imap_data_access.query.assert_has_calls([ + call(instrument='glows', data_level='l3e', version="latest", descriptor=GLOWS_L3E_HI_45_DESCRIPTOR), + call(instrument='glows', data_level='l3e', version="latest", descriptor=GLOWS_L3E_HI_90_DESCRIPTOR), + call(instrument='glows', data_level='l3e', version="latest", descriptor=GLOWS_L3E_LO_DESCRIPTOR), + call(instrument='glows', data_level='l3e', version="latest", descriptor=GLOWS_L3E_ULTRA_SF_DESCRIPTOR), + call(instrument='glows', data_level='l3e', version="latest", descriptor=GLOWS_L3E_ULTRA_HF_DESCRIPTOR) + ]) + + expected_versions_for_hi45_repoint_number = {repoint_number: Version(3, 2) for repoint_number in + updated_repointing_numbers} + expected_versions_for_hi90_repoint_number = {repoint_number: Version(4, 3) for repoint_number in + updated_repointing_numbers} + expected_versions_for_lo_repoint_number = {repoint_number: Version(5, 4) for repoint_number in + updated_repointing_numbers} + expected_versions_for_ultra_sf_repoint_number = {repoint_number: Version(6, 5) for repoint_number in + updated_repointing_numbers} + expected_versions_for_ultra_hf_repoint_number = {repoint_number: Version(7, 6) for repoint_number in + updated_repointing_numbers} + + self.assertCountEqual(updated_repointing_numbers, result.repointing_numbers) + self.assertEqual(expected_versions_for_hi90_repoint_number, result.hi_90_repointings) + self.assertEqual(expected_versions_for_hi45_repoint_number, result.hi_45_repointings) + self.assertEqual(expected_versions_for_lo_repoint_number, result.lo_repointings) + self.assertEqual(expected_versions_for_ultra_sf_repoint_number, result.ultra_sf_repointings) + self.assertEqual(expected_versions_for_ultra_hf_repoint_number, result.ultra_hf_repointings) + + @patch('imap_l3_processing.glows.l3e.glows_l3e_initializer.get_repoint_data') + @patch('imap_l3_processing.glows.l3e.glows_l3e_initializer.get_repoint_numbers_within_cr_window') + @patch('imap_l3_processing.glows.l3e.glows_l3e_initializer.imap_data_access') + def test_identify_versions_for_l3e_output_files_uses_reprocess_info( + self, mock_imap_data_access, mock_get_repoint_numbers_within_cr_window, mock_get_repoint_data + ): + start_cr_of_mission = 2093 + end_cr_of_mission = 2095 + first_cr_updated_in_l3d = None + repointing_path = get_test_data_path("fake_1_day_repointing_file.csv") + version_map = VersionMap({desc: Version(3 + i, 5) for i, desc in enumerate(GLOWS_L3E_DESCRIPTORS)}) + repoint_number = 3700 + mock_reprocess_info = create_autospec(ReprocessInfo, instance=True) + mock_reprocess_info.get_repoints_for_descriptor.side_effect = [ + set(), set(), {repoint_number}, set(), set() + ] + + all_repointing_numbers = set(range(3682, 3763)) + updated_repointing_numbers = set() + + mock_get_repoint_numbers_within_cr_window.side_effect = [ + all_repointing_numbers, + updated_repointing_numbers + ] + + mock_imap_data_access.query.side_effect = [ + create_mock_query_results([ + f'imap_glows_l3e_survival-probability-hi-90_20250101-repoint{repoint:05d}_{Version(3, 1)}.cdf' for + repoint in all_repointing_numbers + ]), + create_mock_query_results([ + f'imap_glows_l3e_survival-probability-hi-45_20250101-repoint{repoint:05d}_{Version(4, 2)}.cdf' for + repoint in all_repointing_numbers + ]), + create_mock_query_results([ + f'imap_glows_l3e_survival-probability-lo_20250101-repoint{repoint:05d}_{Version(5, 3)}.cdf' for repoint + in all_repointing_numbers + ]), + create_mock_query_results([ + f'imap_glows_l3e_survival-probability-ul-sf_20250101-repoint{repoint:05d}_{Version(6, 4)}.cdf' for + repoint in all_repointing_numbers + ]), + create_mock_query_results([ + f'imap_glows_l3e_survival-probability-ul-hf_20250101-repoint{repoint:05d}_{Version(7, 5)}.cdf' for + repoint in all_repointing_numbers + ]) + ] + + result = identify_versions_for_l3e_output_files( + start_cr_of_mission, + end_cr_of_mission, + first_cr_updated_in_l3d, + repointing_path, + version_map, + mock_reprocess_info, + ) + + mock_imap_data_access.query.assert_has_calls([ + call(instrument='glows', data_level='l3e', version="latest", descriptor=GLOWS_L3E_HI_45_DESCRIPTOR), + call(instrument='glows', data_level='l3e', version="latest", descriptor=GLOWS_L3E_HI_90_DESCRIPTOR), + call(instrument='glows', data_level='l3e', version="latest", descriptor=GLOWS_L3E_LO_DESCRIPTOR), + call(instrument='glows', data_level='l3e', version="latest", descriptor=GLOWS_L3E_ULTRA_SF_DESCRIPTOR), + call(instrument='glows', data_level='l3e', version="latest", descriptor=GLOWS_L3E_ULTRA_HF_DESCRIPTOR) + ]) + mock_reprocess_info.get_repoints_for_descriptor.assert_has_calls([ + call(GLOWS_L3E_HI_45_DESCRIPTOR, mock_get_repoint_data.return_value), + call(GLOWS_L3E_HI_90_DESCRIPTOR, mock_get_repoint_data.return_value), + call(GLOWS_L3E_LO_DESCRIPTOR, mock_get_repoint_data.return_value), + call(GLOWS_L3E_ULTRA_SF_DESCRIPTOR, mock_get_repoint_data.return_value), + call(GLOWS_L3E_ULTRA_HF_DESCRIPTOR, mock_get_repoint_data.return_value), + ]) + + expected_versions_for_lo_repoint_number = {repoint_number: Version(5, 4)} + + self.assertEqual([repoint_number], result.repointing_numbers) + self.assertEqual(expected_versions_for_lo_repoint_number, result.lo_repointings) \ No newline at end of file diff --git a/tests/glows/l3e/test_glows_l3e_utils.py b/tests/glows/l3e/test_glows_l3e_utils.py index 8db7dbbf..a213544f 100644 --- a/tests/glows/l3e/test_glows_l3e_utils.py +++ b/tests/glows/l3e/test_glows_l3e_utils.py @@ -2,32 +2,20 @@ import unittest from datetime import datetime, timedelta from pathlib import Path -from unittest.mock import patch, Mock, call, sentinel, create_autospec +from unittest.mock import patch, Mock, call import imap_data_access import numpy as np from imap_data_access import SPICEFilePath -from imap_data_access.file_validation import Version from imap_processing.spice.repoint import get_repoint_data, set_global_repoint_table_paths from spacepy.pycdf import CDF, const -from imap_l3_processing.constants import ONE_SECOND_IN_NANOSECONDS -from imap_l3_processing.glows.descriptors import ( - GLOWS_L3E_DESCRIPTORS, - GLOWS_L3E_HI_45_DESCRIPTOR, - GLOWS_L3E_HI_90_DESCRIPTOR, - GLOWS_L3E_LO_DESCRIPTOR, - GLOWS_L3E_ULTRA_SF_DESCRIPTOR, - GLOWS_L3E_ULTRA_HF_DESCRIPTOR, -) from imap_l3_processing.glows.l3e.glows_l3e_call_arguments import ( GlowsL3eCallArguments, GlowsL3eSpacecraftInfo, ) from imap_l3_processing.glows.l3e.glows_l3e_utils import ( determine_call_args_for_l3e_executable, - identify_versions_for_l3e_output_files, - find_first_updated_cr, get_lo_pivot_angles, get_lo_pivot_angle_from_l1b_file, LoPivotAngle, @@ -36,12 +24,10 @@ determine_spacecraft_info_for_l3e_executable, determine_spacecraft_info_using_predict_if_needed, calculate_energy_deltas, ) -from imap_l3_processing.glows.l3e.reprocess_info import ReprocessInfo, ReprocessTargets from imap_l3_processing.glows.quality_flags import GlowsL3Flags -from imap_l3_processing.models import VersionMap from imap_l3_processing.utils import FurnishMetakernelOutput from tests.integration.integration_test_helpers import mock_imap_data_access, create_metakernel -from tests.test_helpers import get_test_data_path, create_mock_query_results, get_integration_test_data_path +from tests.test_helpers import get_test_data_path, get_integration_test_data_path class TestGlowsL3EUtils(unittest.TestCase): @@ -166,92 +152,6 @@ def test_determine_spacecraft_info_using_predict_if_needed(self): spin_axis_longitude=np.float64(0.9116206658792297), spin_axis_latitude=np.float64(0.0758987849137639)) ,spacecraft_info) - @patch('imap_l3_processing.glows.l3e.glows_l3e_utils.imap_data_access.download') - @patch('imap_l3_processing.glows.l3e.glows_l3e_utils.CDF') - def test_find_first_updated_cr(self, mock_CDF, mock_download): - num_crs = 10 - mock_download.return_value = sentinel.download_old_l3d - - old_l3d = { - 'cr_grid': np.arange(num_crs) + 0.5, - 'lyman_alpha': np.arange(num_crs), - 'phion': np.arange(num_crs), - 'plasma_speed': np.arange(0, 20).reshape((num_crs, 2)), - 'plasma_speed_flag': np.arange(num_crs), - 'proton_density': np.arange(0, 20).reshape((num_crs, 2)), - 'proton_density_flag': np.arange(num_crs), - 'uv_anisotropy': np.arange(0, 20).reshape((num_crs, 2)), - 'uv_anisotropy_flag': np.arange(num_crs), - 'glows_flags': np.arange(num_crs), - } - - new_lyman_alpha = np.arange(num_crs) - new_lyman_alpha[1] = 10 - - new_phion = np.arange(num_crs) - new_phion[2] = 10 - - new_plasma_speed_flag = np.arange(num_crs) - new_plasma_speed_flag[3] = 10 - - new_proton_density_flag = np.arange(num_crs) - new_proton_density_flag[4] = 10 - - new_uv_anisotropy_flag = np.arange(num_crs) - new_uv_anisotropy_flag[5] = 10 - - new_plasma_speed = np.arange(0, 20).reshape((num_crs, 2)) - new_plasma_speed[6, :] = 10 - - new_proton_density = np.arange(0, 20).reshape((num_crs, 2)) - new_proton_density[7, :] = 10 - - new_uv_anisotropy = np.arange(0, 20).reshape((num_crs, 2)) - new_uv_anisotropy[8, :] = 10 - - new_glows_flags = np.arange(num_crs) - new_glows_flags[9] = 10 - - cases = [ - ('cr_grid', np.append(old_l3d['cr_grid'], 10.5), 10), - ('lyman_alpha', new_lyman_alpha, 1), - ('phion', new_phion, 2), - ('plasma_speed_flag', new_plasma_speed_flag, 3), - ('proton_density_flag', new_proton_density_flag, 4), - ('uv_anisotropy_flag', new_uv_anisotropy_flag, 5), - - ('plasma_speed', new_plasma_speed, 6), - ('proton_density', new_proton_density, 7), - ('uv_anisotropy', new_uv_anisotropy, 8), - ('glows_flags', new_glows_flags, 9), - ('no_change', None, None) - ] - - for case, change, expected in cases: - mock_download.reset_mock() - mock_CDF.reset_mock() - - with self.subTest(case=case): - new_l3d = {**old_l3d} - if case != "no_change": - new_l3d[case] = change - - mock_CDF.side_effect = [old_l3d, new_l3d] - - actual_cr = find_first_updated_cr( - sentinel.new_l3d_path, sentinel.old_l3d_filename - ) - - mock_download.assert_called_once_with(sentinel.old_l3d_filename) - mock_CDF.assert_has_calls( - [ - call(str(sentinel.download_old_l3d)), - call(str(sentinel.new_l3d_path)), - ] - ) - - self.assertEqual(actual_cr, expected) - def test_get_lo_pivot_angle_from_l1b_file_real_cdf(self): l1b_file = get_test_data_path( "glows/imap_lo_l1b_nhk_20260318-repoint00189_v003.cdf" @@ -381,7 +281,7 @@ def mock_read_from_cdf(path: Path): def test_get_repoint_numbers_within_cr_window(self): start_cr = 2093 end_cr = 2094 - expected_repoint_numbers = list(range(3682, 3724)) + expected_repoint_numbers = set(range(3682, 3724)) repointing_path = get_test_data_path("fake_1_day_repointing_file.csv") @@ -391,6 +291,7 @@ def test_get_repoint_numbers_within_cr_window(self): actual_repoint_numbers = get_repoint_numbers_within_cr_window(start_cr, end_cr, repointing_data) np.testing.assert_array_equal(actual_repoint_numbers, expected_repoint_numbers) + def test_get_repoint_numbers_within_cr_window_returns_empty_for_none_start(self): start_cr = None end_cr = 2094 @@ -404,280 +305,6 @@ def test_get_repoint_numbers_within_cr_window_returns_empty_for_none_start(self) np.testing.assert_array_equal(actual_repoint_numbers, []) - @patch('imap_l3_processing.glows.l3e.glows_l3e_utils.get_repoint_numbers_within_cr_window') - @patch('imap_l3_processing.glows.l3e.glows_l3e_utils.imap_data_access') - def test_identify_versions_for_l3e_output_files_gives_minor_version_1_for_non_existing_l3e(self, - mock_imap_data_access, - mock_get_repoint_numbers_within_cr_window): - start_cr_of_mission = 2093 - end_cr_of_mission = 2094 - first_cr_updated_in_l3d = None - repointing_path = get_test_data_path("fake_1_day_repointing_file.csv") - version_map = VersionMap({desc: Version(3 + i, 5) for i, desc in enumerate(GLOWS_L3E_DESCRIPTORS)}) - - mock_imap_data_access.query.side_effect = [ - create_mock_query_results([]), - create_mock_query_results([]), - create_mock_query_results([]), - create_mock_query_results([]), - create_mock_query_results([]), - ] - - all_repointing_numbers = list(range(3682, 3736)) - updated_repointing_numbers = list() - mock_get_repoint_numbers_within_cr_window.side_effect = [ - all_repointing_numbers, - updated_repointing_numbers - ] - reprocess_info = ReprocessInfo({}) - - result = identify_versions_for_l3e_output_files(start_cr_of_mission, end_cr_of_mission, first_cr_updated_in_l3d, - repointing_path, version_map, reprocess_info) - - mock_imap_data_access.query.assert_has_calls([ - call(instrument='glows', data_level='l3e', version="latest", descriptor=GLOWS_L3E_HI_45_DESCRIPTOR), - call(instrument='glows', data_level='l3e', version="latest", descriptor=GLOWS_L3E_HI_90_DESCRIPTOR), - call(instrument='glows', data_level='l3e', version="latest", descriptor=GLOWS_L3E_LO_DESCRIPTOR), - call(instrument='glows', data_level='l3e', version="latest", descriptor=GLOWS_L3E_ULTRA_SF_DESCRIPTOR), - call(instrument='glows', data_level='l3e', version="latest", descriptor=GLOWS_L3E_ULTRA_HF_DESCRIPTOR) - ]) - - expected_versions_for_hi45_repoint_number = {repoint_number: Version(3, 1) for repoint_number in - all_repointing_numbers} - expected_versions_for_hi90_repoint_number = {repoint_number: Version(4, 1) for repoint_number in - all_repointing_numbers} - expected_versions_for_lo_repoint_number = {repoint_number: Version(5, 1) for repoint_number in - all_repointing_numbers} - expected_versions_for_ultra_sf_repoint_number = {repoint_number: Version(6, 1) for repoint_number in - all_repointing_numbers} - expected_versions_for_ultra_hf_repoint_number = {repoint_number: Version(7, 1) for repoint_number in - all_repointing_numbers} - - self.assertCountEqual(all_repointing_numbers, result.repointing_numbers) - self.assertEqual(expected_versions_for_hi90_repoint_number, result.hi_90_repointings) - self.assertEqual(expected_versions_for_hi45_repoint_number, result.hi_45_repointings) - self.assertEqual(expected_versions_for_lo_repoint_number, result.lo_repointings) - self.assertEqual( - expected_versions_for_ultra_sf_repoint_number, result.ultra_sf_repointings - ) - self.assertEqual(expected_versions_for_ultra_hf_repoint_number, result.ultra_hf_repointings) - - - @patch('imap_l3_processing.glows.l3e.glows_l3e_utils.get_repoint_numbers_within_cr_window') - @patch('imap_l3_processing.glows.l3e.glows_l3e_utils.imap_data_access') - def test_identify_versions_for_l3e_output_files_increments_major_and_minor_when_given_higher_major_version(self, - mock_imap_data_access, - mock_get_repoint_numbers_within_cr_window): - start_cr_of_mission = 2093 - end_cr_of_mission = 2094 - first_cr_updated_in_l3d = None - repointing_path = get_test_data_path("fake_1_day_repointing_file.csv") - version_map = VersionMap({desc: Version(3 + i, 5) for i, desc in enumerate(GLOWS_L3E_DESCRIPTORS)}) - - all_repointing_numbers = list(range(3682, 3736)) - updated_repointing_numbers = list() - old_major_version = 2 - mock_get_repoint_numbers_within_cr_window.reset_mock() - mock_imap_data_access.reset_mock() - - mock_get_repoint_numbers_within_cr_window.side_effect = [ - all_repointing_numbers, - updated_repointing_numbers - ] - - mock_imap_data_access.query.side_effect = [ - create_mock_query_results([ - f'imap_glows_l3e_survival-probability-hi-90_20250101-repoint03682_{Version(old_major_version, 1)}.cdf', - f'imap_glows_l3e_survival-probability-hi-90_20250101-repoint03683_{Version(3, 1)}.cdf', - f'imap_glows_l3e_survival-probability-hi-90_20250101-repoint03735_{Version(3, 1)}.cdf' - ]), - create_mock_query_results([ - f'imap_glows_l3e_survival-probability-hi-45_20250101-repoint03683_{Version(old_major_version, 2)}.cdf', - f'imap_glows_l3e_survival-probability-hi-45_20250101-repoint03684_{Version(4, 2)}.cdf', - f'imap_glows_l3e_survival-probability-hi-45_20250101-repoint03735_{Version(4, 2)}.cdf' - ]), - create_mock_query_results([ - f'imap_glows_l3e_survival-probability-lo_20250101-repoint03684_{Version(old_major_version, 3)}.cdf', - f'imap_glows_l3e_survival-probability-lo_20250101-repoint03685_{Version(5, 3)}.cdf', - f'imap_glows_l3e_survival-probability-lo_20250101-repoint03735_{Version(5, 3)}.cdf' - ]), - create_mock_query_results([ - f'imap_glows_l3e_survival-probability-ul-sf_20250101-repoint03685_{Version(old_major_version, 4)}.cdf', - f'imap_glows_l3e_survival-probability-ul-sf_20250101-repoint03686_{Version(6, 4)}.cdf', - f'imap_glows_l3e_survival-probability-ul-sf_20250101-repoint03735_{Version(6, 4)}.cdf' - ]), - create_mock_query_results([ - f'imap_glows_l3e_survival-probability-ul-hf_20250101-repoint03686_{Version(old_major_version, 5)}.cdf', - f'imap_glows_l3e_survival-probability-ul-hf_20250101-repoint03687_{Version(7, 5)}.cdf', - f'imap_glows_l3e_survival-probability-ul-hf_20250101-repoint03735_{Version(7, 5)}.cdf' - ]) - ] - - result = identify_versions_for_l3e_output_files(start_cr_of_mission, end_cr_of_mission, - first_cr_updated_in_l3d, repointing_path, - version_map, ReprocessInfo({})) - - mock_imap_data_access.query.assert_has_calls([ - call(instrument='glows', data_level='l3e', version="latest", descriptor=GLOWS_L3E_HI_45_DESCRIPTOR), - call(instrument='glows', data_level='l3e', version="latest", descriptor=GLOWS_L3E_HI_90_DESCRIPTOR), - call(instrument='glows', data_level='l3e', version="latest", descriptor=GLOWS_L3E_LO_DESCRIPTOR), - call(instrument='glows', data_level='l3e', version="latest", descriptor=GLOWS_L3E_ULTRA_SF_DESCRIPTOR), - call(instrument='glows', data_level='l3e', version="latest", descriptor=GLOWS_L3E_ULTRA_HF_DESCRIPTOR) - ]) - - self.assertCountEqual(list(range(3682, 3735)), result.repointing_numbers) - - self.assertNotIn(3683, result.hi_45_repointings) - self.assertNotIn(3684, result.hi_90_repointings) - self.assertNotIn(3685, result.lo_repointings) - self.assertNotIn(3686, result.ultra_sf_repointings) - self.assertNotIn(3687, result.ultra_hf_repointings) - - self.assertEqual(Version(3, 2), result.hi_45_repointings[3682]) - self.assertEqual(Version(4, 3), result.hi_90_repointings[3683]) - self.assertEqual(Version(5, 4), result.lo_repointings[3684]) - self.assertEqual(Version(6, 5), result.ultra_sf_repointings[3685]) - self.assertEqual(Version(7, 6), result.ultra_hf_repointings[3686]) - - @patch('imap_l3_processing.glows.l3e.glows_l3e_utils.get_repoint_numbers_within_cr_window') - @patch('imap_l3_processing.glows.l3e.glows_l3e_utils.imap_data_access') - def test_identify_versions_for_l3e_output_files_increments_minor_when_same_major_and_updated_l3d_covers_pointing( - self, mock_imap_data_access, mock_get_repoint_numbers_within_cr_window - ): - start_cr_of_mission = 2093 - end_cr_of_mission = 2095 - first_cr_updated_in_l3d = None - repointing_path = get_test_data_path("fake_1_day_repointing_file.csv") - version_map = VersionMap({desc: Version(3 + i, 5) for i, desc in enumerate(GLOWS_L3E_DESCRIPTORS)}) - - all_repointing_numbers = list(range(3682, 3763)) - updated_repointing_numbers = list(range(3709, 3763)) - - mock_get_repoint_numbers_within_cr_window.side_effect = [ - all_repointing_numbers, - updated_repointing_numbers - ] - - mock_imap_data_access.query.side_effect = [ - create_mock_query_results([ - f'imap_glows_l3e_survival-probability-hi-90_20250101-repoint{repoint:05d}_{Version(3, 1)}.cdf' for repoint in all_repointing_numbers - ]), - create_mock_query_results([ - f'imap_glows_l3e_survival-probability-hi-45_20250101-repoint{repoint:05d}_{Version(4, 2)}.cdf' for repoint in all_repointing_numbers - ]), - create_mock_query_results([ - f'imap_glows_l3e_survival-probability-lo_20250101-repoint{repoint:05d}_{Version(5, 3)}.cdf' for repoint in all_repointing_numbers - ]), - create_mock_query_results([ - f'imap_glows_l3e_survival-probability-ul-sf_20250101-repoint{repoint:05d}_{Version(6, 4)}.cdf' for repoint in all_repointing_numbers - ]), - create_mock_query_results([ - f'imap_glows_l3e_survival-probability-ul-hf_20250101-repoint{repoint:05d}_{Version(7, 5)}.cdf' for repoint in all_repointing_numbers - ]) - ] - - result = identify_versions_for_l3e_output_files(start_cr_of_mission, end_cr_of_mission, - first_cr_updated_in_l3d, repointing_path, - version_map, ReprocessInfo({})) - - mock_imap_data_access.query.assert_has_calls([ - call(instrument='glows', data_level='l3e', version="latest", descriptor=GLOWS_L3E_HI_45_DESCRIPTOR), - call(instrument='glows', data_level='l3e', version="latest", descriptor=GLOWS_L3E_HI_90_DESCRIPTOR), - call(instrument='glows', data_level='l3e', version="latest", descriptor=GLOWS_L3E_LO_DESCRIPTOR), - call(instrument='glows', data_level='l3e', version="latest", descriptor=GLOWS_L3E_ULTRA_SF_DESCRIPTOR), - call(instrument='glows', data_level='l3e', version="latest", descriptor=GLOWS_L3E_ULTRA_HF_DESCRIPTOR) - ]) - - expected_versions_for_hi45_repoint_number = {repoint_number: Version(3, 2) for repoint_number in - updated_repointing_numbers} - expected_versions_for_hi90_repoint_number = {repoint_number: Version(4, 3) for repoint_number in - updated_repointing_numbers} - expected_versions_for_lo_repoint_number = {repoint_number: Version(5, 4) for repoint_number in - updated_repointing_numbers} - expected_versions_for_ultra_sf_repoint_number = {repoint_number: Version(6, 5) for repoint_number in - updated_repointing_numbers} - expected_versions_for_ultra_hf_repoint_number = {repoint_number: Version(7, 6) for repoint_number in - updated_repointing_numbers} - - self.assertCountEqual(updated_repointing_numbers, result.repointing_numbers) - self.assertEqual(expected_versions_for_hi90_repoint_number, result.hi_90_repointings) - self.assertEqual(expected_versions_for_hi45_repoint_number, result.hi_45_repointings) - self.assertEqual(expected_versions_for_lo_repoint_number, result.lo_repointings) - self.assertEqual(expected_versions_for_ultra_sf_repoint_number, result.ultra_sf_repointings) - self.assertEqual(expected_versions_for_ultra_hf_repoint_number, result.ultra_hf_repointings) - - @patch('imap_l3_processing.glows.l3e.glows_l3e_utils.get_repoint_data') - @patch('imap_l3_processing.glows.l3e.glows_l3e_utils.get_repoint_numbers_within_cr_window') - @patch('imap_l3_processing.glows.l3e.glows_l3e_utils.imap_data_access') - def test_identify_versions_for_l3e_output_files_uses_reprocess_info( - self, mock_imap_data_access, mock_get_repoint_numbers_within_cr_window, mock_get_repoint_data - ): - start_cr_of_mission = 2093 - end_cr_of_mission = 2095 - first_cr_updated_in_l3d = None - repointing_path = get_test_data_path("fake_1_day_repointing_file.csv") - version_map = VersionMap({desc: Version(3 + i, 5) for i, desc in enumerate(GLOWS_L3E_DESCRIPTORS)}) - repoint_number = 3700 - mock_reprocess_info = create_autospec(ReprocessInfo, instance=True) - mock_reprocess_info.get_repoints_for_descriptor.side_effect = [ - [], [], [repoint_number], [], [] - ] - - all_repointing_numbers = list(range(3682, 3763)) - updated_repointing_numbers = list() - - mock_get_repoint_numbers_within_cr_window.side_effect = [ - all_repointing_numbers, - updated_repointing_numbers - ] - - mock_imap_data_access.query.side_effect = [ - create_mock_query_results([ - f'imap_glows_l3e_survival-probability-hi-90_20250101-repoint{repoint:05d}_{Version(3, 1)}.cdf' for repoint in all_repointing_numbers - ]), - create_mock_query_results([ - f'imap_glows_l3e_survival-probability-hi-45_20250101-repoint{repoint:05d}_{Version(4, 2)}.cdf' for repoint in all_repointing_numbers - ]), - create_mock_query_results([ - f'imap_glows_l3e_survival-probability-lo_20250101-repoint{repoint:05d}_{Version(5, 3)}.cdf' for repoint in all_repointing_numbers - ]), - create_mock_query_results([ - f'imap_glows_l3e_survival-probability-ul-sf_20250101-repoint{repoint:05d}_{Version(6, 4)}.cdf' for repoint in all_repointing_numbers - ]), - create_mock_query_results([ - f'imap_glows_l3e_survival-probability-ul-hf_20250101-repoint{repoint:05d}_{Version(7, 5)}.cdf' for repoint in all_repointing_numbers - ]) - ] - - result = identify_versions_for_l3e_output_files( - start_cr_of_mission, - end_cr_of_mission, - first_cr_updated_in_l3d, - repointing_path, - version_map, - mock_reprocess_info, - ) - - mock_imap_data_access.query.assert_has_calls([ - call(instrument='glows', data_level='l3e', version="latest", descriptor=GLOWS_L3E_HI_45_DESCRIPTOR), - call(instrument='glows', data_level='l3e', version="latest", descriptor=GLOWS_L3E_HI_90_DESCRIPTOR), - call(instrument='glows', data_level='l3e', version="latest", descriptor=GLOWS_L3E_LO_DESCRIPTOR), - call(instrument='glows', data_level='l3e', version="latest", descriptor=GLOWS_L3E_ULTRA_SF_DESCRIPTOR), - call(instrument='glows', data_level='l3e', version="latest", descriptor=GLOWS_L3E_ULTRA_HF_DESCRIPTOR) - ]) - mock_reprocess_info.get_repoints_for_descriptor.assert_has_calls([ - call(GLOWS_L3E_HI_45_DESCRIPTOR, mock_get_repoint_data.return_value), - call(GLOWS_L3E_HI_90_DESCRIPTOR, mock_get_repoint_data.return_value), - call(GLOWS_L3E_LO_DESCRIPTOR, mock_get_repoint_data.return_value), - call(GLOWS_L3E_ULTRA_SF_DESCRIPTOR, mock_get_repoint_data.return_value), - call(GLOWS_L3E_ULTRA_HF_DESCRIPTOR, mock_get_repoint_data.return_value), - ]) - - expected_versions_for_lo_repoint_number = {repoint_number: Version(5, 4)} - - self.assertEqual([repoint_number], result.repointing_numbers) - self.assertEqual(expected_versions_for_lo_repoint_number, result.lo_repointings) - def test_calculate_energy_deltas(self): centers = np.array([10, 1000, 100000]) diff --git a/tests/glows/l3e/test_reprocess_info.py b/tests/glows/l3e/test_reprocess_info.py index b129aed5..92bf4c17 100644 --- a/tests/glows/l3e/test_reprocess_info.py +++ b/tests/glows/l3e/test_reprocess_info.py @@ -94,9 +94,9 @@ def test_get_repoints(self): repoints_for_cr2093 = list(range(3682, 3696)) repoints_for_cr2094 = list(range(3710, 3724)) - expected_repoints_lo = [2045, 2046] + repoints_for_cr2093 + [3700] + repoints_for_cr2094 - expected_repoints_hi = [2050] - expected_repoints_ultra = [] + expected_repoints_lo = set([2045, 2046] + repoints_for_cr2093 + [3700] + repoints_for_cr2094) + expected_repoints_hi = {2050} + expected_repoints_ultra = set() self.assertEqual(expected_repoints_lo, repoints_for_lo) self.assertEqual(expected_repoints_hi, repoints_for_hi) self.assertEqual(expected_repoints_ultra, repoints_for_ultra) From 9ec8235bf0b4b9835ff43206c905728a2adc9ada Mon Sep 17 00:00:00 2001 From: Menlo Innovations - CAVA Project Date: Thu, 1 Oct 2026 12:20:04 -0400 Subject: [PATCH 6/7] Harrison - SWYN - 3566 - GH #209 - Correct failing test after pulling logic to determine output filename into InputMetadata --- imap_l3_processing/models.py | 3 +-- tests/test_utils.py | 2 +- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/imap_l3_processing/models.py b/imap_l3_processing/models.py index 6b4861da..c90866d6 100644 --- a/imap_l3_processing/models.py +++ b/imap_l3_processing/models.py @@ -38,7 +38,7 @@ class InputMetadata: def logical_source(self): return f"imap_{self.instrument}_{self.data_level}_{self.descriptor}" - def to_science_file_path(self, cr_number: Optional[int] = None, extension: str = "cdf") -> ScienceFilePath: + def to_science_file_path(self, cr_number: Optional[int] = None) -> ScienceFilePath: assert self.repointing is None or cr_number is None, "You cannot call save_data with both a repointing in the metadata while passing in a CR number" formatted_start_date = self.start_date.strftime("%Y%m%d") version = self.version.lookup(self.descriptor) @@ -50,7 +50,6 @@ def to_science_file_path(self, cr_number: Optional[int] = None, extension: str = start_time=formatted_start_date, major_version=version.major, minor_version=version.minor, - extension=extension, repointing=self.repointing, cr=cr_number, ) diff --git a/tests/test_utils.py b/tests/test_utils.py index dfbed681..0036bc9e 100644 --- a/tests/test_utils.py +++ b/tests/test_utils.py @@ -59,7 +59,7 @@ def tearDown(self) -> None: @patch("imap_l3_processing.utils.ImapAttributeManager") @patch("imap_l3_processing.utils.date") @patch("imap_l3_processing.utils.write_cdf") - @patch("imap_l3_processing.utils.ScienceFilePath") + @patch("imap_l3_processing.models.ScienceFilePath") def test_save_data(self, mock_science_file_path_class, mock_write_cdf, mock_today, mock_attribute_manager): mock_today.today.return_value = date(2024, 9, 16) From 4e82002c8874c644004913d184140a3ac719a721 Mon Sep 17 00:00:00 2001 From: Menlo Innovations - CAVA Project Date: Thu, 1 Oct 2026 13:43:21 -0400 Subject: [PATCH 7/7] Harrison - SWYN/AFAH - 3566 - GH #209 - Fix an issue where L3d was not being uploaded if there was no L3e to produce. --- imap_l3_processing/glows/glows_processor.py | 3 +- tests/glows/test_glows_processor.py | 51 ++++++++++++--------- 2 files changed, 31 insertions(+), 23 deletions(-) diff --git a/imap_l3_processing/glows/glows_processor.py b/imap_l3_processing/glows/glows_processor.py index 1f7de1ec..01611bb1 100644 --- a/imap_l3_processing/glows/glows_processor.py +++ b/imap_l3_processing/glows/glows_processor.py @@ -105,6 +105,8 @@ def process(self): for txt_file in process_l3d_result.l3d_text_file_paths: logger.info(f"Saved L3d text file output to: {txt_file}") + products_list.extend([*process_l3d_result.l3d_text_file_paths, process_l3d_result.l3d_cdf_file_path]) + l3e_initializer_output = GlowsL3EInitializer.get_repointings_to_process( process_l3d_result, old_l3d, @@ -115,7 +117,6 @@ def process(self): if l3e_initializer_output is not None: logger.info(f"Processing L3e for repointings: {l3e_initializer_output.repointings.repointing_numbers}") - products_list.extend([*process_l3d_result.l3d_text_file_paths, process_l3d_result.l3d_cdf_file_path]) l3e_products = process_l3e(l3e_initializer_output) products_list.extend(l3e_products) else: diff --git a/tests/glows/test_glows_processor.py b/tests/glows/test_glows_processor.py index 58d10cbf..f524e15a 100644 --- a/tests/glows/test_glows_processor.py +++ b/tests/glows/test_glows_processor.py @@ -628,6 +628,11 @@ def test_l3bc_uses_all_l3a_file_names_for_l3b_parents(self, mock_save_data, mock self.assertEqual(set(expected_parents), set(l3b_data_product.parent_file_names)) + @patch("imap_l3_processing.glows.glows_processor.GlowsL3EInitializer.get_repointings_to_process") + def test_process_l3d_uploads_even_if_no_l3e_needs_to_be_produced(self, mock_l3e_initializer_get_repoints): + mock_l3e_initializer_get_repoints.return_value = None + self.test_process_l3d() + @patch("imap_l3_processing.glows.glows_processor.process_l3e") @patch("imap_l3_processing.glows.glows_processor.create_glows_l3b_json_file_from_cdf") @patch("imap_l3_processing.glows.glows_processor.create_glows_l3c_json_file_from_cdf") @@ -1141,31 +1146,11 @@ def test_process_l3e(self, mock_process_ultra, mock_process_ultra_hf, mock_proce repointing_midpoint = datetime(2020, 1, 1, 12) mock_get_pointing_date_range.return_value = (start_epoch, end_epoch) - mock_dependencies = Mock() - - mock_dependencies.get_hi_parents.return_value = ["hi_ancillary.dat"] - mock_dependencies.get_lo_parents.return_value = ["lo_ancillary.dat"] - mock_dependencies.get_ul_parents.return_value = ["ul_ancillary.dat"] - - l3d_cdf_path = Path("path/to/l3d.cdf") - initializer_data = GlowsL3EInitializerOutput( - dependencies=mock_dependencies, - repointings=GlowsL3eVersionsForRepointings( - repointing_numbers=[25], - hi_90_repointings={25: Version(None, 1)}, - hi_45_repointings={25: Version(None, 2)}, - lo_repointings={25: Version(None, 3)}, - ultra_sf_repointings={25: Version(None, 4)}, - ultra_hf_repointings={25: Version(None, 4)}, - ), - l3d_cdf_path=l3d_cdf_path, - metakernel_with_predict_ephem=Mock(), - metakernel_without_predict_ephem=Mock(), - ) + initializer_data = self._build_l3e_initializer_output() actual_l3e_products = process_l3e(initializer_data) mock_get_pointing_date_range.assert_called_once_with(25) - mock_compute_glows_flags_for_repoint.assert_called_once_with(l3d_cdf_path, repointing_midpoint) + mock_compute_glows_flags_for_repoint.assert_called_once_with(initializer_data.l3d_cdf_path, repointing_midpoint) mock_determine_spacecraft_info.assert_called_once_with( datetime(2020, 1, 1, 12), initializer_data.metakernel_with_predict_ephem, @@ -1936,6 +1921,28 @@ def test_archive_dependencies(self, mock_zip, mock_json): ]) mock_zip_file.writestr.assert_called_once_with(expected_json_filename, mock_json.dumps.return_value) + def _build_l3e_initializer_output(self) -> GlowsL3EInitializerOutput: + mock_dependencies = Mock() + + mock_dependencies.get_hi_parents.return_value = ["hi_ancillary.dat"] + mock_dependencies.get_lo_parents.return_value = ["lo_ancillary.dat"] + mock_dependencies.get_ul_parents.return_value = ["ul_ancillary.dat"] + + l3d_cdf_path = Path("path/to/l3d.cdf") + return GlowsL3EInitializerOutput( + dependencies=mock_dependencies, + repointings=GlowsL3eVersionsForRepointings( + repointing_numbers=[25], + hi_90_repointings={25: Version(None, 1)}, + hi_45_repointings={25: Version(None, 2)}, + lo_repointings={25: Version(None, 3)}, + ultra_sf_repointings={25: Version(None, 4)}, + ultra_hf_repointings={25: Version(None, 4)}, + ), + l3d_cdf_path=l3d_cdf_path, + metakernel_with_predict_ephem=Mock(), + metakernel_without_predict_ephem=Mock(), + ) if __name__ == '__main__':