diff --git a/imap_l3_processing/glows/glows_processor.py b/imap_l3_processing/glows/glows_processor.py index e72f5293..01611bb1 100644 --- a/imap_l3_processing/glows/glows_processor.py +++ b/imap_l3_processing/glows/glows_processor.py @@ -1,3 +1,4 @@ +import collections import json import logging import os @@ -26,7 +27,7 @@ from imap_l3_processing.glows.l3bc.glows_l3bc_dependencies import GlowsL3BCDependencies from imap_l3_processing.glows.l3bc.glows_l3bc_initializer import GlowsL3BCInitializer, GlowsL3BCInitializerData from imap_l3_processing.glows.l3bc.models import GlowsL3BIonizationRate, GlowsL3CSolarWind, GlowsL3BCProcessorOutput, \ - ExternalDependencies, read_pipeline_settings + ExternalDependencies from imap_l3_processing.glows.l3bc.science.filter_out_bad_days import filter_l3a_files from imap_l3_processing.glows.l3bc.science.generate_l3bc import generate_l3bc from imap_l3_processing.glows.l3bc.utils import get_pointing_date_range @@ -104,6 +105,8 @@ def process(self): for txt_file in process_l3d_result.l3d_text_file_paths: logger.info(f"Saved L3d text file output to: {txt_file}") + products_list.extend([*process_l3d_result.l3d_text_file_paths, process_l3d_result.l3d_cdf_file_path]) + l3e_initializer_output = GlowsL3EInitializer.get_repointings_to_process( process_l3d_result, old_l3d, @@ -114,7 +117,6 @@ def process(self): if l3e_initializer_output is not None: logger.info(f"Processing L3e for repointings: {l3e_initializer_output.repointings.repointing_numbers}") - products_list.extend([*process_l3d_result.l3d_text_file_paths, process_l3d_result.l3d_cdf_file_path]) l3e_products = process_l3e(l3e_initializer_output) products_list.extend(l3e_products) else: @@ -216,12 +218,19 @@ def process_l3d( dependencies: GlowsL3DDependencies, version: Version ) -> Optional[GlowsL3DProcessorOutput]: - [create_glows_l3b_json_file_from_cdf(l3b) for l3b in dependencies.l3b_file_paths] - [create_glows_l3c_json_file_from_cdf(l3c) for l3c in dependencies.l3c_file_paths] + collections.deque((create_glows_l3b_json_file_from_cdf(l3b) for l3b in dependencies.l3b_file_paths), maxlen=0) + collections.deque((create_glows_l3c_json_file_from_cdf(l3c) for l3c in dependencies.l3c_file_paths), maxlen=0) os.makedirs(PATH_TO_L3D_TOOLKIT / "data_l3d", exist_ok=True) os.makedirs(PATH_TO_L3D_TOOLKIT / "data_l3d_txt", exist_ok=True) + start_date = datetime(1947, 3, 3) + data_product_metadata = InputMetadata(instrument="glows", data_level="l3d", descriptor=GLOWS_L3D_DESCRIPTOR, + start_date=start_date, end_date=start_date, + version=VersionMap({GLOWS_L3D_DESCRIPTOR: version})) + + output_l3d_cdf_filename = data_product_metadata.to_science_file_path(cr_number=dependencies.end_cr).construct_path().name + file_manifest = { "external_files": { key: str(val) for key, val in dependencies.external_files.items() @@ -233,6 +242,7 @@ def process_l3d( for key, val in dependencies.ancillary_files["WawHelioIon"].items() }, }, + "l3d_cdf_filename": output_l3d_cdf_filename } last_processed_cr = None @@ -256,7 +266,7 @@ def process_l3d( if output.stdout: last_processed_cr = int(output.stdout.split('= ')[-1]) - if last_processed_cr: + if last_processed_cr == dependencies.end_cr: output_text_files = [] for text_file in os.listdir(PATH_TO_L3D_TOOLKIT / 'data_l3d_txt'): if str(last_processed_cr) in text_file: @@ -269,14 +279,11 @@ def process_l3d( file_name = f'imap_glows_l3d_solar-params-history_19470303-cr0{last_processed_cr}_v00.json' - start_date = datetime(1947, 3, 3) - data_product_metadata = InputMetadata(instrument="glows", data_level="l3d", descriptor=GLOWS_L3D_DESCRIPTOR, - start_date=start_date, end_date=start_date, version=VersionMap({GLOWS_L3D_DESCRIPTOR: version})) parent_file_names = get_parent_file_names_from_l3d_json(PATH_TO_L3D_TOOLKIT / 'data_l3d') l3d_data_product = convert_json_to_l3d_data_product(PATH_TO_L3D_TOOLKIT / 'data_l3d' / file_name, data_product_metadata, parent_file_names) - l3d_data_product_path = save_data(l3d_data_product, cr_number=last_processed_cr) + l3d_data_product_path = save_data(l3d_data_product, cr_number=dependencies.end_cr) return GlowsL3DProcessorOutput(l3d_data_product_path, txt_files_with_correct_version, last_processed_cr) return None diff --git a/imap_l3_processing/glows/l3d/glows_l3d_dependencies.py b/imap_l3_processing/glows/l3d/glows_l3d_dependencies.py index 873f068a..74a5330d 100644 --- a/imap_l3_processing/glows/l3d/glows_l3d_dependencies.py +++ b/imap_l3_processing/glows/l3d/glows_l3d_dependencies.py @@ -17,7 +17,7 @@ class GlowsL3DDependencies: ancillary_files: dict[str, Path | dict[str, Path]] l3b_file_paths: list[Path] l3c_file_paths: list[Path] - end_cr: float + end_cr: int @classmethod def fetch_dependencies(cls, dependencies: ProcessingInputCollection, external_dependencies: ExternalDependencies): diff --git a/imap_l3_processing/glows/l3d/science/generate_l3d.py b/imap_l3_processing/glows/l3d/science/generate_l3d.py index 0aa07b8f..17c53cab 100644 --- a/imap_l3_processing/glows/l3d/science/generate_l3d.py +++ b/imap_l3_processing/glows/l3d/science/generate_l3d.py @@ -22,6 +22,7 @@ ANC_INPUT_FROM_INSTRUMENT_TEAM = dependencies['ancillary_files'] EXT_DEPENDENCIES = dependencies['external_files'] +L3D_CDF_FILENAME = dependencies['l3d_cdf_filename'] # Create list of L3b/c files and read them l3b_fn_list = np.array(sorted(glob.glob('data_l3b/imap_glows_l3b*.json'))) @@ -35,7 +36,8 @@ # Start generating from the initial files (not using the last entry from L3d) -solar_param_hist.header['ancillary_data_files'] = list(fn_initial.values()) +solar_param_hist.header['ancillary_data_files'] = [*fn_initial.values(), ANC_INPUT_FROM_INSTRUMENT_TEAM['pipeline_settings']] +solar_param_hist.header['l3d_cdf_filename'] = L3D_CDF_FILENAME solar_param_hist.generate_initial_history(fn_initial) solar_param_hist.update_solar_params_hist(EXT_DEPENDENCIES, data_l3b, data_l3c, CR_current) diff --git a/imap_l3_processing/glows/l3d/science/toolkit/l3d_SolarParamHistory.py b/imap_l3_processing/glows/l3d/science/toolkit/l3d_SolarParamHistory.py index 21d3a875..58d7e2e8 100644 --- a/imap_l3_processing/glows/l3d/science/toolkit/l3d_SolarParamHistory.py +++ b/imap_l3_processing/glows/l3d/science/toolkit/l3d_SolarParamHistory.py @@ -65,10 +65,10 @@ def __init__(self,anc_input_from_instr_team,ext_dependencies): list of ancillary files used in calculations external_dependeciens : str list of external dependeciens files - l3b_input_filename: str - Input L3b ionization rate profiles file - l3c_input_filename: str - Input L3c solar wind speed and density profiles file + l3b_input_filename: list[str] + All the filenames of input L3b ionization rate profiles file + l3c_input_filename: list[str] + All the filenames of input L3c solar wind speed and density profiles file settings : dict pipeline settings ini_data: dict @@ -108,8 +108,8 @@ def __init__(self,anc_input_from_instr_team,ext_dependencies): 'filename': None, 'ancillary_data_files': None, 'external_dependeciens': ext_dependencies['lya_raw_data'], - 'l3b_input_filename': None, - 'l3c_input_filename': None + 'l3b_input_filename': [], + 'l3c_input_filename': [] } self.ini_data={} @@ -321,7 +321,7 @@ def _generate_hdr_template(self): hdr_temp[0]='lines in header: '+str(N)+'\n' hdr_temp[4]='ground software version: '+VERSION+'\n' hdr_temp[5]='creation date: '+ Time.now().iso +'\n' - hdr_temp[6]='input files: ' + ', '.join(self.header['l3b_input_filename']) + ', ' + ', '.join(self.header['l3c_input_filename']) +'\n' + hdr_temp[6]='input files: ' + self.header['l3d_cdf_filename'] + "\n" hdr_temp[7]='external dependencies: ' + self.header['external_dependeciens'] + '\n' hdr_temp[8]='last CR: '+ str(self.CR_last) + '\n' hdr_temp[9]='fill value: '+ str(self.settings['WawHelioIonGlows_fill_value']) + '\n' @@ -446,8 +446,8 @@ def _update_l3bc_data(self,data_l3b,data_l3c,CR): ''' cr_params, idx_read_b, idx_read_c = self._generate_cr_solar_params(CR, data_l3b, data_l3c) - self.header['l3b_input_filename']=[data_l3b[i]['header']['filename'] for i in idx_read_b] - self.header['l3c_input_filename']=[data_l3c[i]['header']['filename'] for i in idx_read_c] + self.header['l3b_input_filename'].extend(data_l3b[i]['header']['filename'] for i in idx_read_b) + self.header['l3c_input_filename'].extend(data_l3c[i]['header']['filename'] for i in idx_read_c) # add a row with values for current CR for k in self.ini_data['label']: diff --git a/imap_l3_processing/glows/l3e/glows_l3e_initializer.py b/imap_l3_processing/glows/l3e/glows_l3e_initializer.py index bf8b0436..0706ab50 100644 --- a/imap_l3_processing/glows/l3e/glows_l3e_initializer.py +++ b/imap_l3_processing/glows/l3e/glows_l3e_initializer.py @@ -1,25 +1,30 @@ +from __future__ import annotations + import logging from dataclasses import dataclass from pathlib import Path from typing import Optional import imap_data_access +import numpy as np from imap_data_access import ( ProcessingInputCollection, AncillaryInput, ScienceInput, RepointInput, ) +from imap_data_access.file_validation import Version +from imap_processing.spice.repoint import set_global_repoint_table_paths, get_repoint_data +from spacepy.pycdf import CDF +from imap_l3_processing.glows.descriptors import GLOWS_L3E_DESCRIPTORS, GLOWS_L3E_HI_90_DESCRIPTOR, \ + GLOWS_L3E_HI_45_DESCRIPTOR, GLOWS_L3E_LO_DESCRIPTOR, GLOWS_L3E_ULTRA_SF_DESCRIPTOR, GLOWS_L3E_ULTRA_HF_DESCRIPTOR from imap_l3_processing.glows.l3bc.utils import get_pointing_date_range from imap_l3_processing.glows.l3d.models import GlowsL3DProcessorOutput from imap_l3_processing.glows.l3d.utils import get_most_recently_uploaded_ancillary from imap_l3_processing.glows.l3e.glows_l3e_dependencies import GlowsL3EDependencies from imap_l3_processing.glows.l3e.glows_l3e_utils import ( - find_first_updated_cr, - identify_versions_for_l3e_output_files, - GlowsL3eVersionsForRepointings, -) + GlowsL3eVersionsForRepointings, get_repoint_numbers_within_cr_window, ) from imap_l3_processing.glows.l3e.reprocess_info import ReprocessInfo from imap_l3_processing.models import VersionMap from imap_l3_processing.utils import FurnishMetakernelOutput @@ -38,11 +43,11 @@ class GlowsL3EInitializerOutput: class GlowsL3EInitializer: @staticmethod def get_repointings_to_process( - l3d_output: GlowsL3DProcessorOutput, - previous_l3d: Optional[str], - repointing_file_path: Path, - version_map: VersionMap, - reprocess_info: ReprocessInfo, + l3d_output: GlowsL3DProcessorOutput, + previous_l3d: Optional[str], + repointing_file_path: Path, + version_map: VersionMap, + reprocess_info: ReprocessInfo, ) -> Optional[GlowsL3EInitializerOutput]: pipeline_settings_l3bcde = get_most_recently_uploaded_ancillary(imap_data_access.query(table='ancillary', instrument='glows', descriptor='pipeline-settings-l3bcde')) energy_grid_lo = get_most_recently_uploaded_ancillary(imap_data_access.query(table='ancillary', instrument='glows', descriptor='energy-grid-lo')) @@ -106,3 +111,95 @@ def get_repointings_to_process( metakernel_without_predict_ephem=furnished_metakernels[1], ) +def query_existing_l3es_versions(descriptor: str) -> dict[int, Version]: + existing_l3es_for_descriptor = {} + l3e_files = imap_data_access.query(instrument='glows', data_level='l3e', version="latest", descriptor=descriptor) + for l3e in l3e_files: + if 'major_version' in l3e.keys() and 'minor_version' in l3e.keys(): + existing_l3es_for_descriptor[int(l3e['repointing'])] = Version(l3e["major_version"], l3e["minor_version"]) + elif 'version' in l3e.keys(): + existing_l3es_for_descriptor[int(l3e['repointing'])] = Version.from_version(l3e['version']) + else: + continue + return existing_l3es_for_descriptor + +def identify_versions_for_l3e_output_files(start_cr_of_mission: int, end_cr_of_mission: int, first_updated_cr_from_l3d: Optional[int], + repointing_path: Path, version_map: VersionMap, reprocess_info: ReprocessInfo) -> GlowsL3eVersionsForRepointings: + + set_global_repoint_table_paths([repointing_path]) + repointing_data = get_repoint_data() + + all_pointing_numbers = get_repoint_numbers_within_cr_window(start_cr_of_mission, end_cr_of_mission, repointing_data) + pointing_numbers_updated_by_l3d = get_repoint_numbers_within_cr_window(first_updated_cr_from_l3d, end_cr_of_mission, repointing_data) + + updated_pointings_per_instruments = {} + updated_pointing_numbers = {} + + for descriptor in GLOWS_L3E_DESCRIPTORS: + new_major_version = version_map.lookup(descriptor).major + + repointings_to_force_processing = reprocess_info.get_repoints_for_descriptor( + descriptor, repointing_data + ) + + existing_file_versions = query_existing_l3es_versions(descriptor) + out_of_date_repointings = find_out_of_date_l3es(existing_file_versions, all_pointing_numbers, new_major_version) + + repointings_to_process = pointing_numbers_updated_by_l3d | repointings_to_force_processing | out_of_date_repointings + + new_file_versions = {} + for pointing_number in sorted(repointings_to_process): + if previous_version := existing_file_versions.get(pointing_number): + new_version = Version(new_major_version, previous_version.minor + 1) + else: + new_version = Version(new_major_version, 1) + new_file_versions[pointing_number] = new_version + + updated_pointings_per_instruments[descriptor] = new_file_versions + updated_pointing_numbers = updated_pointing_numbers | new_file_versions.keys() + + + return GlowsL3eVersionsForRepointings(list(updated_pointing_numbers), + updated_pointings_per_instruments[GLOWS_L3E_HI_90_DESCRIPTOR], + updated_pointings_per_instruments[GLOWS_L3E_HI_45_DESCRIPTOR], + updated_pointings_per_instruments[GLOWS_L3E_LO_DESCRIPTOR], + updated_pointings_per_instruments[GLOWS_L3E_ULTRA_SF_DESCRIPTOR], + updated_pointings_per_instruments[GLOWS_L3E_ULTRA_HF_DESCRIPTOR], + ) + +def find_out_of_date_l3es(existing_l3es: dict[int, Version], repointings_to_check: set[int], current_major_version: int) -> set[int]: + out_of_date_l3es = set() + for repointing in repointings_to_check: + if existing_l3e_version := existing_l3es.get(repointing): + if existing_l3e_version.major < current_major_version: + out_of_date_l3es.add(repointing) + else: + out_of_date_l3es.add(repointing) + return out_of_date_l3es + + +def find_first_updated_cr(new_l3d: Path, old_l3d: str) -> Optional[int]: + downloaded_old_l3d = imap_data_access.download(old_l3d) + + old_l3d_cdf = CDF(str(downloaded_old_l3d)) + new_l3d_cdf = CDF(str(new_l3d)) + + for i, cr in enumerate(old_l3d_cdf['cr_grid'][...]): + lya_matches = np.isclose(old_l3d_cdf['lyman_alpha'][i], new_l3d_cdf['lyman_alpha'][i]) + phion_matches = np.isclose(old_l3d_cdf['phion'][i], new_l3d_cdf['phion'][i]) + plasma_speed_flag_matches = np.isclose(old_l3d_cdf['plasma_speed_flag'][i], new_l3d_cdf['plasma_speed_flag'][i]) + proton_density_flag_matches = np.isclose(old_l3d_cdf['proton_density_flag'][i], new_l3d_cdf['proton_density_flag'][i]) + uv_anisotropy_flag_matches = np.isclose(old_l3d_cdf['uv_anisotropy_flag'][i], new_l3d_cdf['uv_anisotropy_flag'][i]) + glows_flags_matches = np.isclose(old_l3d_cdf['glows_flags'][i], new_l3d_cdf['glows_flags'][i]) + + plasma_speed_matches = np.all(np.isclose(old_l3d_cdf['plasma_speed'][i], new_l3d_cdf['plasma_speed'][i])) + proton_density_matches = np.all(np.isclose(old_l3d_cdf['proton_density'][i], new_l3d_cdf['proton_density'][i])) + uv_anisotropy_matches = np.all(np.isclose(old_l3d_cdf['uv_anisotropy'][i], new_l3d_cdf['uv_anisotropy'][i])) + + if np.any(np.logical_not([lya_matches, phion_matches, plasma_speed_matches, plasma_speed_flag_matches, proton_density_matches, proton_density_flag_matches, uv_anisotropy_matches, uv_anisotropy_flag_matches, glows_flags_matches])): + return int(cr) + + if old_l3d_cdf['cr_grid'].shape != new_l3d_cdf['cr_grid'].shape: + return int(old_l3d_cdf['cr_grid'][-1]) + 1 + + return None diff --git a/imap_l3_processing/glows/l3e/glows_l3e_utils.py b/imap_l3_processing/glows/l3e/glows_l3e_utils.py index 480849a2..b4495199 100644 --- a/imap_l3_processing/glows/l3e/glows_l3e_utils.py +++ b/imap_l3_processing/glows/l3e/glows_l3e_utils.py @@ -13,21 +13,17 @@ from imap_l3_processing.glows.quality_flags import GlowsL3Flags from imap_data_access.file_validation import Version -from imap_processing.spice.repoint import set_global_repoint_table_paths, get_repoint_data from spacepy.pycdf import CDF from typing import Optional from imap_l3_processing.constants import ONE_AU_IN_KM, TT2000_EPOCH, ONE_SECOND_IN_NANOSECONDS -from imap_l3_processing.glows.descriptors import GLOWS_L3E_ULTRA_HF_DESCRIPTOR, GLOWS_L3E_ULTRA_SF_DESCRIPTOR, \ - GLOWS_L3E_LO_DESCRIPTOR, GLOWS_L3E_HI_45_DESCRIPTOR, GLOWS_L3E_HI_90_DESCRIPTOR, GLOWS_L3E_DESCRIPTORS from imap_l3_processing.glows.l3bc.l3bc_toolkit.funcs import jd_fm_Carrington from imap_l3_processing.glows.l3e.glows_l3e_call_arguments import GlowsL3eCallArguments, GlowsL3eSpacecraftInfo from imap_l3_processing.utils import FurnishMetakernelOutput -from imap_l3_processing.models import VersionMap if typing.TYPE_CHECKING: - from imap_l3_processing.glows.l3e.reprocess_info import ReprocessInfo + pass def determine_call_args_for_l3e_executable(start_date: datetime, repointing_midpoint: datetime, elongation: float, spacecraft_info: GlowsL3eSpacecraftInfo) -> GlowsL3eCallArguments: @@ -99,59 +95,6 @@ class GlowsL3eVersionsForRepointings: ultra_sf_repointings: dict[int, Version] ultra_hf_repointings: dict[int, Version] -def identify_versions_for_l3e_output_files(start_cr_of_mission: int, end_cr_of_mission: int, first_updated_cr_from_l3d: Optional[int], - repointing_path: Path, version_map: VersionMap, reprocess_info: ReprocessInfo) -> GlowsL3eVersionsForRepointings: - - set_global_repoint_table_paths([repointing_path]) - repointing_data = get_repoint_data() - - all_pointing_numbers = get_repoint_numbers_within_cr_window(start_cr_of_mission, end_cr_of_mission, repointing_data) - pointing_numbers_updated_by_l3d = get_repoint_numbers_within_cr_window(first_updated_cr_from_l3d, end_cr_of_mission, repointing_data) - - updated_pointings_per_instruments = {} - updated_pointing_numbers = {} - for descriptor in GLOWS_L3E_DESCRIPTORS: - repointings_to_force_processing = reprocess_info.get_repoints_for_descriptor( - descriptor, repointing_data - ) - repointings_to_process = set(pointing_numbers_updated_by_l3d + repointings_to_force_processing) - l3e_files = imap_data_access.query(instrument='glows', data_level='l3e', version="latest", descriptor=descriptor) - existing_file_versions = {} - for l3e in l3e_files: - if 'major_version' in l3e.keys() and 'minor_version' in l3e.keys(): - existing_file_versions[int(l3e['repointing'])] = Version(l3e["major_version"], l3e["minor_version"]) - elif 'version' in l3e.keys(): - existing_file_versions[int(l3e['repointing'])] = Version.from_version(l3e['version']) - else: - continue - new_file_versions = {} - for pointing_number in all_pointing_numbers: - new_major_version = version_map.lookup(descriptor).major - if pointing_number in existing_file_versions: - previous_version = existing_file_versions[pointing_number] - higher_major_version_given = new_major_version > previous_version.major - - if ( - higher_major_version_given - or pointing_number in repointings_to_process - ): - new_file_versions[pointing_number] = Version(new_major_version, previous_version.minor + 1) - - else: - new_file_versions[pointing_number] = Version(new_major_version, 1) - - updated_pointings_per_instruments[descriptor] = new_file_versions - updated_pointing_numbers = updated_pointing_numbers | new_file_versions.keys() - - - return GlowsL3eVersionsForRepointings(list(updated_pointing_numbers), - updated_pointings_per_instruments[GLOWS_L3E_HI_90_DESCRIPTOR], - updated_pointings_per_instruments[GLOWS_L3E_HI_45_DESCRIPTOR], - updated_pointings_per_instruments[GLOWS_L3E_LO_DESCRIPTOR], - updated_pointings_per_instruments[GLOWS_L3E_ULTRA_SF_DESCRIPTOR], - updated_pointings_per_instruments[GLOWS_L3E_ULTRA_HF_DESCRIPTOR], - ) - def compute_glows_flags_for_repoint(l3d_cdf_path: Path, repoint_midpoint: datetime) -> int: with CDF(str(l3d_cdf_path)) as cdf: @@ -172,33 +115,6 @@ def compute_glows_flags_for_repoint(l3d_cdf_path: Path, repoint_midpoint: dateti return int(np.bitwise_or.reduce(selected.astype(np.uint16), initial=0)) - -def find_first_updated_cr(new_l3d: Path, old_l3d: str) -> Optional[int]: - downloaded_old_l3d = imap_data_access.download(old_l3d) - - old_l3d_cdf = CDF(str(downloaded_old_l3d)) - new_l3d_cdf = CDF(str(new_l3d)) - - for i, cr in enumerate(old_l3d_cdf['cr_grid'][...]): - lya_matches = np.isclose(old_l3d_cdf['lyman_alpha'][i], new_l3d_cdf['lyman_alpha'][i]) - phion_matches = np.isclose(old_l3d_cdf['phion'][i], new_l3d_cdf['phion'][i]) - plasma_speed_flag_matches = np.isclose(old_l3d_cdf['plasma_speed_flag'][i], new_l3d_cdf['plasma_speed_flag'][i]) - proton_density_flag_matches = np.isclose(old_l3d_cdf['proton_density_flag'][i], new_l3d_cdf['proton_density_flag'][i]) - uv_anisotropy_flag_matches = np.isclose(old_l3d_cdf['uv_anisotropy_flag'][i], new_l3d_cdf['uv_anisotropy_flag'][i]) - glows_flags_matches = np.isclose(old_l3d_cdf['glows_flags'][i], new_l3d_cdf['glows_flags'][i]) - - plasma_speed_matches = np.all(np.isclose(old_l3d_cdf['plasma_speed'][i], new_l3d_cdf['plasma_speed'][i])) - proton_density_matches = np.all(np.isclose(old_l3d_cdf['proton_density'][i], new_l3d_cdf['proton_density'][i])) - uv_anisotropy_matches = np.all(np.isclose(old_l3d_cdf['uv_anisotropy'][i], new_l3d_cdf['uv_anisotropy'][i])) - - if np.any(np.logical_not([lya_matches, phion_matches, plasma_speed_matches, plasma_speed_flag_matches, proton_density_matches, proton_density_flag_matches, uv_anisotropy_matches, uv_anisotropy_flag_matches, glows_flags_matches])): - return int(cr) - - if old_l3d_cdf['cr_grid'].shape != new_l3d_cdf['cr_grid'].shape: - return int(old_l3d_cdf['cr_grid'][-1]) + 1 - - return None - def get_lo_pivot_angle_from_l1b_file(path: Path) -> float: with CDF(str(path)) as cdf: epoch = cdf['epoch'][...] @@ -236,9 +152,9 @@ def get_lo_pivot_angles(repointings: list[int]) -> dict[int, LoPivotAngle]: result[repointing] = LoPivotAngle(parent_filename=None, pivot_angle=90.0) return result -def get_repoint_numbers_within_cr_window(start_cr_number: int | None, end_cr_number: int, repointing_data) -> list[int]: +def get_repoint_numbers_within_cr_window(start_cr_number: int | None, end_cr_number: int, repointing_data) -> set[int]: if start_cr_number is None: - return [] + return set() first_carrington_start_date = Time(jd_fm_Carrington(float(start_cr_number)), format='jd') last_cr_end_date = Time(jd_fm_Carrington(float(end_cr_number + 0.5)), format='jd') @@ -251,10 +167,10 @@ def get_repoint_numbers_within_cr_window(start_cr_number: int | None, end_cr_num repoint_ends = vectorized_date_conv(repointing_data["repoint_end_utc"]) repoint_ids = repointing_data["repoint_id"] - repoint_numbers = [] + repoint_numbers = set() for i in range(len(repoint_ids)): if i + 1 < len(repoint_ids) and start_ns < (repoint_starts[i + 1] + repoint_ends[i])/2 < end_ns: - repoint_numbers.append(int(repoint_ids[i])) + repoint_numbers.add(int(repoint_ids[i])) return repoint_numbers diff --git a/imap_l3_processing/glows/l3e/reprocess_info.py b/imap_l3_processing/glows/l3e/reprocess_info.py index efea8156..4583edd8 100644 --- a/imap_l3_processing/glows/l3e/reprocess_info.py +++ b/imap_l3_processing/glows/l3e/reprocess_info.py @@ -79,17 +79,17 @@ def should_reprocess_l3d(self) -> bool: def get_repoints_for_descriptor( self, descriptor: str, repointing_data - ) -> list[int]: + ) -> set[int]: reprocess_targets = self.products_to_reprocess.get(descriptor, None) if reprocess_targets is None: - return [] + return set() repoints = set(reprocess_targets.repoints) for cr in reprocess_targets.carrington_rotations: repoints.update(get_repoint_numbers_within_cr_window(cr, cr, repointing_data)) - return sorted(repoints) + return repoints def fetch_reprocess_info(input_collection: ProcessingInputCollection) -> ReprocessInfo: [reprocessing_file_path] = input_collection.get_file_paths(source="glows", descriptor=GLOWS_REPROCESSING_DESCRIPTOR) diff --git a/imap_l3_processing/models.py b/imap_l3_processing/models.py index 6b1b1b47..c90866d6 100644 --- a/imap_l3_processing/models.py +++ b/imap_l3_processing/models.py @@ -7,7 +7,7 @@ from typing import Union, Optional, TypeVar, Generic import numpy as np -from imap_data_access.file_validation import Version +from imap_data_access.file_validation import Version, ScienceFilePath from imap_l3_processing.data_utils import rebin @@ -38,6 +38,21 @@ class InputMetadata: def logical_source(self): return f"imap_{self.instrument}_{self.data_level}_{self.descriptor}" + def to_science_file_path(self, cr_number: Optional[int] = None) -> ScienceFilePath: + assert self.repointing is None or cr_number is None, "You cannot call save_data with both a repointing in the metadata while passing in a CR number" + formatted_start_date = self.start_date.strftime("%Y%m%d") + version = self.version.lookup(self.descriptor) + + return ScienceFilePath.generate_from_inputs( + instrument=self.instrument, + data_level=self.data_level, + descriptor=self.descriptor, + start_time=formatted_start_date, + major_version=version.major, + minor_version=version.minor, + repointing=self.repointing, + cr=cr_number, + ) @dataclass class DataProductVariable: diff --git a/imap_l3_processing/utils.py b/imap_l3_processing/utils.py index 0c8c8a89..48016f00 100644 --- a/imap_l3_processing/utils.py +++ b/imap_l3_processing/utils.py @@ -66,20 +66,7 @@ class SpiceKernelTypes(enum.Enum): def save_data(data: DataProduct, delete_if_present: bool = False, folder_path: Path = None, cr_number=None) -> Path: - assert data.input_metadata.repointing is None or cr_number is None, "You cannot call save_data with both a repointing in the metadata while passing in a CR number" - formatted_start_date = data.input_metadata.start_date.strftime("%Y%m%d") - version = data.input_metadata.version.lookup(data.input_metadata.descriptor) - science_file_path = ScienceFilePath.generate_from_inputs( - instrument=data.input_metadata.instrument, - data_level=data.input_metadata.data_level, - descriptor=data.input_metadata.descriptor, - start_time=formatted_start_date, - repointing=data.input_metadata.repointing, - cr=cr_number, - major_version=version.major, - minor_version=version.minor, - ) - + science_file_path = data.input_metadata.to_science_file_path(cr_number=cr_number) file_path = science_file_path.construct_path() if folder_path is not None: file_path = folder_path / file_path.name diff --git a/tests/glows/l3e/test_glows_l3e_initializer.py b/tests/glows/l3e/test_glows_l3e_initializer.py index b3e6ca80..b6cd0aac 100644 --- a/tests/glows/l3e/test_glows_l3e_initializer.py +++ b/tests/glows/l3e/test_glows_l3e_initializer.py @@ -1,17 +1,22 @@ import unittest from datetime import datetime from pathlib import Path -from unittest.mock import patch, call, sentinel, Mock +from unittest.mock import patch, call, sentinel, Mock, create_autospec +import numpy as np from imap_data_access import RepointInput from imap_data_access.file_validation import Version -from imap_l3_processing.glows.descriptors import GLOWS_L3BCDE_DESCRIPTORS +from imap_l3_processing.glows.descriptors import GLOWS_L3BCDE_DESCRIPTORS, GLOWS_L3E_HI_45_DESCRIPTOR, \ + GLOWS_L3E_HI_90_DESCRIPTOR, GLOWS_L3E_LO_DESCRIPTOR, GLOWS_L3E_ULTRA_SF_DESCRIPTOR, GLOWS_L3E_ULTRA_HF_DESCRIPTOR, \ + GLOWS_L3E_DESCRIPTORS from imap_l3_processing.glows.l3d.models import GlowsL3DProcessorOutput -from imap_l3_processing.glows.l3e.glows_l3e_initializer import GlowsL3EInitializer, GlowsL3EInitializerOutput +from imap_l3_processing.glows.l3e.glows_l3e_initializer import GlowsL3EInitializer, GlowsL3EInitializerOutput, \ + find_first_updated_cr, identify_versions_for_l3e_output_files from imap_l3_processing.glows.l3e.glows_l3e_utils import GlowsL3eVersionsForRepointings +from imap_l3_processing.glows.l3e.reprocess_info import ReprocessInfo from imap_l3_processing.models import VersionMap -from tests.test_helpers import create_mock_query_results +from tests.test_helpers import create_mock_query_results, get_test_data_path MODULE = 'imap_l3_processing.glows.l3e.glows_l3e_initializer' @@ -265,3 +270,372 @@ def test_get_repointings_to_process_uses_mission_start_cr_when_no_previous_l3d(s sentinel.version_map, sentinel.reprocess_info, ) + + @patch('imap_l3_processing.glows.l3e.glows_l3e_initializer.imap_data_access.download') + @patch('imap_l3_processing.glows.l3e.glows_l3e_initializer.CDF') + def test_find_first_updated_cr(self, mock_CDF, mock_download): + num_crs = 10 + mock_download.return_value = sentinel.download_old_l3d + + old_l3d = { + 'cr_grid': np.arange(num_crs) + 0.5, + 'lyman_alpha': np.arange(num_crs), + 'phion': np.arange(num_crs), + 'plasma_speed': np.arange(0, 20).reshape((num_crs, 2)), + 'plasma_speed_flag': np.arange(num_crs), + 'proton_density': np.arange(0, 20).reshape((num_crs, 2)), + 'proton_density_flag': np.arange(num_crs), + 'uv_anisotropy': np.arange(0, 20).reshape((num_crs, 2)), + 'uv_anisotropy_flag': np.arange(num_crs), + 'glows_flags': np.arange(num_crs), + } + + new_lyman_alpha = np.arange(num_crs) + new_lyman_alpha[1] = 10 + + new_phion = np.arange(num_crs) + new_phion[2] = 10 + + new_plasma_speed_flag = np.arange(num_crs) + new_plasma_speed_flag[3] = 10 + + new_proton_density_flag = np.arange(num_crs) + new_proton_density_flag[4] = 10 + + new_uv_anisotropy_flag = np.arange(num_crs) + new_uv_anisotropy_flag[5] = 10 + + new_plasma_speed = np.arange(0, 20).reshape((num_crs, 2)) + new_plasma_speed[6, :] = 10 + + new_proton_density = np.arange(0, 20).reshape((num_crs, 2)) + new_proton_density[7, :] = 10 + + new_uv_anisotropy = np.arange(0, 20).reshape((num_crs, 2)) + new_uv_anisotropy[8, :] = 10 + + new_glows_flags = np.arange(num_crs) + new_glows_flags[9] = 10 + + cases = [ + ('cr_grid', np.append(old_l3d['cr_grid'], 10.5), 10), + ('lyman_alpha', new_lyman_alpha, 1), + ('phion', new_phion, 2), + ('plasma_speed_flag', new_plasma_speed_flag, 3), + ('proton_density_flag', new_proton_density_flag, 4), + ('uv_anisotropy_flag', new_uv_anisotropy_flag, 5), + + ('plasma_speed', new_plasma_speed, 6), + ('proton_density', new_proton_density, 7), + ('uv_anisotropy', new_uv_anisotropy, 8), + ('glows_flags', new_glows_flags, 9), + ('no_change', None, None) + ] + + for case, change, expected in cases: + mock_download.reset_mock() + mock_CDF.reset_mock() + + with self.subTest(case=case): + new_l3d = {**old_l3d} + if case != "no_change": + new_l3d[case] = change + + mock_CDF.side_effect = [old_l3d, new_l3d] + + actual_cr = find_first_updated_cr( + sentinel.new_l3d_path, sentinel.old_l3d_filename + ) + + mock_download.assert_called_once_with(sentinel.old_l3d_filename) + mock_CDF.assert_has_calls( + [ + call(str(sentinel.download_old_l3d)), + call(str(sentinel.new_l3d_path)), + ] + ) + + self.assertEqual(actual_cr, expected) + + @patch('imap_l3_processing.glows.l3e.glows_l3e_initializer.get_repoint_numbers_within_cr_window') + @patch('imap_l3_processing.glows.l3e.glows_l3e_initializer.imap_data_access') + def test_identify_versions_for_l3e_output_files_gives_minor_version_1_for_non_existing_l3e(self, + mock_imap_data_access, + mock_get_repoint_numbers_within_cr_window): + start_cr_of_mission = 2093 + end_cr_of_mission = 2094 + first_cr_updated_in_l3d = None + repointing_path = get_test_data_path("fake_1_day_repointing_file.csv") + version_map = VersionMap({desc: Version(3 + i, 5) for i, desc in enumerate(GLOWS_L3E_DESCRIPTORS)}) + + mock_imap_data_access.query.side_effect = [ + create_mock_query_results([]), + create_mock_query_results([]), + create_mock_query_results([]), + create_mock_query_results([]), + create_mock_query_results([]), + ] + + all_repointing_numbers = set(range(3682, 3736)) + updated_repointing_numbers = set() + mock_get_repoint_numbers_within_cr_window.side_effect = [ + all_repointing_numbers, + updated_repointing_numbers + ] + reprocess_info = ReprocessInfo({}) + + result = identify_versions_for_l3e_output_files(start_cr_of_mission, end_cr_of_mission, first_cr_updated_in_l3d, + repointing_path, version_map, reprocess_info) + + mock_imap_data_access.query.assert_has_calls([ + call(instrument='glows', data_level='l3e', version="latest", descriptor=GLOWS_L3E_HI_45_DESCRIPTOR), + call(instrument='glows', data_level='l3e', version="latest", descriptor=GLOWS_L3E_HI_90_DESCRIPTOR), + call(instrument='glows', data_level='l3e', version="latest", descriptor=GLOWS_L3E_LO_DESCRIPTOR), + call(instrument='glows', data_level='l3e', version="latest", descriptor=GLOWS_L3E_ULTRA_SF_DESCRIPTOR), + call(instrument='glows', data_level='l3e', version="latest", descriptor=GLOWS_L3E_ULTRA_HF_DESCRIPTOR) + ]) + + expected_versions_for_hi45_repoint_number = {repoint_number: Version(3, 1) for repoint_number in + all_repointing_numbers} + expected_versions_for_hi90_repoint_number = {repoint_number: Version(4, 1) for repoint_number in + all_repointing_numbers} + expected_versions_for_lo_repoint_number = {repoint_number: Version(5, 1) for repoint_number in + all_repointing_numbers} + expected_versions_for_ultra_sf_repoint_number = {repoint_number: Version(6, 1) for repoint_number in + all_repointing_numbers} + expected_versions_for_ultra_hf_repoint_number = {repoint_number: Version(7, 1) for repoint_number in + all_repointing_numbers} + + self.assertCountEqual(all_repointing_numbers, result.repointing_numbers) + self.assertEqual(expected_versions_for_hi90_repoint_number, result.hi_90_repointings) + self.assertEqual(expected_versions_for_hi45_repoint_number, result.hi_45_repointings) + self.assertEqual(expected_versions_for_lo_repoint_number, result.lo_repointings) + self.assertEqual( + expected_versions_for_ultra_sf_repoint_number, result.ultra_sf_repointings + ) + self.assertEqual(expected_versions_for_ultra_hf_repoint_number, result.ultra_hf_repointings) + + @patch('imap_l3_processing.glows.l3e.glows_l3e_initializer.get_repoint_numbers_within_cr_window') + @patch('imap_l3_processing.glows.l3e.glows_l3e_initializer.imap_data_access') + def test_identify_versions_for_l3e_output_files_increments_major_and_minor_when_given_higher_major_version(self, + mock_imap_data_access, + mock_get_repoint_numbers_within_cr_window): + start_cr_of_mission = 2093 + end_cr_of_mission = 2094 + first_cr_updated_in_l3d = None + repointing_path = get_test_data_path("fake_1_day_repointing_file.csv") + version_map = VersionMap({desc: Version(3 + i, 5) for i, desc in enumerate(GLOWS_L3E_DESCRIPTORS)}) + + all_repointing_numbers = set(range(3682, 3736)) + updated_repointing_numbers = set() + old_major_version = 2 + mock_get_repoint_numbers_within_cr_window.reset_mock() + mock_imap_data_access.reset_mock() + + mock_get_repoint_numbers_within_cr_window.side_effect = [ + all_repointing_numbers, + updated_repointing_numbers + ] + + mock_imap_data_access.query.side_effect = [ + create_mock_query_results([ + f'imap_glows_l3e_survival-probability-hi-90_20250101-repoint03682_{Version(old_major_version, 1)}.cdf', + f'imap_glows_l3e_survival-probability-hi-90_20250101-repoint03683_{Version(3, 1)}.cdf', + f'imap_glows_l3e_survival-probability-hi-90_20250101-repoint03735_{Version(3, 1)}.cdf' + ]), + create_mock_query_results([ + f'imap_glows_l3e_survival-probability-hi-45_20250101-repoint03683_{Version(old_major_version, 2)}.cdf', + f'imap_glows_l3e_survival-probability-hi-45_20250101-repoint03684_{Version(4, 2)}.cdf', + f'imap_glows_l3e_survival-probability-hi-45_20250101-repoint03735_{Version(4, 2)}.cdf' + ]), + create_mock_query_results([ + f'imap_glows_l3e_survival-probability-lo_20250101-repoint03684_{Version(old_major_version, 3)}.cdf', + f'imap_glows_l3e_survival-probability-lo_20250101-repoint03685_{Version(5, 3)}.cdf', + f'imap_glows_l3e_survival-probability-lo_20250101-repoint03735_{Version(5, 3)}.cdf' + ]), + create_mock_query_results([ + f'imap_glows_l3e_survival-probability-ul-sf_20250101-repoint03685_{Version(old_major_version, 4)}.cdf', + f'imap_glows_l3e_survival-probability-ul-sf_20250101-repoint03686_{Version(6, 4)}.cdf', + f'imap_glows_l3e_survival-probability-ul-sf_20250101-repoint03735_{Version(6, 4)}.cdf' + ]), + create_mock_query_results([ + f'imap_glows_l3e_survival-probability-ul-hf_20250101-repoint03686_{Version(old_major_version, 5)}.cdf', + f'imap_glows_l3e_survival-probability-ul-hf_20250101-repoint03687_{Version(7, 5)}.cdf', + f'imap_glows_l3e_survival-probability-ul-hf_20250101-repoint03735_{Version(7, 5)}.cdf' + ]) + ] + + result = identify_versions_for_l3e_output_files(start_cr_of_mission, end_cr_of_mission, + first_cr_updated_in_l3d, repointing_path, + version_map, ReprocessInfo({})) + + mock_imap_data_access.query.assert_has_calls([ + call(instrument='glows', data_level='l3e', version="latest", descriptor=GLOWS_L3E_HI_45_DESCRIPTOR), + call(instrument='glows', data_level='l3e', version="latest", descriptor=GLOWS_L3E_HI_90_DESCRIPTOR), + call(instrument='glows', data_level='l3e', version="latest", descriptor=GLOWS_L3E_LO_DESCRIPTOR), + call(instrument='glows', data_level='l3e', version="latest", descriptor=GLOWS_L3E_ULTRA_SF_DESCRIPTOR), + call(instrument='glows', data_level='l3e', version="latest", descriptor=GLOWS_L3E_ULTRA_HF_DESCRIPTOR) + ]) + + self.assertCountEqual(list(range(3682, 3735)), result.repointing_numbers) + + self.assertNotIn(3683, result.hi_45_repointings) + self.assertNotIn(3684, result.hi_90_repointings) + self.assertNotIn(3685, result.lo_repointings) + self.assertNotIn(3686, result.ultra_sf_repointings) + self.assertNotIn(3687, result.ultra_hf_repointings) + + self.assertEqual(Version(3, 2), result.hi_45_repointings[3682]) + self.assertEqual(Version(4, 3), result.hi_90_repointings[3683]) + self.assertEqual(Version(5, 4), result.lo_repointings[3684]) + self.assertEqual(Version(6, 5), result.ultra_sf_repointings[3685]) + self.assertEqual(Version(7, 6), result.ultra_hf_repointings[3686]) + + @patch('imap_l3_processing.glows.l3e.glows_l3e_initializer.get_repoint_numbers_within_cr_window') + @patch('imap_l3_processing.glows.l3e.glows_l3e_initializer.imap_data_access') + def test_identify_versions_for_l3e_output_files_increments_minor_when_same_major_and_updated_l3d_covers_pointing( + self, mock_imap_data_access, mock_get_repoint_numbers_within_cr_window + ): + start_cr_of_mission = 2093 + end_cr_of_mission = 2095 + first_cr_updated_in_l3d = None + repointing_path = get_test_data_path("fake_1_day_repointing_file.csv") + version_map = VersionMap({desc: Version(3 + i, 5) for i, desc in enumerate(GLOWS_L3E_DESCRIPTORS)}) + + all_repointing_numbers = set(range(3682, 3763)) + updated_repointing_numbers = set(range(3709, 3763)) + + mock_get_repoint_numbers_within_cr_window.side_effect = [ + all_repointing_numbers, + updated_repointing_numbers + ] + + mock_imap_data_access.query.side_effect = [ + create_mock_query_results([ + f'imap_glows_l3e_survival-probability-hi-90_20250101-repoint{repoint:05d}_{Version(3, 1)}.cdf' for + repoint in all_repointing_numbers + ]), + create_mock_query_results([ + f'imap_glows_l3e_survival-probability-hi-45_20250101-repoint{repoint:05d}_{Version(4, 2)}.cdf' for + repoint in all_repointing_numbers + ]), + create_mock_query_results([ + f'imap_glows_l3e_survival-probability-lo_20250101-repoint{repoint:05d}_{Version(5, 3)}.cdf' for repoint + in all_repointing_numbers + ]), + create_mock_query_results([ + f'imap_glows_l3e_survival-probability-ul-sf_20250101-repoint{repoint:05d}_{Version(6, 4)}.cdf' for + repoint in all_repointing_numbers + ]), + create_mock_query_results([ + f'imap_glows_l3e_survival-probability-ul-hf_20250101-repoint{repoint:05d}_{Version(7, 5)}.cdf' for + repoint in all_repointing_numbers + ]) + ] + + result = identify_versions_for_l3e_output_files(start_cr_of_mission, end_cr_of_mission, + first_cr_updated_in_l3d, repointing_path, + version_map, ReprocessInfo({})) + + mock_imap_data_access.query.assert_has_calls([ + call(instrument='glows', data_level='l3e', version="latest", descriptor=GLOWS_L3E_HI_45_DESCRIPTOR), + call(instrument='glows', data_level='l3e', version="latest", descriptor=GLOWS_L3E_HI_90_DESCRIPTOR), + call(instrument='glows', data_level='l3e', version="latest", descriptor=GLOWS_L3E_LO_DESCRIPTOR), + call(instrument='glows', data_level='l3e', version="latest", descriptor=GLOWS_L3E_ULTRA_SF_DESCRIPTOR), + call(instrument='glows', data_level='l3e', version="latest", descriptor=GLOWS_L3E_ULTRA_HF_DESCRIPTOR) + ]) + + expected_versions_for_hi45_repoint_number = {repoint_number: Version(3, 2) for repoint_number in + updated_repointing_numbers} + expected_versions_for_hi90_repoint_number = {repoint_number: Version(4, 3) for repoint_number in + updated_repointing_numbers} + expected_versions_for_lo_repoint_number = {repoint_number: Version(5, 4) for repoint_number in + updated_repointing_numbers} + expected_versions_for_ultra_sf_repoint_number = {repoint_number: Version(6, 5) for repoint_number in + updated_repointing_numbers} + expected_versions_for_ultra_hf_repoint_number = {repoint_number: Version(7, 6) for repoint_number in + updated_repointing_numbers} + + self.assertCountEqual(updated_repointing_numbers, result.repointing_numbers) + self.assertEqual(expected_versions_for_hi90_repoint_number, result.hi_90_repointings) + self.assertEqual(expected_versions_for_hi45_repoint_number, result.hi_45_repointings) + self.assertEqual(expected_versions_for_lo_repoint_number, result.lo_repointings) + self.assertEqual(expected_versions_for_ultra_sf_repoint_number, result.ultra_sf_repointings) + self.assertEqual(expected_versions_for_ultra_hf_repoint_number, result.ultra_hf_repointings) + + @patch('imap_l3_processing.glows.l3e.glows_l3e_initializer.get_repoint_data') + @patch('imap_l3_processing.glows.l3e.glows_l3e_initializer.get_repoint_numbers_within_cr_window') + @patch('imap_l3_processing.glows.l3e.glows_l3e_initializer.imap_data_access') + def test_identify_versions_for_l3e_output_files_uses_reprocess_info( + self, mock_imap_data_access, mock_get_repoint_numbers_within_cr_window, mock_get_repoint_data + ): + start_cr_of_mission = 2093 + end_cr_of_mission = 2095 + first_cr_updated_in_l3d = None + repointing_path = get_test_data_path("fake_1_day_repointing_file.csv") + version_map = VersionMap({desc: Version(3 + i, 5) for i, desc in enumerate(GLOWS_L3E_DESCRIPTORS)}) + repoint_number = 3700 + mock_reprocess_info = create_autospec(ReprocessInfo, instance=True) + mock_reprocess_info.get_repoints_for_descriptor.side_effect = [ + set(), set(), {repoint_number}, set(), set() + ] + + all_repointing_numbers = set(range(3682, 3763)) + updated_repointing_numbers = set() + + mock_get_repoint_numbers_within_cr_window.side_effect = [ + all_repointing_numbers, + updated_repointing_numbers + ] + + mock_imap_data_access.query.side_effect = [ + create_mock_query_results([ + f'imap_glows_l3e_survival-probability-hi-90_20250101-repoint{repoint:05d}_{Version(3, 1)}.cdf' for + repoint in all_repointing_numbers + ]), + create_mock_query_results([ + f'imap_glows_l3e_survival-probability-hi-45_20250101-repoint{repoint:05d}_{Version(4, 2)}.cdf' for + repoint in all_repointing_numbers + ]), + create_mock_query_results([ + f'imap_glows_l3e_survival-probability-lo_20250101-repoint{repoint:05d}_{Version(5, 3)}.cdf' for repoint + in all_repointing_numbers + ]), + create_mock_query_results([ + f'imap_glows_l3e_survival-probability-ul-sf_20250101-repoint{repoint:05d}_{Version(6, 4)}.cdf' for + repoint in all_repointing_numbers + ]), + create_mock_query_results([ + f'imap_glows_l3e_survival-probability-ul-hf_20250101-repoint{repoint:05d}_{Version(7, 5)}.cdf' for + repoint in all_repointing_numbers + ]) + ] + + result = identify_versions_for_l3e_output_files( + start_cr_of_mission, + end_cr_of_mission, + first_cr_updated_in_l3d, + repointing_path, + version_map, + mock_reprocess_info, + ) + + mock_imap_data_access.query.assert_has_calls([ + call(instrument='glows', data_level='l3e', version="latest", descriptor=GLOWS_L3E_HI_45_DESCRIPTOR), + call(instrument='glows', data_level='l3e', version="latest", descriptor=GLOWS_L3E_HI_90_DESCRIPTOR), + call(instrument='glows', data_level='l3e', version="latest", descriptor=GLOWS_L3E_LO_DESCRIPTOR), + call(instrument='glows', data_level='l3e', version="latest", descriptor=GLOWS_L3E_ULTRA_SF_DESCRIPTOR), + call(instrument='glows', data_level='l3e', version="latest", descriptor=GLOWS_L3E_ULTRA_HF_DESCRIPTOR) + ]) + mock_reprocess_info.get_repoints_for_descriptor.assert_has_calls([ + call(GLOWS_L3E_HI_45_DESCRIPTOR, mock_get_repoint_data.return_value), + call(GLOWS_L3E_HI_90_DESCRIPTOR, mock_get_repoint_data.return_value), + call(GLOWS_L3E_LO_DESCRIPTOR, mock_get_repoint_data.return_value), + call(GLOWS_L3E_ULTRA_SF_DESCRIPTOR, mock_get_repoint_data.return_value), + call(GLOWS_L3E_ULTRA_HF_DESCRIPTOR, mock_get_repoint_data.return_value), + ]) + + expected_versions_for_lo_repoint_number = {repoint_number: Version(5, 4)} + + self.assertEqual([repoint_number], result.repointing_numbers) + self.assertEqual(expected_versions_for_lo_repoint_number, result.lo_repointings) \ No newline at end of file diff --git a/tests/glows/l3e/test_glows_l3e_utils.py b/tests/glows/l3e/test_glows_l3e_utils.py index 8db7dbbf..a213544f 100644 --- a/tests/glows/l3e/test_glows_l3e_utils.py +++ b/tests/glows/l3e/test_glows_l3e_utils.py @@ -2,32 +2,20 @@ import unittest from datetime import datetime, timedelta from pathlib import Path -from unittest.mock import patch, Mock, call, sentinel, create_autospec +from unittest.mock import patch, Mock, call import imap_data_access import numpy as np from imap_data_access import SPICEFilePath -from imap_data_access.file_validation import Version from imap_processing.spice.repoint import get_repoint_data, set_global_repoint_table_paths from spacepy.pycdf import CDF, const -from imap_l3_processing.constants import ONE_SECOND_IN_NANOSECONDS -from imap_l3_processing.glows.descriptors import ( - GLOWS_L3E_DESCRIPTORS, - GLOWS_L3E_HI_45_DESCRIPTOR, - GLOWS_L3E_HI_90_DESCRIPTOR, - GLOWS_L3E_LO_DESCRIPTOR, - GLOWS_L3E_ULTRA_SF_DESCRIPTOR, - GLOWS_L3E_ULTRA_HF_DESCRIPTOR, -) from imap_l3_processing.glows.l3e.glows_l3e_call_arguments import ( GlowsL3eCallArguments, GlowsL3eSpacecraftInfo, ) from imap_l3_processing.glows.l3e.glows_l3e_utils import ( determine_call_args_for_l3e_executable, - identify_versions_for_l3e_output_files, - find_first_updated_cr, get_lo_pivot_angles, get_lo_pivot_angle_from_l1b_file, LoPivotAngle, @@ -36,12 +24,10 @@ determine_spacecraft_info_for_l3e_executable, determine_spacecraft_info_using_predict_if_needed, calculate_energy_deltas, ) -from imap_l3_processing.glows.l3e.reprocess_info import ReprocessInfo, ReprocessTargets from imap_l3_processing.glows.quality_flags import GlowsL3Flags -from imap_l3_processing.models import VersionMap from imap_l3_processing.utils import FurnishMetakernelOutput from tests.integration.integration_test_helpers import mock_imap_data_access, create_metakernel -from tests.test_helpers import get_test_data_path, create_mock_query_results, get_integration_test_data_path +from tests.test_helpers import get_test_data_path, get_integration_test_data_path class TestGlowsL3EUtils(unittest.TestCase): @@ -166,92 +152,6 @@ def test_determine_spacecraft_info_using_predict_if_needed(self): spin_axis_longitude=np.float64(0.9116206658792297), spin_axis_latitude=np.float64(0.0758987849137639)) ,spacecraft_info) - @patch('imap_l3_processing.glows.l3e.glows_l3e_utils.imap_data_access.download') - @patch('imap_l3_processing.glows.l3e.glows_l3e_utils.CDF') - def test_find_first_updated_cr(self, mock_CDF, mock_download): - num_crs = 10 - mock_download.return_value = sentinel.download_old_l3d - - old_l3d = { - 'cr_grid': np.arange(num_crs) + 0.5, - 'lyman_alpha': np.arange(num_crs), - 'phion': np.arange(num_crs), - 'plasma_speed': np.arange(0, 20).reshape((num_crs, 2)), - 'plasma_speed_flag': np.arange(num_crs), - 'proton_density': np.arange(0, 20).reshape((num_crs, 2)), - 'proton_density_flag': np.arange(num_crs), - 'uv_anisotropy': np.arange(0, 20).reshape((num_crs, 2)), - 'uv_anisotropy_flag': np.arange(num_crs), - 'glows_flags': np.arange(num_crs), - } - - new_lyman_alpha = np.arange(num_crs) - new_lyman_alpha[1] = 10 - - new_phion = np.arange(num_crs) - new_phion[2] = 10 - - new_plasma_speed_flag = np.arange(num_crs) - new_plasma_speed_flag[3] = 10 - - new_proton_density_flag = np.arange(num_crs) - new_proton_density_flag[4] = 10 - - new_uv_anisotropy_flag = np.arange(num_crs) - new_uv_anisotropy_flag[5] = 10 - - new_plasma_speed = np.arange(0, 20).reshape((num_crs, 2)) - new_plasma_speed[6, :] = 10 - - new_proton_density = np.arange(0, 20).reshape((num_crs, 2)) - new_proton_density[7, :] = 10 - - new_uv_anisotropy = np.arange(0, 20).reshape((num_crs, 2)) - new_uv_anisotropy[8, :] = 10 - - new_glows_flags = np.arange(num_crs) - new_glows_flags[9] = 10 - - cases = [ - ('cr_grid', np.append(old_l3d['cr_grid'], 10.5), 10), - ('lyman_alpha', new_lyman_alpha, 1), - ('phion', new_phion, 2), - ('plasma_speed_flag', new_plasma_speed_flag, 3), - ('proton_density_flag', new_proton_density_flag, 4), - ('uv_anisotropy_flag', new_uv_anisotropy_flag, 5), - - ('plasma_speed', new_plasma_speed, 6), - ('proton_density', new_proton_density, 7), - ('uv_anisotropy', new_uv_anisotropy, 8), - ('glows_flags', new_glows_flags, 9), - ('no_change', None, None) - ] - - for case, change, expected in cases: - mock_download.reset_mock() - mock_CDF.reset_mock() - - with self.subTest(case=case): - new_l3d = {**old_l3d} - if case != "no_change": - new_l3d[case] = change - - mock_CDF.side_effect = [old_l3d, new_l3d] - - actual_cr = find_first_updated_cr( - sentinel.new_l3d_path, sentinel.old_l3d_filename - ) - - mock_download.assert_called_once_with(sentinel.old_l3d_filename) - mock_CDF.assert_has_calls( - [ - call(str(sentinel.download_old_l3d)), - call(str(sentinel.new_l3d_path)), - ] - ) - - self.assertEqual(actual_cr, expected) - def test_get_lo_pivot_angle_from_l1b_file_real_cdf(self): l1b_file = get_test_data_path( "glows/imap_lo_l1b_nhk_20260318-repoint00189_v003.cdf" @@ -381,7 +281,7 @@ def mock_read_from_cdf(path: Path): def test_get_repoint_numbers_within_cr_window(self): start_cr = 2093 end_cr = 2094 - expected_repoint_numbers = list(range(3682, 3724)) + expected_repoint_numbers = set(range(3682, 3724)) repointing_path = get_test_data_path("fake_1_day_repointing_file.csv") @@ -391,6 +291,7 @@ def test_get_repoint_numbers_within_cr_window(self): actual_repoint_numbers = get_repoint_numbers_within_cr_window(start_cr, end_cr, repointing_data) np.testing.assert_array_equal(actual_repoint_numbers, expected_repoint_numbers) + def test_get_repoint_numbers_within_cr_window_returns_empty_for_none_start(self): start_cr = None end_cr = 2094 @@ -404,280 +305,6 @@ def test_get_repoint_numbers_within_cr_window_returns_empty_for_none_start(self) np.testing.assert_array_equal(actual_repoint_numbers, []) - @patch('imap_l3_processing.glows.l3e.glows_l3e_utils.get_repoint_numbers_within_cr_window') - @patch('imap_l3_processing.glows.l3e.glows_l3e_utils.imap_data_access') - def test_identify_versions_for_l3e_output_files_gives_minor_version_1_for_non_existing_l3e(self, - mock_imap_data_access, - mock_get_repoint_numbers_within_cr_window): - start_cr_of_mission = 2093 - end_cr_of_mission = 2094 - first_cr_updated_in_l3d = None - repointing_path = get_test_data_path("fake_1_day_repointing_file.csv") - version_map = VersionMap({desc: Version(3 + i, 5) for i, desc in enumerate(GLOWS_L3E_DESCRIPTORS)}) - - mock_imap_data_access.query.side_effect = [ - create_mock_query_results([]), - create_mock_query_results([]), - create_mock_query_results([]), - create_mock_query_results([]), - create_mock_query_results([]), - ] - - all_repointing_numbers = list(range(3682, 3736)) - updated_repointing_numbers = list() - mock_get_repoint_numbers_within_cr_window.side_effect = [ - all_repointing_numbers, - updated_repointing_numbers - ] - reprocess_info = ReprocessInfo({}) - - result = identify_versions_for_l3e_output_files(start_cr_of_mission, end_cr_of_mission, first_cr_updated_in_l3d, - repointing_path, version_map, reprocess_info) - - mock_imap_data_access.query.assert_has_calls([ - call(instrument='glows', data_level='l3e', version="latest", descriptor=GLOWS_L3E_HI_45_DESCRIPTOR), - call(instrument='glows', data_level='l3e', version="latest", descriptor=GLOWS_L3E_HI_90_DESCRIPTOR), - call(instrument='glows', data_level='l3e', version="latest", descriptor=GLOWS_L3E_LO_DESCRIPTOR), - call(instrument='glows', data_level='l3e', version="latest", descriptor=GLOWS_L3E_ULTRA_SF_DESCRIPTOR), - call(instrument='glows', data_level='l3e', version="latest", descriptor=GLOWS_L3E_ULTRA_HF_DESCRIPTOR) - ]) - - expected_versions_for_hi45_repoint_number = {repoint_number: Version(3, 1) for repoint_number in - all_repointing_numbers} - expected_versions_for_hi90_repoint_number = {repoint_number: Version(4, 1) for repoint_number in - all_repointing_numbers} - expected_versions_for_lo_repoint_number = {repoint_number: Version(5, 1) for repoint_number in - all_repointing_numbers} - expected_versions_for_ultra_sf_repoint_number = {repoint_number: Version(6, 1) for repoint_number in - all_repointing_numbers} - expected_versions_for_ultra_hf_repoint_number = {repoint_number: Version(7, 1) for repoint_number in - all_repointing_numbers} - - self.assertCountEqual(all_repointing_numbers, result.repointing_numbers) - self.assertEqual(expected_versions_for_hi90_repoint_number, result.hi_90_repointings) - self.assertEqual(expected_versions_for_hi45_repoint_number, result.hi_45_repointings) - self.assertEqual(expected_versions_for_lo_repoint_number, result.lo_repointings) - self.assertEqual( - expected_versions_for_ultra_sf_repoint_number, result.ultra_sf_repointings - ) - self.assertEqual(expected_versions_for_ultra_hf_repoint_number, result.ultra_hf_repointings) - - - @patch('imap_l3_processing.glows.l3e.glows_l3e_utils.get_repoint_numbers_within_cr_window') - @patch('imap_l3_processing.glows.l3e.glows_l3e_utils.imap_data_access') - def test_identify_versions_for_l3e_output_files_increments_major_and_minor_when_given_higher_major_version(self, - mock_imap_data_access, - mock_get_repoint_numbers_within_cr_window): - start_cr_of_mission = 2093 - end_cr_of_mission = 2094 - first_cr_updated_in_l3d = None - repointing_path = get_test_data_path("fake_1_day_repointing_file.csv") - version_map = VersionMap({desc: Version(3 + i, 5) for i, desc in enumerate(GLOWS_L3E_DESCRIPTORS)}) - - all_repointing_numbers = list(range(3682, 3736)) - updated_repointing_numbers = list() - old_major_version = 2 - mock_get_repoint_numbers_within_cr_window.reset_mock() - mock_imap_data_access.reset_mock() - - mock_get_repoint_numbers_within_cr_window.side_effect = [ - all_repointing_numbers, - updated_repointing_numbers - ] - - mock_imap_data_access.query.side_effect = [ - create_mock_query_results([ - f'imap_glows_l3e_survival-probability-hi-90_20250101-repoint03682_{Version(old_major_version, 1)}.cdf', - f'imap_glows_l3e_survival-probability-hi-90_20250101-repoint03683_{Version(3, 1)}.cdf', - f'imap_glows_l3e_survival-probability-hi-90_20250101-repoint03735_{Version(3, 1)}.cdf' - ]), - create_mock_query_results([ - f'imap_glows_l3e_survival-probability-hi-45_20250101-repoint03683_{Version(old_major_version, 2)}.cdf', - f'imap_glows_l3e_survival-probability-hi-45_20250101-repoint03684_{Version(4, 2)}.cdf', - f'imap_glows_l3e_survival-probability-hi-45_20250101-repoint03735_{Version(4, 2)}.cdf' - ]), - create_mock_query_results([ - f'imap_glows_l3e_survival-probability-lo_20250101-repoint03684_{Version(old_major_version, 3)}.cdf', - f'imap_glows_l3e_survival-probability-lo_20250101-repoint03685_{Version(5, 3)}.cdf', - f'imap_glows_l3e_survival-probability-lo_20250101-repoint03735_{Version(5, 3)}.cdf' - ]), - create_mock_query_results([ - f'imap_glows_l3e_survival-probability-ul-sf_20250101-repoint03685_{Version(old_major_version, 4)}.cdf', - f'imap_glows_l3e_survival-probability-ul-sf_20250101-repoint03686_{Version(6, 4)}.cdf', - f'imap_glows_l3e_survival-probability-ul-sf_20250101-repoint03735_{Version(6, 4)}.cdf' - ]), - create_mock_query_results([ - f'imap_glows_l3e_survival-probability-ul-hf_20250101-repoint03686_{Version(old_major_version, 5)}.cdf', - f'imap_glows_l3e_survival-probability-ul-hf_20250101-repoint03687_{Version(7, 5)}.cdf', - f'imap_glows_l3e_survival-probability-ul-hf_20250101-repoint03735_{Version(7, 5)}.cdf' - ]) - ] - - result = identify_versions_for_l3e_output_files(start_cr_of_mission, end_cr_of_mission, - first_cr_updated_in_l3d, repointing_path, - version_map, ReprocessInfo({})) - - mock_imap_data_access.query.assert_has_calls([ - call(instrument='glows', data_level='l3e', version="latest", descriptor=GLOWS_L3E_HI_45_DESCRIPTOR), - call(instrument='glows', data_level='l3e', version="latest", descriptor=GLOWS_L3E_HI_90_DESCRIPTOR), - call(instrument='glows', data_level='l3e', version="latest", descriptor=GLOWS_L3E_LO_DESCRIPTOR), - call(instrument='glows', data_level='l3e', version="latest", descriptor=GLOWS_L3E_ULTRA_SF_DESCRIPTOR), - call(instrument='glows', data_level='l3e', version="latest", descriptor=GLOWS_L3E_ULTRA_HF_DESCRIPTOR) - ]) - - self.assertCountEqual(list(range(3682, 3735)), result.repointing_numbers) - - self.assertNotIn(3683, result.hi_45_repointings) - self.assertNotIn(3684, result.hi_90_repointings) - self.assertNotIn(3685, result.lo_repointings) - self.assertNotIn(3686, result.ultra_sf_repointings) - self.assertNotIn(3687, result.ultra_hf_repointings) - - self.assertEqual(Version(3, 2), result.hi_45_repointings[3682]) - self.assertEqual(Version(4, 3), result.hi_90_repointings[3683]) - self.assertEqual(Version(5, 4), result.lo_repointings[3684]) - self.assertEqual(Version(6, 5), result.ultra_sf_repointings[3685]) - self.assertEqual(Version(7, 6), result.ultra_hf_repointings[3686]) - - @patch('imap_l3_processing.glows.l3e.glows_l3e_utils.get_repoint_numbers_within_cr_window') - @patch('imap_l3_processing.glows.l3e.glows_l3e_utils.imap_data_access') - def test_identify_versions_for_l3e_output_files_increments_minor_when_same_major_and_updated_l3d_covers_pointing( - self, mock_imap_data_access, mock_get_repoint_numbers_within_cr_window - ): - start_cr_of_mission = 2093 - end_cr_of_mission = 2095 - first_cr_updated_in_l3d = None - repointing_path = get_test_data_path("fake_1_day_repointing_file.csv") - version_map = VersionMap({desc: Version(3 + i, 5) for i, desc in enumerate(GLOWS_L3E_DESCRIPTORS)}) - - all_repointing_numbers = list(range(3682, 3763)) - updated_repointing_numbers = list(range(3709, 3763)) - - mock_get_repoint_numbers_within_cr_window.side_effect = [ - all_repointing_numbers, - updated_repointing_numbers - ] - - mock_imap_data_access.query.side_effect = [ - create_mock_query_results([ - f'imap_glows_l3e_survival-probability-hi-90_20250101-repoint{repoint:05d}_{Version(3, 1)}.cdf' for repoint in all_repointing_numbers - ]), - create_mock_query_results([ - f'imap_glows_l3e_survival-probability-hi-45_20250101-repoint{repoint:05d}_{Version(4, 2)}.cdf' for repoint in all_repointing_numbers - ]), - create_mock_query_results([ - f'imap_glows_l3e_survival-probability-lo_20250101-repoint{repoint:05d}_{Version(5, 3)}.cdf' for repoint in all_repointing_numbers - ]), - create_mock_query_results([ - f'imap_glows_l3e_survival-probability-ul-sf_20250101-repoint{repoint:05d}_{Version(6, 4)}.cdf' for repoint in all_repointing_numbers - ]), - create_mock_query_results([ - f'imap_glows_l3e_survival-probability-ul-hf_20250101-repoint{repoint:05d}_{Version(7, 5)}.cdf' for repoint in all_repointing_numbers - ]) - ] - - result = identify_versions_for_l3e_output_files(start_cr_of_mission, end_cr_of_mission, - first_cr_updated_in_l3d, repointing_path, - version_map, ReprocessInfo({})) - - mock_imap_data_access.query.assert_has_calls([ - call(instrument='glows', data_level='l3e', version="latest", descriptor=GLOWS_L3E_HI_45_DESCRIPTOR), - call(instrument='glows', data_level='l3e', version="latest", descriptor=GLOWS_L3E_HI_90_DESCRIPTOR), - call(instrument='glows', data_level='l3e', version="latest", descriptor=GLOWS_L3E_LO_DESCRIPTOR), - call(instrument='glows', data_level='l3e', version="latest", descriptor=GLOWS_L3E_ULTRA_SF_DESCRIPTOR), - call(instrument='glows', data_level='l3e', version="latest", descriptor=GLOWS_L3E_ULTRA_HF_DESCRIPTOR) - ]) - - expected_versions_for_hi45_repoint_number = {repoint_number: Version(3, 2) for repoint_number in - updated_repointing_numbers} - expected_versions_for_hi90_repoint_number = {repoint_number: Version(4, 3) for repoint_number in - updated_repointing_numbers} - expected_versions_for_lo_repoint_number = {repoint_number: Version(5, 4) for repoint_number in - updated_repointing_numbers} - expected_versions_for_ultra_sf_repoint_number = {repoint_number: Version(6, 5) for repoint_number in - updated_repointing_numbers} - expected_versions_for_ultra_hf_repoint_number = {repoint_number: Version(7, 6) for repoint_number in - updated_repointing_numbers} - - self.assertCountEqual(updated_repointing_numbers, result.repointing_numbers) - self.assertEqual(expected_versions_for_hi90_repoint_number, result.hi_90_repointings) - self.assertEqual(expected_versions_for_hi45_repoint_number, result.hi_45_repointings) - self.assertEqual(expected_versions_for_lo_repoint_number, result.lo_repointings) - self.assertEqual(expected_versions_for_ultra_sf_repoint_number, result.ultra_sf_repointings) - self.assertEqual(expected_versions_for_ultra_hf_repoint_number, result.ultra_hf_repointings) - - @patch('imap_l3_processing.glows.l3e.glows_l3e_utils.get_repoint_data') - @patch('imap_l3_processing.glows.l3e.glows_l3e_utils.get_repoint_numbers_within_cr_window') - @patch('imap_l3_processing.glows.l3e.glows_l3e_utils.imap_data_access') - def test_identify_versions_for_l3e_output_files_uses_reprocess_info( - self, mock_imap_data_access, mock_get_repoint_numbers_within_cr_window, mock_get_repoint_data - ): - start_cr_of_mission = 2093 - end_cr_of_mission = 2095 - first_cr_updated_in_l3d = None - repointing_path = get_test_data_path("fake_1_day_repointing_file.csv") - version_map = VersionMap({desc: Version(3 + i, 5) for i, desc in enumerate(GLOWS_L3E_DESCRIPTORS)}) - repoint_number = 3700 - mock_reprocess_info = create_autospec(ReprocessInfo, instance=True) - mock_reprocess_info.get_repoints_for_descriptor.side_effect = [ - [], [], [repoint_number], [], [] - ] - - all_repointing_numbers = list(range(3682, 3763)) - updated_repointing_numbers = list() - - mock_get_repoint_numbers_within_cr_window.side_effect = [ - all_repointing_numbers, - updated_repointing_numbers - ] - - mock_imap_data_access.query.side_effect = [ - create_mock_query_results([ - f'imap_glows_l3e_survival-probability-hi-90_20250101-repoint{repoint:05d}_{Version(3, 1)}.cdf' for repoint in all_repointing_numbers - ]), - create_mock_query_results([ - f'imap_glows_l3e_survival-probability-hi-45_20250101-repoint{repoint:05d}_{Version(4, 2)}.cdf' for repoint in all_repointing_numbers - ]), - create_mock_query_results([ - f'imap_glows_l3e_survival-probability-lo_20250101-repoint{repoint:05d}_{Version(5, 3)}.cdf' for repoint in all_repointing_numbers - ]), - create_mock_query_results([ - f'imap_glows_l3e_survival-probability-ul-sf_20250101-repoint{repoint:05d}_{Version(6, 4)}.cdf' for repoint in all_repointing_numbers - ]), - create_mock_query_results([ - f'imap_glows_l3e_survival-probability-ul-hf_20250101-repoint{repoint:05d}_{Version(7, 5)}.cdf' for repoint in all_repointing_numbers - ]) - ] - - result = identify_versions_for_l3e_output_files( - start_cr_of_mission, - end_cr_of_mission, - first_cr_updated_in_l3d, - repointing_path, - version_map, - mock_reprocess_info, - ) - - mock_imap_data_access.query.assert_has_calls([ - call(instrument='glows', data_level='l3e', version="latest", descriptor=GLOWS_L3E_HI_45_DESCRIPTOR), - call(instrument='glows', data_level='l3e', version="latest", descriptor=GLOWS_L3E_HI_90_DESCRIPTOR), - call(instrument='glows', data_level='l3e', version="latest", descriptor=GLOWS_L3E_LO_DESCRIPTOR), - call(instrument='glows', data_level='l3e', version="latest", descriptor=GLOWS_L3E_ULTRA_SF_DESCRIPTOR), - call(instrument='glows', data_level='l3e', version="latest", descriptor=GLOWS_L3E_ULTRA_HF_DESCRIPTOR) - ]) - mock_reprocess_info.get_repoints_for_descriptor.assert_has_calls([ - call(GLOWS_L3E_HI_45_DESCRIPTOR, mock_get_repoint_data.return_value), - call(GLOWS_L3E_HI_90_DESCRIPTOR, mock_get_repoint_data.return_value), - call(GLOWS_L3E_LO_DESCRIPTOR, mock_get_repoint_data.return_value), - call(GLOWS_L3E_ULTRA_SF_DESCRIPTOR, mock_get_repoint_data.return_value), - call(GLOWS_L3E_ULTRA_HF_DESCRIPTOR, mock_get_repoint_data.return_value), - ]) - - expected_versions_for_lo_repoint_number = {repoint_number: Version(5, 4)} - - self.assertEqual([repoint_number], result.repointing_numbers) - self.assertEqual(expected_versions_for_lo_repoint_number, result.lo_repointings) - def test_calculate_energy_deltas(self): centers = np.array([10, 1000, 100000]) diff --git a/tests/glows/l3e/test_reprocess_info.py b/tests/glows/l3e/test_reprocess_info.py index b129aed5..92bf4c17 100644 --- a/tests/glows/l3e/test_reprocess_info.py +++ b/tests/glows/l3e/test_reprocess_info.py @@ -94,9 +94,9 @@ def test_get_repoints(self): repoints_for_cr2093 = list(range(3682, 3696)) repoints_for_cr2094 = list(range(3710, 3724)) - expected_repoints_lo = [2045, 2046] + repoints_for_cr2093 + [3700] + repoints_for_cr2094 - expected_repoints_hi = [2050] - expected_repoints_ultra = [] + expected_repoints_lo = set([2045, 2046] + repoints_for_cr2093 + [3700] + repoints_for_cr2094) + expected_repoints_hi = {2050} + expected_repoints_ultra = set() self.assertEqual(expected_repoints_lo, repoints_for_lo) self.assertEqual(expected_repoints_hi, repoints_for_hi) self.assertEqual(expected_repoints_ultra, repoints_for_ultra) diff --git a/tests/glows/test_glows_processor.py b/tests/glows/test_glows_processor.py index 62aafaa9..f524e15a 100644 --- a/tests/glows/test_glows_processor.py +++ b/tests/glows/test_glows_processor.py @@ -628,6 +628,11 @@ def test_l3bc_uses_all_l3a_file_names_for_l3b_parents(self, mock_save_data, mock self.assertEqual(set(expected_parents), set(l3b_data_product.parent_file_names)) + @patch("imap_l3_processing.glows.glows_processor.GlowsL3EInitializer.get_repointings_to_process") + def test_process_l3d_uploads_even_if_no_l3e_needs_to_be_produced(self, mock_l3e_initializer_get_repoints): + mock_l3e_initializer_get_repoints.return_value = None + self.test_process_l3d() + @patch("imap_l3_processing.glows.glows_processor.process_l3e") @patch("imap_l3_processing.glows.glows_processor.create_glows_l3b_json_file_from_cdf") @patch("imap_l3_processing.glows.glows_processor.create_glows_l3c_json_file_from_cdf") @@ -639,13 +644,11 @@ def test_l3bc_uses_all_l3a_file_names_for_l3b_parents(self, mock_save_data, mock @patch('imap_l3_processing.glows.glows_processor.shutil') @patch("imap_l3_processing.glows.glows_processor.os") @patch("imap_l3_processing.glows.glows_processor.GlowsL3DInitializer") - @patch("imap_l3_processing.glows.glows_processor.read_pipeline_settings") - def test_process_l3d(self, mock_read_pipeline_settings, mock_glows_l3d_initializer, mock_os, mock_shutil, mock_run, + def test_process_l3d(self, mock_glows_l3d_initializer, mock_os, mock_shutil, mock_run, mock_convert_json_to_l3d_data_product, mock_get_parent_file_names_from_l3d_json, mock_rename_l3d, mock_save_data, mock_convert_l3c_to_json, mock_convert_l3b_to_json, _): cr_number = 2092 - mock_read_pipeline_settings.return_value = {'start_cr': cr_number} expected_end_cr = cr_number + 1 glows_l3d_dependencies = GlowsL3DDependencies( @@ -671,8 +674,11 @@ def test_process_l3d(self, mock_read_pipeline_settings, mock_glows_l3d_initializ ) old_l3d = Path('imap_glows_l3d_solar-hist_19470303-cr02090_v001.cdf') - input_major_version = 12 - l3d_output_version = Version(input_major_version, 5) + major_version = 12 + l3d_output_version = Version(major_version, 5) + + expected_l3d_cdf_filename = f'imap_glows_l3d_{GLOWS_L3D_DESCRIPTOR}_19470303-cr0{expected_end_cr}_{l3d_output_version}.cdf' + mock_glows_l3d_initializer.should_process_l3d.return_value = ( l3d_output_version, glows_l3d_dependencies, @@ -701,7 +707,7 @@ def test_process_l3d(self, mock_read_pipeline_settings, mock_glows_l3d_initializ { GLOWS_L3B_DESCRIPTOR: Version(2, 1), GLOWS_L3C_DESCRIPTOR: Version(2, 1), - GLOWS_L3D_DESCRIPTOR: Version(input_major_version, 1), + GLOWS_L3D_DESCRIPTOR: Version(major_version, 1), GLOWS_L3E_HI_45_DESCRIPTOR: Version(2, 1), GLOWS_L3E_HI_90_DESCRIPTOR: Version(2, 1), GLOWS_L3E_LO_DESCRIPTOR: Version(2, 1), @@ -718,7 +724,7 @@ def test_process_l3d(self, mock_read_pipeline_settings, mock_glows_l3d_initializ mock_convert_l3b_to_json.assert_has_calls([call(sentinel.l3b_file_1), call(sentinel.l3b_file_2)]) mock_convert_l3c_to_json.assert_has_calls([call(sentinel.l3c_file_1), call(sentinel.l3c_file_2)]) mock_glows_l3d_initializer.should_process_l3d.assert_called_with( - self.mock_external_deps, [], [], self.mock_reprocess_info, input_major_version) + self.mock_external_deps, [], [], self.mock_reprocess_info, major_version) self.mock_fetch_reprocess_info.assert_called_with(processing_input_collection) self.assertEqual([ Path("imap_glows_e-dens_19470303_20100101_v000.dat"), @@ -755,8 +761,9 @@ def test_process_l3d(self, mock_read_pipeline_settings, mock_glows_l3d_initializ 'phion': str(Path('path/to/phion')), 'lya': str(Path('path/to/lya')), 'e-dens': str(Path('path/to/e-dens')) - } - } + }, + }, + "l3d_cdf_filename": expected_l3d_cdf_filename }) expected_working_directory = Path(l3d.__file__).parent / 'science' @@ -892,8 +899,7 @@ def test_process_l3d_adds_parent_file_names_to_output(self, mock_spicepy, mock_r @patch('imap_l3_processing.glows.glows_processor.os') @patch('imap_l3_processing.glows.glows_processor.run') @patch('imap_l3_processing.glows.glows_processor.convert_json_to_l3d_data_product') - @patch('imap_l3_processing.glows.glows_processor.read_pipeline_settings') - def test_process_l3d_handles_unexpected_exception_from_science(self, mock_read_pipeline_settings, + def test_process_l3d_handles_unexpected_exception_from_science(self, mock_convert_json_to_l3d, mock_run, mock_os, _, __): ancillary_files = { @@ -911,7 +917,6 @@ def test_process_l3d_handles_unexpected_exception_from_science(self, mock_read_p external_files = { 'lya_raw_data': Path('path/to/lya'), } - mock_read_pipeline_settings.return_value = {'start_cr': 2091} l3b_file_paths = [] l3c_file_paths = [] expected_cr = 2096 @@ -948,7 +953,7 @@ def test_process_l3d_handles_unexpected_exception_from_science(self, mock_read_p mock_run.side_effect = [CalledProcessError(cmd="", returncode=1, stderr=unexpected_exception)] with self.assertRaises(Exception) as context: - process_l3d(l3d_dependencies, 1) + process_l3d(l3d_dependencies, Version(1, 1)) self.assertEqual(unexpected_exception, context.exception.stderr) mock_convert_json_to_l3d.assert_not_called() @@ -995,6 +1000,26 @@ def test_process_glows_l3d_drift(self, mock_save_data, _): glows_l3d_output = process_l3d(l3d_dependencies, Version(1, 4)) + expected_parents = { + "imap_glows_l3b_ion-rate-profile_20100422_v013.cdf", + "imap_glows_l3b_ion-rate-profile_20100519_v013.cdf", + "imap_glows_l3c_sw-profile_20100422_v012.cdf", + "imap_glows_l3c_sw-profile_20100519_v012.cdf", + "imap_glows_plasma-speed-2010a_20100101_v003.dat", + "imap_glows_proton-density-2010a_20100101_v003.dat", + "imap_glows_uv-anisotropy-2010a_20100101_v003.dat", + "imap_glows_photoion-2010a_20100101_v003.dat", + "imap_glows_lya-2010a_20100101_v003.dat", + "imap_glows_electron-density-2010a_20100101_v003.dat", + "imap_glows_pipeline-settings-l3bcde_20100101_v006.json", + "lyman_alpha_composite.nc", + } + + self.assertEqual(1, mock_save_data.call_count) + [data_product] = mock_save_data.call_args_list[0].args + + self.assertEqual(expected_parents, set(data_product.parent_file_names)) + expected_txt_filenames = ["imap_glows_e-dens_19470303_20100629_v004.dat", "imap_glows_lya_19470303_20100629_v004.dat", "imap_glows_p-dens_19470303_20100629_v004.dat", @@ -1068,7 +1093,14 @@ def test_process_glows_l3d_drift(self, mock_save_data, _): with CDF(str(tmp_dir / expected_cdf_filename)) as actual_cdf: for filename, cdf_var_name, length_of_data, first_line, last_line in test_cases: with self.subTest(msg=filename): - actual = np.loadtxt(PATH_TO_L3D_TOOLKIT / "data_l3d_txt" / filename) + l3d_text_path = PATH_TO_L3D_TOOLKIT / "data_l3d_txt" / filename + actual = np.loadtxt(l3d_text_path) + + actual_input_file_line = next((l for l in l3d_text_path.read_text().split("\n") if l.startswith("# input files:")), None) + self.assertIsNotNone(actual_input_file_line) + + expected_input_file_line = f"# input files: {expected_cdf_filename}" + self.assertEqual(expected_input_file_line, actual_input_file_line) self.assertEqual(length_of_data, len(actual)) self.assertEqual(length_of_data, len(actual_cdf[cdf_var_name][...])) @@ -1114,31 +1146,11 @@ def test_process_l3e(self, mock_process_ultra, mock_process_ultra_hf, mock_proce repointing_midpoint = datetime(2020, 1, 1, 12) mock_get_pointing_date_range.return_value = (start_epoch, end_epoch) - mock_dependencies = Mock() - - mock_dependencies.get_hi_parents.return_value = ["hi_ancillary.dat"] - mock_dependencies.get_lo_parents.return_value = ["lo_ancillary.dat"] - mock_dependencies.get_ul_parents.return_value = ["ul_ancillary.dat"] - - l3d_cdf_path = Path("path/to/l3d.cdf") - initializer_data = GlowsL3EInitializerOutput( - dependencies=mock_dependencies, - repointings=GlowsL3eVersionsForRepointings( - repointing_numbers=[25], - hi_90_repointings={25: Version(None, 1)}, - hi_45_repointings={25: Version(None, 2)}, - lo_repointings={25: Version(None, 3)}, - ultra_sf_repointings={25: Version(None, 4)}, - ultra_hf_repointings={25: Version(None, 4)}, - ), - l3d_cdf_path=l3d_cdf_path, - metakernel_with_predict_ephem=Mock(), - metakernel_without_predict_ephem=Mock(), - ) + initializer_data = self._build_l3e_initializer_output() actual_l3e_products = process_l3e(initializer_data) mock_get_pointing_date_range.assert_called_once_with(25) - mock_compute_glows_flags_for_repoint.assert_called_once_with(l3d_cdf_path, repointing_midpoint) + mock_compute_glows_flags_for_repoint.assert_called_once_with(initializer_data.l3d_cdf_path, repointing_midpoint) mock_determine_spacecraft_info.assert_called_once_with( datetime(2020, 1, 1, 12), initializer_data.metakernel_with_predict_ephem, @@ -1200,7 +1212,6 @@ def test_process_l3e_invoked_from_top_level_process(self, mock_process_l3d, mock mock_process_l3d.return_value = process_l3d_result mock_determine_spacecraft_info.return_value = sentinel.spacecraft_info, GlowsL3Flags.PREDICTIVE_EPHEMERIS, ["spice kernel"] - input_major_version = 5 input_version_map = VersionMap( @@ -1910,6 +1921,28 @@ def test_archive_dependencies(self, mock_zip, mock_json): ]) mock_zip_file.writestr.assert_called_once_with(expected_json_filename, mock_json.dumps.return_value) + def _build_l3e_initializer_output(self) -> GlowsL3EInitializerOutput: + mock_dependencies = Mock() + + mock_dependencies.get_hi_parents.return_value = ["hi_ancillary.dat"] + mock_dependencies.get_lo_parents.return_value = ["lo_ancillary.dat"] + mock_dependencies.get_ul_parents.return_value = ["ul_ancillary.dat"] + + l3d_cdf_path = Path("path/to/l3d.cdf") + return GlowsL3EInitializerOutput( + dependencies=mock_dependencies, + repointings=GlowsL3eVersionsForRepointings( + repointing_numbers=[25], + hi_90_repointings={25: Version(None, 1)}, + hi_45_repointings={25: Version(None, 2)}, + lo_repointings={25: Version(None, 3)}, + ultra_sf_repointings={25: Version(None, 4)}, + ultra_hf_repointings={25: Version(None, 4)}, + ), + l3d_cdf_path=l3d_cdf_path, + metakernel_with_predict_ephem=Mock(), + metakernel_without_predict_ephem=Mock(), + ) if __name__ == '__main__': diff --git a/tests/periodically_run_tests.json b/tests/periodically_run_tests.json index 2eb85e0c..906e9ac0 100644 --- a/tests/periodically_run_tests.json +++ b/tests/periodically_run_tests.json @@ -1,13 +1,13 @@ { - "test_spectral_fit_against_validation_data": "2026-09-25T14:46:51.493678", + "test_spectral_fit_against_validation_data": "2026-10-01T08:50:41.360170", "test_codice_lo_3d_distributions": "2026-05-28T09:43:22.990231", - "test_swe_processor_with_local_data": "2026-09-21T13:49:18.260259", - "test_ultra_all_sp_maps": "2026-09-25T14:47:48.096216", - "test_ultra_combined_nsp_maps": "2026-09-21T13:48:32.412322", - "test_ultra_combined_sp_maps": "2026-09-21T13:48:49.416421", + "test_swe_processor_with_local_data": "2026-10-01T08:53:28.218357", + "test_ultra_all_sp_maps": "2026-10-01T08:52:29.438105", + "test_ultra_combined_nsp_maps": "2026-10-01T08:52:41.585975", + "test_ultra_combined_sp_maps": "2026-10-01T08:52:58.910527", "test_glows_l3abcde_from_l2": "2026-09-20T09:28:16.106080", "test_l3bcde_automatic_reprocessing": "2026-09-20T10:25:40.483342", "test_l3bcde_first_time_processing": "2026-09-20T11:20:19.798340", "test_proton_normalized_errors_are_standard": "2026-09-25T14:48:27.874117", - "test_alpha_normalized_errors_are_standard": "2026-09-21T13:50:04.172539" + "test_alpha_normalized_errors_are_standard": "2026-10-01T08:54:13.637480" } \ No newline at end of file diff --git a/tests/test_utils.py b/tests/test_utils.py index dfbed681..0036bc9e 100644 --- a/tests/test_utils.py +++ b/tests/test_utils.py @@ -59,7 +59,7 @@ def tearDown(self) -> None: @patch("imap_l3_processing.utils.ImapAttributeManager") @patch("imap_l3_processing.utils.date") @patch("imap_l3_processing.utils.write_cdf") - @patch("imap_l3_processing.utils.ScienceFilePath") + @patch("imap_l3_processing.models.ScienceFilePath") def test_save_data(self, mock_science_file_path_class, mock_write_cdf, mock_today, mock_attribute_manager): mock_today.today.return_value = date(2024, 9, 16)