GLOWS L3d parents include the full set of input files

imap_l3_processing/glows/glows_processor.py

@@ -1,3 +1,4 @@
+import collections
 import json
 import logging
 import os
@@ -26,7 +27,7 @@
 from imap_l3_processing.glows.l3bc.glows_l3bc_dependencies import GlowsL3BCDependencies
 from imap_l3_processing.glows.l3bc.glows_l3bc_initializer import GlowsL3BCInitializer, GlowsL3BCInitializerData
 from imap_l3_processing.glows.l3bc.models import GlowsL3BIonizationRate, GlowsL3CSolarWind, GlowsL3BCProcessorOutput, \
-    ExternalDependencies, read_pipeline_settings
+    ExternalDependencies
 from imap_l3_processing.glows.l3bc.science.filter_out_bad_days import filter_l3a_files
 from imap_l3_processing.glows.l3bc.science.generate_l3bc import generate_l3bc
 from imap_l3_processing.glows.l3bc.utils import get_pointing_date_range
@@ -104,6 +105,8 @@ def process(self):
             for txt_file in process_l3d_result.l3d_text_file_paths:
                 logger.info(f"Saved L3d text file output to: {txt_file}")
 
+            products_list.extend([*process_l3d_result.l3d_text_file_paths, process_l3d_result.l3d_cdf_file_path])
+
             l3e_initializer_output = GlowsL3EInitializer.get_repointings_to_process(
                 process_l3d_result,
                 old_l3d,
@@ -114,7 +117,6 @@ def process(self):
 
             if l3e_initializer_output is not None:
                 logger.info(f"Processing L3e for repointings: {l3e_initializer_output.repointings.repointing_numbers}")
-                products_list.extend([*process_l3d_result.l3d_text_file_paths, process_l3d_result.l3d_cdf_file_path])
                 l3e_products = process_l3e(l3e_initializer_output)
                 products_list.extend(l3e_products)
             else:
@@ -216,12 +218,19 @@ def process_l3d(
     dependencies: GlowsL3DDependencies, version: Version
 ) -> Optional[GlowsL3DProcessorOutput]:
 
-    [create_glows_l3b_json_file_from_cdf(l3b) for l3b in dependencies.l3b_file_paths]
-    [create_glows_l3c_json_file_from_cdf(l3c) for l3c in dependencies.l3c_file_paths]
+    collections.deque((create_glows_l3b_json_file_from_cdf(l3b) for l3b in dependencies.l3b_file_paths), maxlen=0)
+    collections.deque((create_glows_l3c_json_file_from_cdf(l3c) for l3c in dependencies.l3c_file_paths), maxlen=0)
 
     os.makedirs(PATH_TO_L3D_TOOLKIT / "data_l3d", exist_ok=True)
     os.makedirs(PATH_TO_L3D_TOOLKIT / "data_l3d_txt", exist_ok=True)
 
+    start_date = datetime(1947, 3, 3)
+    data_product_metadata = InputMetadata(instrument="glows", data_level="l3d", descriptor=GLOWS_L3D_DESCRIPTOR,
+                                          start_date=start_date, end_date=start_date,
+                                          version=VersionMap({GLOWS_L3D_DESCRIPTOR: version}))
+
+    output_l3d_cdf_filename = data_product_metadata.to_science_file_path(cr_number=dependencies.end_cr).construct_path().name
+
     file_manifest = {
         "external_files": {
             key: str(val) for key, val in dependencies.external_files.items()
@@ -233,6 +242,7 @@ def process_l3d(
                 for key, val in dependencies.ancillary_files["WawHelioIon"].items()
             },
         },
+        "l3d_cdf_filename": output_l3d_cdf_filename
     }
 
     last_processed_cr = None
@@ -256,7 +266,7 @@ def process_l3d(
     if output.stdout:
         last_processed_cr = int(output.stdout.split('= ')[-1])
 
-    if last_processed_cr:
+    if last_processed_cr == dependencies.end_cr:
         output_text_files = []
         for text_file in os.listdir(PATH_TO_L3D_TOOLKIT / 'data_l3d_txt'):
             if str(last_processed_cr) in text_file:
@@ -269,14 +279,11 @@ def process_l3d(
 
         file_name = f'imap_glows_l3d_solar-params-history_19470303-cr0{last_processed_cr}_v00.json'
 
-        start_date = datetime(1947, 3, 3)
-        data_product_metadata = InputMetadata(instrument="glows", data_level="l3d", descriptor=GLOWS_L3D_DESCRIPTOR,
-                                              start_date=start_date, end_date=start_date, version=VersionMap({GLOWS_L3D_DESCRIPTOR: version}))
         parent_file_names = get_parent_file_names_from_l3d_json(PATH_TO_L3D_TOOLKIT / 'data_l3d')
 
         l3d_data_product = convert_json_to_l3d_data_product(PATH_TO_L3D_TOOLKIT / 'data_l3d' / file_name,
                                                             data_product_metadata, parent_file_names)
-        l3d_data_product_path = save_data(l3d_data_product, cr_number=last_processed_cr)
+        l3d_data_product_path = save_data(l3d_data_product, cr_number=dependencies.end_cr)
 
         return GlowsL3DProcessorOutput(l3d_data_product_path, txt_files_with_correct_version, last_processed_cr)
     return None

imap_l3_processing/glows/l3d/glows_l3d_dependencies.py

@@ -17,7 +17,7 @@ class GlowsL3DDependencies:
     ancillary_files: dict[str, Path | dict[str, Path]]
     l3b_file_paths: list[Path]
     l3c_file_paths: list[Path]
-    end_cr: float
+    end_cr: int
 
     @classmethod
     def fetch_dependencies(cls, dependencies: ProcessingInputCollection, external_dependencies: ExternalDependencies):

imap_l3_processing/glows/l3d/science/generate_l3d.py

@@ -22,6 +22,7 @@
 
 ANC_INPUT_FROM_INSTRUMENT_TEAM = dependencies['ancillary_files']
 EXT_DEPENDENCIES = dependencies['external_files']
+L3D_CDF_FILENAME = dependencies['l3d_cdf_filename']
 
 # Create list of L3b/c files and read them
 l3b_fn_list = np.array(sorted(glob.glob('data_l3b/imap_glows_l3b*.json')))
@@ -35,7 +36,8 @@
 
 # Start generating from the initial files (not using the last entry from L3d)
 
-solar_param_hist.header['ancillary_data_files'] = list(fn_initial.values())
+solar_param_hist.header['ancillary_data_files'] = [*fn_initial.values(), ANC_INPUT_FROM_INSTRUMENT_TEAM['pipeline_settings']]
+solar_param_hist.header['l3d_cdf_filename'] = L3D_CDF_FILENAME
 solar_param_hist.generate_initial_history(fn_initial)
 
 solar_param_hist.update_solar_params_hist(EXT_DEPENDENCIES, data_l3b, data_l3c, CR_current)

imap_l3_processing/glows/l3d/science/toolkit/l3d_SolarParamHistory.py

@@ -65,10 +65,10 @@ def __init__(self,anc_input_from_instr_team,ext_dependencies):
                 list of ancillary files used in calculations
             external_dependeciens : str
                 list of external dependeciens files
-            l3b_input_filename: str
-                Input L3b ionization rate profiles file
-            l3c_input_filename: str
-                Input L3c solar wind speed and density profiles file
+            l3b_input_filename: list[str]
+                All the filenames of input L3b ionization rate profiles file
+            l3c_input_filename: list[str]
+                All the filenames of input L3c solar wind speed and density profiles file
         settings : dict
             pipeline settings
         ini_data: dict
@@ -108,8 +108,8 @@ def __init__(self,anc_input_from_instr_team,ext_dependencies):
             'filename': None,
             'ancillary_data_files': None,
             'external_dependeciens': ext_dependencies['lya_raw_data'],
-            'l3b_input_filename': None,
-            'l3c_input_filename': None
+            'l3b_input_filename': [],
+            'l3c_input_filename': []
         }
 
         self.ini_data={}
@@ -321,7 +321,7 @@ def _generate_hdr_template(self):
         hdr_temp[0]='lines in header: '+str(N)+'\n'
         hdr_temp[4]='ground software version: '+VERSION+'\n'
         hdr_temp[5]='creation date: '+ Time.now().iso +'\n'
-        hdr_temp[6]='input files: ' + ', '.join(self.header['l3b_input_filename']) + ', ' + ', '.join(self.header['l3c_input_filename']) +'\n'
+        hdr_temp[6]='input files: ' + self.header['l3d_cdf_filename'] + "\n"
         hdr_temp[7]='external dependencies: ' + self.header['external_dependeciens'] + '\n'
         hdr_temp[8]='last CR: '+ str(self.CR_last) + '\n'
         hdr_temp[9]='fill value: '+ str(self.settings['WawHelioIonGlows_fill_value']) + '\n'
@@ -446,8 +446,8 @@ def _update_l3bc_data(self,data_l3b,data_l3c,CR):
         '''
         cr_params, idx_read_b, idx_read_c = self._generate_cr_solar_params(CR, data_l3b, data_l3c)
 
-        self.header['l3b_input_filename']=[data_l3b[i]['header']['filename'] for i in idx_read_b]
-        self.header['l3c_input_filename']=[data_l3c[i]['header']['filename'] for i in idx_read_c]
+        self.header['l3b_input_filename'].extend(data_l3b[i]['header']['filename'] for i in idx_read_b)
+        self.header['l3c_input_filename'].extend(data_l3c[i]['header']['filename'] for i in idx_read_c)
 
         # add a row with values for current CR
         for k in self.ini_data['label']:

imap_l3_processing/glows/l3e/glows_l3e_initializer.py

@@ -1,25 +1,30 @@
+from __future__ import annotations
+
 import logging
 from dataclasses import dataclass
 from pathlib import Path
 from typing import Optional
 
 import imap_data_access
+import numpy as np
 from imap_data_access import (
     ProcessingInputCollection,
     AncillaryInput,
     ScienceInput,
     RepointInput,
 )
+from imap_data_access.file_validation import Version
+from imap_processing.spice.repoint import set_global_repoint_table_paths, get_repoint_data
+from spacepy.pycdf import CDF
 
+from imap_l3_processing.glows.descriptors import GLOWS_L3E_DESCRIPTORS, GLOWS_L3E_HI_90_DESCRIPTOR, \
+    GLOWS_L3E_HI_45_DESCRIPTOR, GLOWS_L3E_LO_DESCRIPTOR, GLOWS_L3E_ULTRA_SF_DESCRIPTOR, GLOWS_L3E_ULTRA_HF_DESCRIPTOR
 from imap_l3_processing.glows.l3bc.utils import get_pointing_date_range
 from imap_l3_processing.glows.l3d.models import GlowsL3DProcessorOutput
 from imap_l3_processing.glows.l3d.utils import get_most_recently_uploaded_ancillary
 from imap_l3_processing.glows.l3e.glows_l3e_dependencies import GlowsL3EDependencies
 from imap_l3_processing.glows.l3e.glows_l3e_utils import (
-    find_first_updated_cr,
-    identify_versions_for_l3e_output_files,
-    GlowsL3eVersionsForRepointings,
-)
+    GlowsL3eVersionsForRepointings, get_repoint_numbers_within_cr_window, )
 from imap_l3_processing.glows.l3e.reprocess_info import ReprocessInfo
 from imap_l3_processing.models import VersionMap
 from imap_l3_processing.utils import FurnishMetakernelOutput
@@ -38,11 +43,11 @@ class GlowsL3EInitializerOutput:
 class GlowsL3EInitializer:
     @staticmethod
     def get_repointings_to_process(
-            l3d_output: GlowsL3DProcessorOutput,
-            previous_l3d: Optional[str],
-            repointing_file_path: Path,
-            version_map: VersionMap,
-            reprocess_info: ReprocessInfo,
+        l3d_output: GlowsL3DProcessorOutput,
+        previous_l3d: Optional[str],
+        repointing_file_path: Path,
+        version_map: VersionMap,
+        reprocess_info: ReprocessInfo,
     ) -> Optional[GlowsL3EInitializerOutput]:
         pipeline_settings_l3bcde = get_most_recently_uploaded_ancillary(imap_data_access.query(table='ancillary', instrument='glows', descriptor='pipeline-settings-l3bcde'))
         energy_grid_lo = get_most_recently_uploaded_ancillary(imap_data_access.query(table='ancillary', instrument='glows', descriptor='energy-grid-lo'))
@@ -106,3 +111,95 @@ def get_repointings_to_process(
             metakernel_without_predict_ephem=furnished_metakernels[1],
         )
 
+def query_existing_l3es_versions(descriptor: str) -> dict[int, Version]:
+    existing_l3es_for_descriptor = {}
+    l3e_files = imap_data_access.query(instrument='glows', data_level='l3e', version="latest", descriptor=descriptor)
+    for l3e in l3e_files:
+        if 'major_version' in l3e.keys() and 'minor_version' in l3e.keys():
+            existing_l3es_for_descriptor[int(l3e['repointing'])] = Version(l3e["major_version"], l3e["minor_version"])
+        elif 'version' in l3e.keys():
+            existing_l3es_for_descriptor[int(l3e['repointing'])] = Version.from_version(l3e['version'])
+        else:
+            continue
+    return existing_l3es_for_descriptor
+
+def identify_versions_for_l3e_output_files(start_cr_of_mission: int, end_cr_of_mission: int, first_updated_cr_from_l3d: Optional[int],
+                                           repointing_path: Path, version_map: VersionMap, reprocess_info: ReprocessInfo) -> GlowsL3eVersionsForRepointings:
+
+    set_global_repoint_table_paths([repointing_path])
+    repointing_data = get_repoint_data()
+
+    all_pointing_numbers = get_repoint_numbers_within_cr_window(start_cr_of_mission, end_cr_of_mission, repointing_data)
+    pointing_numbers_updated_by_l3d = get_repoint_numbers_within_cr_window(first_updated_cr_from_l3d, end_cr_of_mission, repointing_data)
+
+    updated_pointings_per_instruments = {}
+    updated_pointing_numbers = {}
+
+    for descriptor in GLOWS_L3E_DESCRIPTORS:
+        new_major_version = version_map.lookup(descriptor).major
+
+        repointings_to_force_processing = reprocess_info.get_repoints_for_descriptor(
+            descriptor, repointing_data
+        )
+
+        existing_file_versions = query_existing_l3es_versions(descriptor)
+        out_of_date_repointings = find_out_of_date_l3es(existing_file_versions, all_pointing_numbers, new_major_version)
+
+        repointings_to_process = pointing_numbers_updated_by_l3d | repointings_to_force_processing | out_of_date_repointings
+
+        new_file_versions = {}
+        for pointing_number in sorted(repointings_to_process):
+            if previous_version := existing_file_versions.get(pointing_number):
+                new_version = Version(new_major_version, previous_version.minor + 1)
+            else:
+                new_version = Version(new_major_version, 1)
+            new_file_versions[pointing_number] = new_version
+
+        updated_pointings_per_instruments[descriptor] = new_file_versions
+        updated_pointing_numbers = updated_pointing_numbers | new_file_versions.keys()
+
+
+    return GlowsL3eVersionsForRepointings(list(updated_pointing_numbers),
+                                          updated_pointings_per_instruments[GLOWS_L3E_HI_90_DESCRIPTOR],
+                                          updated_pointings_per_instruments[GLOWS_L3E_HI_45_DESCRIPTOR],
+                                          updated_pointings_per_instruments[GLOWS_L3E_LO_DESCRIPTOR],
+                                          updated_pointings_per_instruments[GLOWS_L3E_ULTRA_SF_DESCRIPTOR],
+                                          updated_pointings_per_instruments[GLOWS_L3E_ULTRA_HF_DESCRIPTOR],
+                                          )
+
+def find_out_of_date_l3es(existing_l3es: dict[int, Version], repointings_to_check: set[int], current_major_version: int) -> set[int]:
+    out_of_date_l3es = set()
+    for repointing in repointings_to_check:
+        if existing_l3e_version := existing_l3es.get(repointing):
+            if existing_l3e_version.major < current_major_version:
+                out_of_date_l3es.add(repointing)
+        else:
+            out_of_date_l3es.add(repointing)
+    return out_of_date_l3es
+
+
+def find_first_updated_cr(new_l3d: Path, old_l3d: str) -> Optional[int]:
+    downloaded_old_l3d = imap_data_access.download(old_l3d)
+
+    old_l3d_cdf = CDF(str(downloaded_old_l3d))
+    new_l3d_cdf = CDF(str(new_l3d))
+
+    for i, cr in enumerate(old_l3d_cdf['cr_grid'][...]):
+        lya_matches = np.isclose(old_l3d_cdf['lyman_alpha'][i], new_l3d_cdf['lyman_alpha'][i])
+        phion_matches = np.isclose(old_l3d_cdf['phion'][i], new_l3d_cdf['phion'][i])
+        plasma_speed_flag_matches = np.isclose(old_l3d_cdf['plasma_speed_flag'][i], new_l3d_cdf['plasma_speed_flag'][i])
+        proton_density_flag_matches = np.isclose(old_l3d_cdf['proton_density_flag'][i], new_l3d_cdf['proton_density_flag'][i])
+        uv_anisotropy_flag_matches = np.isclose(old_l3d_cdf['uv_anisotropy_flag'][i], new_l3d_cdf['uv_anisotropy_flag'][i])
+        glows_flags_matches = np.isclose(old_l3d_cdf['glows_flags'][i], new_l3d_cdf['glows_flags'][i])
+
+        plasma_speed_matches = np.all(np.isclose(old_l3d_cdf['plasma_speed'][i], new_l3d_cdf['plasma_speed'][i]))
+        proton_density_matches = np.all(np.isclose(old_l3d_cdf['proton_density'][i], new_l3d_cdf['proton_density'][i]))
+        uv_anisotropy_matches = np.all(np.isclose(old_l3d_cdf['uv_anisotropy'][i], new_l3d_cdf['uv_anisotropy'][i]))
+
+        if np.any(np.logical_not([lya_matches, phion_matches, plasma_speed_matches, plasma_speed_flag_matches, proton_density_matches, proton_density_flag_matches, uv_anisotropy_matches, uv_anisotropy_flag_matches, glows_flags_matches])):
+            return int(cr)
+
+    if old_l3d_cdf['cr_grid'].shape != new_l3d_cdf['cr_grid'].shape:
+        return int(old_l3d_cdf['cr_grid'][-1]) + 1
+
+    return None