From 12a643d030b71f194c5f85f92acf6a295b374145 Mon Sep 17 00:00:00 2001 From: Nicky Hochmuth Date: Wed, 17 Dec 2025 11:44:43 +0100 Subject: [PATCH 01/10] mixings in columns --- stixcore/io/FlareListManager.py | 45 ++-- .../io/product_processors/fits/processors.py | 20 +- stixcore/processing/FLtoFL.py | 15 +- stixcore/processing/FlareListL3.py | 2 +- stixcore/processing/pipeline_daily.py | 106 +++++---- stixcore/products/__init__.py | 1 + stixcore/products/level3/flarelist.py | 122 ++++++----- stixcore/products/level3/flarelistproduct.py | 14 +- stixcore/products/level3/processing.py | 203 ++++++++++++++++++ stixcore/products/product.py | 2 +- 10 files changed, 392 insertions(+), 138 deletions(-) create mode 100644 stixcore/products/level3/processing.py diff --git a/stixcore/io/FlareListManager.py b/stixcore/io/FlareListManager.py index 38359be6..e9d32e23 100644 --- a/stixcore/io/FlareListManager.py +++ b/stixcore/io/FlareListManager.py @@ -12,6 +12,7 @@ from astropy.time import Time from stixcore.config.config import CONFIG +from stixcore.io.product_processors.fits.processors import CreateUtcColumn from stixcore.products.level3.flarelist import FlarelistSC, FlarelistSDC from stixcore.products.product import Product from stixcore.util.logging import get_logger @@ -188,12 +189,17 @@ def get_data(self, *, start, end, fido_client): data["flare_id"] = Column( mt["flare_id"].astype(int), description=f"unique flare id for flarelist {self.flarelistname}" ) - data["start_UTC"] = Column(0, description="start time of flare") - data["start_UTC"] = [Time(d, format="isot", scale="utc") for d in mt["start_UTC"]] + CreateUtcColumn( + data, + [Time(d, format="isot", scale="utc") for d in mt["start_UTC"]], + "start_UTC", + description="start time of flare", + ) + data["duration"] = Column(mt["duration"].astype(float) * u.s, description="duration of flare") - data["end_UTC"] = Column(0, description="end time of flare") + data["end_UTC"] = CreateUtcColumn(description="end time of flare") data["end_UTC"] = [Time(d, format="isot", scale="utc") for d in mt["end_UTC"]] - data["peak_UTC"] = Column(0, description="flare peak time") + data["peak_UTC"] = CreateUtcColumn(description="flare peak time") data["peak_UTC"] = [Time(d, format="isot", scale="utc") for d in mt["peak_UTC"]] data["att_in"] = Column(mt["att_in"].astype(bool), description="was attenuator in during flare") data["bkg_baseline"] = Column(mt["LC0_BKG"] * u.ct, description="background baseline at 4-10 keV") @@ -277,7 +283,7 @@ def get_data(self, *, start, end, fido_client): data.add_index("flare_id") - # add energy axis for the lightcurve peek time data for each flare + # add energy axis for the lightcurve peak time data for each flare # the energy bins are taken from the daily ql-lightcurve products # as the definition of the lc energy chanel's are will change only very seldom # the ql-lightcurve products assume a constant definition for an entire day. @@ -439,13 +445,28 @@ def get_data(self, *, start, end, fido_client): data["flare_id"] = Column( mt["flare_id"].astype(int), description=f"unique flare id for flarelist {self.flarelistname}" ) - data["start_UTC"] = Column(0, description="start time of flare") - data["start_UTC"] = [Time(d, format="isot", scale="utc") for d in mt["start_UTC"]] + + CreateUtcColumn( + data, + [Time(d, format="isot", scale="utc") for d in mt["start_UTC"]], + "start_UTC", + description="start time of flare", + ) data["duration"] = Column(mt["duration"].astype(float) * u.s, description="duration of flare") - data["end_UTC"] = Column(0, description="end time of flare") - data["end_UTC"] = [Time(d, format="isot", scale="utc") for d in mt["end_UTC"]] - data["peak_UTC"] = Column(0, description="flare peak time") - data["peak_UTC"] = [Time(d, format="isot", scale="utc") for d in mt["peak_UTC"]] + + CreateUtcColumn( + data, + [Time(d, format="isot", scale="utc") for d in mt["end_UTC"]], + "end_UTC", + description="end time of flare", + ) + CreateUtcColumn( + data, + [Time(d, format="isot", scale="utc") for d in mt["peak_UTC"]], + "peak_UTC", + description="flare peak time", + ) + data["att_in"] = Column(mt["att_in"].astype(bool), description="was attenuator in during flare") data["bkg_baseline"] = Column(mt["LC0_BKG"] * u.ct, description="background baseline at 4-10 keV") data["GOES_class"] = Column( @@ -528,7 +549,7 @@ def get_data(self, *, start, end, fido_client): data.add_index("flare_id") - # add energy axis for the lightcurve peek time data for each flare + # add energy axis for the lightcurve peak time data for each flare # the energy bins are taken from the daily ql-lightcurve products # as the definition of the lc energy chanel's are will change only very seldom # the ql-lightcurve products assume a constant definition for an entire day. diff --git a/stixcore/io/product_processors/fits/processors.py b/stixcore/io/product_processors/fits/processors.py index 90630eea..7ca925b2 100644 --- a/stixcore/io/product_processors/fits/processors.py +++ b/stixcore/io/product_processors/fits/processors.py @@ -58,6 +58,24 @@ def set_bscale_unsigned(table_hdu): return table_hdu +def CreateUtcColumn(table, data, colname, description="UTC Time"): + """ + Create UTC time column for FITS tables. + + Parameters + ---------- + description : `str` + Description for the column + + Returns + ------- + `astropy.table.Column` + Column representing UTC time + """ + table[colname] = data + table[colname].info.description = description + + def add_default_tuint(table_hdu): """ Add a default empty string tunit if not already defined @@ -1139,7 +1157,7 @@ def write_fits(self, prod, *, version=0): # Add comment and history [primary_hdu.header.add_comment(com) for com in prod.comment] [primary_hdu.header.add_history(com) for com in prod.history] - primary_hdu.header.update({"HISTORY": "Processed by STIXCore ANC"}) + primary_hdu.header.update({"HISTORY": "Processed by STIXCore L3"}) if hasattr(prod, "maps") and len(prod.maps) > 0: # fig = plt.figure(figsize=(12, 6)) diff --git a/stixcore/processing/FLtoFL.py b/stixcore/processing/FLtoFL.py index 087f0d36..7f08acf1 100644 --- a/stixcore/processing/FLtoFL.py +++ b/stixcore/processing/FLtoFL.py @@ -15,7 +15,7 @@ ) from stixcore.products.level3.flarelist import ( FlareList, - FlarePeekPreviewMixin, + FlarePeakPreviewMixin, FlarePositionMixin, FlareSOOPMixin, ) @@ -107,7 +107,7 @@ def test_for_processing( """ try: c_header = fits.getheader(candidate) - f_data_end = datetime.fromisoformat(c_header["DATE-END"]) + # f_data_end = datetime.fromisoformat(c_header["DATE-END"]) f_create_date = datetime.fromisoformat(c_header["DATE"]) cfn = get_complete_file_name_and_path(candidate) @@ -128,8 +128,9 @@ def test_for_processing( # safety margin of 1day until we process higher products with position and pointing # only use flown spice kernels not predicted once as pointing information # can be "very off" - if f_data_end > (Spice.instance.get_mk_date(meta_kernel_type="flown") - timedelta(hours=24)): - return TestForProcessingResult.NotSuitable + # TODO redo + # if f_data_end > (Spice.instance.get_mk_date(meta_kernel_type="flown") - timedelta(hours=24)): + # return TestForProcessingResult.NotSuitable # safety margin of x until we start with processing the list files if f_create_date >= (datetime.now() - self.cadence): @@ -187,9 +188,9 @@ def process_fits_files( if issubclass(out_product, FlareSOOPMixin) and not issubclass(in_product, FlareSOOPMixin): out_product.add_soop(data) - # add peek preview images if not already present - if issubclass(out_product, FlarePeekPreviewMixin) and not issubclass(in_product, FlarePeekPreviewMixin): - out_product.add_peek_preview(data, energy, file_path.name, fido_client, img_processor, month=month) + # add peak preview images if not already present + if issubclass(out_product, FlarePeakPreviewMixin) and not issubclass(in_product, FlarePeakPreviewMixin): + out_product.add_peak_preview(data, energy, file_path.name, fido_client, img_processor, month=month) out_prod = out_product(control=control, data=data, month=month, energy=energy) out_prod.parent = file_path.name diff --git a/stixcore/processing/FlareListL3.py b/stixcore/processing/FlareListL3.py index 89e6d5c8..855bdde2 100644 --- a/stixcore/processing/FlareListL3.py +++ b/stixcore/processing/FlareListL3.py @@ -24,7 +24,7 @@ class FlareListL3(SingleProductProcessingStepMixin): """Processing step from a FlareListManager to monthly solo_L3_stix-flarelist-*.fits file.""" - STARTDATE = date(2024, 1, 1) + STARTDATE = date(2025, 1, 1) def __init__(self, flm: FlareListManager, output_dir: Path): """Crates a new Processor. diff --git a/stixcore/processing/pipeline_daily.py b/stixcore/processing/pipeline_daily.py index 09b19181..0b1a5a20 100644 --- a/stixcore/processing/pipeline_daily.py +++ b/stixcore/processing/pipeline_daily.py @@ -9,34 +9,27 @@ from stixcore.config.config import CONFIG from stixcore.ephemeris.manager import Spice, SpiceKernelManager +from stixcore.io.FlareListManager import SDCFlareListManager from stixcore.io.ProcessingHistoryStorage import ProcessingHistoryStorage from stixcore.io.product_processors.fits.processors import ( # FitsANCProcessor,; FitsL3Processor, + FitsANCProcessor, FitsL2Processor, + FitsL3Processor, ) from stixcore.io.product_processors.plots.processors import PlotProcessor from stixcore.io.RidLutManager import RidLutManager from stixcore.processing.AspectANC import AspectANC +from stixcore.processing.FlareListL3 import FlareListL3 +from stixcore.processing.FLtoFL import FLtoFL from stixcore.processing.LL import LL03QL from stixcore.processing.pipeline import PipelineStatus from stixcore.processing.SingleStep import SingleProcessingStepResult from stixcore.products.level1.quicklookL1 import LightCurve +from stixcore.products.level3.flarelist import FlarelistSDC, FlarelistSDCLoc from stixcore.products.lowlatency.quicklookLL import LightCurveL3 from stixcore.soop.manager import SOOPManager from stixcore.util.logging import STX_LOGGER_DATE_FORMAT, STX_LOGGER_FORMAT, get_logger -# from stixpy.net.client import STIXClient -# from stixcore.io.FlareListManager import SCFlareListManager, SDCFlareListManager -# from stixcore.processing.FlareListL3 import FlareListL3 -# from stixcore.processing.FLtoFL import FLtoFL -# from stixcore.products.level3.flarelist import ( -# FlarelistSC, -# FlarelistSCLoc, -# FlarelistSCLocImg, -# FlarelistSDC, -# FlarelistSDCLoc, -# FlarelistSDCLocImg, -# ) - logger = get_logger(__name__) @@ -215,8 +208,8 @@ def run_daily_pipeline(args): # SCFlareListManager.instance = SCFlareListManager(flare_lut_file, fido_client, update=True) # TODO reactivate once flarelist processing is finalized - # flare_lut_file = Path(CONFIG.get("Pipeline", "flareid_sdc_lut_file")) - # SDCFlareListManager.instance = SDCFlareListManager(flare_lut_file, update=False) + flare_lut_file = Path(CONFIG.get("Pipeline", "flareid_sdc_lut_file")) + SDCFlareListManager.instance = SDCFlareListManager(flare_lut_file, update=False) RidLutManager.instance = RidLutManager(Path(CONFIG.get("Publish", "rid_lut_file")), update=False) @@ -249,19 +242,19 @@ def run_daily_pipeline(args): aspect_anc_processor = AspectANC(fits_in_dir, fits_out_dir) # TODO reactivate once flarelist processing is finalized - # flarelist_sdc = FlareListL3(SDCFlareListManager.instance, fits_out_dir) + flarelist_sdc = FlareListL3(SDCFlareListManager.instance, fits_out_dir) # flarelist_sc = FlareListL3(SCFlareListManager.instance, fits_out_dir) - # fl_to_fl = FLtoFL( - # fits_in_dir, - # fits_out_dir, - # products_in_out=[ - # (FlarelistSDC, FlarelistSDCLoc), - # (FlarelistSDCLoc, FlarelistSDCLocImg), - # (FlarelistSC, FlarelistSCLoc), - # (FlarelistSCLoc, FlarelistSCLocImg), - # ], - # cadence=timedelta(seconds=1), - # ) + fl_to_fl = FLtoFL( + fits_in_dir, + fits_out_dir, + products_in_out=[ + (FlarelistSDC, FlarelistSDCLoc), + # (FlarelistSDCLoc, FlarelistSDCLocImg), + # (FlarelistSC, FlarelistSCLoc), + # (FlarelistSCLoc, FlarelistSCLocImg), + ], + cadence=timedelta(seconds=1), + ) ll03ql = LL03QL( fits_in_dir, fits_out_dir, in_product=LightCurve, out_product=LightCurveL3, cadence=timedelta(seconds=1) @@ -270,24 +263,25 @@ def run_daily_pipeline(args): plot_writer = PlotProcessor(fits_out_dir) l2_fits_writer = FitsL2Processor(fits_out_dir) # TODO reactivate once flarelist processing is finalized - # l3_fits_writer = FitsL3Processor(fits_out_dir) - # anc_fits_writer = FitsANCProcessor(fits_out_dir) + l3_fits_writer = FitsL3Processor(fits_out_dir) + anc_fits_writer = FitsANCProcessor(fits_out_dir) - hk_in_files = aspect_anc_processor.get_processing_files(phs) + # hk_in_files = aspect_anc_processor.get_processing_files(phs) + hk_in_files = [] - ll_candidates = ll03ql.get_processing_files(phs) - # ll_candidates = [] + # ll_candidates = ll03ql.get_processing_files(phs) + ll_candidates = [] # TODO reactivate once flarelist processing is finalized - # fl_sdc_months = flarelist_sdc.find_processing_months(phs) - # fl_sdc_months = [] + fl_sdc_months = flarelist_sdc.find_processing_months(phs) + fl_sdc_months = [] # TODO reactivate once flarelist processing is finalized # fl_sc_months = flarelist_sc.find_processing_months(phs) # fl_sc_months = [] # TODO reactivate once flarelist processing is finalized - # fl_to_fl_files = fl_to_fl.get_processing_files(phs) + fl_to_fl_files = fl_to_fl.get_processing_files(phs) # fl_to_fl_files = [] # all processing files should be terminated before the next step as the different @@ -306,16 +300,16 @@ def run_daily_pipeline(args): ) ) # TODO reactivate once flarelist processing is finalized - # jobs.append( - # executor.submit( - # flarelist_sdc.process_fits_files, - # fl_sdc_months, - # soopmanager=SOOPManager.instance, - # spice_kernel_path=Spice.instance.meta_kernel_path, - # processor=l3_fits_writer, - # config=CONFIG, - # ) - # ) + jobs.append( + executor.submit( + flarelist_sdc.process_fits_files, + fl_sdc_months, + soopmanager=SOOPManager.instance, + spice_kernel_path=Spice.instance.meta_kernel_path, + processor=l3_fits_writer, + config=CONFIG, + ) + ) # jobs.append( # executor.submit( @@ -330,17 +324,17 @@ def run_daily_pipeline(args): # # TODO a owen processing step for each flarelist file? # # for fl_to_fl_file in fl_to_fl_files: - # jobs.append( - # executor.submit( - # fl_to_fl.process_fits_files, - # fl_to_fl_files, - # soopmanager=SOOPManager.instance, - # spice_kernel_path=Spice.instance.meta_kernel_path, - # fl_processor=anc_fits_writer, - # img_processor=l3_fits_writer, - # config=CONFIG, - # ) - # ) + jobs.append( + executor.submit( + fl_to_fl.process_fits_files, + fl_to_fl_files, + soopmanager=SOOPManager.instance, + spice_kernel_path=Spice.instance.meta_kernel_path, + fl_processor=anc_fits_writer, + img_processor=l3_fits_writer, + config=CONFIG, + ) + ) jobs.append( executor.submit( diff --git a/stixcore/products/__init__.py b/stixcore/products/__init__.py index 7863b1bf..974838ed 100644 --- a/stixcore/products/__init__.py +++ b/stixcore/products/__init__.py @@ -10,5 +10,6 @@ from stixcore.products.level2.housekeepingL2 import * from stixcore.products.level2.quicklookL2 import * from stixcore.products.level2.scienceL2 import * +from stixcore.products.level3.flarelist import * from stixcore.products.levelb.binary import LevelB from stixcore.products.lowlatency.quicklookLL import * diff --git a/stixcore/products/level3/flarelist.py b/stixcore/products/level3/flarelist.py index 188f6193..80e30601 100644 --- a/stixcore/products/level3/flarelist.py +++ b/stixcore/products/level3/flarelist.py @@ -24,7 +24,8 @@ from stixcore.config.config import CONFIG from stixcore.ephemeris.manager import Spice -from stixcore.products.level3.flarelistproduct import PeekPreviewImage +from stixcore.products.level3.flarelistproduct import PeakPreviewImage +from stixcore.products.level3.processing import estimate_stix_flare_location from stixcore.products.product import CountDataMixin, GenericProduct, L2Mixin, read_qtable from stixcore.soop.manager import SOOPManager from stixcore.time import SCETime, SCETimeRange @@ -39,7 +40,7 @@ "FlareSOOPMixin", "FlareList", "FlarelistSDCLocImg", - "FlarePeekPreviewMixin", + "FlarePeakPreviewMixin", "FlarelistSC", "FlarelistSCLoc", "FlarelistSCLocImg", @@ -85,13 +86,16 @@ def add_flare_position( fido_client: STIXClient, *, filter_function=lambda x: True, - peek_time_colname="peak_UTC", + peak_time_colname="peak_UTC", start_time_colname="start_UTC", end_time_colname="end_UTC", keep_all_flares=True, month=None, ): - data["flare_position"] = [SkyCoord(0, 0, frame="icrs", unit="deg") for i in range(0, len(data))] + # helio_frame = Helioprojective(observer="earth") + # SkyCoord(HeliographicStonyhurst(0 * u.deg, 0 * u.deg)) + # SkyCoord(0 * u.deg, 0 * u.deg, frame=helio_frame) + data["flare_position"] = [SkyCoord(HeliographicStonyhurst(0 * u.deg, 0 * u.deg)) for i in range(0, len(data))] data["anc_ephemeris_path"] = Column(" " * 500, dtype=str, description="TDB") data["cpd_path"] = Column(" " * 500, dtype=str, description="TDB") @@ -106,11 +110,11 @@ def add_flare_position( total_flares = len(data) day_asp_ephemeris_cache = dict() - + flare_positions = [] for i, row in enumerate(data): - if filter_function(row): + if filter_function(row) and i < 200: pass_filter += 1 - peak_time = row[peek_time_colname] + peak_time = row[peak_time_colname] start_time = row[start_time_colname] end_time = row[end_time_colname] @@ -187,15 +191,26 @@ def add_flare_position( best_cpd_idx = 0 data[i]["cpd_path"] = cpd_res["path"][best_cpd_idx] - # do the calculations with stixpy + try: + stixpy_cpd = STIXPYProduct(Path(data[i]["cpd_path"])) + coord, map = estimate_stix_flare_location(stixpy_cpd) + + roll, solo_xyz, pointing = get_hpc_info(start_time, end_time) + solo = HeliographicStonyhurst(*solo_xyz, obstime=peak_time, representation_type="cartesian") - data[i]["flare_position"] = SkyCoord(1, 1, frame="icrs", unit="deg") - data[i]["_position_status"] = True - data[i]["_position_message"] = "OK" + # data[i]["flare_position"] = coord.transform_to(Helioprojective(observer=solo)) + flare_positions.append(coord.transform_to(Helioprojective(observer=solo))) + data[i]["_position_status"] = True + data[i]["_position_message"] = "OK" + except Exception as e: + flare_positions.append(None) + data[i]["_position_message"] = f"Error: {type(e)}" else: to_remove.append(i) + flare_positions.append(None) + data["flare_position"] = flare_positions if not keep_all_flares: data.remove_rows(to_remove) @@ -203,7 +218,8 @@ def add_flare_position( f"Flare position calculated for month {month} with {total_flares} flares, " f"passed filter: {pass_filter} no ephemeris data found for {no_ephemeris} " f"flares, no CPD data found for {no_cpd} flares, many CPD data found for " - f"{many_cpd} flares, one CPD data found for {one_cpd} flares" + f"{many_cpd} flares, one CPD data found for {one_cpd} flares." + f"finally {len(data)} flares remaining" ) @@ -212,14 +228,14 @@ class FlareSOOPMixin: @classmethod def add_soop( - self, data, *, peek_time_colname="peak_UTC", start_time_colname="start_UTC", end_time_colname="end_UTC" + self, data, *, peak_time_colname="peak_UTC", start_time_colname="start_UTC", end_time_colname="end_UTC" ): soop_encoded_type = list() soop_id = list() soop_type = list() for row in data: - soops = SOOPManager.instance.find_soops(start=row[peek_time_colname]) + soops = SOOPManager.instance.find_soops(start=row[peak_time_colname]) if soops: soop = soops[0] soop_encoded_type.append(soop.encodedSoopType) @@ -235,13 +251,13 @@ def add_soop( data["soop_type"] = Column(soop_type, dtype=str, description="name of the SOOP campaign") -class FlarePeekPreviewMixin: - """Mixin class to add peek preview images to flare list products. - This class provides a method to generate and add peek preview images +class FlarePeakPreviewMixin: + """Mixin class to add peak preview images to flare list products. + This class provides a method to generate and add peak preview images to the flare list data. The images are generated based on the flare's peak time, start time, and end time, using the STIXPy library for visibility calculations and image reconstruction. - The generated images are stored in the 'peek_preview_path' column of the data. + The generated images are stored in the 'peak_preview_path' column of the data. The method also updates the status and message columns to indicate the success or failure of the image generation process. @@ -249,7 +265,7 @@ class FlarePeekPreviewMixin: """ @classmethod - def add_peek_preview( + def add_peak_preview( cls, data, energies, @@ -257,7 +273,7 @@ def add_peek_preview( fido_client: STIXClient, img_processor, *, - peek_time_colname="peak_UTC", + peak_time_colname="peak_UTC", start_time_colname="start_UTC", end_time_colname="end_UTC", anc_ephemeris_path_colname="anc_ephemeris_path", @@ -266,17 +282,17 @@ def add_peek_preview( keep_all_flares=True, month=None, ): - data["peek_preview_path"] = Column(" " * 500, dtype=str, description="TDB") - data["preview_start_UTC"] = [Time(d, format="isot", scale="utc") for d in data[peek_time_colname]] - data["preview_end_UTC"] = [Time(d, format="isot", scale="utc") for d in data[peek_time_colname]] - data["_peek_preview_status"] = Column(False, dtype=bool, description="TDB") - data["_peek_preview_message"] = Column(" " * 500, dtype=str, description="TDB") + data["peak_preview_path"] = Column(" " * 500, dtype=str, description="TDB") + data["preview_start_UTC"] = [Time(d, format="isot", scale="utc") for d in data[peak_time_colname]] + data["preview_end_UTC"] = [Time(d, format="isot", scale="utc") for d in data[peak_time_colname]] + data["_peak_preview_status"] = Column(False, dtype=bool, description="TDB") + data["_peak_preview_message"] = Column(" " * 500, dtype=str, description="TDB") to_remove = [] products = [] images = 0 for i, row in enumerate(data): - peak_time = row[peek_time_colname] + peak_time = row[peak_time_colname] row[start_time_colname] row[end_time_colname] @@ -286,8 +302,8 @@ def add_peek_preview( status = False message = "" - peek_preview_start = row[peek_time_colname] - peek_preview_end = row[peek_time_colname] + peak_preview_start = row[peak_time_colname] + peak_preview_end = row[peak_time_colname] if anc_ephemeris_path.exists() and cpd_path.exists(): try: @@ -295,18 +311,18 @@ def add_peek_preview( # do the imaging with stixpy preview_data = data[i : i + 1] - del preview_data["peek_preview_path"] - del preview_data["_peek_preview_status"] - del preview_data["_peek_preview_message"] + del preview_data["peak_preview_path"] + del preview_data["_peak_preview_status"] + del preview_data["_peak_preview_message"] - peek_preview_start = row[peek_time_colname] - 10 * u.s - peek_preview_end = row[peek_time_colname] + 10 * u.s + peak_preview_start = row[peak_time_colname] - 10 * u.s + peak_preview_end = row[peak_time_colname] + 10 * u.s - preview_data["preview_start_UTC"] = peek_preview_start - preview_data["preview_end_UTC"] = peek_preview_end + preview_data["preview_start_UTC"] = peak_preview_start + preview_data["preview_end_UTC"] = peak_preview_end cpd_sci = STIXPYProduct(cpd_path) - time_range_sci = [peek_preview_start, peek_preview_end] + time_range_sci = [peak_preview_start, peak_preview_end] maps = [] for energy_range in [[4, 20], [20, 120]] * u.keV: # flare_position = preview_data['flare_position'][0] @@ -388,7 +404,7 @@ def add_peek_preview( maps.append((map_with_erange, header)) - ppi = PeekPreviewImage( + ppi = PeakPreviewImage( control=QTable(), data=preview_data, month=month, @@ -407,18 +423,18 @@ def add_peek_preview( status = False message = str(e) - data[i]["preview_start_UTC"] = peek_preview_start - data[i]["preview_end_UTC"] = peek_preview_end - data[i]["peek_preview_path"] = "test" - data[i]["_peek_preview_status"] = status - data[i]["_peek_preview_message"] = message + data[i]["preview_start_UTC"] = peak_preview_start + data[i]["preview_end_UTC"] = peak_preview_end + data[i]["peak_preview_path"] = "test" + data[i]["_peak_preview_status"] = status + data[i]["_peak_preview_message"] = message if not keep_all_flares: data.remove_rows(to_remove) logger.info( f"Flare images created for month {month} with {len(data)} flares, " - f"{len(products)} peek previews created, with total {images} images" + f"{len(products)} peak previews created, with total {images} images" ) return products @@ -558,7 +574,7 @@ def add_flare_position(cls, data, fido_client: STIXClient, *, month=None): data, fido_client, filter_function=cls.filter_flare_function, - peek_time_colname="peak_UTC", + peak_time_colname="peak_UTC", start_time_colname="start_UTC", end_time_colname="end_UTC", keep_all_flares=False, @@ -570,7 +586,7 @@ def is_datasource_for(cls, *, service_type, service_subtype, ssid, **kwargs): return kwargs["level"] == "L3" and service_type == 0 and service_subtype == 0 and ssid == 3 -class FlarelistSDCLocImg(FlarelistSDCLoc, FlarePeekPreviewMixin): +class FlarelistSDCLocImg(FlarelistSDCLoc, FlarePeakPreviewMixin): """Flarelist product class for StixDataCenter flares. In ANC product format. @@ -589,14 +605,14 @@ def enhance_from_product(self, in_prod: GenericProduct): pass @classmethod - def add_peek_preview(cls, data, energies, parent, fido_client: STIXClient, img_processor, *, month=None): - super().add_peek_preview( + def add_peak_preview(cls, data, energies, parent, fido_client: STIXClient, img_processor, *, month=None): + super().add_peak_preview( data, energies, parent, fido_client, img_processor, - peek_time_colname="peak_UTC", + peak_time_colname="peak_UTC", start_time_colname="start_UTC", end_time_colname="end_UTC", anc_ephemeris_path_colname="anc_ephemeris_path", @@ -695,7 +711,7 @@ def add_flare_position(cls, data, fido_client: STIXClient, *, month=None): data, fido_client, filter_function=cls.filter_flare_function, - peek_time_colname="peak_UTC", + peak_time_colname="peak_UTC", start_time_colname="start_UTC", end_time_colname="end_UTC", keep_all_flares=False, @@ -707,7 +723,7 @@ def is_datasource_for(cls, *, service_type, service_subtype, ssid, **kwargs): return kwargs["level"] == "L3" and service_type == 0 and service_subtype == 0 and ssid == 7 -class FlarelistSCLocImg(FlarelistSCLoc, FlarePeekPreviewMixin): +class FlarelistSCLocImg(FlarelistSCLoc, FlarePeakPreviewMixin): """Flarelist product class for StixCore flares. In ANC product format. @@ -726,14 +742,14 @@ def enhance_from_product(self, in_prod: GenericProduct): pass @classmethod - def add_peek_preview(cls, data, energies, parent, fido_client: STIXClient, img_processor, *, month=None): - super().add_peek_preview( + def add_peak_preview(cls, data, energies, parent, fido_client: STIXClient, img_processor, *, month=None): + super().add_peak_preview( data, energies, parent, fido_client, img_processor, - peek_time_colname="peak_UTC", + peak_time_colname="peak_UTC", start_time_colname="start_UTC", end_time_colname="end_UTC", anc_ephemeris_path_colname="anc_ephemeris_path", diff --git a/stixcore/products/level3/flarelistproduct.py b/stixcore/products/level3/flarelistproduct.py index ae7cb0d4..032864e1 100644 --- a/stixcore/products/level3/flarelistproduct.py +++ b/stixcore/products/level3/flarelistproduct.py @@ -6,7 +6,7 @@ from stixcore.time.datetime import SCETime, SCETimeRange from stixcore.util.logging import get_logger -__all__ = ["FlareListProduct", "PeekPreviewImage"] +__all__ = ["FlareListProduct", "PeakPreviewImage"] logger = get_logger(__name__) @@ -22,17 +22,17 @@ def from_timerange(cls, timerange: SCETimeRange, *, flarelistparent: str = ""): pass -class PeekPreviewImage(FlareListProduct): +class PeakPreviewImage(FlareListProduct): PRODUCT_PROCESSING_VERSION = 1 Level = "L3" Type = "sci" - Name = "peekpreviewimg" + Name = "peakpreviewimg" def __init__(self, control, data, energy, maps, parents, *, product_name_suffix="", **kwargs): super().__init__(service_type=0, service_subtype=0, ssid=5, control=control, data=data, energy=energy, **kwargs) - self.name = f"{PeekPreviewImage.Name}-{product_name_suffix}" - self.level = PeekPreviewImage.Level - self.type = PeekPreviewImage.Type + self.name = f"{PeakPreviewImage.Name}-{product_name_suffix}" + self.level = PeakPreviewImage.Level + self.type = PeakPreviewImage.Type self.energy = energy self.maps = maps self.parents = parents @@ -62,4 +62,4 @@ def split_to_files(self): @classmethod def is_datasource_for(cls, *, service_type, service_subtype, ssid, **kwargs): - return kwargs["level"] == PeekPreviewImage.Level and service_type == 0 and service_subtype == 0 and ssid == 5 + return kwargs["level"] == PeakPreviewImage.Level and service_type == 0 and service_subtype == 0 and ssid == 5 diff --git a/stixcore/products/level3/processing.py b/stixcore/products/level3/processing.py new file mode 100644 index 00000000..e6225a1e --- /dev/null +++ b/stixcore/products/level3/processing.py @@ -0,0 +1,203 @@ +######################################################### +### Temporary code should be come from stixpy finally ### +######################################################### + +import numpy as np +import stixpy.calibration.visibility +import stixpy.coordinates.transforms +import sunpy.map +import sunpy.time +import xrayvision.imaging +from stixpy.coordinates.transforms import STIXImaging +from sunpy.coordinates import HeliographicStonyhurst + +import astropy.units as u +from astropy.coordinates import SkyCoord +from astropy.time import Time + + +def construct_stix_calibrated_visibilities( + cpd_sci, + flare_location, + time_range=None, + energy_range=None, + subcollimators=None, + cpd_bkg=None, + time_range_bkg=None, + **kwargs, +): + """ + Constructs calibrated STIX visibilities from STIX compressed pixel data. + + Extra kwargs are passed to `stixpy.calibration.visibility.create_meta_pixels` + + Parameters + ---------- + cpd_sci: `stixpy.product.Product` + The STIX pixel data. Assumed to be already background subtracted. + flare_location: `astropy.coordinates.SkyCoord` + The flare location. Frame must be convertible to `stixpy.coordinates.transdforms.STIXImaging`. + time_range: `sunpy.time.TimeRange` (optional) + The time range over which to estimate the flare location. + Default is all times in cpd_sci. + energy_range: `astropy.units.Quantity` in spectral units (optional) + Length-2 quantity giving the lower and upper bounds of the energy range to use for imaging. + Default is all finite energies in cpd_sci. + subcollimators: `iterable` of `str` + The labels of the subcollimators to use in estimating the flare locations, e.g. + ``['10a', '10b', '10c',...]`` + Default is all subcollimators in cpd_sci. + cpd_bkg: stixpy.product.Product` (optional) + The background to subtract from the pixel data before determining the flare location. + If None and time_range_bkg is also None, no background is subtracted. + time_range_bkg: `sunpy.time.TimeRange` + The time range within cpd_bkg to use for the background subtraction. + If None, entire time range of cpd_bkg is used to determine background. + If not None, and cpd_bkg is None, the background is determined from this time range + applied to cpd_sci. + + Returns + ------- + vis: `xrayvision.visibility.Visibilities` + The calibrated STIX visibilities. + """ + # Sanitze inputs. + if time_range is None: + time_range = cpd_sci.time_range + times = Time([time_range.start, time_range.end]) + if energy_range is None: + energy_range = u.Quantity( + [ + cpd_sci.energies["e_low"][np.isfinite(cpd_sci.energies["e_low"])][0], + cpd_sci.energies["e_high"][np.isfinite(cpd_sci.energies["e_high"])][-1], + ] + ) + no_shadowing = kwargs.pop("no_shadowing", True) + # Generate meta_pixels from pixel_data. + meta_pixels = stixpy.calibration.visibility.create_meta_pixels( + cpd_sci, + time_range=times, + energy_range=energy_range, + flare_location=flare_location, + no_shadowing=no_shadowing, + **kwargs, + ) + # Subtract background if background pixel data provided. + if cpd_bkg is None and time_range_bkg is not None: + cpd_bkg = cpd_sci + if cpd_bkg is not None: + if time_range_bkg is None: + time_range_bkg = cpd_bkg.time_range + times_bkg = Time([time_range_bkg.start, time_range_bkg.end]) + meta_pixels_bkg = stixpy.calibration.visibility.create_meta_pixels( + cpd_bkg, + time_range=times_bkg, + energy_range=energy_range, + flare_location=[0, 0] * u.arcsec, + no_shadowing=no_shadowing, + **kwargs, + ) + meta_pixels = _subtract_background_from_stix_meta_pixels(meta_pixels, meta_pixels_bkg) + # Generate and calibrate visibilities. + vis = stixpy.calibration.visibility.create_visibility(meta_pixels) + vis = stixpy.calibration.visibility.calibrate_visibility(vis, flare_location=SkyCoord(flare_location)) + if subcollimators is not None: + idx_subcol = np.argwhere(np.isin(vis.meta["vis_labels"], subcollimators)).ravel() + vis = vis[idx_subcol] + return vis + + +def estimate_stix_flare_location( + cpd_sci, time_range=None, energy_range=None, subcollimators=None, cpd_bkg=None, time_range_bkg=None +): + """ + Estimates flare location from STIX compressed pixel data. + + FLare location is assumed to be the location of the brightest pixel in the backprojection + map calculated from the input STIX pixel data. + + Parameters + ---------- + cpd_sci: `stixpy.product.Product` + The STIX pixel data. Assumed to be already background subtracted. + time_range: `sunpy.time.TimeRange` (optional) + The time range over which to estimate the flare location. + Default is all times in cpd_sci. + energy_range: `astropy.units.Quantity` in spectral units (optional) + Length-2 quantity giving the lower and upper bounds of the energy range to use for imaging. + Default is all finite energies in cpd_sci. + subcollimators: `iterable` of `str` (optional) + The labels of the subcollimators to include in the output Visibilities object, e.g. + ``['10a', '10b', '10c',...]`` + Default is all subcollimators in cpd_sci + cpd_bkg: stixpy.product.Product` (optional) + The background to subtract from the pixel data before determining the flare location. + Passed to construct_stix_calibrated_visibilities(). + time_range_bkg: `sunpy.time.TimeRange` + The time range within cpd_bkg to use for the background subtraction. + Passed to construct_stix_calibrated_visibilities(). + + Returns + ------- + flare_loc: `astropy.coordinates.SkyCoord` + The estimated flare location. + map_bp: `sunpy.map.Map` + The backprojection map from whose brightest pixel the flare location was estimated. + """ + if time_range is None: + time_range = cpd_sci.time_range + if subcollimators is None: + subcollimators = ["10a", "10b", "10c", "9a", "9b", "9c", "8a", "8b", "8c", "7a", "7b", "7c"] + # Construct STIX location and centre of FOV. + roll, solo_xyz, pointing = stixpy.coordinates.transforms.get_hpc_info(time_range.start, time_range.end) + solo = HeliographicStonyhurst(*solo_xyz, obstime=time_range.center, representation_type="cartesian") + fov_centre = STIXImaging( + 0 * u.arcsec, 0 * u.arcsec, obstime=time_range.start, obstime_end=time_range.end, observer=solo + ) + # Generate calibrated visibilities using coarser subcollimators. + vis = construct_stix_calibrated_visibilities( + cpd_sci, fov_centre, time_range=time_range, energy_range=energy_range, subcollimators=subcollimators + ) + # Produce backprojected image and find brightest pixel. Use this for flare location. + imsize = [512, 512] * u.pixel # number of pixels of the map to reconstruct + plate_scale = [10, 10] * u.arcsec / u.pixel # pixel size in arcsec + bp_image = xrayvision.imaging.vis_to_image(vis, imsize, pixel_size=plate_scale) + max_idx = np.argwhere(bp_image == bp_image.max()).ravel() + # Calculate WCS for backprojected image in STIXImaging and HPC frames. + # Recalculate STIX HPC info as slightly different times will have been used + # than input times due to onboard STIX time binning. + vis_tr = sunpy.time.TimeRange(vis.meta["time_range"]) + roll, solo_xyz, pointing = stixpy.coordinates.transforms.get_hpc_info(vis_tr.start, vis_tr.end) + solo = HeliographicStonyhurst(*solo_xyz, obstime=vis_tr.center, representation_type="cartesian") + coord = STIXImaging(0 * u.arcsec, 0 * u.arcsec, obstime=vis_tr.start, obstime_end=vis_tr.end, observer=solo) + header_bp = sunpy.map.make_fitswcs_header( + bp_image, coord, telescope="STIX", observatory="Solar Orbiter", scale=plate_scale + ) + map_bp = sunpy.map.Map(bp_image, header_bp) + wcs_bp = map_bp.wcs + # Estimate flare location from brightest pixel in backprojection image + flare_loc = wcs_bp.array_index_to_world(*max_idx) + return flare_loc, map_bp + + +def _subtract_background_from_stix_meta_pixels(meta_pixels_sci, meta_pixels_bkg): + """ + Estimates flare location from STIX pixel data. + + Parameters + ---------- + meta_pixels_sci: `dict` + The STIX meta pixel representing the observations. Format must be + same as output from `stixpy.calibration.visibility.create_meta_pixels`. + meta_pixels_bkg: `dict` + The STIX meta pixels representing the background. Format must be + same as output from `stixpy.calibration.visibility.create_meta_pixels`. + """ + meta_pixels_bkg_sub = { + **meta_pixels_sci, + "abcd_rate_kev_cm": meta_pixels_sci["abcd_rate_kev_cm"] - meta_pixels_bkg["abcd_rate_kev_cm"], + "abcd_rate_error_kev_cm": np.sqrt( + meta_pixels_sci["abcd_rate_error_kev_cm"] ** 2 + meta_pixels_bkg["abcd_rate_error_kev_cm"] ** 2 + ), + } + return meta_pixels_bkg_sub diff --git a/stixcore/products/product.py b/stixcore/products/product.py index 02712266..27415375 100644 --- a/stixcore/products/product.py +++ b/stixcore/products/product.py @@ -74,7 +74,7 @@ def read_qtable(file, hdu, hdul=None): `astropy.table.QTable` The corrected QTable with correct data types """ - qtable = QTable.read(file, hdu) + qtable = QTable.read(file, hdu, astropy_native=True) if hdul is None: hdul = fits.open(file) From 0b5f6c4a1c9904d434d95e5e190c6b783cabc0c2 Mon Sep 17 00:00:00 2001 From: Nicky Hochmuth Date: Mon, 26 Jan 2026 17:57:32 +0100 Subject: [PATCH 02/10] skyCoords components as single columns --- stixcore/processing/FlareListL3.py | 2 +- stixcore/products/level3/flarelist.py | 119 +++++++++++++++++++++++--- stixcore/soop/manager.py | 2 + 3 files changed, 110 insertions(+), 13 deletions(-) diff --git a/stixcore/processing/FlareListL3.py b/stixcore/processing/FlareListL3.py index 855bdde2..5a6e8b3b 100644 --- a/stixcore/processing/FlareListL3.py +++ b/stixcore/processing/FlareListL3.py @@ -24,7 +24,7 @@ class FlareListL3(SingleProductProcessingStepMixin): """Processing step from a FlareListManager to monthly solo_L3_stix-flarelist-*.fits file.""" - STARTDATE = date(2025, 1, 1) + STARTDATE = date(2025, 7, 1) def __init__(self, flm: FlareListManager, output_dir: Path): """Crates a new Processor. diff --git a/stixcore/products/level3/flarelist.py b/stixcore/products/level3/flarelist.py index 80e30601..9379f02f 100644 --- a/stixcore/products/level3/flarelist.py +++ b/stixcore/products/level3/flarelist.py @@ -1,5 +1,6 @@ from pathlib import Path from datetime import datetime +from itertools import groupby import numpy as np from stixpy.calibration.visibility import ( @@ -10,7 +11,7 @@ from stixpy.coordinates.transforms import get_hpc_info from stixpy.net.client import STIXClient from stixpy.product import Product as STIXPYProduct -from sunpy.coordinates import HeliographicStonyhurst, Helioprojective +from sunpy.coordinates import HeliographicStonyhurst, Helioprojective, SphericalScreen from sunpy.map import make_fitswcs_header from sunpy.net import attrs as a from sunpy.time import TimeRange @@ -95,12 +96,26 @@ def add_flare_position( # helio_frame = Helioprojective(observer="earth") # SkyCoord(HeliographicStonyhurst(0 * u.deg, 0 * u.deg)) # SkyCoord(0 * u.deg, 0 * u.deg, frame=helio_frame) - data["flare_position"] = [SkyCoord(HeliographicStonyhurst(0 * u.deg, 0 * u.deg)) for i in range(0, len(data))] + + n = len(data) + + data["flareposition_obs_hgs_x"] = Column( + np.zeros(n, dtype=float) * u.km, description="HeliographicStonyhurst X of observer" + ) + data["flareposition_obs_hgs_y"] = Column( + np.zeros(n, dtype=float) * u.km, description="HeliographicStonyhurst Y of observer" + ) + data["flareposition_obs_hgs_z"] = Column( + np.zeros(n, dtype=float) * u.km, description="HeliographicStonyhurst Z of observer" + ) + data["flareposition_hp_tx"] = Column(np.zeros(n, dtype=float) * u.arcsec, description="Helioprojective Tx") + data["flareposition_hp_ty"] = Column(np.zeros(n, dtype=float) * u.arcsec, description="Helioprojective Ty") data["anc_ephemeris_path"] = Column(" " * 500, dtype=str, description="TDB") data["cpd_path"] = Column(" " * 500, dtype=str, description="TDB") data["_position_status"] = Column(False, dtype=bool, description="TDB") data["_position_message"] = Column(" " * 500, dtype=str, description="TDB") + to_remove = [] pass_filter = 0 no_ephemeris = 0 @@ -110,9 +125,9 @@ def add_flare_position( total_flares = len(data) day_asp_ephemeris_cache = dict() - flare_positions = [] + for i, row in enumerate(data): - if filter_function(row) and i < 200: + if filter_function(row): # and i < 200: pass_filter += 1 peak_time = row[peak_time_colname] start_time = row[start_time_colname] @@ -183,7 +198,7 @@ def add_flare_position( cpd_res["duration"][i] = header["OBT_END"] - header["OBT_BEG"] # TODO: add more criteria to select the best CPD file - cpd_res.sort(["tbins", "duration"]) + cpd_res.sort(["inc_peak", "tbins", "duration"], reverse=True) # cpd_res.pprint() best_cpd_idx = 0 else: @@ -193,24 +208,63 @@ def add_flare_position( try: stixpy_cpd = STIXPYProduct(Path(data[i]["cpd_path"])) - coord, map = estimate_stix_flare_location(stixpy_cpd) + time_range = TimeRange(max(peak_time - 20 * u.s, start_time), min(peak_time + 20 * u.s, end_time)) + overlaps = calculate_overlap(stixpy_cpd.time_range, time_range) + if overlaps is None: + logger.warning( + f"CPD data does not cover time range around peak time {time_range.start} to {time_range.end}" + ) + time_range = stixpy_cpd.time_range + contains_peak_time = False + else: + contains_peak_time = True + time_range = overlaps + + mask = (stixpy_cpd.data["time"] >= time_range.start) & (stixpy_cpd.data["time"] <= time_range.end) + data_at_peak = stixpy_cpd.data[mask] + if len(np.unique(data_at_peak["rcr"])) > 1: + logger.warning( + f"Multiple rcr values found for flare at time {time_range.start} : {time_range.end}" + ) + # allow a larger time range for finding a constant rcr sequence + if contains_peak_time: + time_range = TimeRange( + max(peak_time - 40 * u.s, start_time), min(peak_time + 40 * u.s, end_time) + ) + mask = (stixpy_cpd.data["time"] >= time_range.start) & ( + stixpy_cpd.data["time"] <= time_range.end + ) + data_at_peak = stixpy_cpd.data[mask] + length, start_idx, rcr = longest_constant_sequence(data_at_peak["rcr"].value) + time_range = TimeRange( + data_at_peak["time"][start_idx], data_at_peak["time"][start_idx + length - 1] + ) + logger.info( + f"Using time range {time_range.start} to {time_range.end} for flare at around {peak_time} with constant rcr={rcr}" + ) + + coord, map = estimate_stix_flare_location(stixpy_cpd, time_range=time_range) roll, solo_xyz, pointing = get_hpc_info(start_time, end_time) solo = HeliographicStonyhurst(*solo_xyz, obstime=peak_time, representation_type="cartesian") + with SphericalScreen(solo, only_off_disk=True): + center_hpc = coord.transform_to(Helioprojective(observer=solo)) + + data[i]["flareposition_obs_hgs_x"] = solo_xyz[0].to(u.km) + data[i]["flareposition_obs_hgs_y"] = solo_xyz[1].to(u.km) + data[i]["flareposition_obs_hgs_z"] = solo_xyz[2].to(u.km) + data[i]["flareposition_hp_tx"] = center_hpc.Tx.to(u.arcsec) + data[i]["flareposition_hp_ty"] = center_hpc.Ty.to(u.arcsec) - # data[i]["flare_position"] = coord.transform_to(Helioprojective(observer=solo)) - flare_positions.append(coord.transform_to(Helioprojective(observer=solo))) data[i]["_position_status"] = True data[i]["_position_message"] = "OK" except Exception as e: - flare_positions.append(None) + data[i]["_position_status"] = False data[i]["_position_message"] = f"Error: {type(e)}" - + logger.warn(f"Error calculating flare position for flare at time {start_time} : {end_time}: {e}") else: to_remove.append(i) - flare_positions.append(None) - data["flare_position"] = flare_positions if not keep_all_flares: data.remove_rows(to_remove) @@ -762,3 +816,44 @@ def add_peak_preview(cls, data, energies, parent, fido_client: STIXClient, img_p @classmethod def is_datasource_for(cls, *, service_type, service_subtype, ssid, **kwargs): return kwargs["level"] == "L3" and service_type == 0 and service_subtype == 0 and ssid == 8 + + +def longest_constant_sequence(state_array): + """Find the longest sequence where state is constant. + In case of equal length, prefer the one with the lower state value.""" + if len(state_array) == 0: + return 0, None, None + + max_length = 0 + max_state = None + max_start_idx = None + current_idx = 0 + + for state, group in groupby(state_array): + length = len(list(group)) + # Update if longer, OR if equal length but lower state value + if length > max_length or (length == max_length and (max_state is None or state < max_state)): + max_length = length + max_state = state + max_start_idx = current_idx + current_idx += length + + return max_length, max_start_idx, max_state + + +def calculate_overlap(range1, range2): + """Calculate the overlap between two TimeRanges. + Returns the overlap duration and the overlapping TimeRange, or None if no overlap.""" + + # Check if they intersect first + if not range1.intersects(range2): + return None + + # Calculate intersection boundaries + overlap_start = max(range1.start, range2.start) + overlap_end = min(range1.end, range2.end) + + # Create the overlapping TimeRange + overlap_range = TimeRange(overlap_start, overlap_end) + + return overlap_range diff --git a/stixcore/soop/manager.py b/stixcore/soop/manager.py index b3dd242e..142bfa42 100644 --- a/stixcore/soop/manager.py +++ b/stixcore/soop/manager.py @@ -529,6 +529,8 @@ def add_soop_file_to_index(self, path, *, rebuild_index=True, **args): all_soop_file = Path(CONFIG.get("SOOP", "soop_files_download")) / f"{plan}.{version}.all.json" if not all_soop_file.exists(): + # TODO remove ove SOOP API is working reliably + return self.download_all_soops_from_api(plan, version, all_soop_file) with open(all_soop_file) as f_all: From 1e3f10ca0f8abd7625fb011d63e3f3c17bc7e054 Mon Sep 17 00:00:00 2001 From: Nicky Hochmuth Date: Mon, 23 Mar 2026 10:29:33 +0100 Subject: [PATCH 03/10] SkyCoord for flarePosition Fits de/serialize hook for converting into fits serializeable icrs frame --- .../io/product_processors/fits/processors.py | 3 +- stixcore/products/level3/flarelist.py | 104 +++++++++++++----- stixcore/products/product.py | 15 +++ stixcore/products/tests/test_flarelist.py | 86 +++++++++++++++ 4 files changed, 180 insertions(+), 28 deletions(-) create mode 100644 stixcore/products/tests/test_flarelist.py diff --git a/stixcore/io/product_processors/fits/processors.py b/stixcore/io/product_processors/fits/processors.py index 7ca925b2..a9887344 100644 --- a/stixcore/io/product_processors/fits/processors.py +++ b/stixcore/io/product_processors/fits/processors.py @@ -1144,7 +1144,8 @@ def write_fits(self, prod, *, version=0): elif fitspath_complete.exists(): logger.warning("Complete Fits file %s exists will be overridden", fitspath.name) - data = prod.data + data = prod.data.copy() + prod.on_serialize(data) primary_header, header_override = self.generate_primary_header(filename, prod, version=version) diff --git a/stixcore/products/level3/flarelist.py b/stixcore/products/level3/flarelist.py index 9379f02f..ee4e750b 100644 --- a/stixcore/products/level3/flarelist.py +++ b/stixcore/products/level3/flarelist.py @@ -97,24 +97,12 @@ def add_flare_position( # SkyCoord(HeliographicStonyhurst(0 * u.deg, 0 * u.deg)) # SkyCoord(0 * u.deg, 0 * u.deg, frame=helio_frame) - n = len(data) - - data["flareposition_obs_hgs_x"] = Column( - np.zeros(n, dtype=float) * u.km, description="HeliographicStonyhurst X of observer" - ) - data["flareposition_obs_hgs_y"] = Column( - np.zeros(n, dtype=float) * u.km, description="HeliographicStonyhurst Y of observer" - ) - data["flareposition_obs_hgs_z"] = Column( - np.zeros(n, dtype=float) * u.km, description="HeliographicStonyhurst Z of observer" - ) - data["flareposition_hp_tx"] = Column(np.zeros(n, dtype=float) * u.arcsec, description="Helioprojective Tx") - data["flareposition_hp_ty"] = Column(np.zeros(n, dtype=float) * u.arcsec, description="Helioprojective Ty") - data["anc_ephemeris_path"] = Column(" " * 500, dtype=str, description="TDB") data["cpd_path"] = Column(" " * 500, dtype=str, description="TDB") data["_position_status"] = Column(False, dtype=bool, description="TDB") data["_position_message"] = Column(" " * 500, dtype=str, description="TDB") + tx_list, ty_list = [], [] + solo_x_list, solo_y_list, solo_z_list, peak_time_list = [], [], [], [] to_remove = [] pass_filter = 0 @@ -127,12 +115,11 @@ def add_flare_position( day_asp_ephemeris_cache = dict() for i, row in enumerate(data): - if filter_function(row): # and i < 200: + peak_time = row[peak_time_colname] + start_time = row[start_time_colname] + end_time = row[end_time_colname] + if filter_function(row): # and i < 60: pass_filter += 1 - peak_time = row[peak_time_colname] - start_time = row[start_time_colname] - end_time = row[end_time_colname] - day = peak_time.to_datetime().date() if day in day_asp_ephemeris_cache: @@ -249,21 +236,54 @@ def add_flare_position( solo = HeliographicStonyhurst(*solo_xyz, obstime=peak_time, representation_type="cartesian") with SphericalScreen(solo, only_off_disk=True): center_hpc = coord.transform_to(Helioprojective(observer=solo)) - - data[i]["flareposition_obs_hgs_x"] = solo_xyz[0].to(u.km) - data[i]["flareposition_obs_hgs_y"] = solo_xyz[1].to(u.km) - data[i]["flareposition_obs_hgs_z"] = solo_xyz[2].to(u.km) - data[i]["flareposition_hp_tx"] = center_hpc.Tx.to(u.arcsec) - data[i]["flareposition_hp_ty"] = center_hpc.Ty.to(u.arcsec) + tx_list.append(center_hpc.Tx) + ty_list.append(center_hpc.Ty) + solo_x_list.append(solo.cartesian.x) + solo_y_list.append(solo.cartesian.y) + solo_z_list.append(solo.cartesian.z) + peak_time_list.append(peak_time) data[i]["_position_status"] = True data[i]["_position_message"] = "OK" except Exception as e: data[i]["_position_status"] = False data[i]["_position_message"] = f"Error: {type(e)}" - logger.warn(f"Error calculating flare position for flare at time {start_time} : {end_time}: {e}") + logger.warning(f"Error calculating flare position for flare at time {start_time} : {end_time}: {e}") + tx_list.append(np.nan * u.arcsec) + ty_list.append(np.nan * u.arcsec) + solo_x_list.append(np.nan * u.km) + solo_y_list.append(np.nan * u.km) + solo_z_list.append(np.nan * u.km) + peak_time_list.append(peak_time) else: to_remove.append(i) + tx_list.append(np.nan * u.arcsec) + ty_list.append(np.nan * u.arcsec) + solo_x_list.append(np.nan * u.km) + solo_y_list.append(np.nan * u.km) + solo_z_list.append(np.nan * u.km) + peak_time_list.append(peak_time) + data[i]["_position_status"] = False + data[i]["_position_message"] = "flare did not pass the filter function" + + solo_times = Time(peak_time_list) + hgs_coords = SkyCoord( + u.Quantity(solo_x_list), + u.Quantity(solo_y_list), + u.Quantity(solo_z_list), + frame=HeliographicStonyhurst(obstime=solo_times), + representation_type="cartesian", + ) + + hp_coords = SkyCoord( + u.Quantity(tx_list), u.Quantity(ty_list), frame=Helioprojective(obstime=solo_times, observer=hgs_coords) + ) + + data["location_hgs"] = hgs_coords + # description="Flare location in Heliographic Stonyhurst coordinates" + + data["location_hp"] = hp_coords + # description="Flare location in Helioprojective coordinates" if not keep_all_flares: data.remove_rows(to_remove) @@ -276,6 +296,34 @@ def add_flare_position( f"finally {len(data)} flares remaining" ) + def on_serialize(self, data): + for col_name in ("location_hgs", "location_hp"): + if col_name in data.colnames: + icrs = data[col_name].icrs + icrs_coord = SkyCoord(icrs.ra, icrs.dec, icrs.distance, frame="icrs") + col_idx = data.colnames.index(col_name) + data.remove_column(col_name) + data.add_column(icrs_coord, name=col_name, index=col_idx) + s = super() + if hasattr(s, "on_serialize"): + s.on_serialize(data) + + def on_deserialize(self, data, *, peak_time_colname=None): + peak_col = peak_time_colname or self.peak_time_colname + if peak_col not in data.colnames: + logger.warning(f"on_deserialize: column '{peak_col}' not found, skipping location transform") + else: + obstime = Time(data[peak_col]) + if "location_hgs" in data.colnames: + data["location_hgs"] = data["location_hgs"].transform_to(HeliographicStonyhurst(obstime=obstime)) + if "location_hp" in data.colnames: + data["location_hp"] = data["location_hp"].transform_to( + Helioprojective(obstime=obstime, observer=data["location_hgs"]) + ) + s = super() + if hasattr(s, "on_deserialize"): + s.on_deserialize(data) + class FlareSOOPMixin: """_summary_""" @@ -614,6 +662,7 @@ def __init__(self, *, service_type=0, service_subtype=0, ssid=3, data, month, ** self.name = FlarelistSDCLoc.NAME self.ssid = 3 + self.peak_time_colname = "peak_UTC" def enhance_from_product(self, in_prod: GenericProduct): pass @@ -631,7 +680,7 @@ def add_flare_position(cls, data, fido_client: STIXClient, *, month=None): peak_time_colname="peak_UTC", start_time_colname="start_UTC", end_time_colname="end_UTC", - keep_all_flares=False, + keep_all_flares=True, month=month, ) @@ -751,6 +800,7 @@ def __init__(self, *, service_type=0, service_subtype=0, ssid=7, data, month, ** self.name = FlarelistSCLoc.NAME self.ssid = 7 + self.peak_time_colname = "peak_UTC" def enhance_from_product(self, in_prod: GenericProduct): pass diff --git a/stixcore/products/product.py b/stixcore/products/product.py index 27415375..7df9c3e8 100644 --- a/stixcore/products/product.py +++ b/stixcore/products/product.py @@ -336,6 +336,9 @@ def __call__(self, *args, **kwargs): month=month, ) + if hasattr(p, "on_deserialize") and callable(getattr(p, "on_deserialize")): + p.on_deserialize(p.data) + if hasattr(p, "get_additional_extensions") and data is not None: for _, name in p.get_additional_extensions(): # read the additional extension data @@ -609,6 +612,18 @@ def max_exposure(self): # default for FITS HEADER return 0.0 + def on_serialize(self, data): + """Hook called before writing data to FITS. Mixins override and chain via super().""" + s = super() + if hasattr(s, "on_serialize"): + s.on_serialize(data) + + def on_deserialize(self, data): + """Hook called after reading data from FITS. Mixins override and chain via super().""" + s = super() + if hasattr(s, "on_deserialize"): + s.on_deserialize(data) + def find_parent_products(self, root): """ Convenient way to get access to the parent products. diff --git a/stixcore/products/tests/test_flarelist.py b/stixcore/products/tests/test_flarelist.py new file mode 100644 index 00000000..ff847b6c --- /dev/null +++ b/stixcore/products/tests/test_flarelist.py @@ -0,0 +1,86 @@ +from datetime import date + +import numpy as np +import pytest +from sunpy.coordinates import HeliographicStonyhurst, Helioprojective + +import astropy.units as u +from astropy.coordinates import SkyCoord +from astropy.io import fits +from astropy.table import QTable +from astropy.time import Time + +from stixcore.io.product_processors.fits.processors import FitsL3Processor +from stixcore.products.level3.flarelist import FlarelistSDCLoc +from stixcore.products.product import Product + +N = 10 + + +@pytest.fixture +def flare_data(): + peak_times = Time("2022-01-01T12:00:00") + np.arange(N) * 600 * u.s + + hgs_coords = SkyCoord( + lon=np.linspace(0, 30, N) * u.deg, + lat=np.linspace(-5, 5, N) * u.deg, + radius=np.ones(N) * 1.0 * u.AU, + frame=HeliographicStonyhurst(obstime=peak_times), + ) + + hp_coords = SkyCoord( + Tx=np.linspace(-300, 300, N) * u.arcsec, + Ty=np.linspace(-200, 200, N) * u.arcsec, + frame=Helioprojective(obstime=peak_times, observer=hgs_coords), + ) + + data = QTable() + data["peak_UTC"] = peak_times + data["start_UTC"] = peak_times - 60 * u.s + data["end_UTC"] = peak_times + 60 * u.s + data["duration"] = np.ones(N) * 120 * u.s + data["lc_peak"] = np.ones((N, 5)) * u.ct / u.s + data["location_hgs"] = hgs_coords + data["location_hp"] = hp_coords + + return data + + +def test_flarelist_sdcloc_location_roundtrip(flare_data, tmp_path): + prod = FlarelistSDCLoc( + data=flare_data, + month=date(2022, 1, 1), + control=QTable(), + ) + + # minimal header bypasses the Spice-dependent header generation chain + header = fits.Header() + header["LEVEL"] = "L3" + header["STYPE"] = 0 + header["SSTYPE"] = 0 + header["SSID"] = 3 + header["DATE-BEG"] = "2022-01-01T00:00:00" + prod.fits_header = header + + # energy/additional_header_keywords are not set for freshly created products + prod.energy = None + prod._additional_header_keywords = [] + + orig_hgs_lon = prod.data["location_hgs"].lon.copy() + orig_hgs_lat = prod.data["location_hgs"].lat.copy() + orig_hp_tx = prod.data["location_hp"].Tx.copy() + orig_hp_ty = prod.data["location_hp"].Ty.copy() + + # write via FitsL3Processor — calls on_serialize internally, prod.data unchanged + writer = FitsL3Processor(tmp_path) + written_file_name = writer.write_fits(prod) + assert len(written_file_name) == 1 + + # read back via Product factory — calls on_deserialize internally + recovered = Product(written_file_name[0]) + + assert isinstance(recovered, FlarelistSDCLoc) + assert u.allclose(recovered.data["location_hgs"].lon, orig_hgs_lon, atol=1e-6 * u.deg) + assert u.allclose(recovered.data["location_hgs"].lat, orig_hgs_lat, atol=1e-6 * u.deg) + assert u.allclose(recovered.data["location_hp"].Tx, orig_hp_tx, atol=1e-3 * u.arcsec) + assert u.allclose(recovered.data["location_hp"].Ty, orig_hp_ty, atol=1e-3 * u.arcsec) From a888dcb38a47be708dfa3491e2a9f47fe1249171 Mon Sep 17 00:00:00 2001 From: Nicky Hochmuth Date: Thu, 26 Mar 2026 09:45:09 +0100 Subject: [PATCH 04/10] change on_deserialize flow --- .../io/product_processors/fits/processors.py | 5 +- stixcore/products/level3/flarelist.py | 124 ++++++++++-------- stixcore/products/product.py | 21 +-- stixcore/products/tests/test_flarelist.py | 67 ++++++---- 4 files changed, 128 insertions(+), 89 deletions(-) diff --git a/stixcore/io/product_processors/fits/processors.py b/stixcore/io/product_processors/fits/processors.py index a9887344..fda79ca1 100644 --- a/stixcore/io/product_processors/fits/processors.py +++ b/stixcore/io/product_processors/fits/processors.py @@ -823,10 +823,11 @@ def generate_primary_header(self, filename, product, *, version=0): if default[0] not in soop_key_names: soop_headers += tuple([default]) + scet_range = product.scet_timerange time_headers = ( # Name, Value, Comment - ("OBT_BEG", product.scet_timerange.start.as_float().value, "Start of acquisition time in OBT"), - ("OBT_END", product.scet_timerange.end.as_float().value, "End of acquisition time in OBT"), + ("OBT_BEG", scet_range.start.as_float().value, "Start of acquisition time in OBT"), + ("OBT_END", scet_range.end.as_float().value, "End of acquisition time in OBT"), ("TIMESYS", "UTC", "System used for time keywords"), ("LEVEL", "L1", "Processing level of the data"), ("DATE-OBS", product.utc_timerange.start.fits, "Start of acquisition time in UTC"), diff --git a/stixcore/products/level3/flarelist.py b/stixcore/products/level3/flarelist.py index ee4e750b..5abcc611 100644 --- a/stixcore/products/level3/flarelist.py +++ b/stixcore/products/level3/flarelist.py @@ -11,7 +11,7 @@ from stixpy.coordinates.transforms import get_hpc_info from stixpy.net.client import STIXClient from stixpy.product import Product as STIXPYProduct -from sunpy.coordinates import HeliographicStonyhurst, Helioprojective, SphericalScreen +from sunpy.coordinates import HeliographicStonyhurst, Helioprojective from sunpy.map import make_fitswcs_header from sunpy.net import attrs as a from sunpy.time import TimeRange @@ -77,7 +77,21 @@ def make_stix_fitswcs_header(data, flare_position, *, scale, exposure, rotation_ return header -class FlarePositionMixin: +class _SerializeMixin: + """No-op chain terminator for on_serialize/on_deserialize. + + Functional mixins inherit from this so super() calls always land safely + instead of hitting object and raising AttributeError. + """ + + def on_serialize(self, data): + pass + + def on_deserialize(self, data, **kwargs): + pass + + +class FlarePositionMixin(_SerializeMixin): """_summary_""" @classmethod @@ -101,8 +115,8 @@ def add_flare_position( data["cpd_path"] = Column(" " * 500, dtype=str, description="TDB") data["_position_status"] = Column(False, dtype=bool, description="TDB") data["_position_message"] = Column(" " * 500, dtype=str, description="TDB") - tx_list, ty_list = [], [] - solo_x_list, solo_y_list, solo_z_list, peak_time_list = [], [], [], [] + # tx_list, ty_list = [], [] + solo_cartesian_list = [] to_remove = [] pass_filter = 0 @@ -118,6 +132,7 @@ def add_flare_position( peak_time = row[peak_time_colname] start_time = row[start_time_colname] end_time = row[end_time_colname] + logger.info(f"Processing flare {i}/{len(data)} at time {start_time} : {end_time} (peak at {peak_time})") if filter_function(row): # and i < 60: pass_filter += 1 day = peak_time.to_datetime().date() @@ -137,6 +152,8 @@ def add_flare_position( logger.warning(f"No ephemeris data found for flare at time {start_time} : {end_time}") data[i]["_position_message"] = "no ephemeris data found" no_ephemeris += 1 + solo_cartesian_list.append((np.nan * u.km, np.nan * u.km, np.nan * u.km)) + continue data[i]["anc_ephemeris_path"] = anc_res["path"][0] @@ -153,6 +170,8 @@ def add_flare_position( logger.warning(f"No CPD data found for flare at time {start_time} : {end_time}") data[i]["_position_message"] = "no CPD data found" no_cpd += 1 + solo_cartesian_list.append((np.nan * u.km, np.nan * u.km, np.nan * u.km)) + continue if len(cpd_res) > 1: logger.debug(f"Many CPD data found for flare at time {start_time} : {end_time}") @@ -234,14 +253,11 @@ def add_flare_position( roll, solo_xyz, pointing = get_hpc_info(start_time, end_time) solo = HeliographicStonyhurst(*solo_xyz, obstime=peak_time, representation_type="cartesian") - with SphericalScreen(solo, only_off_disk=True): - center_hpc = coord.transform_to(Helioprojective(observer=solo)) - tx_list.append(center_hpc.Tx) - ty_list.append(center_hpc.Ty) - solo_x_list.append(solo.cartesian.x) - solo_y_list.append(solo.cartesian.y) - solo_z_list.append(solo.cartesian.z) - peak_time_list.append(peak_time) + # with SphericalScreen(solo, only_off_disk=True): + # center_hpc = coord.transform_to(Helioprojective(observer=solo)) + # tx_list.append(center_hpc.Tx) + # ty_list.append(center_hpc.Ty) + solo_cartesian_list.append((solo.cartesian.x, solo.cartesian.y, solo.cartesian.z)) data[i]["_position_status"] = True data[i]["_position_message"] = "OK" @@ -249,40 +265,36 @@ def add_flare_position( data[i]["_position_status"] = False data[i]["_position_message"] = f"Error: {type(e)}" logger.warning(f"Error calculating flare position for flare at time {start_time} : {end_time}: {e}") - tx_list.append(np.nan * u.arcsec) - ty_list.append(np.nan * u.arcsec) - solo_x_list.append(np.nan * u.km) - solo_y_list.append(np.nan * u.km) - solo_z_list.append(np.nan * u.km) - peak_time_list.append(peak_time) + # tx_list.append(np.nan * u.arcsec) + # ty_list.append(np.nan * u.arcsec) + solo_cartesian_list.append((np.nan * u.km, np.nan * u.km, np.nan * u.km)) + else: to_remove.append(i) - tx_list.append(np.nan * u.arcsec) - ty_list.append(np.nan * u.arcsec) - solo_x_list.append(np.nan * u.km) - solo_y_list.append(np.nan * u.km) - solo_z_list.append(np.nan * u.km) - peak_time_list.append(peak_time) + # tx_list.append(np.nan * u.arcsec) + # ty_list.append(np.nan * u.arcsec) + solo_cartesian_list.append((np.nan * u.km, np.nan * u.km, np.nan * u.km)) data[i]["_position_status"] = False data[i]["_position_message"] = "flare did not pass the filter function" - solo_times = Time(peak_time_list) + solo_times = Time(data[peak_time_colname]) + solo_x, solo_y, solo_z = zip(*solo_cartesian_list) hgs_coords = SkyCoord( - u.Quantity(solo_x_list), - u.Quantity(solo_y_list), - u.Quantity(solo_z_list), + u.Quantity(solo_x), + u.Quantity(solo_y), + u.Quantity(solo_z), frame=HeliographicStonyhurst(obstime=solo_times), representation_type="cartesian", ) - hp_coords = SkyCoord( - u.Quantity(tx_list), u.Quantity(ty_list), frame=Helioprojective(obstime=solo_times, observer=hgs_coords) - ) + # hp_coords = SkyCoord( + # u.Quantity(tx_list), u.Quantity(ty_list), frame=Helioprojective(obstime=solo_times, observer=hgs_coords) + # ) data["location_hgs"] = hgs_coords # description="Flare location in Heliographic Stonyhurst coordinates" - data["location_hp"] = hp_coords + # data["location_hp"] = hp_coords # description="Flare location in Helioprojective coordinates" if not keep_all_flares: @@ -297,35 +309,31 @@ def add_flare_position( ) def on_serialize(self, data): - for col_name in ("location_hgs", "location_hp"): - if col_name in data.colnames: - icrs = data[col_name].icrs - icrs_coord = SkyCoord(icrs.ra, icrs.dec, icrs.distance, frame="icrs") - col_idx = data.colnames.index(col_name) - data.remove_column(col_name) - data.add_column(icrs_coord, name=col_name, index=col_idx) - s = super() - if hasattr(s, "on_serialize"): - s.on_serialize(data) - - def on_deserialize(self, data, *, peak_time_colname=None): + logger.info("FlarePositionMixin on_serialize called, transforming location columns to ICRS for serialization") + + if "location_hgs" in data.colnames: + icrs = data["location_hgs"].icrs + icrs_coord = SkyCoord(icrs.ra, icrs.dec, icrs.distance, frame="icrs") + col_idx = data.colnames.index("location_hgs") + data.remove_column("location_hgs") + data.add_column(icrs_coord, name="location_icrs", index=col_idx) + super().on_serialize(data) + + def on_deserialize(self, data, *, peak_time_colname=None, **kwargs): + logger.info( + "FlarePositionMixin on_deserialize called, transforming location columns back to heliographic coordinates" + ) peak_col = peak_time_colname or self.peak_time_colname if peak_col not in data.colnames: logger.warning(f"on_deserialize: column '{peak_col}' not found, skipping location transform") else: obstime = Time(data[peak_col]) - if "location_hgs" in data.colnames: - data["location_hgs"] = data["location_hgs"].transform_to(HeliographicStonyhurst(obstime=obstime)) - if "location_hp" in data.colnames: - data["location_hp"] = data["location_hp"].transform_to( - Helioprojective(obstime=obstime, observer=data["location_hgs"]) - ) - s = super() - if hasattr(s, "on_deserialize"): - s.on_deserialize(data) + if "location_icrs" in data.colnames: + data["location_hgs"] = data["location_icrs"].transform_to(HeliographicStonyhurst(obstime=obstime)) + super().on_deserialize(data, **kwargs) -class FlareSOOPMixin: +class FlareSOOPMixin(_SerializeMixin): """_summary_""" @classmethod @@ -352,6 +360,14 @@ def add_soop( data["soop_id"] = Column(soop_id, dtype=str, description="SOOP ID") data["soop_type"] = Column(soop_type, dtype=str, description="name of the SOOP campaign") + # def on_serialize(self, data): + # logger.info("FlareSOOPMixin on_serialize called, but no special handling implemented for SOOP data") + # super().on_serialize(data) + + # def on_deserialize(self, data, **kwargs): + # logger.info("FlareSOOPMixin on_deserialize called, but no special handling implemented for SOOP data") + # super().on_deserialize(data, **kwargs) + class FlarePeakPreviewMixin: """Mixin class to add peak preview images to flare list products. diff --git a/stixcore/products/product.py b/stixcore/products/product.py index 7df9c3e8..3dce6632 100644 --- a/stixcore/products/product.py +++ b/stixcore/products/product.py @@ -613,16 +613,21 @@ def max_exposure(self): return 0.0 def on_serialize(self, data): - """Hook called before writing data to FITS. Mixins override and chain via super().""" - s = super() - if hasattr(s, "on_serialize"): - s.on_serialize(data) + """Hook called before writing data to FITS. Mixins override and chain via super(). - def on_deserialize(self, data): + Uses getattr so plain products without functional mixins are safe — + the functional mixins (FlarePositionMixin etc.) appear after GenericProduct + in the MRO, so pass would stop the chain before reaching them. + """ + serialize = getattr(super(), "on_serialize", None) + if serialize is not None: + serialize(data) + + def on_deserialize(self, data, **kwargs): """Hook called after reading data from FITS. Mixins override and chain via super().""" - s = super() - if hasattr(s, "on_deserialize"): - s.on_deserialize(data) + deserialize = getattr(super(), "on_deserialize", None) + if deserialize is not None: + deserialize(data, **kwargs) def find_parent_products(self, root): """ diff --git a/stixcore/products/tests/test_flarelist.py b/stixcore/products/tests/test_flarelist.py index ff847b6c..842e3960 100644 --- a/stixcore/products/tests/test_flarelist.py +++ b/stixcore/products/tests/test_flarelist.py @@ -2,12 +2,13 @@ import numpy as np import pytest -from sunpy.coordinates import HeliographicStonyhurst, Helioprojective +from sunpy.coordinates import HeliographicStonyhurst import astropy.units as u from astropy.coordinates import SkyCoord from astropy.io import fits from astropy.table import QTable +from astropy.tests.helper import assert_quantity_allclose from astropy.time import Time from stixcore.io.product_processors.fits.processors import FitsL3Processor @@ -21,19 +22,19 @@ def flare_data(): peak_times = Time("2022-01-01T12:00:00") + np.arange(N) * 600 * u.s + lon = np.linspace(0, 30, N) + lat = np.linspace(-5, 5, N) + # mark same rows fully NaN so the entire SkyCoord row is invalid + lon[2] = lat[2] = np.nan + lon[7] = lat[7] = np.nan + hgs_coords = SkyCoord( - lon=np.linspace(0, 30, N) * u.deg, - lat=np.linspace(-5, 5, N) * u.deg, + lon=lon * u.deg, + lat=lat * u.deg, radius=np.ones(N) * 1.0 * u.AU, frame=HeliographicStonyhurst(obstime=peak_times), ) - hp_coords = SkyCoord( - Tx=np.linspace(-300, 300, N) * u.arcsec, - Ty=np.linspace(-200, 200, N) * u.arcsec, - frame=Helioprojective(obstime=peak_times, observer=hgs_coords), - ) - data = QTable() data["peak_UTC"] = peak_times data["start_UTC"] = peak_times - 60 * u.s @@ -41,12 +42,12 @@ def flare_data(): data["duration"] = np.ones(N) * 120 * u.s data["lc_peak"] = np.ones((N, 5)) * u.ct / u.s data["location_hgs"] = hgs_coords - data["location_hp"] = hp_coords return data -def test_flarelist_sdcloc_location_roundtrip(flare_data, tmp_path): +@pytest.fixture +def written_fits(flare_data, tmp_path): prod = FlarelistSDCLoc( data=flare_data, month=date(2022, 1, 1), @@ -61,26 +62,42 @@ def test_flarelist_sdcloc_location_roundtrip(flare_data, tmp_path): header["SSID"] = 3 header["DATE-BEG"] = "2022-01-01T00:00:00" prod.fits_header = header - - # energy/additional_header_keywords are not set for freshly created products prod.energy = None prod._additional_header_keywords = [] + writer = FitsL3Processor(tmp_path) + written = writer.write_fits(prod) + assert len(written) == 1 + + return prod, written[0] + + +def test_flarelist_sdcloc_location_roundtrip(written_fits): + prod, fits_path = written_fits orig_hgs_lon = prod.data["location_hgs"].lon.copy() orig_hgs_lat = prod.data["location_hgs"].lat.copy() - orig_hp_tx = prod.data["location_hp"].Tx.copy() - orig_hp_ty = prod.data["location_hp"].Ty.copy() - - # write via FitsL3Processor — calls on_serialize internally, prod.data unchanged - writer = FitsL3Processor(tmp_path) - written_file_name = writer.write_fits(prod) - assert len(written_file_name) == 1 # read back via Product factory — calls on_deserialize internally - recovered = Product(written_file_name[0]) + recovered = Product(fits_path) assert isinstance(recovered, FlarelistSDCLoc) - assert u.allclose(recovered.data["location_hgs"].lon, orig_hgs_lon, atol=1e-6 * u.deg) - assert u.allclose(recovered.data["location_hgs"].lat, orig_hgs_lat, atol=1e-6 * u.deg) - assert u.allclose(recovered.data["location_hp"].Tx, orig_hp_tx, atol=1e-3 * u.arcsec) - assert u.allclose(recovered.data["location_hp"].Ty, orig_hp_ty, atol=1e-3 * u.arcsec) + assert_quantity_allclose(recovered.data["location_hgs"].lon, orig_hgs_lon, atol=1e-6 * u.deg, equal_nan=True) + assert_quantity_allclose(recovered.data["location_hgs"].lat, orig_hgs_lat, atol=1e-6 * u.deg, equal_nan=True) + + +def test_flarelist_sdcloc_fits_stores_icrs(written_fits): + prod, fits_path = written_fits + orig_hgs_lon = prod.data["location_hgs"].lon.copy() + orig_hgs_lat = prod.data["location_hgs"].lat.copy() + + # read the DATA extension directly — no on_deserialize, raw FITS content + raw = QTable.read(fits_path, hdu="DATA", astropy_native=True) + + assert "location_hgs" not in raw.colnames, "HGS column should not be stored in FITS" + assert "location_icrs" in raw.colnames, "ICRS column should be present in FITS" + + # manually transform ICRS back to HGS and compare with original + obstime = Time(raw["peak_UTC"]) + hgs = raw["location_icrs"].transform_to(HeliographicStonyhurst(obstime=obstime)) + assert_quantity_allclose(hgs.lon, orig_hgs_lon, atol=1e-6 * u.deg, equal_nan=True) + assert_quantity_allclose(hgs.lat, orig_hgs_lat, atol=1e-6 * u.deg, equal_nan=True) From 4bbc2e85d8fe45d6945f797451114905d32bebbd Mon Sep 17 00:00:00 2001 From: Nicky Hochmuth Date: Thu, 26 Mar 2026 09:55:48 +0100 Subject: [PATCH 05/10] enable soop update --- stixcore/soop/manager.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/stixcore/soop/manager.py b/stixcore/soop/manager.py index 142bfa42..b3dd242e 100644 --- a/stixcore/soop/manager.py +++ b/stixcore/soop/manager.py @@ -529,8 +529,6 @@ def add_soop_file_to_index(self, path, *, rebuild_index=True, **args): all_soop_file = Path(CONFIG.get("SOOP", "soop_files_download")) / f"{plan}.{version}.all.json" if not all_soop_file.exists(): - # TODO remove ove SOOP API is working reliably - return self.download_all_soops_from_api(plan, version, all_soop_file) with open(all_soop_file) as f_all: From 776b72132a3cf393037dd461ff3bee92c258cae8 Mon Sep 17 00:00:00 2001 From: Nicky Hochmuth Date: Fri, 27 Mar 2026 14:36:33 +0100 Subject: [PATCH 06/10] experimental fix for reading aspect burst data --- stixcore/products/product.py | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/stixcore/products/product.py b/stixcore/products/product.py index 3dce6632..01c29478 100644 --- a/stixcore/products/product.py +++ b/stixcore/products/product.py @@ -74,7 +74,10 @@ def read_qtable(file, hdu, hdul=None): `astropy.table.QTable` The corrected QTable with correct data types """ - qtable = QTable.read(file, hdu, astropy_native=True) + astropy_native = True + if (hdu.upper() == "DATA") and (file.name.startswith("solo_L0_stix-sci-aspect-burst")): + astropy_native = False + qtable = QTable.read(file, hdu, astropy_native=astropy_native) if hdul is None: hdul = fits.open(file) @@ -91,11 +94,12 @@ def read_qtable(file, hdu, hdul=None): if hasattr(dtype, "subdtype"): dtype = dtype.base - + # qtable[col.name] = qtable[col.name].astype(dtype) if col.coord_type != "UTC": qtable[col.name] = qtable[col.name].astype(dtype) else: - qtable[col.name].format = "isot" + # qtable[col.name].format = "isot" + pass return qtable From 46ad35cc255e162ebb62119491db10ff124722ef Mon Sep 17 00:00:00 2001 From: Nicky Hochmuth Date: Fri, 10 Apr 2026 16:32:28 +0200 Subject: [PATCH 07/10] added time_shift and disc_size --- stixcore/ephemeris/manager.py | 14 + .../io/product_processors/fits/processors.py | 3 +- stixcore/processing/FlareListL3.py | 2 +- stixcore/processing/pipeline_daily.py | 5 +- stixcore/products/level3/flarelist.py | 246 ++++++++++++++---- stixcore/products/level3/processing.py | 2 +- stixcore/products/product.py | 3 +- stixcore/products/tests/test_flarelist.py | 3 +- 8 files changed, 214 insertions(+), 64 deletions(-) diff --git a/stixcore/ephemeris/manager.py b/stixcore/ephemeris/manager.py index 787e9490..adee19ff 100644 --- a/stixcore/ephemeris/manager.py +++ b/stixcore/ephemeris/manager.py @@ -388,6 +388,20 @@ def get_sun_disc_size(self, *, date): return rsun_arc + def get_earth_solo_time_shift(self, *, date): + """gets Time(Sun to Earth) - Time(Sun to S/C) + + Returns + ------- + `astropy.units.Quantity` + Time difference between Sun to Earth and Sun to S/C in seconds + """ + et = spiceypy.scs2e(SOLAR_ORBITER_ID, str(date)) + solo_sun_hg, sun_solo_lt = spiceypy.spkezr("SOLO", et, "SUN_EARTH_CEQU", "None", "Sun") + sun_earth_hee, sun_earth_lt = spiceypy.spkezr("Earth", et, "SOLO_HEE", "None", "Sun") + + return (sun_earth_lt - sun_solo_lt) * u.s + def get_position(self, *, date, frame): """ Get the position of SolarOrbiter at the given date in the given coordinate frame. diff --git a/stixcore/io/product_processors/fits/processors.py b/stixcore/io/product_processors/fits/processors.py index fda79ca1..b843f369 100644 --- a/stixcore/io/product_processors/fits/processors.py +++ b/stixcore/io/product_processors/fits/processors.py @@ -1068,7 +1068,8 @@ def write_fits(self, prod, *, version=0): elif fitspath_complete.exists(): logger.warning("Complete Fits file %s exists will be overridden", fitspath.name) - data = prod.data + data = prod.data.copy() + prod.on_serialize(data) primary_header, header_override = self.generate_primary_header(filename, prod, version=version) primary_hdu = fits.PrimaryHDU() diff --git a/stixcore/processing/FlareListL3.py b/stixcore/processing/FlareListL3.py index 5a6e8b3b..c75b4f87 100644 --- a/stixcore/processing/FlareListL3.py +++ b/stixcore/processing/FlareListL3.py @@ -24,7 +24,7 @@ class FlareListL3(SingleProductProcessingStepMixin): """Processing step from a FlareListManager to monthly solo_L3_stix-flarelist-*.fits file.""" - STARTDATE = date(2025, 7, 1) + STARTDATE = date(2022, 1, 1) def __init__(self, flm: FlareListManager, output_dir: Path): """Crates a new Processor. diff --git a/stixcore/processing/pipeline_daily.py b/stixcore/processing/pipeline_daily.py index 0b1a5a20..e720f170 100644 --- a/stixcore/processing/pipeline_daily.py +++ b/stixcore/processing/pipeline_daily.py @@ -274,7 +274,7 @@ def run_daily_pipeline(args): # TODO reactivate once flarelist processing is finalized fl_sdc_months = flarelist_sdc.find_processing_months(phs) - fl_sdc_months = [] + # fl_sdc_months = [] # TODO reactivate once flarelist processing is finalized # fl_sc_months = flarelist_sc.find_processing_months(phs) @@ -282,8 +282,11 @@ def run_daily_pipeline(args): # TODO reactivate once flarelist processing is finalized fl_to_fl_files = fl_to_fl.get_processing_files(phs) + # fl_to_fl_files = fl_to_fl_files[2:-1] # fl_to_fl_files = [] + fl_to_fl_files = [f for f in fl_to_fl_files if "/2023/" in str(f[2])] + # all processing files should be terminated before the next step as the different # processing steeps might create new candidates # let each processing "task" run in its own process diff --git a/stixcore/products/level3/flarelist.py b/stixcore/products/level3/flarelist.py index 5abcc611..563af41f 100644 --- a/stixcore/products/level3/flarelist.py +++ b/stixcore/products/level3/flarelist.py @@ -11,7 +11,7 @@ from stixpy.coordinates.transforms import get_hpc_info from stixpy.net.client import STIXClient from stixpy.product import Product as STIXPYProduct -from sunpy.coordinates import HeliographicStonyhurst, Helioprojective +from sunpy.coordinates import HeliographicStonyhurst, Helioprojective, SphericalScreen from sunpy.map import make_fitswcs_header from sunpy.net import attrs as a from sunpy.time import TimeRange @@ -19,6 +19,7 @@ import astropy.units as u from astropy.coordinates import SkyCoord +from astropy.coordinates.representation import CartesianRepresentation from astropy.io import fits from astropy.table import Column, QTable from astropy.time import Time @@ -104,18 +105,14 @@ def add_flare_position( peak_time_colname="peak_UTC", start_time_colname="start_UTC", end_time_colname="end_UTC", + location_time_colname="location_time_UTC", keep_all_flares=True, month=None, ): - # helio_frame = Helioprojective(observer="earth") - # SkyCoord(HeliographicStonyhurst(0 * u.deg, 0 * u.deg)) - # SkyCoord(0 * u.deg, 0 * u.deg, frame=helio_frame) - - data["anc_ephemeris_path"] = Column(" " * 500, dtype=str, description="TDB") - data["cpd_path"] = Column(" " * 500, dtype=str, description="TDB") - data["_position_status"] = Column(False, dtype=bool, description="TDB") - data["_position_message"] = Column(" " * 500, dtype=str, description="TDB") - # tx_list, ty_list = [], [] + anc_ephemeris_paths = [] + cpd_paths = [] + position_statuses = [] + position_messages = [] solo_cartesian_list = [] to_remove = [] @@ -129,6 +126,10 @@ def add_flare_position( day_asp_ephemeris_cache = dict() for i, row in enumerate(data): + _anc_path = "" + _cpd_path = "" + _status = False + _message = "" peak_time = row[peak_time_colname] start_time = row[start_time_colname] end_time = row[end_time_colname] @@ -150,12 +151,25 @@ def add_flare_position( if len(anc_res) < 1: logger.warning(f"No ephemeris data found for flare at time {start_time} : {end_time}") - data[i]["_position_message"] = "no ephemeris data found" + _message = "no ephemeris data found" no_ephemeris += 1 - solo_cartesian_list.append((np.nan * u.km, np.nan * u.km, np.nan * u.km)) - + solo_cartesian_list.append( + ( + np.nan * u.km, + np.nan * u.km, + np.nan * u.km, + peak_time, + np.nan * u.km, + np.nan * u.km, + np.nan * u.km, + ) + ) + anc_ephemeris_paths.append(_anc_path) + cpd_paths.append(_cpd_path) + position_statuses.append(_status) + position_messages.append(_message) continue - data[i]["anc_ephemeris_path"] = anc_res["path"][0] + _anc_path = str(anc_res["path"][0]) if start_time.datetime.hour < 2: start_time = start_time - 2 * u.hour @@ -168,10 +182,23 @@ def add_flare_position( if len(cpd_res) < 1: logger.warning(f"No CPD data found for flare at time {start_time} : {end_time}") - data[i]["_position_message"] = "no CPD data found" + _message = "no CPD data found" no_cpd += 1 - solo_cartesian_list.append((np.nan * u.km, np.nan * u.km, np.nan * u.km)) - + solo_cartesian_list.append( + ( + np.nan * u.km, + np.nan * u.km, + np.nan * u.km, + peak_time, + np.nan * u.km, + np.nan * u.km, + np.nan * u.km, + ) + ) + anc_ephemeris_paths.append(_anc_path) + cpd_paths.append(_cpd_path) + position_statuses.append(_status) + position_messages.append(_message) continue if len(cpd_res) > 1: logger.debug(f"Many CPD data found for flare at time {start_time} : {end_time}") @@ -210,10 +237,10 @@ def add_flare_position( else: one_cpd += 1 best_cpd_idx = 0 - data[i]["cpd_path"] = cpd_res["path"][best_cpd_idx] + _cpd_path = str(cpd_res["path"][best_cpd_idx]) try: - stixpy_cpd = STIXPYProduct(Path(data[i]["cpd_path"])) + stixpy_cpd = STIXPYProduct(Path(_cpd_path)) time_range = TimeRange(max(peak_time - 20 * u.s, start_time), min(peak_time + 20 * u.s, end_time)) overlaps = calculate_overlap(stixpy_cpd.time_range, time_range) if overlaps is None: @@ -249,37 +276,82 @@ def add_flare_position( f"Using time range {time_range.start} to {time_range.end} for flare at around {peak_time} with constant rcr={rcr}" ) - coord, map = estimate_stix_flare_location(stixpy_cpd, time_range=time_range) + center_time = time_range.center + flare_loc, _, solo = estimate_stix_flare_location(stixpy_cpd, time_range=time_range) - roll, solo_xyz, pointing = get_hpc_info(start_time, end_time) - solo = HeliographicStonyhurst(*solo_xyz, obstime=peak_time, representation_type="cartesian") - # with SphericalScreen(solo, only_off_disk=True): - # center_hpc = coord.transform_to(Helioprojective(observer=solo)) - # tx_list.append(center_hpc.Tx) - # ty_list.append(center_hpc.Ty) - solo_cartesian_list.append((solo.cartesian.x, solo.cartesian.y, solo.cartesian.z)) + with SphericalScreen(solo, only_off_disk=True): + center_hgs = flare_loc.transform_to(HeliographicStonyhurst(obstime=center_time)).cartesian + solo_cartesian_list.append( + (center_hgs.x, center_hgs.y, center_hgs.z, center_time, solo.x, solo.y, solo.z) + ) - data[i]["_position_status"] = True - data[i]["_position_message"] = "OK" + _status = True + _message = "OK" except Exception as e: - data[i]["_position_status"] = False - data[i]["_position_message"] = f"Error: {type(e)}" + _status = False + _message = f"Error: {type(e)}" logger.warning(f"Error calculating flare position for flare at time {start_time} : {end_time}: {e}") - # tx_list.append(np.nan * u.arcsec) - # ty_list.append(np.nan * u.arcsec) - solo_cartesian_list.append((np.nan * u.km, np.nan * u.km, np.nan * u.km)) + solo_cartesian_list.append( + ( + np.nan * u.km, + np.nan * u.km, + np.nan * u.km, + peak_time, + np.nan * u.km, + np.nan * u.km, + np.nan * u.km, + ) + ) + anc_ephemeris_paths.append(_anc_path) + cpd_paths.append(_cpd_path) + position_statuses.append(_status) + position_messages.append(_message) else: to_remove.append(i) - # tx_list.append(np.nan * u.arcsec) - # ty_list.append(np.nan * u.arcsec) - solo_cartesian_list.append((np.nan * u.km, np.nan * u.km, np.nan * u.km)) - data[i]["_position_status"] = False - data[i]["_position_message"] = "flare did not pass the filter function" - - solo_times = Time(data[peak_time_colname]) - solo_x, solo_y, solo_z = zip(*solo_cartesian_list) + solo_cartesian_list.append( + ( + np.nan * u.km, + np.nan * u.km, + np.nan * u.km, + peak_time, + np.nan * u.km, + np.nan * u.km, + np.nan * u.km, + ) + ) + anc_ephemeris_paths.append(_anc_path) + cpd_paths.append(_cpd_path) + position_statuses.append(False) + position_messages.append("flare did not pass the filter function") + + primer = fido_client.baseurl.replace(fido_client.datapath, "") + primer = primer[7:] if primer.startswith("file://") else primer + + data["anc_ephemeris_path"] = [v.replace(primer, "") for v in anc_ephemeris_paths] + data["anc_ephemeris_path"].info.description = "Path to the daily ancillary ephemeris file" + + data["cpd_path"] = [v.replace(primer, "") for v in cpd_paths] + data["cpd_path"].info.description = "Path to the CPD file used for flare location estimation" + + data["_position_status"] = position_statuses + data["_position_status"].info.description = "Status of the flare position calculation" + + data["_position_message"] = position_messages + data["_position_message"].info.description = "Message describing the status of the flare position calculation" + + flare_x, flare_y, flare_z, solo_times, solo_x, solo_y, solo_z = zip(*solo_cartesian_list) + solo_times = Time(solo_times) + hgs_coords = SkyCoord( + u.Quantity(flare_x), + u.Quantity(flare_y), + u.Quantity(flare_z), + frame=HeliographicStonyhurst(obstime=solo_times), + representation_type="cartesian", + ) + + solo_coords = SkyCoord( u.Quantity(solo_x), u.Quantity(solo_y), u.Quantity(solo_z), @@ -287,15 +359,42 @@ def add_flare_position( representation_type="cartesian", ) - # hp_coords = SkyCoord( - # u.Quantity(tx_list), u.Quantity(ty_list), frame=Helioprojective(obstime=solo_times, observer=hgs_coords) - # ) + # hgc_coords = hgs_coords.transform_to(HeliographicCarrington(obstime=solo_times, observer="Earth")) + hp_coords = hgs_coords.transform_to(Helioprojective(obstime=solo_times, observer="Earth")) data["location_hgs"] = hgs_coords - # description="Flare location in Heliographic Stonyhurst coordinates" + data["location_hgs"].info.description = "Flare location in Heliographic Stonyhurst coordinates" + + data["solo_location_hgs"] = solo_coords + data["solo_location_hgs"].info.description = "SOLO location in Heliographic Stonyhurst coordinates" + + # data["location_hgc"] = hgc_coords + # data["location_hgc"].info.description = "Flare location in Heliographic Carrington coordinates seen from Earth" + + data["visible_from_earth"] = FlarePositionMixin.is_visible(hp_coords) + data[ + "visible_from_earth" + ].info.description = "Whether the flare location is visible from Earth (not occulted by the Sun)" + + data[location_time_colname] = solo_times + data[location_time_colname].info.description = "time used for flare location estimation in UTC" + + ( + time_shift, + disc_size, + ) = zip( + *[ + (Spice.instance.get_earth_solo_time_shift(date=scet), Spice.instance.get_sun_disc_size(date=scet)) + for t in solo_times + for scet in (Spice.instance.datetime_to_scet(t),) + ] + ) - # data["location_hp"] = hp_coords - # description="Flare location in Helioprojective coordinates" + data["time_shift"] = time_shift + data["time_shift"].info.description = "Time(Sun to Earth) - Time(Sun to S/C)" + + data["sun_disc_size"] = disc_size + data["sun_disc_size"].info.description = "Apparent photospheric solar radius" if not keep_all_flares: data.remove_rows(to_remove) @@ -305,11 +404,13 @@ def add_flare_position( f"passed filter: {pass_filter} no ephemeris data found for {no_ephemeris} " f"flares, no CPD data found for {no_cpd} flares, many CPD data found for " f"{many_cpd} flares, one CPD data found for {one_cpd} flares." - f"finally {len(data)} flares remaining" + f"finally {len(data) - len(to_remove)} flare locations found" ) def on_serialize(self, data): - logger.info("FlarePositionMixin on_serialize called, transforming location columns to ICRS for serialization") + logger.warning( + "FlarePositionMixin on_serialize called, transforming location columns to ICRS for serialization" + ) if "location_hgs" in data.colnames: icrs = data["location_hgs"].icrs @@ -317,21 +418,52 @@ def on_serialize(self, data): col_idx = data.colnames.index("location_hgs") data.remove_column("location_hgs") data.add_column(icrs_coord, name="location_icrs", index=col_idx) + if "solo_location_hgs" in data.colnames: + icrs = data["solo_location_hgs"].icrs + icrs_coord = SkyCoord(icrs.ra, icrs.dec, icrs.distance, frame="icrs") + col_idx = data.colnames.index("solo_location_hgs") + data.remove_column("solo_location_hgs") + data.add_column(icrs_coord, name="solo_location_icrs", index=col_idx) super().on_serialize(data) - def on_deserialize(self, data, *, peak_time_colname=None, **kwargs): - logger.info( + def on_deserialize(self, data, *, location_time_colname=None, **kwargs): + logger.warning( "FlarePositionMixin on_deserialize called, transforming location columns back to heliographic coordinates" ) - peak_col = peak_time_colname or self.peak_time_colname - if peak_col not in data.colnames: - logger.warning(f"on_deserialize: column '{peak_col}' not found, skipping location transform") + time_col = location_time_colname or self.location_time_colname + if time_col not in data.colnames: + logger.warning(f"on_deserialize: column '{time_col}' not found, skipping location transform") else: - obstime = Time(data[peak_col]) + obstime = Time(data[time_col]) if "location_icrs" in data.colnames: data["location_hgs"] = data["location_icrs"].transform_to(HeliographicStonyhurst(obstime=obstime)) + if "solo_location_icrs" in data.colnames: + data["solo_location_hgs"] = data["solo_location_icrs"].transform_to( + HeliographicStonyhurst(obstime=obstime) + ) + super().on_deserialize(data, **kwargs) + @classmethod + def is_visible(cls, coord): + """ + Returns whether the coordinate is on the visible side of the Sun. + This function is a modified version of PR#7118 + """ + + coord = coord.make_3d() + data = coord.cartesian + data_to_sun = coord.observer.radius * CartesianRepresentation(1, 0, 0) - data + + is_behind = data.x < 0 + # print(data.x.to(u.AU)) + is_beyond_limb = np.sqrt(1 - (data.x / data.norm()) ** 2) > coord.rsun / coord.observer.radius + # is_above_surface = data_to_sun.norm() >= coord.rsun + + is_on_near_side = data.dot(data_to_sun) >= 0 + + return is_behind | is_beyond_limb | (is_on_near_side) + class FlareSOOPMixin(_SerializeMixin): """_summary_""" @@ -678,7 +810,7 @@ def __init__(self, *, service_type=0, service_subtype=0, ssid=3, data, month, ** self.name = FlarelistSDCLoc.NAME self.ssid = 3 - self.peak_time_colname = "peak_UTC" + self.location_time_colname = "location_time_UTC" def enhance_from_product(self, in_prod: GenericProduct): pass diff --git a/stixcore/products/level3/processing.py b/stixcore/products/level3/processing.py index e6225a1e..c274a45c 100644 --- a/stixcore/products/level3/processing.py +++ b/stixcore/products/level3/processing.py @@ -177,7 +177,7 @@ def estimate_stix_flare_location( wcs_bp = map_bp.wcs # Estimate flare location from brightest pixel in backprojection image flare_loc = wcs_bp.array_index_to_world(*max_idx) - return flare_loc, map_bp + return flare_loc, map_bp, solo def _subtract_background_from_stix_meta_pixels(meta_pixels_sci, meta_pixels_bkg): diff --git a/stixcore/products/product.py b/stixcore/products/product.py index 01c29478..4d9dbf7d 100644 --- a/stixcore/products/product.py +++ b/stixcore/products/product.py @@ -98,8 +98,7 @@ def read_qtable(file, hdu, hdul=None): if col.coord_type != "UTC": qtable[col.name] = qtable[col.name].astype(dtype) else: - # qtable[col.name].format = "isot" - pass + qtable[col.name].format = "isot" return qtable diff --git a/stixcore/products/tests/test_flarelist.py b/stixcore/products/tests/test_flarelist.py index 842e3960..001e0911 100644 --- a/stixcore/products/tests/test_flarelist.py +++ b/stixcore/products/tests/test_flarelist.py @@ -37,6 +37,7 @@ def flare_data(): data = QTable() data["peak_UTC"] = peak_times + data["location_time_UTC"] = peak_times data["start_UTC"] = peak_times - 60 * u.s data["end_UTC"] = peak_times + 60 * u.s data["duration"] = np.ones(N) * 120 * u.s @@ -97,7 +98,7 @@ def test_flarelist_sdcloc_fits_stores_icrs(written_fits): assert "location_icrs" in raw.colnames, "ICRS column should be present in FITS" # manually transform ICRS back to HGS and compare with original - obstime = Time(raw["peak_UTC"]) + obstime = Time(raw["location_time_UTC"]) hgs = raw["location_icrs"].transform_to(HeliographicStonyhurst(obstime=obstime)) assert_quantity_allclose(hgs.lon, orig_hgs_lon, atol=1e-6 * u.deg, equal_nan=True) assert_quantity_allclose(hgs.lat, orig_hgs_lat, atol=1e-6 * u.deg, equal_nan=True) From 83f53ee0a7ee7ccfd12cb37851c489589d4b3ef9 Mon Sep 17 00:00:00 2001 From: Nicky Hochmuth Date: Mon, 22 Jun 2026 13:17:48 +0200 Subject: [PATCH 08/10] added time_shift and disc_size --- stixcore/processing/pipeline_daily.py | 2 +- stixcore/products/level3/flarelist.py | 68 +++++-- stixcore/products/level3/processing.py | 228 +++++++++++++++++++++- stixcore/products/tests/test_flarelist.py | 89 ++++++++- 4 files changed, 366 insertions(+), 21 deletions(-) diff --git a/stixcore/processing/pipeline_daily.py b/stixcore/processing/pipeline_daily.py index e720f170..642ed876 100644 --- a/stixcore/processing/pipeline_daily.py +++ b/stixcore/processing/pipeline_daily.py @@ -285,7 +285,7 @@ def run_daily_pipeline(args): # fl_to_fl_files = fl_to_fl_files[2:-1] # fl_to_fl_files = [] - fl_to_fl_files = [f for f in fl_to_fl_files if "/2023/" in str(f[2])] + fl_to_fl_files = [f for f in fl_to_fl_files if "/2024/" in str(f[2])] # all processing files should be terminated before the next step as the different # processing steeps might create new candidates diff --git a/stixcore/products/level3/flarelist.py b/stixcore/products/level3/flarelist.py index 563af41f..3279d5b6 100644 --- a/stixcore/products/level3/flarelist.py +++ b/stixcore/products/level3/flarelist.py @@ -27,7 +27,7 @@ from stixcore.config.config import CONFIG from stixcore.ephemeris.manager import Spice from stixcore.products.level3.flarelistproduct import PeakPreviewImage -from stixcore.products.level3.processing import estimate_stix_flare_location +from stixcore.products.level3.processing import stx_estimate_flare_location from stixcore.products.product import CountDataMixin, GenericProduct, L2Mixin, read_qtable from stixcore.soop.manager import SOOPManager from stixcore.time import SCETime, SCETimeRange @@ -159,9 +159,12 @@ def add_flare_position( np.nan * u.km, np.nan * u.km, peak_time, + 0 * u.s, np.nan * u.km, np.nan * u.km, np.nan * u.km, + 0, + np.nan, ) ) anc_ephemeris_paths.append(_anc_path) @@ -190,9 +193,12 @@ def add_flare_position( np.nan * u.km, np.nan * u.km, peak_time, + 0 * u.s, np.nan * u.km, np.nan * u.km, np.nan * u.km, + 0, + np.nan, ) ) anc_ephemeris_paths.append(_anc_path) @@ -253,8 +259,12 @@ def add_flare_position( contains_peak_time = True time_range = overlaps - mask = (stixpy_cpd.data["time"] >= time_range.start) & (stixpy_cpd.data["time"] <= time_range.end) + _times = stixpy_cpd.data["time"] + _half_bin = stixpy_cpd.data["timedel"] / 2 + mask = (_times + _half_bin >= time_range.start) & (_times - _half_bin <= time_range.end) data_at_peak = stixpy_cpd.data[mask] + energy_range = [4, 16] * u.keV + if len(np.unique(data_at_peak["rcr"])) > 1: logger.warning( f"Multiple rcr values found for flare at time {time_range.start} : {time_range.end}" @@ -264,9 +274,7 @@ def add_flare_position( time_range = TimeRange( max(peak_time - 40 * u.s, start_time), min(peak_time + 40 * u.s, end_time) ) - mask = (stixpy_cpd.data["time"] >= time_range.start) & ( - stixpy_cpd.data["time"] <= time_range.end - ) + mask = (_times + _half_bin >= time_range.start) & (_times - _half_bin <= time_range.end) data_at_peak = stixpy_cpd.data[mask] length, start_idx, rcr = longest_constant_sequence(data_at_peak["rcr"].value) time_range = TimeRange( @@ -276,13 +284,31 @@ def add_flare_position( f"Using time range {time_range.start} to {time_range.end} for flare at around {peak_time} with constant rcr={rcr}" ) - center_time = time_range.center - flare_loc, _, solo = estimate_stix_flare_location(stixpy_cpd, time_range=time_range) + rcr_at_peak = data_at_peak["rcr"].max() + if rcr_at_peak > 0: + energy_range = [4, 25] * u.keV + + _, flare_loc, sidelobe, solo, img_time_range = stx_estimate_flare_location( + stixpy_cpd, time_range, energy_range + ) with SphericalScreen(solo, only_off_disk=True): - center_hgs = flare_loc.transform_to(HeliographicStonyhurst(obstime=center_time)).cartesian + center_hgs = flare_loc.transform_to( + HeliographicStonyhurst(obstime=img_time_range.center) + ).cartesian solo_cartesian_list.append( - (center_hgs.x, center_hgs.y, center_hgs.z, center_time, solo.x, solo.y, solo.z) + ( + center_hgs.x, + center_hgs.y, + center_hgs.z, + img_time_range.center, + img_time_range.seconds, + solo.x, + solo.y, + solo.z, + rcr_at_peak, + sidelobe, + ) ) _status = True @@ -297,9 +323,12 @@ def add_flare_position( np.nan * u.km, np.nan * u.km, peak_time, + 0 * u.s, np.nan * u.km, np.nan * u.km, np.nan * u.km, + 0, + np.nan, ) ) anc_ephemeris_paths.append(_anc_path) @@ -315,9 +344,12 @@ def add_flare_position( np.nan * u.km, np.nan * u.km, peak_time, + 0 * u.s, np.nan * u.km, np.nan * u.km, np.nan * u.km, + 0, + np.nan, ) ) anc_ephemeris_paths.append(_anc_path) @@ -340,7 +372,9 @@ def add_flare_position( data["_position_message"] = position_messages data["_position_message"].info.description = "Message describing the status of the flare position calculation" - flare_x, flare_y, flare_z, solo_times, solo_x, solo_y, solo_z = zip(*solo_cartesian_list) + flare_x, flare_y, flare_z, solo_times, duration, solo_x, solo_y, solo_z, rcr_at_peak, sidelobe = zip( + *solo_cartesian_list + ) solo_times = Time(solo_times) hgs_coords = SkyCoord( @@ -368,8 +402,13 @@ def add_flare_position( data["solo_location_hgs"] = solo_coords data["solo_location_hgs"].info.description = "SOLO location in Heliographic Stonyhurst coordinates" - # data["location_hgc"] = hgc_coords - # data["location_hgc"].info.description = "Flare location in Heliographic Carrington coordinates seen from Earth" + data["sidelobes_ratio"] = sidelobe + data["sidelobes_ratio"].info.description = "Ratio of sidelobes in the STIX image used to assess imaging quality" + + data["rcr_at_peak"] = rcr_at_peak + data[ + "rcr_at_peak" + ].info.description = "max rcr level at flare location estimation time range, > 0 attenuator in place" data["visible_from_earth"] = FlarePositionMixin.is_visible(hp_coords) data[ @@ -377,7 +416,10 @@ def add_flare_position( ].info.description = "Whether the flare location is visible from Earth (not occulted by the Sun)" data[location_time_colname] = solo_times - data[location_time_colname].info.description = "time used for flare location estimation in UTC" + data[location_time_colname].info.description = "time center used for flare location estimation in UTC" + + data["location_duration"] = duration + data["location_duration"].info.description = "duration of the flare location estimation time range" ( time_shift, diff --git a/stixcore/products/level3/processing.py b/stixcore/products/level3/processing.py index c274a45c..73106521 100644 --- a/stixcore/products/level3/processing.py +++ b/stixcore/products/level3/processing.py @@ -6,17 +6,233 @@ import stixpy.calibration.visibility import stixpy.coordinates.transforms import sunpy.map +import sunpy.sun.constants as sun_const import sunpy.time import xrayvision.imaging -from stixpy.coordinates.transforms import STIXImaging -from sunpy.coordinates import HeliographicStonyhurst +from stixpy.calibration.visibility import ( + calibrate_visibility, + create_meta_pixels, + create_visibility, +) +from stixpy.coordinates.frames import STIXImaging +from stixpy.coordinates.transforms import get_hpc_info +from sunpy.coordinates import HeliographicStonyhurst, SphericalScreen, frames +from sunpy.time import TimeRange +from xrayvision.imaging import vis_to_image -import astropy.units as u +from astropy import units as u from astropy.coordinates import SkyCoord +from astropy.coordinates.representation import CartesianRepresentation from astropy.time import Time -def construct_stix_calibrated_visibilities( +def get_rsun_obs(observer): + """ + Get the observed radius of the Sun from an observer location. + """ + + rsun_obs = ((sun_const.radius / (observer.spherical.distance - sun_const.radius)).decompose() * u.radian).to( + u.arcsec + ) + return rsun_obs + + +def get_distance_off_limb(coord): + theta_x = coord.Tx + theta_y = coord.Ty + rsun_obs = get_rsun_obs(coord.observer) + distance_off_limb = np.sqrt(theta_x**2 + theta_y**2) - rsun_obs + distance_r_sun = np.sqrt(theta_x**2 + theta_y**2) / rsun_obs + + return distance_off_limb, distance_r_sun + + +def generate_blank_map(date_obs, observer): + """ + Given a date and an observer create a blank map + + """ + data = np.full((12, 12), np.nan) + + # Define a reference coordinate and create a header using sunpy.map.make_fitswcs_header + skycoord = SkyCoord(0 * u.arcsec, 0 * u.arcsec, frame=frames.Helioprojective(observer=observer, obstime=date_obs)) + + # Scale set to the following for solar limb to be in the field of view + header = sunpy.map.make_fitswcs_header(data, skycoord, scale=[600, 600] * u.arcsec / u.pixel) + + # Use sunpy.map.Map to create the blank map + blank_map = sunpy.map.Map(data, header) + return blank_map + + +def is_visible(coord): + """ + Returns whether the coordinate is on the visible side of the Sun. + This function is a modified version of PR#7118 + """ + + coord = coord.make_3d() + data = coord.cartesian + data_to_sun = coord.observer.radius * CartesianRepresentation(1, 0, 0) - data + + is_behind = data.x < 0 + # print(data.x.to(u.AU)) + is_beyond_limb = np.sqrt(1 - (data.x / data.norm()) ** 2) > coord.rsun / coord.observer.radius + # is_above_surface = data_to_sun.norm() >= coord.rsun + + is_on_near_side = data.dot(data_to_sun) >= 0 + + return is_behind | is_beyond_limb | (is_on_near_side) + + +def stx_estimate_flare_location(cpd_sci, time_range, energy_range): + """ + Estimate the flare location using STIX imaging data. + + This function processes the imaging data from the STIX instrument on Solar Orbiter to estimate the location of a solar flare. + It is based on the IDL software `stx_estimate_flare_location`. + + It creates back-projected images in both STIX imaging coordinates and Helioprojective coordinates, and finds the maximum location of the pixel. + + Optionally, it plots the results showing the maximum pixel locations in both coordinate systems. + + Parameters + ---------- + cpd_sci : CPDProduct + the STIX pixel data product. + time_range : `sunpy.time.TimeRange` + The time range over which to estimate the flare location. + energy_range : `astropy.units.Quantity` + The energy range (e.g., in keV) for the analysis. + + Returns + ------- + max_stix : `astropy.coordinates.SkyCoord` + The estimated flare location in STIX imaging coordinates. + max_hpc : `astropy.coordinates.SkyCoord` + The estimated flare location in Helioprojective Cartesian coordinates. + + Notes + ----- + The function involves the following steps: + - Reading STIX pixel data and generating meta pixels for a given time and energy range. + - Creating visibility data from the meta pixels. + - Obtaining solar observer coordinates and converting them to the Heliographic Stonyhurst frame. + - Creating a back-projected image from the visibility data. + - Transforming the coordinates of the maximum pixel in the image to Helioprojective coordinates. + + """ + + meta_pixels_sci = create_meta_pixels( + cpd_sci, + time_range=[time_range.start, time_range.end], + energy_range=energy_range, + flare_location=[0, 0] * u.arcsec, + no_shadowing=True, + ) + + # create visibilities + vis = create_visibility(meta_pixels_sci) + vis_tr = TimeRange(vis.meta["time_range"]) + + roll, solo_xyz, pointing = get_hpc_info(vis_tr.start, vis_tr.end) + solo = frames.HeliographicStonyhurst(*solo_xyz, obstime=vis_tr.center, representation_type="cartesian") + + center_map = SkyCoord(0 * u.arcsec, 0 * u.arcsec, frame=frames.Helioprojective(observer=solo, obstime=solo.obstime)) + center_coord = center_map.transform_to(STIXImaging(obstime=vis_tr.start, obstime_end=vis_tr.end, observer=solo)) + + # get calibrated visibilities - use center of Sun as phase center + cal_vis = calibrate_visibility(vis, flare_location=center_coord) + + # order by sub-collimator e.g. 10a, 10b, 10c, 9a, 9b, 9c .... + isc_10_7 = [3, 20, 22, 16, 14, 32, 21, 26, 4, 24, 8, 28] + idx = np.argwhere(np.isin(cal_vis.meta["isc"], isc_10_7)).ravel() + + # only use subcolimators 7 - 10 + vis10_7 = cal_vis[idx] + + # set up image size + imsize = [512, 512] * u.pixel + + # to make sure the full Sun is within FOV - the 2.6 is taken to be the same as the IDL software + pixel = get_rsun_obs(solo) * 2.6 / imsize + + # get back projection image + bp_image = vis_to_image(vis10_7, imsize, pixel_size=pixel) + + # Make a sunpy map from the bp_image, in STIX imaging frame + header = sunpy.map.make_fitswcs_header( + bp_image, center_coord, telescope="STIX", observatory="Solar Orbiter", scale=pixel + ) + fd_bp_map = sunpy.map.Map((bp_image, header)) + + sidelobes_ratio = calculate_sidelobes_ratio(fd_bp_map) + + # Make a sunpy map from the bp_image, in HPC from STIX observer + hpc_ref = center_coord.transform_to(frames.Helioprojective(observer=solo, obstime=vis_tr.center)) + header_hp = sunpy.map.make_fitswcs_header(bp_image, hpc_ref, scale=pixel, rotation_angle=90 * u.deg + roll) + hp_map = sunpy.map.Map((bp_image, header_hp)) + + # get the position of the max pixel + max_pixel = np.argwhere(fd_bp_map.data == fd_bp_map.data.max()).ravel() * u.pixel + # get the world coord of the max pixel - (note WCS axes and array are reversed) + max_stix = fd_bp_map.pixel_to_world(max_pixel[1], max_pixel[0]) + + # get the coordinate of the max pixel in HPC - if coordinate is off limb, assume spherical screen for transform + with SphericalScreen(hp_map.observer_coordinate, only_off_disk=True): + max_hpc = max_stix.transform_to(hp_map.coordinate_frame) + + vis_time_range = TimeRange(vis.meta["time_range"][0], vis.meta["time_range"][1]) + return max_stix, max_hpc, sidelobes_ratio, solo, vis_time_range + + +def calculate_sidelobes_ratio(bp_nat_map, threshold=200 * u.arcsec): + """ + + Calculate the sidelobes ratio for a back-projected image map. + + The sidelobes ratio is a measure of the relative strength of the sidelobes compared to the main peak of the image. + This ratio helps determine the reliability of the flare location. A sidelobes ratio close to or above 0.9 suggests + that the flare location may not be reliable due to significant sidelobe interference. + + Parameters + ---------- + bp_nat_map : `sunpy.map.Map` + The back-projected image map (in natural units) to analyze. This map is typically generated from visibility data + and contains the image of the flare. + threshold : `astropy.units.Quantity`, optional + The angular separation threshold (in arcseconds) around the peak within which sidelobes are excluded from the calculation. + Default is 200 arcseconds. + + Returns + ------- + sidelobes_ratio : float + The ratio of the maximum sidelobe intensity to the peak intensity in the back-projected image. + A value close to 1 indicates significant sidelobes, potentially making the flare location unreliable. + + Notes + ----- + - This is based upon the methodology in the STIX-GSW IDL software. + """ + max_bp = np.max(bp_nat_map.data) + ind_max = np.unravel_index(np.argmax(bp_nat_map.data, axis=None), bp_nat_map.data.shape) + max_bp_coord = bp_nat_map.pixel_to_world(ind_max[1] * u.pix, ind_max[0] * u.pix) + + yy, xx = np.indices(bp_nat_map.data.shape) + world_coords = bp_nat_map.pixel_to_world(xx * u.pix, yy * u.pix) + + distance_wrt_peak = world_coords.separation(max_bp_coord) + + bp_image_masked = np.copy(bp_nat_map.data) + mask = distance_wrt_peak <= threshold + bp_image_masked[mask] = 0 + + sidelobes_ratio = np.max(bp_image_masked) / max_bp + + return sidelobes_ratio + + +def _construct_stix_calibrated_visibilities( cpd_sci, flare_location, time_range=None, @@ -107,7 +323,7 @@ def construct_stix_calibrated_visibilities( return vis -def estimate_stix_flare_location( +def _estimate_stix_flare_location( cpd_sci, time_range=None, energy_range=None, subcollimators=None, cpd_bkg=None, time_range_bkg=None ): """ @@ -155,7 +371,7 @@ def estimate_stix_flare_location( 0 * u.arcsec, 0 * u.arcsec, obstime=time_range.start, obstime_end=time_range.end, observer=solo ) # Generate calibrated visibilities using coarser subcollimators. - vis = construct_stix_calibrated_visibilities( + vis = _construct_stix_calibrated_visibilities( cpd_sci, fov_centre, time_range=time_range, energy_range=energy_range, subcollimators=subcollimators ) # Produce backprojected image and find brightest pixel. Use this for flare location. diff --git a/stixcore/products/tests/test_flarelist.py b/stixcore/products/tests/test_flarelist.py index 001e0911..07522646 100644 --- a/stixcore/products/tests/test_flarelist.py +++ b/stixcore/products/tests/test_flarelist.py @@ -3,6 +3,7 @@ import numpy as np import pytest from sunpy.coordinates import HeliographicStonyhurst +from sunpy.time import TimeRange import astropy.units as u from astropy.coordinates import SkyCoord @@ -12,7 +13,11 @@ from astropy.time import Time from stixcore.io.product_processors.fits.processors import FitsL3Processor -from stixcore.products.level3.flarelist import FlarelistSDCLoc +from stixcore.products.level3.flarelist import ( + FlarelistSDCLoc, + calculate_overlap, + longest_constant_sequence, +) from stixcore.products.product import Product N = 10 @@ -102,3 +107,85 @@ def test_flarelist_sdcloc_fits_stores_icrs(written_fits): hgs = raw["location_icrs"].transform_to(HeliographicStonyhurst(obstime=obstime)) assert_quantity_allclose(hgs.lon, orig_hgs_lon, atol=1e-6 * u.deg, equal_nan=True) assert_quantity_allclose(hgs.lat, orig_hgs_lat, atol=1e-6 * u.deg, equal_nan=True) + + +# --- longest_constant_sequence --- + + +def test_lcs_empty(): + length, start, state = longest_constant_sequence([]) + assert length == 0 + assert start is None + assert state is None + + +def test_lcs_single_element(): + length, start, state = longest_constant_sequence([5]) + assert length == 1 + assert start == 0 + assert state == 5 + + +def test_lcs_all_same(): + length, start, state = longest_constant_sequence([3, 3, 3, 3]) + assert length == 4 + assert start == 0 + assert state == 3 + + +def test_lcs_clear_winner(): + length, start, state = longest_constant_sequence([1, 2, 2, 2, 3, 3]) + assert length == 3 + assert start == 1 + assert state == 2 + + +def test_lcs_tie_prefers_lower_state(): + # two runs of length 2: state=1 at index 0, state=3 at index 2 + length, start, state = longest_constant_sequence([1, 1, 3, 3]) + assert length == 2 + assert start == 0 + assert state == 1 + + +def test_lcs_numpy_array(): + arr = np.array([0, 0, 1, 1, 1, 0]) + length, start, state = longest_constant_sequence(arr) + assert length == 3 + assert start == 2 + assert state == 1 + + +# --- calculate_overlap --- + + +def test_overlap_no_intersection(): + r1 = TimeRange("2024-01-01T00:00:00", "2024-01-01T01:00:00") + r2 = TimeRange("2024-01-01T02:00:00", "2024-01-01T03:00:00") + assert calculate_overlap(r1, r2) is None + + +def test_overlap_partial(): + r1 = TimeRange("2024-01-01T00:00:00", "2024-01-01T02:00:00") + r2 = TimeRange("2024-01-01T01:00:00", "2024-01-01T03:00:00") + result = calculate_overlap(r1, r2) + assert result is not None + assert result.start == Time("2024-01-01T01:00:00") + assert result.end == Time("2024-01-01T02:00:00") + + +def test_overlap_contained(): + r1 = TimeRange("2024-01-01T00:00:00", "2024-01-01T04:00:00") + r2 = TimeRange("2024-01-01T01:00:00", "2024-01-01T03:00:00") + result = calculate_overlap(r1, r2) + assert result is not None + assert result.start == Time("2024-01-01T01:00:00") + assert result.end == Time("2024-01-01T03:00:00") + + +def test_overlap_identical(): + r1 = TimeRange("2024-01-01T00:00:00", "2024-01-01T01:00:00") + result = calculate_overlap(r1, r1) + assert result is not None + assert result.start == r1.start + assert result.end == r1.end From 5e59133f891f59a79e0d4ef21e2b605279b7cd2a Mon Sep 17 00:00:00 2001 From: Nicky Hochmuth Date: Tue, 4 Aug 2026 11:34:15 +0200 Subject: [PATCH 09/10] add find quiet background files --- .gitignore | 1 + stixcore/data/stixcore.ini | 12 + stixcore/data/test/publish/rid_lut.csv | 4 +- stixcore/io/FlareListManager.py | 644 ++++++++++++++++++--- stixcore/io/RidLutManager.py | 149 ++++- stixcore/io/tests/test_flarelistmanager.py | 609 +++++++++++++++++++ stixcore/processing/FlareListL3.py | 2 +- stixcore/products/level3/flarelist.py | 4 +- 8 files changed, 1348 insertions(+), 77 deletions(-) create mode 100644 stixcore/io/tests/test_flarelistmanager.py diff --git a/.gitignore b/.gitignore index c942678b..d4d6d599 100644 --- a/.gitignore +++ b/.gitignore @@ -166,3 +166,4 @@ stixcore/data/test/idb/v2.26.38/idb.sqlite .python-version sunpy/_version.py stixcore/_version.py +.claude/* diff --git a/stixcore/data/stixcore.ini b/stixcore/data/stixcore.ini index cdc8b347..3d7afdd7 100644 --- a/stixcore/data/stixcore.ini +++ b/stixcore/data/stixcore.ini @@ -36,3 +36,15 @@ soop_files_download = ./stixcore/data/soop ecc_path = /opt/stix_det_cal/bin/ [Processing] flarelist_sdc_min_count = 1000 +# background-data-file lookup (find_background_file_for_time) +flarelist_bkg_window_past_days = 21 +flarelist_bkg_window_future_days = 21 +flarelist_bkg_min_duration_s = 500 +# require the background file to share the flare's on-board ELUT configuration (ELUTManager lookup) +flarelist_bkg_require_same_elut = True +# optional stricter selection (default OFF - all three move common cases, not just edge cases): +# purpose preference: prefer purpose=="Background" unless a subject-only match is this many days closer +# (0.0 disables); exclude "elevated" backgrounds; drop requests whose comment references a flare id +flarelist_bkg_purpose_penalty_days = 0.0 +flarelist_bkg_exclude_elevated = False +flarelist_bkg_exclude_flare_comment = True diff --git a/stixcore/data/test/publish/rid_lut.csv b/stixcore/data/test/publish/rid_lut.csv index a6f03173..1eae888e 100644 --- a/stixcore/data/test/publish/rid_lut.csv +++ b/stixcore/data/test/publish/rid_lut.csv @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:2ba33aa45a7d045dcbc08d1ddddf8bdf8d01dde93e01fdf4db0c12e1dfb0b4bf -size 1005 +oid sha256:577485ea766335f15fb1ac4e40f1803d96e82919fd77164c429029a11f7afcac +size 1383 diff --git a/stixcore/io/FlareListManager.py b/stixcore/io/FlareListManager.py index e9d32e23..86f492e4 100644 --- a/stixcore/io/FlareListManager.py +++ b/stixcore/io/FlareListManager.py @@ -1,29 +1,357 @@ import sys import time +from pathlib import Path from datetime import datetime, timedelta +from collections import namedtuple import numpy as np import pandas as pd from stixdcpy.net import Request as stixdcpy_req +from stixpy.calibration.livetime import get_livetime_fraction +from stixpy.product import Product as STIXPYProduct from sunpy.net import attrs as a import astropy.units as u from astropy.table import Column, QTable, vstack from astropy.time import Time +from stixcore.calibration.elut_manager import ELUTManager from stixcore.config.config import CONFIG from stixcore.io.product_processors.fits.processors import CreateUtcColumn +from stixcore.io.RidLutManager import ( + DEFAULT_BKG_EXCLUDE_KEYWORDS, + DEFAULT_BKG_KEYWORDS, + RidLutManager, + search_background_candidates, +) from stixcore.products.level3.flarelist import FlarelistSC, FlarelistSDC from stixcore.products.product import Product from stixcore.util.logging import get_logger from stixcore.util.singleton import Singleton from stixcore.util.util import url_to_path -__all__ = ["FlareListManager", "SDCFlareListManager", "SCFlareListManager"] +__all__ = [ + "FlareListManager", + "SDCFlareListManager", + "SCFlareListManager", + "compute_ql_count_rate", + "build_month_timeline", + "nearest_bin_index", + "max_rcr_in_window", + "find_background_file_for_time", + "BackgroundSelection", +] logger = get_logger(__name__) +def compute_ql_count_rate(counts, timedel, triggers, energy_delta, *, n_detectors): + """Reproduce stixpy's QL count-rate normalization -> ``ct / (s * keV)``. + + Mirrors ``stixpy.timeseries.quicklook`` (lightcurve uses ``n_detectors=16``, + background uses ``n_detectors=1``). Pure, no I/O. + + Parameters + ---------- + counts : `~astropy.units.Quantity` + Raw counts, shape ``(N, 5)`` in ``ct``. + timedel : `~astropy.units.Quantity` + Bin durations, shape ``(N,)``. + triggers : array-like + Trigger counts, shape ``(N,)`` or ``(N, 1)``. + energy_delta : `~astropy.units.Quantity` + Channel widths, shape ``(5,)`` in ``keV``. + n_detectors : int + 16 for the lightcurve, 1 for the background detector. + + Returns + ------- + `~astropy.units.Quantity` + Count rate, shape ``(N, 5)`` in ``ct / (s * keV)``. + """ + timedel = timedel.to(u.s) + trig = np.asarray(triggers).reshape(-1) + live_frac, *_ = get_livetime_fraction(trig / (n_detectors * timedel)) + return counts / ((timedel * live_frac).reshape(-1, 1) * energy_delta) + + +def build_month_timeline(daily_data_tables): + """Stack per-day QTables into one time-sorted timeline with unique timestamps. + + ``None``/empty inputs are ignored; an empty list yields an empty ``QTable``. + After sorting by ``time`` duplicate timestamps (day-boundary overlaps) are + dropped so the timeline is strictly increasing. + """ + tables = [t for t in daily_data_tables if t is not None and len(t) > 0] + if not tables: + return QTable() + timeline = vstack(tables, metadata_conflicts="silent") + timeline.sort("time") + if len(timeline) > 1: + keep = np.ones(len(timeline), dtype=bool) + keep[1:] = np.diff(timeline["time"].jd) > 0 + timeline = timeline[keep] + return timeline + + +def nearest_bin_index(times, target, tol): + """Index of the bin in ``times`` nearest ``target``. + + Returns ``None`` if ``times`` is empty or the nearest gap exceeds ``tol`` + (both ``target`` and ``times`` are `~astropy.time.Time`, ``tol`` a duration). + """ + if times is None or len(times) == 0: + return None + dt = np.abs((times - target).to_value(u.s)) + j = int(np.argmin(dt)) + if dt[j] > tol.to_value(u.s): + return None + return j + + +def max_rcr_in_window(times, rcr, start, end, *, fallback): + """Highest ``rcr`` for bins with ``start <= time <= end``; ``fallback`` if none.""" + if times is None or len(times) == 0: + return fallback + mask = (times >= start) & (times <= end) + if not np.any(mask): + return fallback + return int(np.asarray(rcr)[mask].max()) + + +#: Result of :func:`find_background_file_for_time`. ``path`` is a `~pathlib.Path` +#: (or ``None`` when nothing qualifies), ``rid`` the selected BSD request id +#: (``-1`` when none), and ``valid_from``/``valid_to`` the `~astropy.time.Time` +#: interval over which this selection stays valid for a time-ordered caller. +BackgroundSelection = namedtuple("BackgroundSelection", ["path", "rid", "valid_from", "valid_to"]) + + +def _rid_from_filename(path): + """Parse the BSD request id embedded in a science FITS filename. + + Mirrors ``stixcore.processing.publish`` (the 6th ``_``-separated segment is + ``-``). Returns ``None`` when the name doesn't carry one. + """ + parts = Path(path).name.split("_") + if len(parts) <= 5: + return None + try: + return int(parts[5].replace(".fits", "").split("-")[0]) + except ValueError: + return None + + +def _elut_id(time): + """Identifier of the ELUT active at ``time`` (the resolved ELUT filename), or + ``None`` when the ELUT index has no unambiguous entry for it. Metadata-only + lookup via `~stixcore.calibration.elut_manager.ELUTManager` — reads no FITS.""" + try: + return ELUTManager.instance._find_elut_file(time.to_datetime()) + except Exception as e: + logger.debug(f"no ELUT resolved for {getattr(time, 'isot', time)}: {e}") + return None + + +def _effective_crossover(t0, sa, pa, sb, pb): + """Earliest ``t >= t0`` (all in JD days) at which candidate ``b``'s effective + distance ``|t - sb| + pb`` drops to/below ``a``'s ``|t - sa| + pa``; ``None`` if + it never does. Bounds the nearest-in-time validity interval under the purpose + penalty (piecewise-linear, breakpoints at the two starts).""" + + def val(x): + return abs(x - sb) - abs(x - sa) + (pb - pa) + + if val(t0) < 0: + return t0 + lefts = [t0] + sorted(x for x in (sa, sb) if x > t0) + for i, left in enumerate(lefts): + right = lefts[i + 1] if i + 1 < len(lefts) else None + v = val(left) + if v < 0: + return left + probe = (left + right) / 2 if right is not None else left + 1.0 + slope = ((abs(probe - sb) - abs(probe - sa)) - (abs(left - sb) - abs(left - sa))) / (probe - left) + if slope < 0: + tc = left + v / (-slope) + if right is None or tc <= right: + return tc + return None + + +def find_background_file_for_time( + time, + *, + fido_client, + rid_lut=None, + window_past=None, + window_future=None, + min_duration=None, + require_same_elut=None, + purpose_penalty=None, + keywords=DEFAULT_BKG_KEYWORDS, + exclude_keywords=None, + exclude_flare_comment=None, +): + """Find the best quiet-time background CPD file applicable at ``time``. + + Strategy (see :func:`stixcore.io.RidLutManager.search_background_candidates`): + rank background requests from the RID LUT **nearest-in-time first** (closest + request start, past or future, within the separate ``window_past`` / + ``window_future`` bounds). For each candidate resolve the real ``sci_xray_cpd`` + L1 file(s) via FIDO, keep those whose filename carries the candidate rid, and + accept the first one that passes the "good background" checks (requested long + enough, attenuator out over the whole interval). + + Three optional stricter filters exist but default **off** (they move common + cases, not just edge cases): a ``purpose == "Background"`` preference + (``purpose_penalty`` > 0), dropping "elevated" backgrounds (``exclude_keywords``), + and dropping requests whose comment references a specific flare id + (``exclude_flare_comment``). See the ``[Processing]`` config keys. + + Alongside the winning file a validity interval ``[valid_from, valid_to]`` is + returned so a time-ordered caller (monthly flare processing) can reuse the + result for every later time inside the interval. ``valid_to`` is the earliest + time at which another candidate's effective distance would overtake the chosen + one, or the chosen start plus ``window_past`` when there is none — capped at + ``time + window_future`` so newly-reachable candidates are re-scanned. A + "no file found" result is cached until the next candidate could appear. + + Parameters + ---------- + time : `~astropy.time.Time` or str + The query time (e.g. a flare peak). + fido_client : `~stixpy.net.client.STIXClient` + Client used for the ``sci_xray_cpd`` L1 search. + rid_lut : `~astropy.table.Table`, optional + The RID LUT; defaults to ``RidLutManager.instance.rid_lut``. + window_past, window_future : `~astropy.units.Quantity`, optional + Search windows; default to the ``[Processing]`` config keys + ``flarelist_bkg_window_past_days`` / ``flarelist_bkg_window_future_days``. + min_duration : `~astropy.units.Quantity`, optional + Minimum requested integration time; defaults to + ``flarelist_bkg_min_duration_s``. + require_same_elut : bool, optional + If True, a candidate is only accepted when the ELUT active at its request + start (per `~stixcore.calibration.elut_manager.ELUTManager`) matches the + one active at ``time`` — i.e. the background was taken under the same + on-board ELUT configuration as the flare. Defaults to the ``[Processing]`` + config key ``flarelist_bkg_require_same_elut``. + purpose_penalty : `~astropy.units.Quantity`, optional + Distance penalty for non-``Background``-purpose candidates; defaults to the + ``[Processing]`` config key ``flarelist_bkg_purpose_penalty_days``. + keywords, exclude_keywords : tuple of str, optional + Positive / negative background keywords for the candidate search. + exclude_flare_comment : bool, optional + Drop candidates whose comment references a specific flare id. + + Returns + ------- + BackgroundSelection + """ + if window_past is None: + window_past = CONFIG.getfloat("Processing", "flarelist_bkg_window_past_days", fallback=30.0) * u.day + if window_future is None: + window_future = CONFIG.getfloat("Processing", "flarelist_bkg_window_future_days", fallback=7.0) * u.day + if min_duration is None: + min_duration = CONFIG.getfloat("Processing", "flarelist_bkg_min_duration_s", fallback=1200.0) * u.s + if require_same_elut is None: + require_same_elut = CONFIG.getboolean("Processing", "flarelist_bkg_require_same_elut", fallback=True) + if purpose_penalty is None: + purpose_penalty = CONFIG.getfloat("Processing", "flarelist_bkg_purpose_penalty_days", fallback=0.0) * u.day + if exclude_keywords is None: + exclude_keywords = ( + DEFAULT_BKG_EXCLUDE_KEYWORDS + if CONFIG.getboolean("Processing", "flarelist_bkg_exclude_elevated", fallback=False) + else () + ) + if exclude_flare_comment is None: + exclude_flare_comment = CONFIG.getboolean("Processing", "flarelist_bkg_exclude_flare_comment", fallback=False) + + t = time if isinstance(time, Time) else Time(time) + if rid_lut is None: + rid_lut = RidLutManager.instance.rid_lut + + # ELUT active at the flare time; only enforced when it can be resolved + flare_elut = _elut_id(t) if require_same_elut else None + + candidates = search_background_candidates( + rid_lut, + t, + window_past=window_past, + window_future=window_future, + keywords=keywords, + exclude_keywords=exclude_keywords, + exclude_flare_comment=exclude_flare_comment, + purpose_penalty=purpose_penalty, + ) + + t_jd = t.to_value("jd") + penalty_days = purpose_penalty.to_value(u.day) + + def _valid_to(chosen): + # the selection holds until another candidate's *effective* distance overtakes + # the chosen one's (accounts for the purpose penalty), or the chosen leaves the + # past window if none does. Capped at time + window_future so newly-reachable + # candidates get re-scanned. + pa = 0.0 if chosen.is_background else penalty_days + sa = chosen.start.to_value("jd") + edge = sa + window_past.to_value(u.day) + for j in candidates: + if j.rid == chosen.rid: + continue + pb = 0.0 if j.is_background else penalty_days + tc = _effective_crossover(t_jd, sa, pa, j.start.to_value("jd"), pb) + if tc is not None: + edge = min(edge, tc) + return min(Time(edge, format="jd", scale="utc"), t + window_future) + + for cand in candidates: + if (cand.end - cand.start) < min_duration: + logger.debug(f"bkg candidate rid {cand.rid}: requested duration below {min_duration}; skipping") + continue + if flare_elut is not None and _elut_id(cand.start) != flare_elut: + logger.debug(f"bkg candidate rid {cand.rid}: different ELUT configuration; skipping") + continue + try: + res = fido_client.search( + a.Time(cand.start, cand.end), + a.Instrument.stix, + a.stix.DataProduct.sci_xray_cpd, + a.Level("L1"), + ) + except Exception as e: + logger.warning(f"bkg candidate rid {cand.rid}: CPD search failed: {e}") + continue + if len(res) == 0: + continue + res.filter_for_latest_version() + url_to_path(res) + if "path" not in res.columns: + continue + for path in res["path"]: + if path is None: + continue + if _rid_from_filename(path) != cand.rid: + continue + try: + p = STIXPYProduct(path) + except Exception as e: + logger.warning(f"bkg candidate rid {cand.rid}: could not load {path}: {e}") + continue + rcr = p.data["rcr"] if "rcr" in p.data.colnames else None + if rcr is None or np.any(np.asarray(rcr) != 0): + logger.debug(f"bkg candidate rid {cand.rid}: attenuator in (rcr != 0); skipping {path}") + continue + valid_to = _valid_to(cand) + logger.info(f"selected background file {path} (rid {cand.rid}, {cand.side}) for {t.isot}") + return BackgroundSelection(path=Path(str(path)), rid=cand.rid, valid_from=t, valid_to=valid_to) + + later = [c.start for c in candidates if c.start > t] + valid_to = min(later) if later else t + window_future + logger.info(f"no usable background file found for {t.isot}") + return BackgroundSelection(path=None, rid=-1, valid_from=t, valid_to=valid_to) + + class FlareListManager: @property def flarelist(self): @@ -41,6 +369,229 @@ def flarelistname(self): def productCls(self): return self._product_cls + def _build_ql_month_timeline(self, *, start, end, fido_client, data_product, n_detectors, track_energy): + """Search + load all L1 QL files of ``data_product`` for ``[start, end)`` and + stack them into one time-sorted timeline with a per-bin count-rate column. + + Each daily file is opened exactly once (used for both the energy-table + lookup and the counts), so the whole month costs one open per day. + + Returns + ------- + (timeline, energy, date_to_eidx) + ``timeline`` : QTable with columns ``time``, ``counts`` (raw ``ct``, + ``(N, 5)``), ``counts_rate`` (``(N, 5)``) and, for the lightcurve, + ``rcr``. ``energy`` and ``date_to_eidx`` are only populated when + ``track_energy`` is True. + """ + energy = QTable() + energy_look_up = {} + date_to_eidx = {} + daily_tables = [] + + try: + res = fido_client.search(a.Time(start, end), a.Instrument.stix, data_product, a.Level("L1")) + if len(res) > 0: + res.filter_for_latest_version() + url_to_path(res) + except Exception as e: + logger.error(f"error searching L1 QL {data_product} files for month {start}: {e}") + return QTable(), energy, date_to_eidx + + if len(res) == 0 or "path" not in res.columns: + return QTable(), energy, date_to_eidx + + for path in res["path"]: + if path is None: + continue + try: + p = STIXPYProduct(path) + except Exception as e: + logger.warning(f"could not load QL product {path}: {e}") + continue + + energies = getattr(p, "_energies", None) + if energies is None: + logger.warning(f"QL product {path} has no energies table; skipping") + continue + + counts = p.data["counts"] + if counts.shape[1] != 5: + logger.warning(f"QL product {path} has {counts.shape[1]} channels (expected 5); skipping") + continue + + energy_delta = energies["e_high"] - energies["e_low"] + rate = compute_ql_count_rate( + counts, p.data["timedel"], p.data["triggers"], energy_delta, n_detectors=n_detectors + ) + + daily = QTable() + daily["time"] = p.data["time"] + daily["counts"] = counts + daily["counts_rate"] = rate + if "rcr" in p.data.colnames: + daily["rcr"] = np.asarray(p.data["rcr"]).astype(np.int16) + daily_tables.append(daily) + + if track_energy: + e_sub = QTable() + e_sub["channel"] = energies["channel"] + e_sub["e_low"] = energies["e_low"] + e_sub["e_high"] = energies["e_high"] + e_hash = frozenset(pd.core.util.hashing.hash_array(e_sub.as_array())) + if e_hash not in energy_look_up: + e_idx = len(energy_look_up) + energy_look_up[e_hash] = e_idx + e_sub["index"] = Column(e_idx, description="energy edge table index", dtype=np.int8) + energy = vstack([energy, e_sub]) + if e_idx > 0: + logger.warning(f"multiple energy ql-lc tables found for month {start}") + eidx = energy_look_up[e_hash] + for d in {t.to_datetime().date() for t in p.data["time"]}: + date_to_eidx.setdefault(d, eidx) + logger.info(f"loaded QL product {path} with {len(p.data)} bins and {len(energies)} energy channels") + + return build_month_timeline(daily_tables), energy, date_to_eidx + + def add_lc_bkg_columns(self, data, *, start, end, fido_client): + """Populate LC/BKG peak counts + rates, RCR and ``att_in`` on ``data`` from the + real L1 QL lightcurve + background products, and return the energy QTable. + + ``data`` must already carry ``flare_id`` and the astropy ``Time`` columns + ``start_UTC`` / ``end_UTC`` / ``peak_UTC``. Columns are added in place: + ``lc_peak``, ``lc_peak_rate``, ``lc_bgk_peak``, ``lc_bgk_peak_rate``, + ``rcr_at_peak``, ``rcr_max``, ``att_in``, ``energy_index``. + """ + n = len(data) + tol = CONFIG.getfloat("Processing", "flarelist_peak_max_dist_s", fallback=60.0) * u.s + + lc_timeline, energy, date_to_eidx = self._build_ql_month_timeline( + start=start, + end=end, + fido_client=fido_client, + data_product=a.stix.DataProduct.ql_lightcurve, + n_detectors=16, + track_energy=True, + ) + bkg_timeline, _, _ = self._build_ql_month_timeline( + start=start, + end=end, + fido_client=fido_client, + data_product=a.stix.DataProduct.ql_background, + n_detectors=1, + track_energy=False, + ) + + lc_peak = np.zeros((n, 5), dtype=np.int64) + lc_peak_rate = np.zeros((n, 5), dtype=np.float64) + lc_bgk_peak = np.zeros((n, 5), dtype=np.int64) + lc_bgk_peak_rate = np.zeros((n, 5), dtype=np.float64) + rcr_at_peak = np.full(n, -1, dtype=np.int8) + rcr_max = np.full(n, -1, dtype=np.int8) + energy_index = np.zeros(n, dtype=np.int8) + + rate_unit = u.ct / (u.s * u.keV) + lc_has = len(lc_timeline) > 0 + bkg_has = len(bkg_timeline) > 0 + if not lc_has: + logger.warning(f"No L1 QL lightcurve data found for month {start}") + if not bkg_has: + logger.warning(f"No L1 QL background data found for month {start}") + + for i, row in enumerate(data): + peak = row["peak_UTC"] + fid = row["flare_id"] + if lc_has: + j = nearest_bin_index(lc_timeline["time"], peak, tol) + if j is not None: + lc_peak[i] = lc_timeline["counts"][j].to_value(u.ct) + lc_peak_rate[i] = lc_timeline["counts_rate"][j].to_value(rate_unit) + rcr_at_peak[i] = int(lc_timeline["rcr"][j]) + rcr_max[i] = max_rcr_in_window( + lc_timeline["time"], + lc_timeline["rcr"], + row["start_UTC"], + row["end_UTC"], + fallback=int(rcr_at_peak[i]), + ) + energy_index[i] = date_to_eidx.get(peak.to_datetime().date(), 0) + else: + logger.warning(f"flare {fid}: no LC bin within {tol} of peak {peak.isot}") + if bkg_has: + k = nearest_bin_index(bkg_timeline["time"], peak, tol) + if k is not None: + lc_bgk_peak[i] = bkg_timeline["counts"][k].to_value(u.ct) + lc_bgk_peak_rate[i] = bkg_timeline["counts_rate"][k].to_value(rate_unit) + else: + logger.warning(f"flare {fid}: no BKG bin within {tol} of peak {peak.isot}") + + data["lc_peak"] = Column( + lc_peak * u.ct, + description="raw counts at the L1 QL lightcurve bin nearest the flare peak (5 energy channels)", + dtype=np.int64, + ) + data["lc_peak_rate"] = Column( + lc_peak_rate * rate_unit, + description="livetime-corrected count rate at the peak lightcurve bin (5 energy channels)", + ) + data["lc_bgk_peak"] = Column( + lc_bgk_peak * u.ct, + description="raw background counts at the L1 QL background bin nearest the flare peak (5 energy channels)", + dtype=np.int64, + ) + data["lc_bgk_peak_rate"] = Column( + lc_bgk_peak_rate * rate_unit, + description="livetime-corrected background count rate at the peak bin (5 energy channels)", + ) + data["rcr_at_peak"] = Column( + rcr_at_peak, description="rate control regime at the peak bin (>0 attenuator in)", dtype=np.int8 + ) + data["rcr_max"] = Column( + rcr_max, description="max rate control regime over the flare start..end window", dtype=np.int8 + ) + data["att_in"] = Column(rcr_max > 0, description="was attenuator in during flare (rcr_max > 0)") + data["energy_index"] = Column(energy_index, description="energy band index", dtype=np.int8) + + return energy + + def add_background_file_column(self, data, *, fido_client): + """Add ``bkg_file`` / ``bkg_rid`` columns: the best quiet-time background + CPD file for each flare peak. + + ``data`` rows are assumed to be peak-time ascending (as produced by the + source flare list), so each per-time background search + (:func:`find_background_file_for_time`) is cached with its validity + interval and only re-run once a flare peak crosses out of that period. + """ + n = len(data) + bkg_files = [""] * n + bkg_rids = np.full(n, -1, dtype=np.int64) + + primer = "" + baseurl = getattr(fido_client, "baseurl", None) + datapath = getattr(fido_client, "datapath", None) + if baseurl is not None and datapath is not None: + primer = baseurl.replace(datapath, "") + primer = primer[7:] if primer.startswith("file://") else primer + + selection = None + searches = 0 + for i, row in enumerate(data): + peak = row["peak_UTC"] + if selection is None or not (selection.valid_from <= peak <= selection.valid_to): + selection = find_background_file_for_time(peak, fido_client=fido_client) + searches += 1 + if selection.path is not None: + bkg_files[i] = str(selection.path).replace(primer, "") + bkg_rids[i] = selection.rid + + data["bkg_file"] = Column(bkg_files, description="path to the quiet-time background CPD file") + data["bkg_rid"] = Column( + bkg_rids, description="BSD request id of the selected background file (-1 if none)", dtype=np.int64 + ) + logger.info(f"background file search ran {searches}x for {n} flares") + return data + class SCFlareListManager(FlareListManager, metaclass=Singleton): """Manages a local copy of the flarelist provided by STIXCore or runs the flare detection @@ -202,7 +753,6 @@ def get_data(self, *, start, end, fido_client): data["peak_UTC"] = CreateUtcColumn(description="flare peak time") data["peak_UTC"] = [Time(d, format="isot", scale="utc") for d in mt["peak_UTC"]] data["att_in"] = Column(mt["att_in"].astype(bool), description="was attenuator in during flare") - data["bkg_baseline"] = Column(mt["LC0_BKG"] * u.ct, description="background baseline at 4-10 keV") data["GOES_class"] = Column( mt["GOES_class"].astype(str), description="GOES class of the GOES XRS data at time of flare" @@ -226,15 +776,15 @@ def get_data(self, *, start, end, fido_client): "flare isn't visible to Earth", ) data["goes_min_flux_est"] = Column( - mt["goes_estimated_min_flux"].astype(float) * u.W / u.m**2, + (10 ** mt["goes_estimated_min_flux"].astype(float)) * u.W / u.m**2, description="min GOES flux estimate derived from STIX data", ) data["goes_max_flux_est"] = Column( - mt["goes_estimated_max_flux"].astype(float) * u.W / u.m**2, + (10 ** mt["goes_estimated_max_flux"].astype(float)) * u.W / u.m**2, description="max GOES flux estimate derived from STIX data", ) data["goes_mean_flux_est"] = Column( - mt["goes_estimated_mean_flux"].astype(float) * u.W / u.m**2, + (10 ** mt["goes_estimated_mean_flux"].astype(float)) * u.W / u.m**2, description="mean GOES flux estimate derived from STIX data", ) @@ -467,8 +1017,6 @@ def get_data(self, *, start, end, fido_client): description="flare peak time", ) - data["att_in"] = Column(mt["att_in"].astype(bool), description="was attenuator in during flare") - data["bkg_baseline"] = Column(mt["LC0_BKG"] * u.ct, description="background baseline at 4-10 keV") data["GOES_class"] = Column( mt["GOES_class"].astype(str), description="GOES class of the GOES XRS data at time of flare" @@ -492,15 +1040,15 @@ def get_data(self, *, start, end, fido_client): "flare isn't visible to Earth", ) data["goes_min_flux_est"] = Column( - mt["goes_estimated_min_flux"].astype(float) * u.W / u.m**2, + (10 ** mt["goes_estimated_min_flux"].astype(float)) * u.W / u.m**2, description="min GOES flux estimate derived from STIX data", ) data["goes_max_flux_est"] = Column( - mt["goes_estimated_max_flux"].astype(float) * u.W / u.m**2, + (10 ** mt["goes_estimated_max_flux"].astype(float)) * u.W / u.m**2, description="max GOES flux estimate derived from STIX data", ) data["goes_mean_flux_est"] = Column( - mt["goes_estimated_mean_flux"].astype(float) * u.W / u.m**2, + (10 ** mt["goes_estimated_mean_flux"].astype(float)) * u.W / u.m**2, description="mean GOES flux estimate derived from STIX data", ) @@ -511,24 +1059,13 @@ def get_data(self, *, start, end, fido_client): # description="coarse flare location in y direction provided by" # "onboard algorithm. (0,0) represents disk center") - data["lc_peak"] = Column( - ( - np.vstack( - ( - mt["LC0_PEAK_COUNTS_4S"].value, - mt["LC1_PEAK_COUNTS_4S"].value, - mt["LC2_PEAK_COUNTS_4S"].value, - mt["LC3_PEAK_COUNTS_4S"].value, - mt["LC4_PEAK_COUNTS_4S"].value, - ) - ).T - * u.ct - ).astype(int), - description="counts in 4s peak window from quicklook lightcurve", - dtype=np.int64, + # background estimates carried over from the source flare list (distinct from the + # QL-derived at-peak background added below) + data["bkg_baseline"] = Column( + mt["LC0_BKG"].astype(float) * u.ct, + description="median value of the fitted baseline", ) - - data["lc_bgk_peak"] = Column( + data["bkg_quiet_period"] = Column( ( np.vstack( ( @@ -540,55 +1077,20 @@ def get_data(self, *, start, end, fido_client): ) ).T * u.ct - ).astype(int), - description="background counts in 4s peak windowfrom quicklook lightcurve", + ).astype(np.int64), + description="background counts per QL energy channel, median value for the most recent quiet period", dtype=np.int64, ) - data["energy_index"] = Column(0, description="energy band index", dtype=np.int8) - - data.add_index("flare_id") - - # add energy axis for the lightcurve peak time data for each flare - # the energy bins are taken from the daily ql-lightcurve products - # as the definition of the lc energy chanel's are will change only very seldom - # the ql-lightcurve products assume a constant definition for an entire day. - # So we do the lookup also just grouped by peak day in order to save file lookups - - energy_look_up = {} - data["peak_day"] = [d.datetime.day for d in data["peak_UTC"]] - data_by_day = data.group_by("peak_day") - - for day, flares in zip(data_by_day.groups.keys, data_by_day.groups): - time = flares["peak_UTC"][0] - lc_data = fido_client.search(a.Time(time, time), a.Instrument.stix, a.stix.DataProduct.ql_lightcurve) - lc_data.filter_for_latest_version() - url_to_path(lc_data) - - if len(lc_data) == 0: - logger.warning(f"No lightcurve data found for flare at time {time}") - continue - lc = Product(lc_data["path"][0]) + # LC/BKG peak counts+rates, RCR, att_in and energy_index are derived from the + # real L1 QL lightcurve + background products (the source CSV values are + # unreliable) using one monthly timeline per product built once. + energy = self.add_lc_bkg_columns(data, start=start, end=end, fido_client=fido_client) - energy_table_hash = frozenset(pd.core.util.hashing.hash_array(lc.energies.as_array())) + # select the best quiet-time background data file for each flare peak + self.add_background_file_column(data, fido_client=fido_client) - # add the energy table to the energy table list if not already - # present and define a new index number - if energy_table_hash not in energy_look_up: - e_idx = len(energy_look_up.keys()) - energy_look_up[energy_table_hash] = e_idx - lc.energies["index"] = Column(e_idx, description="energy edge table index", dtype=np.int8) - energy = vstack([energy, lc.energies]) - if e_idx > 0: - logger.warning(f"multiple energy ql-lc tables found for month {start}") - - # add the energy index to the flare data to all flares of the same day - # https://docs.astropy.org/en/latest/table/modify_table.html#caveats - replace = data.loc[flares["flare_id"]] - replace["energy_index"] = energy_look_up[energy_table_hash] - data.loc[flares["flare_id"]] = replace - - del data["peak_day"] + data.add_index("flare_id") return data, control, energy diff --git a/stixcore/io/RidLutManager.py b/stixcore/io/RidLutManager.py index bd1f9d26..154956b8 100644 --- a/stixcore/io/RidLutManager.py +++ b/stixcore/io/RidLutManager.py @@ -1,23 +1,163 @@ +import re import sys import time import tempfile import urllib.request from datetime import date, datetime, timedelta +from collections import namedtuple import numpy as np +import astropy.units as u from astropy.io import ascii from astropy.table import Table from astropy.table.operations import unique, vstack +from astropy.time import Time from stixcore.config.config import CONFIG from stixcore.util.logging import get_logger from stixcore.util.singleton import Singleton -__all__ = ["RidLutManager"] +__all__ = ["RidLutManager", "BackgroundCandidate", "search_background_candidates"] logger = get_logger(__name__) +#: Keywords (case-insensitive) that mark a BSD request as a background/quiet +#: observation in the descriptive columns of the RID LUT. +DEFAULT_BKG_KEYWORDS = ("bkg", "quiet", "background", "non-flaring") + +#: Negative keywords: a candidate whose descriptive text contains any of these is +#: rejected (e.g. "elevated" background is not a clean quiet baseline). +DEFAULT_BKG_EXCLUDE_KEYWORDS = ("elevated",) + +#: Matches a specific flare id referenced in a comment, e.g. "for Flare 2309081508". +#: Such rows are flare data requests, not dedicated backgrounds. +_FLARE_REF_RE = re.compile(r"flare\s*\d{3,}") + +#: A single background-request candidate from the RID LUT. ``start``/``end``/``mid`` +#: are `~astropy.time.Time`, ``side`` is ``"past"``/``"future"`` relative to the query +#: time, and ``is_background`` is True when ``purpose == "Background"`` (a clean +#: background request, preferred over subject-only keyword matches). +BackgroundCandidate = namedtuple("BackgroundCandidate", ["rid", "start", "end", "mid", "side", "is_background"]) + + +def _col_as_lower_str(tbl, name): + """Return column ``name`` of ``tbl`` as a lower-cased ``str`` numpy array.""" + col = tbl[name] + try: + col = col.filled("") + except (AttributeError, TypeError): + pass + return np.char.lower(np.asarray(col, dtype=str)) + + +def search_background_candidates( + rid_lut, + time, + *, + window_past, + window_future, + keywords=DEFAULT_BKG_KEYWORDS, + exclude_keywords=DEFAULT_BKG_EXCLUDE_KEYWORDS, + exclude_flare_comment=True, + purpose_penalty=1.0 * u.day, +): + """Find background-request candidates in a RID LUT near ``time``. + + Rows are recognised as background requests by a case-insensitive keyword + match (``keywords``) over the ``subject``/``purpose``/``comment`` columns, + then filtered and ranked: + + * **exclude keywords** — a row whose text contains any of ``exclude_keywords`` + (default ``"elevated"``) is dropped. + * **flare-id comment** — if ``exclude_flare_comment`` a row whose comment + references a specific flare id (e.g. "for Flare 2309081508") is dropped. + * **ranking** — **nearest-in-time first** by ``|time - start|`` (past preferred + on a tie), but a request with ``purpose == "Background"`` is preferred over a + subject-only keyword match unless the latter is more than ``purpose_penalty`` + closer. This is implemented as an *effective* distance + ``|time - start| + purpose_penalty`` for non-Background rows. + + ``window_past`` / ``window_future`` remain separate bounds on how far a + candidate's start may lie in each direction. + + Parameters + ---------- + rid_lut : `~astropy.table.Table` + The RID LUT (as produced by `RidLutManager.read_rid_lut`). + time : `~astropy.time.Time` or str + The query time (e.g. a flare peak). + window_past, window_future : `~astropy.units.Quantity` + How far back / forward from ``time`` a candidate's start may lie. + keywords, exclude_keywords : tuple of str, optional + Positive / negative case-insensitive keywords. + exclude_flare_comment : bool, optional + Drop rows whose comment references a specific flare id. + purpose_penalty : `~astropy.units.Quantity`, optional + Distance penalty added to non-``Background``-purpose candidates so a clean + Background request wins unless a subject-only match is clearly closer. + + Returns + ------- + list of BackgroundCandidate + Ranked by ascending effective distance (past preferred on a tie). Empty + if the LUT has no matching rows in range. + """ + if rid_lut is None or len(rid_lut) == 0: + return [] + t = time if isinstance(time, Time) else Time(time) + + subj = _col_as_lower_str(rid_lut, "subject") + purp = _col_as_lower_str(rid_lut, "purpose") + comm = _col_as_lower_str(rid_lut, "comment") + haystack = np.char.add(np.char.add(subj, " "), np.char.add(purp, np.char.add(" ", comm))) + + mask = np.zeros(len(rid_lut), dtype=bool) + for kw in keywords: + mask |= np.char.find(haystack, kw.lower()) >= 0 + for kw in exclude_keywords: # negative keywords drop the row + mask &= np.char.find(haystack, kw.lower()) < 0 + if not np.any(mask): + return [] + + sub = rid_lut[mask] + purp_sub = purp[mask] + comm_sub = comm[mask] + starts = Time(np.asarray(sub["start_utc"], dtype=str), format="isot", scale="utc") + durations = np.asarray(sub["duration"], dtype=float) * u.s + ends = starts + durations + mids = starts + durations / 2 + rids = np.asarray(sub["unique_id"]).astype(np.int64) + + wp = window_past.to_value(u.day) + wf = window_future.to_value(u.day) + penalty = purpose_penalty.to_value(u.day) + kept = [] + for i in range(len(sub)): + if exclude_flare_comment and _FLARE_REF_RE.search(comm_sub[i]): + continue + d = (t - starts[i]).to_value(u.day) # > 0 starts before ``time`` (past), < 0 after (future) + if d >= 0: + if d > wp: + continue + side = "past" + else: + if -d > wf: + continue + side = "future" + is_bg = purp_sub[i].strip() == "background" + effective = abs(d) + (0.0 if is_bg else penalty) # prefer Background at equal-ish distance + kept.append( + ( + effective, + 0 if side == "past" else 1, + BackgroundCandidate(int(rids[i]), starts[i], ends[i], mids[i], side, is_bg), + ) + ) + + kept.sort(key=lambda x: (x[0], x[1])) + return [c for _, _, c in kept] + class RidLutManager(metaclass=Singleton): """Manages metadata for BSD requests @@ -78,6 +218,13 @@ def get_reason(self, rid): logger.warning("can't get request purpose: no request founds for rid: {rid}") return "" + def find_background_candidates(self, time, *, window_past, window_future, keywords=DEFAULT_BKG_KEYWORDS): + """Find background-request candidates near ``time`` (see + :func:`search_background_candidates`).""" + return search_background_candidates( + self.rid_lut, time, window_past=window_past, window_future=window_future, keywords=keywords + ) + def get_scaling_factor(self, rid): """Gets the trigger descaling factor connected to the BSD request. diff --git a/stixcore/io/tests/test_flarelistmanager.py b/stixcore/io/tests/test_flarelistmanager.py new file mode 100644 index 00000000..95b9dd93 --- /dev/null +++ b/stixcore/io/tests/test_flarelistmanager.py @@ -0,0 +1,609 @@ +from pathlib import Path + +import numpy as np +import pytest + +import astropy.units as u +from astropy.table import QTable, Table +from astropy.time import Time + +import stixcore.io.FlareListManager as flm_mod +from stixcore.io.FlareListManager import ( + BackgroundSelection, + FlareListManager, + build_month_timeline, + compute_ql_count_rate, + find_background_file_for_time, + max_rcr_in_window, + nearest_bin_index, +) +from stixcore.io.RidLutManager import RidLutManager, search_background_candidates + +RATE_UNIT = u.ct / (u.s * u.keV) + + +# --- helpers to build synthetic QL data --------------------------------------- + + +def _energies(): + e = QTable() + e["channel"] = np.arange(5, dtype=np.uint8) + e["e_low"] = [4, 10, 15, 25, 50] * u.keV + e["e_high"] = [10, 15, 25, 50, 84] * u.keV + return e + + +def _ql_data(t0, n, *, rcr=None, counts_value=100, with_rcr=True): + """Build a synthetic QL product ``.data`` QTable on a 4 s grid.""" + times = Time(t0) + np.arange(n) * 4 * u.s + data = QTable() + data["time"] = times + data["timedel"] = np.full(n, 4.0) * u.s + data["triggers"] = np.zeros(n) # zero triggers -> live_frac == 1 (deterministic) + data["counts"] = (np.full((n, 5), counts_value)).astype(int) * u.ct + if with_rcr: + data["rcr"] = np.zeros(n, dtype=np.ubyte) if rcr is None else np.asarray(rcr, dtype=np.ubyte) + return data + + +class FakeProduct: + def __init__(self, data, energies): + self.data = data + self._energies = energies + + +class FakeResponse: + """Minimal stand-in for a StixQueryResponse carrying a ``path`` column.""" + + def __init__(self, paths): + self._paths = list(paths) + + def __len__(self): + return len(self._paths) + + @property + def columns(self): + return ["path"] if self._paths else [] + + def filter_for_latest_version(self): + pass + + def __getitem__(self, key): + assert key == "path" + return self._paths + + +class FakeFido: + def __init__(self, lc_paths, bkg_paths): + self.lc_paths = lc_paths + self.bkg_paths = bkg_paths + + def search(self, time, instrument, data_product, level): + name = getattr(data_product, "value", str(data_product)) + if "background" in name: + return FakeResponse(self.bkg_paths) + return FakeResponse(self.lc_paths) + + +# --- compute_ql_count_rate ---------------------------------------------------- + + +def test_count_rate_units_and_shape(): + data = _ql_data("2024-06-15T12:00:00", 3) + ed = _energies()["e_high"] - _energies()["e_low"] + rate = compute_ql_count_rate(data["counts"], data["timedel"], data["triggers"], ed, n_detectors=16) + assert rate.shape == (3, 5) + assert rate.unit.is_equivalent(RATE_UNIT) + + +def test_count_rate_zero_triggers_is_deterministic(): + # zero triggers => live_frac == 1 => rate == counts / (timedel * energy_delta), + # independent of n_detectors, so LC and BKG agree exactly. + data = _ql_data("2024-06-15T12:00:00", 2, counts_value=200) + ed = _energies()["e_high"] - _energies()["e_low"] + lc = compute_ql_count_rate(data["counts"], data["timedel"], data["triggers"], ed, n_detectors=16) + bkg = compute_ql_count_rate(data["counts"], data["timedel"], data["triggers"], ed, n_detectors=1) + expected = data["counts"] / ((data["timedel"]).reshape(-1, 1) * ed) + assert u.allclose(lc, expected.to(RATE_UNIT)) + assert u.allclose(lc, bkg) + + +def test_count_rate_more_detectors_gives_smaller_rate_when_triggers_nonzero(): + # nonzero triggers: larger n_detectors -> lower trigger rate -> higher live_frac + # -> larger denominator -> smaller count rate. + data = _ql_data("2024-06-15T12:00:00", 1) + data["triggers"] = np.array([5000.0]) + ed = _energies()["e_high"] - _energies()["e_low"] + lc = compute_ql_count_rate(data["counts"], data["timedel"], data["triggers"], ed, n_detectors=16) + bkg = compute_ql_count_rate(data["counts"], data["timedel"], data["triggers"], ed, n_detectors=1) + assert np.all(lc.to_value(RATE_UNIT) < bkg.to_value(RATE_UNIT)) + + +# --- build_month_timeline ----------------------------------------------------- + + +def test_build_timeline_sorts_and_dedups_overlap(): + d1 = _ql_data("2024-06-15T12:00:00", 3) # 12:00:00, :04, :08 + d2 = _ql_data("2024-06-15T12:00:08", 3) # :08 (dup), :12, :16 + timeline = build_month_timeline([d2, d1]) # pass out of order + t = timeline["time"] + assert len(timeline) == 5 # 6 bins minus 1 duplicate at :08 + assert np.all(np.diff(t.jd) > 0) # strictly increasing + + +def test_build_timeline_empty(): + assert len(build_month_timeline([])) == 0 + assert len(build_month_timeline([None, QTable()])) == 0 + + +# --- nearest_bin_index -------------------------------------------------------- + + +def test_nearest_bin_exact(): + times = Time("2024-06-15T12:00:00") + np.arange(5) * 4 * u.s + assert nearest_bin_index(times, times[2], 2 * u.s) == 2 + + +def test_nearest_bin_within_tol(): + times = Time("2024-06-15T12:00:00") + np.arange(5) * 4 * u.s + target = times[3] + 1 * u.s + assert nearest_bin_index(times, target, 2 * u.s) == 3 + + +def test_nearest_bin_beyond_tol_returns_none(): + times = Time("2024-06-15T12:00:00") + np.arange(5) * 4 * u.s + target = times[-1] + 1 * u.h + assert nearest_bin_index(times, target, 60 * u.s) is None + + +def test_nearest_bin_empty_returns_none(): + assert nearest_bin_index(Time([], format="isot"), Time("2024-06-15T12:00:00"), 60 * u.s) is None + + +# --- max_rcr_in_window -------------------------------------------------------- + + +def test_max_rcr_over_window(): + times = Time("2024-06-15T12:00:00") + np.arange(6) * 4 * u.s + rcr = np.array([0, 0, 1, 2, 1, 0]) + assert max_rcr_in_window(times, rcr, times[1], times[4], fallback=-1) == 2 + + +def test_max_rcr_peak_only_window(): + times = Time("2024-06-15T12:00:00") + np.arange(6) * 4 * u.s + rcr = np.array([0, 0, 1, 2, 1, 0]) + assert max_rcr_in_window(times, rcr, times[0], times[0], fallback=-1) == 0 + + +def test_max_rcr_empty_window_returns_fallback(): + times = Time("2024-06-15T12:00:00") + np.arange(6) * 4 * u.s + rcr = np.array([0, 0, 1, 2, 1, 0]) + before = Time("2024-06-15T11:00:00") + assert max_rcr_in_window(times, rcr, before, before, fallback=7) == 7 + + +# --- add_lc_bkg_columns (integration, monkeypatched) -------------------------- + + +@pytest.fixture +def flare_data(): + # bin grid starts 2024-06-15T12:00:00, 4 s cadence; bins 5-7 attenuated (rcr=1) + peaks = Time( + [ + "2024-06-15T12:00:20", # bin 5 (attenuated), window covers attenuated bins + "2024-06-15T12:00:00", # bin 0, no attenuation + "2024-06-15T13:00:00", # far away -> beyond tolerance + ] + ) + data = QTable() + data["flare_id"] = [1, 2, 3] + data["start_UTC"] = peaks - 8 * u.s + data["end_UTC"] = peaks + 8 * u.s + data["peak_UTC"] = peaks + return data + + +def _patch_fido(monkeypatch): + n = 20 + rcr = np.zeros(n, dtype=np.ubyte) + rcr[5:8] = 1 + lc = FakeProduct(_ql_data("2024-06-15T12:00:00", n, rcr=rcr), _energies()) + bkg = FakeProduct(_ql_data("2024-06-15T12:00:00", n, with_rcr=False), _energies()) + + products = {"lc": lc, "bkg": bkg} + monkeypatch.setattr("stixcore.io.FlareListManager.STIXPYProduct", lambda path: products[path]) + return FakeFido(lc_paths=["lc"], bkg_paths=["bkg"]) + + +def test_add_lc_bkg_columns(flare_data, monkeypatch): + fido = _patch_fido(monkeypatch) + from datetime import date + + energy = FlareListManager().add_lc_bkg_columns( + flare_data, start=date(2024, 6, 1), end=date(2024, 7, 1), fido_client=fido + ) + + # shapes / units (QTable stores unit-bearing columns as float64 Quantity, + # same as the pre-existing lc_peak column; values stay integral counts) + assert flare_data["lc_peak"].shape == (3, 5) + assert flare_data["lc_peak"].unit == u.ct + assert np.all(flare_data["lc_peak"].value == np.round(flare_data["lc_peak"].value)) + assert flare_data["lc_peak_rate"].unit.is_equivalent(RATE_UNIT) + assert flare_data["lc_bgk_peak"].shape == (3, 5) + assert flare_data["lc_bgk_peak"].unit == u.ct + assert flare_data["att_in"].dtype == bool + assert flare_data["energy_index"].dtype == np.int8 + + # rcr semantics + assert np.all(flare_data["rcr_max"] >= flare_data["rcr_at_peak"]) + assert list(flare_data["att_in"]) == [True, False, False] + assert flare_data["rcr_max"][0] == 1 # window covers attenuated bins + assert flare_data["rcr_at_peak"][0] == 1 + assert flare_data["rcr_at_peak"][1] == 0 + + # far flare -> beyond tolerance -> zero filled, rcr -1 + assert np.all(flare_data["lc_peak"][2].to_value(u.ct) == 0) + assert flare_data["rcr_at_peak"][2] == -1 + assert flare_data["rcr_max"][2] == -1 + + # counts pulled from the real timeline for in-range flares + assert np.all(flare_data["lc_peak"][0].to_value(u.ct) == 100) + assert np.all(flare_data["lc_bgk_peak"][0].to_value(u.ct) == 100) + + # returned energy table schema + assert set(energy.colnames) == {"channel", "e_low", "e_high", "index"} + assert len(energy) == 5 + + +def test_add_lc_bkg_columns_no_files(flare_data, monkeypatch): + from datetime import date + + fido = FakeFido(lc_paths=[], bkg_paths=[]) + energy = FlareListManager().add_lc_bkg_columns( + flare_data, start=date(2024, 6, 1), end=date(2024, 7, 1), fido_client=fido + ) + + assert np.all(flare_data["lc_peak"].to_value(u.ct) == 0) + assert np.all(flare_data["lc_bgk_peak"].to_value(u.ct) == 0) + assert np.all(flare_data["rcr_at_peak"] == -1) + assert np.all(flare_data["rcr_max"] == -1) + assert not np.any(flare_data["att_in"]) + assert len(energy) == 0 + + +# --- background candidate search (RID LUT) ------------------------------------ + + +def _bkg_lut(rows): + """Build a minimal RID LUT ``Table`` from ``(rid, start_iso, duration_s)`` rows.""" + t = Table() + t["unique_id"] = [r[0] for r in rows] + t["start_utc"] = [r[1] for r in rows] + t["duration"] = [r[2] for r in rows] + t["subject"] = ["BKG quiet"] * len(rows) + t["purpose"] = ["Background"] * len(rows) + t["comment"] = [""] * len(rows) + return t + + +def test_candidates_nearest_in_time_regardless_of_side(): + lut = _bkg_lut( + [ + (3001, "2023-06-13T00:00:00", 3600), # past 2 d + (3003, "2023-06-18T00:00:00", 3600), # future 3 d + (3002, "2023-06-05T00:00:00", 3600), # past 10 d + (3004, "2023-04-15T00:00:00", 3600), # past 61 d -> outside 30 d window + ] + ) + t = Time("2023-06-15T00:00:00") + cands = search_background_candidates(lut, t, window_past=30 * u.day, window_future=7 * u.day) + # nearest start first regardless of side: 2 d past, 3 d future, 10 d past + assert [c.rid for c in cands] == [3001, 3003, 3002] + assert [c.side for c in cands] == ["past", "future", "past"] + + # widening the past window pulls in the legacy request, ranked by its (large) distance + cands90 = search_background_candidates(lut, t, window_past=90 * u.day, window_future=7 * u.day) + assert [c.rid for c in cands90] == [3001, 3003, 3002, 3004] + + +def test_candidates_tie_prefers_past(): + # equal distance past vs future -> past wins the tie + lut = _bkg_lut([(3001, "2023-06-10T00:00:00", 3600), (3003, "2023-06-20T00:00:00", 3600)]) + t = Time("2023-06-15T00:00:00") + cands = search_background_candidates(lut, t, window_past=30 * u.day, window_future=30 * u.day) + assert [c.rid for c in cands] == [3001, 3003] + assert cands[0].side == "past" + + +def _bkg_lut_full(rows): + """RID LUT from ``(rid, start_iso, duration_s, subject, purpose, comment)`` rows.""" + t = Table() + t["unique_id"] = [r[0] for r in rows] + t["start_utc"] = [r[1] for r in rows] + t["duration"] = [r[2] for r in rows] + t["subject"] = [r[3] for r in rows] + t["purpose"] = [r[4] for r in rows] + t["comment"] = [r[5] for r in rows] + return t + + +def test_candidates_exclude_elevated(): + lut = _bkg_lut_full( + [ + (1, "2023-06-14T00:00:00", 3600, "BKG elevated", "Background", ""), # closer but excluded + (2, "2023-06-10T00:00:00", 3600, "BKG quiet", "Background", ""), + ] + ) + cands = search_background_candidates( + lut, Time("2023-06-15T00:00:00"), window_past=30 * u.day, window_future=7 * u.day + ) + assert [c.rid for c in cands] == [2] + + +def test_candidates_exclude_flare_comment(): + lut = _bkg_lut_full( + [ + ( + 1, + "2023-06-14T00:00:00", + 3600, + "non-flaring AR?", + "Solar Flare", + "CL1 data request for Flare 2309081508", + ), # closer but flare-referenced + (2, "2023-06-10T00:00:00", 3600, "BKG quiet", "Background", ""), + ] + ) + cands = search_background_candidates( + lut, Time("2023-06-15T00:00:00"), window_past=30 * u.day, window_future=7 * u.day + ) + assert [c.rid for c in cands] == [2] + # keeping flare-referenced rows brings the closer one back to the front + keep = search_background_candidates( + lut, Time("2023-06-15T00:00:00"), window_past=30 * u.day, window_future=7 * u.day, exclude_flare_comment=False + ) + assert [c.rid for c in keep] == [1, 2] + + +def test_candidates_prefer_background_purpose(): + P = 1.0 * u.day + t = Time("2023-06-15T00:00:00") + # subject-only match 0.5 d closer than the Background request -> Background still wins (within penalty) + lut1 = _bkg_lut_full( + [ + (1, "2023-06-13T00:00:00", 3600, "BKG quiet", "Background", ""), # 2.0 d -> eff 2.0 + (2, "2023-06-13T12:00:00", 3600, "quiet region", "obs", ""), # 1.5 d -> eff 2.5 + ] + ) + c1 = search_background_candidates(lut1, t, window_past=30 * u.day, window_future=7 * u.day, purpose_penalty=P) + assert [c.rid for c in c1] == [1, 2] + assert c1[0].is_background + + # subject-only match clearly closer (>penalty) -> it wins + lut2 = _bkg_lut_full( + [ + (1, "2023-06-13T00:00:00", 3600, "BKG quiet", "Background", ""), # 2.0 d -> eff 2.0 + (2, "2023-06-14T12:00:00", 3600, "quiet region", "obs", ""), # 0.5 d -> eff 1.5 + ] + ) + c2 = search_background_candidates(lut2, t, window_past=30 * u.day, window_future=7 * u.day, purpose_penalty=P) + assert [c.rid for c in c2] == [2, 1] + assert not c2[0].is_background + + +def test_candidates_future_window_excludes_far_future(): + lut = _bkg_lut([(4001, "2023-06-25T00:00:00", 3600)]) # 10 d in the future + t = Time("2023-06-15T00:00:00") + assert search_background_candidates(lut, t, window_past=30 * u.day, window_future=7 * u.day) == [] + got = search_background_candidates(lut, t, window_past=30 * u.day, window_future=14 * u.day) + assert [c.rid for c in got] == [4001] + + +def test_candidates_keyword_match_only(): + lut = Table() + lut["unique_id"] = [1, 2] + lut["start_utc"] = ["2023-06-10T00:00:00", "2023-06-11T00:00:00"] + lut["duration"] = [3600, 3600] + lut["subject"] = ["Solar Flare", "some quiet interval"] # only row 2 matches + lut["purpose"] = ["flare", "obs"] + lut["comment"] = ["", ""] + cands = search_background_candidates( + lut, Time("2023-06-15T00:00:00"), window_past=30 * u.day, window_future=7 * u.day + ) + assert [c.rid for c in cands] == [2] + + +def test_candidates_empty_lut(): + assert ( + search_background_candidates(Table(), Time("2023-06-15"), window_past=30 * u.day, window_future=7 * u.day) == [] + ) + + +def test_ridlutmanager_singleton_find_background_candidates(): + # exercises the method against the shipped test LUT (rows 3001-3004) + cands = RidLutManager.instance.find_background_candidates( + Time("2023-06-15T00:00:00"), window_past=30 * u.day, window_future=7 * u.day + ) + # 3001 (5 d past) and 3003 (5 d future) are equidistant -> past wins tie, then 3002 (14 d past) + assert [c.rid for c in cands] == [3001, 3003, 3002] + + +# --- find_background_file_for_time -------------------------------------------- + + +def _cpd_name(rid): + return f"solo_L1_stix-sci-xray-cpd_20230101T000000-20230101T010000_V01_{rid:010d}-00001.fits" + + +class FakeCpdFido: + """Returns the same CPD file list for every search; the rid-in-filename filter + inside ``find_background_file_for_time`` selects the per-candidate file.""" + + def __init__(self, paths): + self._paths = list(paths) + self.searches = 0 + + def search(self, time, instrument, data_product, level): + self.searches += 1 + return FakeResponse(self._paths) + + +def _patch_cpd_products(monkeypatch, spec): + """``spec``: {rid: rcr_array_or_None}. Builds CPD filenames + FakeProducts and + monkeypatches STIXPYProduct to resolve them.""" + products = {} + paths = [] + for rid, rcr in spec.items(): + name = _cpd_name(rid) + paths.append(name) + with_rcr = rcr is not None + products[name] = FakeProduct(_ql_data("2023-06-10T00:00:00", 5, rcr=rcr, with_rcr=with_rcr), _energies()) + monkeypatch.setattr("stixcore.io.FlareListManager.STIXPYProduct", lambda path: products[path]) + return FakeCpdFido(paths) + + +def test_find_bkg_picks_closest_usable(monkeypatch): + # 3001 starts 5 d before, 3002 starts 7 d after -> 3001 is nearest in time + lut = _bkg_lut([(3001, "2023-06-10T00:00:00", 3600), (3002, "2023-06-22T00:00:00", 3600)]) + fido = _patch_cpd_products(monkeypatch, {3001: np.zeros(5), 3002: np.zeros(5)}) + t = Time("2023-06-15T00:00:00") + sel = find_background_file_for_time( + t, fido_client=fido, rid_lut=lut, min_duration=600 * u.s, require_same_elut=False + ) + assert sel.rid == 3001 # nearest-in-time wins + assert sel.path == Path(_cpd_name(3001)) + assert sel.valid_from == t + # valid until the midpoint to the next later start: (2023-06-10 + 2023-06-22) / 2 = 2023-06-16 + assert sel.valid_to == Time("2023-06-16T00:00:00") + + +def test_find_bkg_skips_attenuator_in(monkeypatch): + # 3001 (closer past) has attenuator in (rcr>0) -> fall through to 3002 + lut = _bkg_lut([(3001, "2023-06-10T00:00:00", 3600), (3002, "2023-06-05T00:00:00", 3600)]) + fido = _patch_cpd_products(monkeypatch, {3001: np.ones(5), 3002: np.zeros(5)}) + sel = find_background_file_for_time( + Time("2023-06-15T00:00:00"), + fido_client=fido, + rid_lut=lut, + min_duration=600 * u.s, + require_same_elut=False, + ) + assert sel.rid == 3002 + + +def test_find_bkg_skips_too_short(monkeypatch): + # 3001 requested only 300 s (< min_duration) -> skipped without even a search + lut = _bkg_lut([(3001, "2023-06-10T00:00:00", 300), (3002, "2023-06-05T00:00:00", 3600)]) + fido = _patch_cpd_products(monkeypatch, {3001: np.zeros(5), 3002: np.zeros(5)}) + sel = find_background_file_for_time( + Time("2023-06-15T00:00:00"), + fido_client=fido, + rid_lut=lut, + min_duration=600 * u.s, + require_same_elut=False, + ) + assert sel.rid == 3002 + + +def test_find_bkg_future_fallback(monkeypatch): + # no past candidate -> the future one is used + lut = _bkg_lut([(3003, "2023-06-18T00:00:00", 3600)]) + fido = _patch_cpd_products(monkeypatch, {3003: np.zeros(5)}) + sel = find_background_file_for_time( + Time("2023-06-15T00:00:00"), + fido_client=fido, + rid_lut=lut, + min_duration=600 * u.s, + require_same_elut=False, + ) + assert sel.rid == 3003 + + +def test_find_bkg_none_when_nothing_qualifies(monkeypatch): + # only candidate has attenuator in -> no file, but a valid period is still returned + lut = _bkg_lut([(3001, "2023-06-10T00:00:00", 3600)]) + fido = _patch_cpd_products(monkeypatch, {3001: np.ones(5)}) + t = Time("2023-06-15T00:00:00") + sel = find_background_file_for_time( + t, + fido_client=fido, + rid_lut=lut, + window_future=7 * u.day, + min_duration=600 * u.s, + require_same_elut=False, + ) + assert sel.path is None + assert sel.rid == -1 + assert sel.valid_from == t + assert sel.valid_to == t + 7 * u.day # no later candidate -> t + window_future + + +# --- same-ELUT criterion ------------------------------------------------------ + + +class _FakeELUTInstance: + def __init__(self, fn): + self._fn = fn + + def _find_elut_file(self, dt): + return self._fn(dt) + + +def _patch_elut(monkeypatch, fn): + import types + + monkeypatch.setattr(flm_mod, "ELUTManager", types.SimpleNamespace(instance=_FakeELUTInstance(fn))) + + +def test_find_bkg_requires_same_elut(monkeypatch): + # 3001 (closer past) is under a DIFFERENT ELUT than the flare; 3002 matches it + lut = _bkg_lut([(3001, "2023-06-10T00:00:00", 3600), (3002, "2023-06-05T00:00:00", 3600)]) + fido = _patch_cpd_products(monkeypatch, {3001: np.zeros(5), 3002: np.zeros(5)}) + # flare(15)->'A', 3001(10)->'B' (different), 3002(5)->'A' (same) + _patch_elut(monkeypatch, lambda dt: "B" if dt.day == 10 else "A") + t = Time("2023-06-15T00:00:00") + + sel = find_background_file_for_time( + t, fido_client=fido, rid_lut=lut, min_duration=600 * u.s, require_same_elut=True + ) + assert sel.rid == 3002 # 3001 skipped: different ELUT configuration + + sel_off = find_background_file_for_time( + t, fido_client=fido, rid_lut=lut, min_duration=600 * u.s, require_same_elut=False + ) + assert sel_off.rid == 3001 # criterion off -> closest past wins + + +def test_find_bkg_same_elut_skipped_when_flare_elut_unknown(monkeypatch): + # ELUT can't be resolved for the flare -> criterion cannot be enforced -> not restrictive + lut = _bkg_lut([(3001, "2023-06-10T00:00:00", 3600)]) + fido = _patch_cpd_products(monkeypatch, {3001: np.zeros(5)}) + _patch_elut(monkeypatch, lambda dt: None) + sel = find_background_file_for_time( + Time("2023-06-15T00:00:00"), fido_client=fido, rid_lut=lut, min_duration=600 * u.s, require_same_elut=True + ) + assert sel.rid == 3001 + + +# --- add_background_file_column (valid-period cache) -------------------------- + + +def test_add_background_file_column_uses_valid_period_cache(monkeypatch): + calls = [] + + def fake_find(time, *, fido_client, **kwargs): + calls.append(time) + # each selection stays valid for 2 days from the queried time + return BackgroundSelection(path=Path("bkg.fits"), rid=42, valid_from=time, valid_to=time + 2 * u.day) + + monkeypatch.setattr(flm_mod, "find_background_file_for_time", fake_find) + + data = QTable() + data["peak_UTC"] = Time( + ["2023-06-15T00:00:00", "2023-06-16T00:00:00", "2023-06-18T00:00:00"] # 3rd is beyond the 1st period + ) + FlareListManager().add_background_file_column(data, fido_client=object()) + + assert len(calls) == 2 # 1st + 3rd flare trigger a search; 2nd reuses the cache + assert list(data["bkg_rid"]) == [42, 42, 42] + assert list(data["bkg_file"]) == ["bkg.fits"] * 3 diff --git a/stixcore/processing/FlareListL3.py b/stixcore/processing/FlareListL3.py index c75b4f87..76f5c070 100644 --- a/stixcore/processing/FlareListL3.py +++ b/stixcore/processing/FlareListL3.py @@ -24,7 +24,7 @@ class FlareListL3(SingleProductProcessingStepMixin): """Processing step from a FlareListManager to monthly solo_L3_stix-flarelist-*.fits file.""" - STARTDATE = date(2022, 1, 1) + STARTDATE = date(2023, 1, 1) def __init__(self, flm: FlareListManager, output_dir: Path): """Crates a new Processor. diff --git a/stixcore/products/level3/flarelist.py b/stixcore/products/level3/flarelist.py index 3279d5b6..e38f5cf2 100644 --- a/stixcore/products/level3/flarelist.py +++ b/stixcore/products/level3/flarelist.py @@ -788,7 +788,7 @@ class FlarelistSDC(FlareList, FlareSOOPMixin): In L3 product format. """ - PRODUCT_PROCESSING_VERSION = 2 + PRODUCT_PROCESSING_VERSION = 3 NAME = "sdc" def __init__(self, *, service_type=0, service_subtype=0, ssid=2, data, month, **kwargs): @@ -844,7 +844,7 @@ class FlarelistSDCLoc(FlarelistSDC, FlarePositionMixin): In ANC product format. """ - PRODUCT_PROCESSING_VERSION = 2 + PRODUCT_PROCESSING_VERSION = 3 NAME = "sdcloc" def __init__(self, *, service_type=0, service_subtype=0, ssid=3, data, month, **kwargs): From 10d188db372215173adadb924f469748fa6979eb Mon Sep 17 00:00:00 2001 From: Nicky Hochmuth Date: Thu, 17 Sep 2026 12:37:37 +0200 Subject: [PATCH 10/10] prepare for review rework documentation structure --- changelog/000.doc.rst | 1 + docs/code_ref/io.rst | 3 + docs/code_ref/products.rst | 40 +- docs/code_ref/products/anc.rst | 6 + docs/code_ref/products/cal.rst | 6 + docs/code_ref/products/l0.rst | 10 + docs/code_ref/products/l1.rst | 10 + docs/code_ref/products/l2.rst | 10 + docs/code_ref/products/l3.rst | 10 + docs/code_ref/products/l3_flarelist.rst | 20 + docs/code_ref/products/lb.rst | 6 + docs/code_ref/products/ll.rst | 6 + docs/index.rst | 1 + docs/products/flarelist.rst | 215 +++++ notebooks/flarelist_sdcloc_catalog.md | 777 +++++++++++++++++++ stixcore/io/FlareListManager.py | 498 ++++++++++-- stixcore/io/tests/test_flarelistmanager.py | 331 +++++++- stixcore/processing/FlareListL3.py | 8 +- stixcore/processing/pipeline_daily.py | 16 +- stixcore/products/level3/flarelist.py | 449 +++++++---- stixcore/products/level3/flarelistproduct.py | 26 + stixcore/products/level3/processing.py | 9 + stixcore/products/tests/test_flarelist.py | 105 ++- 23 files changed, 2317 insertions(+), 246 deletions(-) create mode 100644 changelog/000.doc.rst create mode 100644 docs/code_ref/products/anc.rst create mode 100644 docs/code_ref/products/cal.rst create mode 100644 docs/code_ref/products/l0.rst create mode 100644 docs/code_ref/products/l1.rst create mode 100644 docs/code_ref/products/l2.rst create mode 100644 docs/code_ref/products/l3.rst create mode 100644 docs/code_ref/products/l3_flarelist.rst create mode 100644 docs/code_ref/products/lb.rst create mode 100644 docs/code_ref/products/ll.rst create mode 100644 docs/products/flarelist.rst create mode 100644 notebooks/flarelist_sdcloc_catalog.md diff --git a/changelog/000.doc.rst b/changelog/000.doc.rst new file mode 100644 index 00000000..4bc0ffef --- /dev/null +++ b/changelog/000.doc.rst @@ -0,0 +1 @@ +Documented the SDC level-3 flare list: `~stixcore.products.level3.flarelist` and `~stixcore.io.FlareListManager` are now part of the API reference, and a new guide describes how the flare list is built from the STIX Data Center flare list and enriched (quiet-time background selection, CPD-file selection and flare-location imaging). diff --git a/docs/code_ref/io.rst b/docs/code_ref/io.rst index 5962f501..e3b8cac6 100644 --- a/docs/code_ref/io.rst +++ b/docs/code_ref/io.rst @@ -6,6 +6,9 @@ to certain directories. .. automodapi:: stixcore.io +.. automodapi:: stixcore.io.FlareListManager + :skip: SCFlareListManager + .. automodapi:: stixcore.io.product_processors.fits.processors .. automodapi:: stixcore.io.product_processors.plots.processors diff --git a/docs/code_ref/products.rst b/docs/code_ref/products.rst index e43497d0..21d69d18 100644 --- a/docs/code_ref/products.rst +++ b/docs/code_ref/products.rst @@ -2,32 +2,30 @@ STIXCore Products ***************** The ``products`` submodule contains processing classes representing high level -data products create from multiple packets with additional checks. +data products created from multiple packets with additional checks. +The products are organized by processing level (``LB`` raw binary, ``L0``-``L3``) +and by product category (``ANC`` ancillary, ``CAL`` calibration): -.. automodapi:: stixcore.products +.. toctree:: + :maxdepth: 2 -.. automodapi:: stixcore.products.common - -.. automodapi:: stixcore.products.product - :include-all-objects: - -.. automodapi:: stixcore.products.levelb - -.. automodapi:: stixcore.products.levelb.binary + products/lb + products/l0 + products/l1 + products/l2 + products/l3 + products/ll + products/anc + products/cal -.. automodapi:: stixcore.products.level0.quicklookL0 -.. automodapi:: stixcore.products.level1.quicklookL1 +Base classes +============ -.. automodapi:: stixcore.products.level0.housekeepingL0 +Shared base classes and helpers used by all product levels. -.. automodapi:: stixcore.products.level1.housekeepingL1 - -.. automodapi:: stixcore.products.level0.scienceL0 - -.. automodapi:: stixcore.products.level1.scienceL1 - -.. automodapi:: stixcore.products.level2.housekeepingL2 +.. automodapi:: stixcore.products.product + :include-all-objects: -.. automodapi:: stixcore.products.level2.quicklookL2 +.. automodapi:: stixcore.products.common diff --git a/docs/code_ref/products/anc.rst b/docs/code_ref/products/anc.rst new file mode 100644 index 00000000..ebddf3e8 --- /dev/null +++ b/docs/code_ref/products/anc.rst @@ -0,0 +1,6 @@ +Ancillary (ANC) +*************** + +Ancillary products such as the aspect / ephemeris data. + +.. automodapi:: stixcore.products.ANC.aspect diff --git a/docs/code_ref/products/cal.rst b/docs/code_ref/products/cal.rst new file mode 100644 index 00000000..357d49e0 --- /dev/null +++ b/docs/code_ref/products/cal.rst @@ -0,0 +1,6 @@ +Calibration (CAL) +***************** + +Calibration products such as the energy calibration. + +.. automodapi:: stixcore.products.CAL.energy diff --git a/docs/code_ref/products/l0.rst b/docs/code_ref/products/l0.rst new file mode 100644 index 00000000..887889da --- /dev/null +++ b/docs/code_ref/products/l0.rst @@ -0,0 +1,10 @@ +Level 0 (L0) +************ + +Decommutated, uncalibrated products in engineering/raw units. + +.. automodapi:: stixcore.products.level0.quicklookL0 + +.. automodapi:: stixcore.products.level0.housekeepingL0 + +.. automodapi:: stixcore.products.level0.scienceL0 diff --git a/docs/code_ref/products/l1.rst b/docs/code_ref/products/l1.rst new file mode 100644 index 00000000..c3f148ec --- /dev/null +++ b/docs/code_ref/products/l1.rst @@ -0,0 +1,10 @@ +Level 1 (L1) +************ + +Calibrated products in physical units with applied corrections. + +.. automodapi:: stixcore.products.level1.quicklookL1 + +.. automodapi:: stixcore.products.level1.housekeepingL1 + +.. automodapi:: stixcore.products.level1.scienceL1 diff --git a/docs/code_ref/products/l2.rst b/docs/code_ref/products/l2.rst new file mode 100644 index 00000000..3d78288c --- /dev/null +++ b/docs/code_ref/products/l2.rst @@ -0,0 +1,10 @@ +Level 2 (L2) +************ + +Higher-level calibrated products. + +.. automodapi:: stixcore.products.level2.quicklookL2 + +.. automodapi:: stixcore.products.level2.housekeepingL2 + +.. automodapi:: stixcore.products.level2.scienceL2 diff --git a/docs/code_ref/products/l3.rst b/docs/code_ref/products/l3.rst new file mode 100644 index 00000000..a01773d5 --- /dev/null +++ b/docs/code_ref/products/l3.rst @@ -0,0 +1,10 @@ +Level 3 (L3) +************ + +Level-3 products are higher-level science products derived from the lower levels. +All current L3 products belong to the flare list. + +.. toctree:: + :maxdepth: 1 + + l3_flarelist diff --git a/docs/code_ref/products/l3_flarelist.rst b/docs/code_ref/products/l3_flarelist.rst new file mode 100644 index 00000000..17a89691 --- /dev/null +++ b/docs/code_ref/products/l3_flarelist.rst @@ -0,0 +1,20 @@ +FlareList +========= + +The L3 flare-list products: the enriched SDC flare list, the per-flare peak-preview +image product, and the flare-location imaging helpers. + +.. seealso:: + + :doc:`/products/flarelist` — how the SDC flare list is built from the STIX Data Center + flare list and enriched (quiet-time background selection, CPD-file selection and + flare-location imaging). See also `~stixcore.io.FlareListManager` for the source manager. + +.. automodapi:: stixcore.products.level3.flarelist + :skip: FlarelistSC + :skip: FlarelistSCLocation + :skip: FlarelistSCLocationImage + +.. automodapi:: stixcore.products.level3.flarelistproduct + +.. automodapi:: stixcore.products.level3.processing diff --git a/docs/code_ref/products/lb.rst b/docs/code_ref/products/lb.rst new file mode 100644 index 00000000..1f9eaf9e --- /dev/null +++ b/docs/code_ref/products/lb.rst @@ -0,0 +1,6 @@ +Level B (LB) — raw binary +************************* + +Raw binary telemetry products, the entry point of the processing pipeline. + +.. automodapi:: stixcore.products.levelb.binary diff --git a/docs/code_ref/products/ll.rst b/docs/code_ref/products/ll.rst new file mode 100644 index 00000000..578c192f --- /dev/null +++ b/docs/code_ref/products/ll.rst @@ -0,0 +1,6 @@ +Low Latency (LL) +**************** + +Low-latency quicklook products. + +.. automodapi:: stixcore.products.lowlatency.quicklookLL diff --git a/docs/index.rst b/docs/index.rst index 4082f323..c09e796e 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -8,6 +8,7 @@ This is the documentation for STIXCore. whatsnew/index users + products/flarelist developers code_ref/index pipelineconfiguration diff --git a/docs/products/flarelist.rst b/docs/products/flarelist.rst new file mode 100644 index 00000000..e75185c1 --- /dev/null +++ b/docs/products/flarelist.rst @@ -0,0 +1,215 @@ +.. _flarelist_sdc: + +**************************** +SDC Flare List (Level 3) +**************************** + +The SDC flare list is an enriched, level-3 flare catalogue. It starts from the operational +flare list produced by the STIX Data Center (SDC) and adds STIX-derived quantities that the +source list does not provide reliably: quicklook lightcurve / background counts at the flare +peak, a quiet-time background spectrum, and — for the higher product levels — a flare +location and peak-preview images. + +This page describes where the raw data comes from and each enrichment step, including the +decisions taken when several inputs are available (which background period, which CPD file, +which imaging parameters). For the class and column reference see +:doc:`/code_ref/products/l3_flarelist` (`~stixcore.products.level3.flarelist`, +`~stixcore.products.level3.flarelistproduct`) and :doc:`/code_ref/io` +(`~stixcore.io.FlareListManager`). + +.. note:: + + A parallel ``FlarelistSC*`` family (a flare list detected inside STIXCore rather than + mirrored from the SDC) is planned but not yet in use; it is intentionally omitted here. + + +Product chain +============= + +The SDC products form a chain in which each level adds one enrichment step (built by +`~stixcore.processing.FlareListL3.FlareListL3` for the first level and upgraded +level-to-level by `~stixcore.processing.FLtoFL.FLtoFL`): + +.. list-table:: + :header-rows: 1 + :widths: 30 10 60 + + * - Product + - ssid + - Adds + * - `~stixcore.products.level3.flarelist.FlarelistSDC` + - 2 + - Base list: id, times, GOES class/flux, QL peak & quiet-time background, SOOP campaign. + * - `~stixcore.products.level3.flarelist.FlarelistSDCLocation` + - 3 + - Flare location from CPD imaging (+ SOLO position, imaging-quality metric, 1-AU fluxes). + * - `~stixcore.products.level3.flarelist.FlarelistSDCLocationImage` + - 4 + - Per-flare peak-preview CLEAN images (`~stixcore.products.level3.flarelistproduct.PeakPreviewImage`). + + +Raw data in — the STIX Data Center flare list +============================================= + +`~stixcore.io.FlareListManager.SDCFlareListManager` maintains a local CSV mirror of the SDC +flare list. ``read_flarelist`` fetches it with ``stixdcpy.fetch_flare_list(start, end)`` in +roughly monthly chunks from 2020-01-01 up to *now* — the API is batched by month and +throttled (~2 calls/s, so the loop sleeps between chunks), and incremental updates re-fetch +the last ~60 days to catch late revisions. The combined list is de-duplicated, sorted by +``peak_UTC`` and cached to CSV. ``get_data`` then slices out the requested month. + +The following source fields are consumed (see +`~stixcore.io.FlareListManager.SDCFlareListManager.get_data`): + +.. list-table:: + :header-rows: 1 + :widths: 45 55 + + * - Source (CSV) field + - Product column + * - ``flare_id`` + - ``flare_id`` + * - ``start_UTC``, ``duration``, ``end_UTC``, ``peak_UTC`` + - same names (UTC time columns / duration in s) + * - ``GOES_class``, ``goes_estimated_{min,max,mean}_class`` + - ``GOES_class``, ``goes_{min,max,mean}_class_est`` + * - ``GOES_flux``, ``goes_estimated_{min,max,mean}_flux`` + - ``GOES_flux``, ``goes_{min,max,mean}_flux_est`` (stored as ``10**value`` W/m²) + +.. warning:: + + The ``GOES_*`` columns describe the Earth-viewed GOES/XRS flux and are **not** derived + from STIX — they are meaningless when the flare is occulted from Earth. The source list's + own at-peak lightcurve counts and attenuator flag are **unreliable** and are deliberately + re-derived from real STIX data (next section). + + +Enrichment step — QL lightcurve & background at peak +==================================================== + +`~stixcore.io.FlareListManager.FlareListManager.add_lc_bkg_columns` builds one monthly +timeline each for the ``ql_lightcurve`` and ``ql_background`` products (via +`~stixcore.io.FlareListManager.build_month_timeline`, so the daily files are opened only +once), then for every flare picks the bin nearest the peak +(`~stixcore.io.FlareListManager.nearest_bin_index`). It records, for the five QL energy +channels, the raw counts (``lc_peak``, ``lc_bkg_peak``) and the livetime- and +area-normalized flux (``lc_peak_flux``, ``lc_bkg_peak_flux``, in +``ct s⁻¹ keV⁻¹ cm⁻²``), together with ``rcr_at_peak`` / ``rcr_max``, the derived ``att_in`` +flag and the ``energy_index`` into the energy table. This replaces the unreliable +source-CSV values with quantities taken straight from the L1 quicklook data. + + +Enrichment step — quiet-time background file +============================================ + +For the background *spectrum* a quiet-time science file has to be chosen. +`~stixcore.io.FlareListManager.find_background_file_for_time` implements the selection: + +* **Candidate ranking.** Background requests are taken from the RID look-up table + (``search_background_candidates``) and ranked **nearest-in-time first** — the request whose + start is closest to the flare, past or future — within separate look-back / look-ahead + windows (``window_past``, default 30 d; ``window_future``, default 7 d). +* **Acceptance checks.** A candidate is accepted only if its requested integration is long + enough (``min_duration``, default 1200 s), its real ``sci_xray_cpd`` L1 file can be resolved + (and the filename's request id matches), and the **attenuator is out for the whole file** + (``rcr == 0``). The first candidate that passes wins. +* **Optional stricter filters** (all default *off* except same-ELUT, because they move common + cases rather than edge cases), each backed by a ``[Processing]`` config key: + + .. list-table:: + :header-rows: 1 + :widths: 55 45 + + * - Filter + - Config key + * - Require the same ELUT as the flare time + - ``flarelist_bkg_require_same_elut`` (default on) + * - Prefer ``purpose == "Background"`` requests + - ``flarelist_bkg_purpose_penalty_days`` + * - Drop "elevated" backgrounds + - ``flarelist_bkg_exclude_elevated`` + * - Drop requests whose comment references a specific flare + - ``flarelist_bkg_exclude_flare_comment`` + +* **Validity-interval caching.** The result carries a ``[valid_from, valid_to]`` interval, so + the monthly (time-ordered) loop reuses one selection for every later flare until a different + candidate would become effectively closer (or the window edge is reached). This is why the + same background file is not re-resolved flare by flare. + +The spectrum itself (`~stixcore.io.FlareListManager.background_spectrum_from_cpd`) is the +**median over the file's time bins** — the "most recent quiet period" — summed over the 30 +imaging detectors and their pixels. It is stored as native-channel counts (``bkg_spec``) and +flux (``bkg_spec_flux``), and rebinned to the QL lightcurve bands as counts (``bkg_spec_ql``) +and flux (``bkg_spec_flux_ql``); the file's native (32-channel) binning is appended to the +energy table and referenced by ``bkg_energy_index``. + + +Enrichment step — flare location +================================ + +`~stixcore.products.level3.flarelist.FlarePositionMixin.add_flare_position` (products at +ssid ≥ 3) adds a location for each flare. Per flare it searches for the daily ancillary +``asp_ephemeris`` file (cached per day) and for ``sci_xray_cpd`` files over +``[start, end]``, recording ``anc_ephemeris_path`` and ``cpd_path``. + +CPD-file selection +------------------ + +When several CPD files cover a flare, each candidate is read as a full product +(`~stixpy.product.Product`) and scored on four criteria (in priority order):: + + cpd_res.sort(["inc_peak", "inc_flare", "_neg_min_dt", "ebins"], reverse=True) # take the top row + +* ``inc_peak`` — whether the flare **peak** time falls inside the file (preferred first), +* ``inc_flare`` — **percentage** of the flare ``start..end`` duration covered by the file (higher preferred next), +* **min time resolution** — the shortest ``timedel`` among the data-table bins overlapping the flare + (**shorter is better**; sorted via its negative ``_neg_min_dt``, in deciseconds), +* ``ebins`` — number of **energy bins** in the energy table (**more is better**). + +(A ``TODO`` notes more criteria may be added.) **The CPD file chosen here is the same file later used to +make the peak-preview images**, so the choice serves both the location and the imaging. + +Fit time & energy range +----------------------- + +The fit uses ``[peak − 20 s, peak + 20 s]`` clamped to ``[start, end]`` (falling back to the +file's own time range if there is no overlap). If ``rcr`` is not constant across that window, +the window is widened to ±40 s and the **longest constant-``rcr`` sub-sequence** +(`~stixcore.products.level3.flarelist.longest_constant_sequence`) is used instead. The energy +range is ``[4, 16] keV``, widening to ``[4, 25] keV`` when the attenuator is in +(``rcr > 0``). + +Imaging parameters +------------------ + +The location is estimated by +`~stixcore.products.level3.processing.stx_estimate_flare_location` (a port of the STIX-GSW +IDL routine of the same name): + +* meta pixels via ``create_meta_pixels(no_shadowing=True, flare_location=[0, 0]″)``, then + ``create_visibility`` and ``calibrate_visibility`` with the **Sun centre** as phase centre; +* only the **coarse sub-collimators 7–10** are used + (``isc = [3, 20, 22, 16, 14, 32, 21, 26, 4, 24, 8, 28]``); +* a back-projection map of ``512 × 512`` pixels with plate scale + ``pixel = rsun_obs · 2.6 / imsize`` (the factor 2.6 keeps the full solar disc in the field + of view, matching the IDL implementation); the flare location is the brightest pixel, + transformed to Helioprojective coordinates (a spherical screen is assumed for off-disk + positions); +* an imaging-quality metric ``sidelobes_ratio`` is computed as the strongest sidelobe outside + a 200″ radius relative to the peak — a value close to (≳ 0.9) or above the peak indicates the + location is unreliable. It is surfaced as the ``sidelobes_ratio`` column. + +.. note:: + + The peak-preview *image* product + (`~stixcore.products.level3.flarelist.FlarePeakPreviewMixin`, ssid 4) runs a fuller CLEAN + reconstruction in the ``[4, 20]`` and ``[20, 120] keV`` bands on the **same selected CPD + file**. Its output columns are not yet finalised and are out of scope for this page. + + +Outputs +======= + +The per-flare columns (units, shapes, meanings) are documented as the column descriptions in +the API reference — see `~stixcore.products.level3.flarelist` and +`~stixcore.io.FlareListManager`. A consolidated data dictionary is planned as a follow-up. diff --git a/notebooks/flarelist_sdcloc_catalog.md b/notebooks/flarelist_sdcloc_catalog.md new file mode 100644 index 00000000..05b85356 --- /dev/null +++ b/notebooks/flarelist_sdcloc_catalog.md @@ -0,0 +1,777 @@ +# SDC-Location flare-catalog column reference +Auto-generated by `notebooks/generate_flarelist_catalog_md.py` from a real product read via the stixcore `Product(path)` reader (no direct FITS access). +- **product class:** `FlarelistSDCLocation` +- **file:** `solo_L3_stix-flarelist-sdcloc_20230101T000000-20230131T231535_V04.fits` +- **rows:** 1649 (example values below are from row index 1, a fully-populated flare) +- **columns:** 53 + +**Review notes on the file itself:** +- Column descriptions **are** persisted, via astropy's serialized-column metadata (a YAML block in the header `COMMENT` cards — the standard astropy unit/`meta` framework), not in the classic per-column `TCOMMn` keyword. They round-trip through a plain `astropy` `QTable.read` as well as the stixcore `Product` reader. +- 3 columns have **no** stored description (`energy_index`, and the ICRS location columns `location_icrs` / `solo_location_icrs`); those are supplied here from a small override map. +- On read, the location is provided both as ICRS (`location_icrs`, `solo_location_icrs`, the on-disk form) and as HGS (`location_hgs`, `solo_location_hgs`). +- `source` legend: **API** = one-to-one copy from the SDC API/CSV; **stixcore** = reprocessed / newly generated by code in the stixcore package; **stixpy** = substantive computation whose code lives in the stixpy package. (In this catalog no column is **stixpy** yet: the reprocessing code all lives in stixcore, which uses stixpy/sunpy/astropy/xrayvision as libraries — noted per column where relevant. Some of these methods are intended to move into stixpy in the future, at which point those columns become **stixpy**.) + +--- + +## SDC (FlarelistSDC) + +### 1. Source CSV — [`SDCFlareListManager.get_data`](../stixcore/io/FlareListManager.py#L1361-L1499) + +Base flare definitions sliced from the local SDC flare-list CSV mirror (fetched from the STIX Data Center API). One-to-one copies unless noted. + +#### flare_id + +- **comment:** unique flare id for flarelist SDCFlareListManager +- **unit:** — +- **example:** `2301010034` +- **source:** API + +**review comment:** unique identifier for the flare maintained by stix datacenter + +#### start_UTC + +- **comment:** start time of flare +- **unit:** UTC (Time) +- **example:** `2023-01-01T00:32:16.484 (UTC)` +- **source:** API + +**review comment:** _(to be filled in via `notebooks/flarelist_sdcloc_review_comments.yaml`)_ + +#### duration + +- **comment:** duration of flare +- **unit:** s +- **example:** `1240 s` +- **source:** API + +**review comment:** _(to be filled in via `notebooks/flarelist_sdcloc_review_comments.yaml`)_ + +#### end_UTC + +- **comment:** end time of flare +- **unit:** UTC (Time) +- **example:** `2023-01-01T00:52:56.486 (UTC)` +- **source:** API + +**review comment:** _(to be filled in via `notebooks/flarelist_sdcloc_review_comments.yaml`)_ + +#### peak_UTC + +- **comment:** flare peak time +- **unit:** UTC (Time) +- **example:** `2023-01-01T00:34:44.484 (UTC)` +- **source:** API + +**review comment:** _(to be filled in via `notebooks/flarelist_sdcloc_review_comments.yaml`)_ + +#### GOES_class + +- **comment:** GOES class of the GOES XRS data at time of flare - not derived from STIX data. Do not use when flare isn't visible to Earth +- **unit:** — +- **example:** `C1.7` +- **source:** API + +**review comment:** _(to be filled in via `notebooks/flarelist_sdcloc_review_comments.yaml`)_ + +#### goes_min_class_est + +- **comment:** min GOES class estimate derived from STIX data +- **unit:** — +- **example:** `B6` +- **source:** API + +**review comment:** _(to be filled in via `notebooks/flarelist_sdcloc_review_comments.yaml`)_ + +#### goes_max_class_est + +- **comment:** max GOES class estimate derived from STIX data +- **unit:** — +- **example:** `C1` +- **source:** API + +**review comment:** _(to be filled in via `notebooks/flarelist_sdcloc_review_comments.yaml`)_ + +#### goes_mean_class_est + +- **comment:** mean GOES class estimate derived from STIX data +- **unit:** — +- **example:** `B8` +- **source:** API + +**review comment:** _(to be filled in via `notebooks/flarelist_sdcloc_review_comments.yaml`)_ + +#### GOES_flux + +- **comment:** GOES flux of the GOES XRS data at time of flare- not derived from STIX data. Do not use when the flare isn't visible to Earth +- **unit:** W / m2 +- **example:** `1.67984e-06 W / m2` +- **source:** API + +**review comment:** _(to be filled in via `notebooks/flarelist_sdcloc_review_comments.yaml`)_ + +#### goes_min_flux_est + +- **comment:** min GOES flux estimate derived from STIX data _(stored as 10**; otherwise a one-to-one copy from the SDC flare-list CSV.)_ +- **unit:** W / m2 +- **example:** `nan W / m2` +- **source:** API + +**review comment:** _(to be filled in via `notebooks/flarelist_sdcloc_review_comments.yaml`)_ + +#### goes_max_flux_est + +- **comment:** max GOES flux estimate derived from STIX data _(stored as 10**; otherwise a one-to-one copy from the SDC flare-list CSV.)_ +- **unit:** W / m2 +- **example:** `nan W / m2` +- **source:** API + +**review comment:** _(to be filled in via `notebooks/flarelist_sdcloc_review_comments.yaml`)_ + +#### goes_mean_flux_est + +- **comment:** mean GOES flux estimate derived from STIX data _(stored as 10**; otherwise a one-to-one copy from the SDC flare-list CSV.)_ +- **unit:** W / m2 +- **example:** `nan W / m2` +- **source:** API + +**review comment:** _(to be filled in via `notebooks/flarelist_sdcloc_review_comments.yaml`)_ + +### 2. QL peak & background — [`FlareListManager.add_lc_bkg_columns`](../stixcore/io/FlareListManager.py#L708-L826) + +At-peak lightcurve / background-detector counts and fluxes, rcr and attenuator state, derived from the real L1 QL products (the source-CSV values are unreliable). + +#### lc_peak + +- **comment:** raw counts at the L1 QL lightcurve bin nearest the flare peak (5 energy channels) +- **unit:** ct +- **example:** `[1087., 99., 75., 735., 431.] ct` +- **source:** stixcore + +**calculation summary:** Raw counts at the QL lightcurve bin nearest the flare peak (5 QL channels), from a monthly ql_lightcurve timeline built once. +Processing: [`FlareListManager.add_lc_bkg_columns`](../stixcore/io/FlareListManager.py#L708-L826), [`FlareListManager._build_ql_month_timeline`](../stixcore/io/FlareListManager.py#L618-L706), [`nearest_bin_index`](../stixcore/io/FlareListManager.py#L106-L118). + +**review comment:** _(to be filled in via `notebooks/flarelist_sdcloc_review_comments.yaml`)_ + +#### lc_peak_flux + +- **comment:** livetime- and area-corrected flux at the peak lightcurve bin (ct/s/keV/cm2, 5 energy channels) +- **unit:** ct / (keV s cm2) +- **example:** `[1.7785, 0.1944, 0.0736, 0.2886, 0.1244] ct / (keV s cm2)` +- **source:** stixcore +- **uses:** `lc_peak` + +**calculation summary:** Livetime- and area-corrected flux at the peak bin. Flux normalization = counts / (livetime x energy-width x collecting-area) -> ct/s/keV/cm2. Livetime via stixpy `get_livetime_fraction`; collecting area from the pixel/detector masks. +Processing: [`compute_ql_count_rate`](../stixcore/io/FlareListManager.py#L57-L84), [`collecting_area_cm2`](../stixcore/io/FlareListManager.py#L164-L173). +From [`FlareListManager.py#L57-L84`](../stixcore/io/FlareListManager.py#L57-L84): + +```python +def compute_ql_count_rate(counts, timedel, triggers, energy_delta, *, n_detectors): + """Reproduce stixpy's QL count-rate normalization -> ``ct / (s * keV)``. + + Mirrors ``stixpy.timeseries.quicklook`` (lightcurve uses ``n_detectors=16``, + background uses ``n_detectors=1``). Pure, no I/O. + + Parameters + ---------- + counts : `~astropy.units.Quantity` + Raw counts, shape ``(N, 5)`` in ``ct``. + timedel : `~astropy.units.Quantity` + Bin durations, shape ``(N,)``. + triggers : array-like + Trigger counts, shape ``(N,)`` or ``(N, 1)``. + energy_delta : `~astropy.units.Quantity` + Channel widths, shape ``(5,)`` in ``keV``. + n_detectors : int + 16 for the lightcurve, 1 for the background detector. + + Returns + ------- + `~astropy.units.Quantity` + Count rate, shape ``(N, 5)`` in ``ct / (s * keV)``. + """ + timedel = timedel.to(u.s) + trig = np.asarray(triggers).reshape(-1) + live_frac, *_ = get_livetime_fraction(trig / (n_detectors * timedel)) + return counts / ((timedel * live_frac).reshape(-1, 1) * energy_delta) +``` + +**review comment:** _(to be filled in via `notebooks/flarelist_sdcloc_review_comments.yaml`)_ + +#### lc_bkg_peak + +- **comment:** raw counts from the STIX background detector at the L1 QL background bin nearest the flare peak (5 energy channels) — unmodulated by imaging subcollimators and not affected by the attenuator +- **unit:** ct +- **example:** `[24., 3., 5., 45., 28.] ct` +- **source:** stixcore + +**calculation summary:** Raw counts from the STIX background-monitor detector at the QL background bin nearest the peak (unmodulated by the imaging grids, unaffected by the attenuator). +Processing: [`FlareListManager.add_lc_bkg_columns`](../stixcore/io/FlareListManager.py#L708-L826), [`FlareListManager._build_ql_month_timeline`](../stixcore/io/FlareListManager.py#L618-L706). + +**review comment:** _(to be filled in via `notebooks/flarelist_sdcloc_review_comments.yaml`)_ + +#### lc_bkg_peak_flux + +- **comment:** livetime- and area-corrected STIX background detector flux at the peak bin (ct/s/keV/cm2, 5 energy channels) +- **unit:** ct / (keV s cm2) +- **example:** `[0.5893, 0.0884, 0.0737, 0.2652, 0.1213] ct / (keV s cm2)` +- **source:** stixcore +- **uses:** `lc_bkg_peak` + +**calculation summary:** As lc_peak_flux but for the background-monitor detector (open-detector area). Flux normalization = counts / (livetime x energy-width x collecting-area) -> ct/s/keV/cm2. +Processing: [`compute_ql_count_rate`](../stixcore/io/FlareListManager.py#L57-L84), [`collecting_area_cm2`](../stixcore/io/FlareListManager.py#L164-L173). + +**review comment:** _(to be filled in via `notebooks/flarelist_sdcloc_review_comments.yaml`)_ + +#### rcr_at_peak + +- **comment:** max rcr level at flare location estimation time range, > 0 attenuator in place +- **unit:** — +- **example:** `0` +- **source:** stixcore + +**calculation summary:** Rate-control-regime at the peak bin. NOTE: first written here, then OVERWRITTEN at the sdcloc level by add_flare_position with the max rcr over the imaging window. +Processing: [`FlareListManager.add_lc_bkg_columns`](../stixcore/io/FlareListManager.py#L708-L826), [`FlarePositionMixin.add_flare_position`](../stixcore/products/level3/flarelist.py#L213-L550). + +**review comment:** _(to be filled in via `notebooks/flarelist_sdcloc_review_comments.yaml`)_ + +#### rcr_max + +- **comment:** max rate control regime over the flare start..end window +- **unit:** — +- **example:** `False` +- **source:** stixcore + +**calculation summary:** Maximum rate-control-regime over the flare start..end window (stored as bool when the month only has rcr in {0,1}). +Processing: [`FlareListManager.add_lc_bkg_columns`](../stixcore/io/FlareListManager.py#L708-L826), [`max_rcr_in_window`](../stixcore/io/FlareListManager.py#L121-L128). + +**review comment:** _(to be filled in via `notebooks/flarelist_sdcloc_review_comments.yaml`)_ + +#### att_in + +- **comment:** was attenuator in during flare (rcr_max > 0) +- **unit:** — +- **example:** `False` +- **source:** stixcore +- **uses:** `rcr_max` + +**calculation summary:** Whether the attenuator was in during the flare, derived as rcr_max > 0. +Processing: [`FlareListManager.add_lc_bkg_columns`](../stixcore/io/FlareListManager.py#L708-L826). + +**review comment:** _(to be filled in via `notebooks/flarelist_sdcloc_review_comments.yaml`)_ + +#### energy_index + +- **comment:** index into the ENERGIES table of the QL energy binning used for this flare (no description is stored on the column itself) +- **unit:** — +- **example:** `False` +- **source:** stixcore + +**calculation summary:** Index into the ENERGIES table for the QL binning used (honors a QL binning that changes over time). +Processing: [`FlareListManager.add_lc_bkg_columns`](../stixcore/io/FlareListManager.py#L708-L826). + +**review comment:** _(to be filled in via `notebooks/flarelist_sdcloc_review_comments.yaml`)_ + +### 3. Quiet-time background spectrum — [`FlareListManager.add_background_file_column`](../stixcore/io/FlareListManager.py#L828-L969) + +Median quiet-period background spectrum from the selected background CPD file, native and QL-rebinned. + +#### bkg_file + +- **comment:** path to the quiet-time background CPD file +- **unit:** — +- **example:** `L1/2023/01/01/SCI/solo_L1_stix-sci-xray-cpd_20230101T110501-20230101T115341_V02_2301018231-56312.fits` +- **source:** stixcore + +**calculation summary:** Path of the selected quiet-time background CPD file (nearest-in-time candidate that passes the duration/attenuator checks). +Processing: [`FlareListManager.add_background_file_column`](../stixcore/io/FlareListManager.py#L828-L969), [`find_background_file_for_time`](../stixcore/io/FlareListManager.py#L414-L585). + +**review comment:** _(to be filled in via `notebooks/flarelist_sdcloc_review_comments.yaml`)_ + +#### bkg_rid + +- **comment:** BSD request id of the selected background file (-1 if none) +- **unit:** — +- **example:** `2301018231` +- **source:** stixcore + +**calculation summary:** BSD request id of the selected background file (-1 if none). +Processing: [`FlareListManager.add_background_file_column`](../stixcore/io/FlareListManager.py#L828-L969), [`find_background_file_for_time`](../stixcore/io/FlareListManager.py#L414-L585). + +**review comment:** _(to be filled in via `notebooks/flarelist_sdcloc_review_comments.yaml`)_ + +#### bkg_spec + +- **comment:** median quiet-period background counts per science energy channel (30 imaging detectors) +- **unit:** ct +- **example:** `[ 45704., 20467., 15673., 53039., 24553., 6801., 5492., 10407., 8293., 4180., 3357., 3404., + 7143., 7481., 8093., 16228., 26373., 382036., 87009., 14315., 12837., 15052., 32817., 38605., + 29864., 34574., 145977., 40422., 226197., 372446., 175127., nan] ct` +- **source:** stixcore + +**calculation summary:** Median quiet-period background counts per native science channel, summed over the 30 imaging detectors and their pixels. +Processing: [`background_spectrum_from_cpd`](../stixcore/io/FlareListManager.py#L221-L267). + +**review comment:** _(to be filled in via `notebooks/flarelist_sdcloc_review_comments.yaml`)_ + +#### bkg_spec_ql + +- **comment:** median quiet-period background counts rebinned to the QL lightcurve energy bands +- **unit:** ct +- **example:** `[166237., 31729., 42349., 537622., 281837.] ct` +- **source:** stixcore +- **uses:** `bkg_spec`, `ENERGIES table` + +**calculation summary:** bkg_spec rebinned to the flare's QL energy bands (whole-channel sum). +Processing: [`rebin_spectrum_to_ql`](../stixcore/io/FlareListManager.py#L282-L302). +From [`FlareListManager.py#L282-L302`](../stixcore/io/FlareListManager.py#L282-L302): + +```python +def rebin_spectrum_to_ql(counts_32, energies_32, ql_block): + """Rebin a science *counts* spectrum onto the QL energy bands of ``ql_block``. + + ``ql_block`` carries the target band edges (``e_low``/``e_high``) read from the + energy table for a given flare (never hardcoded, so a QL binning that changes + over time is honored). A science channel contributes to a band when its + ``[e_low, e_high]`` lies within the band; whole channels are summed (the edges + align). Summing preserves the input unit, so the result carries ``counts_32``'s unit. + """ + e_low = _edges_kev(energies_32["e_low"]) + e_high = _edges_kev(energies_32["e_high"]) + b_low = _edges_kev(ql_block["e_low"]) + b_high = _edges_kev(ql_block["e_high"]) + unit = getattr(counts_32, "unit", None) + counts = np.asarray(getattr(counts_32, "value", counts_32), dtype=float) + out = np.full(len(b_low), np.nan) + for b in range(len(b_low)): + chans = (e_low >= b_low[b]) & (e_high <= b_high[b]) + if np.any(chans): + out[b] = np.nansum(counts[chans]) + return out * unit if unit is not None else out +``` + +**review comment:** _(to be filled in via `notebooks/flarelist_sdcloc_review_comments.yaml`)_ + +#### bkg_spec_flux + +- **comment:** median quiet-period background flux per science channel (ct/s/keV/cm2, 30 imaging detectors) +- **unit:** ct / (keV s cm2) +- **example:** `[0.6145, 0.2752, 0.2107, 0.7131, 0.3301, 0.0914, 0.0738, 0.1399, 0.1115, 0.0562, 0.0451, 0.0458, 0.048 , + 0.0503, 0.0544, 0.0727, 0.1182, 1.2841, 0.2925, 0.0481, 0.0345, 0.0405, 0.0735, 0.0741, 0.0574, 0.0775, + 0.2453, 0.034 , 0.1521, 0.1669, nan, nan] ct / (keV s cm2)` +- **source:** stixcore + +**calculation summary:** Livetime- and area-normalized background flux per native channel. Flux normalization = counts / (livetime x energy-width x collecting-area) -> ct/s/keV/cm2. +Processing: [`background_spectrum_from_cpd`](../stixcore/io/FlareListManager.py#L221-L267). + +**review comment:** _(to be filled in via `notebooks/flarelist_sdcloc_review_comments.yaml`)_ + +#### bkg_spec_flux_ql + +- **comment:** median quiet-period background flux rebinned to the QL lightcurve energy bands (ct/s/keV/cm2) +- **unit:** ct / (keV s cm2) +- **example:** `[0.3725, 0.0853, 0.0569, 0.2891, 0.1114] ct / (keV s cm2)` +- **source:** stixcore +- **uses:** `bkg_spec_flux`, `ENERGIES table` + +**calculation summary:** bkg_spec_flux rebinned to the QL bands (dE-weighted mean, a density). +Processing: [`rebin_flux_to_ql`](../stixcore/io/FlareListManager.py#L305-L325). +From [`FlareListManager.py#L305-L325`](../stixcore/io/FlareListManager.py#L305-L325): + +```python +def rebin_flux_to_ql(flux_32, energies_32, ql_block): + """Rebin a science *flux* spectrum (per keV) onto the QL bands of ``ql_block``. + + Flux is a density, so bands combine as the dE-weighted mean + ``flux_band = Σ(flux_ch·dE_ch) / Σ dE_ch`` over each band's channels (edges align). + The dE weights cancel dimensionally, so the result carries ``flux_32``'s unit. + """ + e_low = _edges_kev(energies_32["e_low"]) + e_high = _edges_kev(energies_32["e_high"]) + dE = e_high - e_low + b_low = _edges_kev(ql_block["e_low"]) + b_high = _edges_kev(ql_block["e_high"]) + unit = getattr(flux_32, "unit", None) + flux = np.asarray(getattr(flux_32, "value", flux_32), dtype=float) + out = np.full(len(b_low), np.nan) + for b in range(len(b_low)): + chans = (e_low >= b_low[b]) & (e_high <= b_high[b]) + denom = np.nansum(dE[chans]) + if np.any(chans) and denom > 0: + out[b] = np.nansum(flux[chans] * dE[chans]) / denom + return out * unit if unit is not None else out +``` + +**review comment:** _(to be filled in via `notebooks/flarelist_sdcloc_review_comments.yaml`)_ + +#### bkg_energy_index + +- **comment:** energy table index of the native background binning (-1 if no background file) +- **unit:** — +- **example:** `1` +- **source:** stixcore + +**calculation summary:** Index into the ENERGIES table of the native (32-ch) background binning (-1 if no file). +Processing: [`FlareListManager.add_background_file_column`](../stixcore/io/FlareListManager.py#L828-L969). + +**review comment:** _(to be filled in via `notebooks/flarelist_sdcloc_review_comments.yaml`)_ + +#### bkg_solo_sun_distance + +- **comment:** SOLO-Sun distance at the background CPD's observation time (from DSUN_OBS; NaN if none) +- **unit:** km +- **example:** `1.41632e+08 km` +- **source:** stixcore + +**calculation summary:** SOLO-Sun distance at the background CPD's observation time, from its DSUN_OBS header keyword. +Processing: [`FlareListManager.add_background_file_column`](../stixcore/io/FlareListManager.py#L828-L969). + +**review comment:** _(to be filled in via `notebooks/flarelist_sdcloc_review_comments.yaml`)_ + +### 4. SOOP campaign — [`FlareSOOPMixin.add_soop`](../stixcore/products/level3/flarelist.py#L618-L649) + +Solar Orbiter observing-campaign metadata active at the flare peak. + +#### soop_encoded_type + +- **comment:** campaign ID +- **unit:** — +- **example:** `None` +- **source:** stixcore + +**calculation summary:** Encoded SOOP campaign type active at the flare peak. +Processing: [`FlareSOOPMixin.add_soop`](../stixcore/products/level3/flarelist.py#L618-L649). + +**review comment:** _(to be filled in via `notebooks/flarelist_sdcloc_review_comments.yaml`)_ + +#### soop_id + +- **comment:** SOOP ID +- **unit:** — +- **example:** `None` +- **source:** stixcore + +**calculation summary:** SOOP campaign instance id active at the flare peak. +Processing: [`FlareSOOPMixin.add_soop`](../stixcore/products/level3/flarelist.py#L618-L649). + +**review comment:** _(to be filled in via `notebooks/flarelist_sdcloc_review_comments.yaml`)_ + +#### soop_type + +- **comment:** name of the SOOP campaign +- **unit:** — +- **example:** `None` +- **source:** stixcore + +**calculation summary:** SOOP campaign name active at the flare peak. +Processing: [`FlareSOOPMixin.add_soop`](../stixcore/products/level3/flarelist.py#L618-L649). + +**review comment:** _(to be filled in via `notebooks/flarelist_sdcloc_review_comments.yaml`)_ + +--- + +## SDCLOC additions (FlarelistSDCLocation) + +### 5. Flare position — [`FlarePositionMixin.add_flare_position`](../stixcore/products/level3/flarelist.py#L213-L550) + +Flare location by back-projection imaging (stixpy/xrayvision) plus the CPD/ephemeris bookkeeping, geometry and quality metrics. Added when the sdc product is upgraded to sdcloc. + +#### anc_ephemeris_path + +- **comment:** Path to the daily ancillary ephemeris file +- **unit:** — +- **example:** `ANC/2023/01/01/ASP/solo_ANC_stix-asp-ephemeris_20230101_V02.fits` +- **source:** stixcore + +**calculation summary:** Path of the daily ancillary aspect-ephemeris file found via FIDO for the flare peak. +Processing: [`FlarePositionMixin.add_flare_position`](../stixcore/products/level3/flarelist.py#L213-L550). + +**review comment:** _(to be filled in via `notebooks/flarelist_sdcloc_review_comments.yaml`)_ + +#### cpd_path + +- **comment:** Path to the CPD file used for flare location estimation +- **unit:** — +- **example:** `L1/2023/01/01/SCI/solo_L1_stix-sci-xray-cpd_20230101T003107-20230101T004058_V02_2301015683-59282.fits` +- **source:** stixcore + +**calculation summary:** Path of the CPD file selected for the location (scored inc_peak/inc_flare/min-dt/ebins); the same file feeds the peak-preview imaging. +Processing: [`FlarePositionMixin.add_flare_position`](../stixcore/products/level3/flarelist.py#L213-L550). + +**review comment:** _(to be filled in via `notebooks/flarelist_sdcloc_review_comments.yaml`)_ + +#### _position_status + +- **comment:** Status of the flare position calculation +- **unit:** — +- **example:** `True` +- **source:** stixcore + +**calculation summary:** Internal QC flag: True when a location was successfully estimated. +Processing: [`FlarePositionMixin.add_flare_position`](../stixcore/products/level3/flarelist.py#L213-L550). + +**review comment:** _(to be filled in via `notebooks/flarelist_sdcloc_review_comments.yaml`)_ + +#### _position_message + +- **comment:** Message describing the status of the flare position calculation +- **unit:** — +- **example:** `OK` +- **source:** stixcore + +**calculation summary:** Internal QC message (OK / why the location was skipped or failed). +Processing: [`FlarePositionMixin.add_flare_position`](../stixcore/products/level3/flarelist.py#L213-L550). + +**review comment:** _(to be filled in via `notebooks/flarelist_sdcloc_review_comments.yaml`)_ + +#### location_hgs + +- **comment:** flare location in Heliographic Stonyhurst coordinates (restored from the on-disk ICRS on read) +- **unit:** SkyCoord (heliographic_stonyhurst) +- **example:** `heliographic_stonyhurst lon/Tx≈1.9506 deg, lat/Ty≈17.2736 deg, distance=6.957e+05 km` +- **source:** stixcore + +**calculation summary:** Flare location estimated by `stx_estimate_flare_location` (stixcore): builds a back-projection image from STIX visibilities, takes the brightest pixel as the source, and transforms it to Heliographic Stonyhurst via sunpy/astropy. The visibility helpers it calls live in stixpy (`create_meta_pixels`, `create_visibility`, `calibrate_visibility`, `get_hpc_info`) and xrayvision (`vis_to_image`), but the algorithm assembly and transforms are stixcore code. +Processing: [`stx_estimate_flare_location`](../stixcore/products/level3/processing.py#L97-L195), [`FlarePositionMixin.add_flare_position`](../stixcore/products/level3/flarelist.py#L213-L550). + +**review comment:** the flare location calculation will come in from STIXPY PR: https://github.com/TCDSolar/stixpy/pull/234 + +#### location_icrs + +- **comment:** on-disk ICRS serialization of the flare location (see location_hgs) +- **unit:** SkyCoord (icrs) +- **example:** `ICRS ra=161.3131 deg, dec=18.3517 deg, distance=1.627e+06 km` +- **source:** stixcore + +**calculation summary:** On-disk ICRS serialization of location_hgs (converted on write, back to HGS on read) by the stixcore serialize hook. +Processing: [`FlarePositionMixin.on_serialize`](../stixcore/products/level3/flarelist.py#L552-L569). + +**review comment:** the flare location calculation will come in from STIXPY PR: https://github.com/TCDSolar/stixpy/pull/234 + +#### solo_location_hgs + +- **comment:** Solar Orbiter location in Heliographic Stonyhurst coordinates (restored from ICRS on read) +- **unit:** SkyCoord (heliographic_stonyhurst) +- **example:** `heliographic_stonyhurst lon/Tx≈-21.1655 deg, lat/Ty≈3.8127 deg, distance=1.415e+08 km` +- **source:** stixcore + +**calculation summary:** Solar Orbiter position assembled in `stx_estimate_flare_location` / `add_flare_position` (stixcore) and stored in HGS; the raw pointing/ephemeris comes from stixpy `get_hpc_info`. +Processing: [`stx_estimate_flare_location`](../stixcore/products/level3/processing.py#L97-L195), [`FlarePositionMixin.add_flare_position`](../stixcore/products/level3/flarelist.py#L213-L550). + +**review comment:** the flare location calculation will come in from STIXPY PR: https://github.com/TCDSolar/stixpy/pull/234 + +#### solo_location_icrs + +- **comment:** on-disk ICRS serialization of the SOLO location (see solo_location_hgs) +- **unit:** SkyCoord (icrs) +- **example:** `ICRS ra=77.3106 deg, dec=27.1101 deg, distance=1.413e+08 km` +- **source:** stixcore + +**calculation summary:** On-disk ICRS serialization of solo_location_hgs by the stixcore serialize hook. +Processing: [`FlarePositionMixin.on_serialize`](../stixcore/products/level3/flarelist.py#L552-L569). + +**review comment:** the flare location calculation will come in from STIXPY PR: https://github.com/TCDSolar/stixpy/pull/234 + +#### solo_sun_distance + +- **comment:** distance of Solar Orbiter to Sun center +- **unit:** km +- **example:** `1.41521e+08 km` +- **source:** stixcore +- **uses:** `solo_location_hgs` + +**calculation summary:** Norm of the SOLO HGS cartesian position (km) at the imaging time. +Processing: [`FlarePositionMixin.add_flare_position`](../stixcore/products/level3/flarelist.py#L213-L550). + +**review comment:** via spice + +#### sidelobes_ratio + +- **comment:** Ratio of sidelobes in the STIX image used to assess imaging quality +- **unit:** — +- **example:** `0.785695` +- **source:** stixcore + +**calculation summary:** Imaging-quality metric: strongest back-projection sidelobe outside a 200 arcsec radius relative to the peak (>~0.9 => unreliable). +Processing: [`calculate_sidelobes_ratio`](../stixcore/products/level3/processing.py#L198-L241). + +**review comment:** the flare location calculation will come in from STIXPY PR: https://github.com/TCDSolar/stixpy/pull/234 + +#### visible_from_earth + +- **comment:** Whether the flare location is visible from Earth (not occulted by the Sun) +- **unit:** — +- **example:** `True` +- **source:** stixcore + +**calculation summary:** Whether the flare location is on the Earth-facing side of the Sun. +Processing: [`is_visible`](../stixcore/products/level3/processing.py#L77-L94), [`FlarePositionMixin.add_flare_position`](../stixcore/products/level3/flarelist.py#L213-L550). + +**review comment:** is_visible also moving to STIXPY + +#### location_time_UTC + +- **comment:** time center used for flare location estimation in UTC +- **unit:** UTC (Time) +- **example:** `2023-01-01T00:34:45.133 (UTC)` +- **source:** stixcore + +**calculation summary:** Time center of the imaging window used for the location. +Processing: [`FlarePositionMixin.add_flare_position`](../stixcore/products/level3/flarelist.py#L213-L550). + +**review comment:** the actual time period (center time) on which the location is calculated is defined first in stixcore. The visibility code might (stixpy) might modify the time period to be used for the location calculation + +#### location_duration + +- **comment:** duration of the flare location estimation time range +- **unit:** s +- **example:** `63.3 s` +- **source:** stixcore + +**calculation summary:** Length of the imaging window used for the location. +Processing: [`FlarePositionMixin.add_flare_position`](../stixcore/products/level3/flarelist.py#L213-L550). + +**review comment:** duration of location_time_UTC + +#### min_exposure + +- **comment:** minimum CPD per-bin exposure ds (timedel) over the flare start..end window +- **unit:** ds +- **example:** `599 ds` +- **source:** stixcore + +**calculation summary:** Shortest CPD time-step (ds) among bins overlapping the flare start..end window. +Processing: [`cpd_timedel_range`](../stixcore/products/level3/flarelist.py#L168-L181). +From [`flarelist.py#L168-L181`](../stixcore/products/level3/flarelist.py#L168-L181): + +```python +def cpd_timedel_range(times, timedels, start, end): + """Min and max CPD time-step ``ds`` (``timedel``) for bins overlapping ``[start, end]``. + + Uses the same half-bin overlap test as the peak-window selection so partially covered + windows still contribute. Returns ``(min_exposure, max_exposure)`` as + `~astropy.units.Quantity` in deciseconds (``u.ds``, STIX's native ``timedel`` unit); + ``(NaN ds, NaN ds)`` if no bin overlaps. + """ + half = timedels / 2 + mask = (times + half >= start) & (times - half <= end) + sel = timedels[mask] + if len(sel) == 0: + return np.nan * u.ds, np.nan * u.ds + return sel.min().to(u.ds), sel.max().to(u.ds) +``` + +**review comment:** _(to be filled in via `notebooks/flarelist_sdcloc_review_comments.yaml`)_ + +#### max_exposure + +- **comment:** maximum CPD per-bin exposure ds (timedel) over the flare start..end window +- **unit:** ds +- **example:** `792 ds` +- **source:** stixcore + +**calculation summary:** Longest CPD time-step (ds) among bins overlapping the flare start..end window. +Processing: [`cpd_timedel_range`](../stixcore/products/level3/flarelist.py#L168-L181). + +**review comment:** _(to be filled in via `notebooks/flarelist_sdcloc_review_comments.yaml`)_ + +#### time_shift + +- **comment:** Time(Sun to Earth) - Time(Sun to S/C) +- **unit:** s +- **example:** `18.6269 s` +- **source:** stixcore + +**calculation summary:** Time(Sun->Earth) - Time(Sun->S/C) light-travel-time difference from Spice. +Processing: [`Spice.get_earth_solo_time_shift`](../stixcore/ephemeris/manager.py#L391-L403). + +**review comment:** _(to be filled in via `notebooks/flarelist_sdcloc_review_comments.yaml`)_ + +#### sun_disc_size + +- **comment:** Apparent photospheric solar radius +- **unit:** arcsec +- **example:** `1013.98 arcsec` +- **source:** stixcore + +**calculation summary:** Apparent photospheric solar radius (arcsec) at the flare, from Spice. +Processing: [`Spice.get_sun_disc_size`](../stixcore/ephemeris/manager.py#L374-L389). + +**review comment:** _(to be filled in via `notebooks/flarelist_sdcloc_review_comments.yaml`)_ + +### 6. 1-AU normalization — [`add_distance_normalized_flux`](../stixcore/products/level3/flarelist.py#L94-L110) + +Distance-normalized twins of the flux columns (called at the end of add_flare_position). + +**Calculation summary (shared by all columns in this step):** All columns in this step are produced by a single call to [`add_distance_normalized_flux`](../stixcore/products/level3/flarelist.py#L94-L110), each scaling its flux column by ``(r/1AU)**2`` using its mapped distance column (see **uses** per column). Flux ~ 1/r**2, so the factor is dimensionless and the unit is unchanged; the original flux columns are kept. + +From [`flarelist.py#L94-L110`](../stixcore/products/level3/flarelist.py#L94-L110): + +```python +def add_distance_normalized_flux(data, mapping=_FLUX_DISTANCE_COLS): + """Add ``_at_1au`` = `` * (r/1AU)**2`` for each present flux column, using the + per-flare distance in its mapped column (flux scales as 1/r**2). + + The existing flux columns are kept unchanged; the scale factor is dimensionless so the + ``*_at_1au`` columns keep the flux unit (ct/s/keV/cm2). NaN distance -> NaN; a flux column + (or its distance column) that is absent is skipped. + """ + for col, dist_col in mapping.items(): + if col not in data.colnames or dist_col not in data.colnames: + continue + factor = (data[dist_col] / (1 * u.AU)).decompose().value ** 2 # (N,), dimensionless + f = data[col] + data[col + "_at_1au"] = f * (factor[:, None] if f.ndim == 2 else factor) + data[col + "_at_1au"].info.description = ( + f"{col} scaled to what would be seen at 1 AU (x (r_solo/1AU)**2, r_solo from {dist_col})" + ) +``` + +#### lc_peak_flux_at_1au + +- **comment:** lc_peak_flux scaled to what would be seen at 1 AU (x (r_solo/1AU)**2, r_solo from solo_sun_distance) +- **unit:** ct / (keV s cm2) +- **example:** `[1.5917, 0.174 , 0.0659, 0.2583, 0.1114] ct / (keV s cm2)` +- **source:** stixcore +- **uses:** `lc_peak_flux`, `solo_sun_distance` + +**calculation summary:** see the shared step summary above. + +**review comment:** _(to be filled in via `notebooks/flarelist_sdcloc_review_comments.yaml`)_ + +#### lc_bkg_peak_flux_at_1au + +- **comment:** lc_bkg_peak_flux scaled to what would be seen at 1 AU (x (r_solo/1AU)**2, r_solo from solo_sun_distance) +- **unit:** ct / (keV s cm2) +- **example:** `[0.5274, 0.0791, 0.0659, 0.2373, 0.1086] ct / (keV s cm2)` +- **source:** stixcore +- **uses:** `lc_bkg_peak_flux`, `solo_sun_distance` + +**calculation summary:** see the shared step summary above. + +**review comment:** _(to be filled in via `notebooks/flarelist_sdcloc_review_comments.yaml`)_ + +#### bkg_spec_flux_at_1au + +- **comment:** bkg_spec_flux scaled to what would be seen at 1 AU (x (r_solo/1AU)**2, r_solo from bkg_solo_sun_distance) +- **unit:** ct / (keV s cm2) +- **example:** `[0.5508, 0.2466, 0.1889, 0.6392, 0.2959, 0.082 , 0.0662, 0.1254, 0.0999, 0.0504, 0.0405, 0.041 , 0.043 , + 0.0451, 0.0488, 0.0652, 0.1059, 1.151 , 0.2621, 0.0431, 0.0309, 0.0363, 0.0659, 0.0665, 0.0514, 0.0694, + 0.2199, 0.0304, 0.1363, 0.1496, nan, nan] ct / (keV s cm2)` +- **source:** stixcore +- **uses:** `bkg_spec_flux`, `bkg_solo_sun_distance` + +**calculation summary:** see the shared step summary above. + +**review comment:** _(to be filled in via `notebooks/flarelist_sdcloc_review_comments.yaml`)_ + +#### bkg_spec_flux_ql_at_1au + +- **comment:** bkg_spec_flux_ql scaled to what would be seen at 1 AU (x (r_solo/1AU)**2, r_solo from bkg_solo_sun_distance) +- **unit:** ct / (keV s cm2) +- **example:** `[0.3339, 0.0765, 0.051 , 0.2592, 0.0999] ct / (keV s cm2)` +- **source:** stixcore +- **uses:** `bkg_spec_flux_ql`, `bkg_solo_sun_distance` + +**calculation summary:** see the shared step summary above. + +**review comment:** _(to be filled in via `notebooks/flarelist_sdcloc_review_comments.yaml`)_ diff --git a/stixcore/io/FlareListManager.py b/stixcore/io/FlareListManager.py index 86f492e4..b854b2f9 100644 --- a/stixcore/io/FlareListManager.py +++ b/stixcore/io/FlareListManager.py @@ -40,10 +40,19 @@ "max_rcr_in_window", "find_background_file_for_time", "BackgroundSelection", + "background_spectrum_from_cpd", + "rebin_spectrum_to_ql", + "rebin_flux_to_ql", + "collecting_area_cm2", ] logger = get_logger(__name__) +#: Expected unit of the livetime- and area-corrected flux columns. Computations carry +#: astropy units through and validate against this with ``.to(FLUX_UNIT)`` rather than +#: labelling bare floats. +FLUX_UNIT = u.ct / (u.s * u.keV * u.cm**2) + def compute_ql_count_rate(counts, timedel, triggers, energy_delta, *, n_detectors): """Reproduce stixpy's QL count-rate normalization -> ``ct / (s * keV)``. @@ -119,6 +128,230 @@ def max_rcr_in_window(times, rcr, start, end, *, fallback): return int(np.asarray(rcr)[mask].max()) +#: 0-based detector indices used for a background spectrum: all 32 science +#: sub-collimators except the Coarse Flare Locator (idx 8) and the Background +#: monitor (idx 9), i.e. the 30 imaging detectors. +_IMAGING_DETECTORS = [d for d in range(32) if d not in (8, 9)] + + +def _edges_kev(col): + """Return an energy-edge column as a plain float ndarray in keV.""" + return np.asarray(col.to_value(u.keV) if hasattr(col, "to_value") else col, dtype=float) + + +_PIXEL_AREA_CACHE = None + + +def _pixel_area_cm2(): + """12-pixel active-area vector (cm^2) from stixcore's ``stx_subc_params`` config. + + STIX Caliste layout is 8 large pixels then 4 small; pixel sizes are identical + across detectors so one 12-vector applies to all. Cached after first read. + """ + global _PIXEL_AREA_CACHE + if _PIXEL_AREA_CACHE is None: + import stixcore.config + from stixcore.config.reader import read_subc_params + + path = Path(stixcore.config.__file__).parent / "data" / "common" / "detector" / "stx_subc_params.csv" + t = read_subc_params(path) + large = float(t["L Pixel Xsize"][0]) * float(t["L Pixel Ysize"][0]) # mm^2 + small = float(t["S Pixel Xsize"][0]) * float(t["S Pixel Ysize"][0]) # mm^2 + _PIXEL_AREA_CACHE = np.array([large] * 8 + [small] * 4, dtype=float) / 100.0 # mm^2 -> cm^2 + return _PIXEL_AREA_CACHE + + +def collecting_area_cm2(product): + """Geometric collecting area (cm^2) of a detector+pixel-summed product = + ``detector_mask.sum() * sum(pixel_area[pixel_mask])``. Returns ``None`` when the + product's control has no detector/pixel mask (e.g. the QL background monitor).""" + ctrl = getattr(product, "control", None) + if ctrl is None or "detector_mask" not in ctrl.colnames or "pixel_mask" not in ctrl.colnames: + return None + det = np.asarray(ctrl["detector_mask"][0], dtype=bool) + pix = np.asarray(ctrl["pixel_mask"][0], dtype=bool) + return float(det.sum()) * float(_pixel_area_cm2()[pix].sum()) + + +def _bkg_monitor_area_cm2(): + """Fixed area (cm^2) of the QL background monitor: one open (grid-less) detector + over all 12 pixels.""" + return float(_pixel_area_cm2().sum()) + + +def _cpd_present_channel_mask(cpd): + """Boolean mask (length 32) of which science channels are present in a CPD's + ``counts`` when only a telemetered subset is stored, or ``None`` if unknown.""" + ctrl = getattr(cpd, "control", None) + if ctrl is None: + return None + if "energy_bin_mask" in ctrl.colnames: + m = np.asarray(ctrl["energy_bin_mask"][0], dtype=bool).ravel() + return m if m.size == 32 else None + if "energy_bin_edge_mask" in ctrl.colnames: + edges = np.asarray(ctrl["energy_bin_edge_mask"][0], dtype=bool).ravel() + if edges.size == 33: # 33 edges -> a channel is present when both its edges are set + return edges[:-1] & edges[1:] + return None + + +def _align_to_energies(spec, nE, energies, cpd): + """Pad/align a per-channel ``spec`` (length ``nE``) to the energies-table length. + + Preserves an astropy unit on ``spec`` (the NaN padding is created in the same unit, + so assignment stays unit-checked).""" + ne_en = len(energies) + if nE == ne_en: + return spec + unit = getattr(spec, "unit", None) + aligned = np.full(ne_en, np.nan) + if unit is not None: + aligned = aligned * unit + mask = _cpd_present_channel_mask(cpd) + if mask is not None and mask.size == ne_en and int(mask.sum()) == nE: + aligned[mask] = spec + elif nE < ne_en: + aligned[:nE] = spec + else: + logger.warning(f"CPD counts have {nE} channels but energies has {ne_en}; truncating") + aligned[:] = spec[:ne_en] + return aligned + + +def background_spectrum_from_cpd(cpd): + """Median quiet-period background spectrum (counts and flux) from a CPD product. + + Sums ``counts`` over the 30 imaging detectors (excl. CFL idx 8, BKG monitor idx 9) + and all pixels, then takes the median over the file's time bins (the whole quiet + period). ``flux`` is the livetime- and area-normalized rate density + ``ct s^-1 keV^-1 cm^-2`` (per-bin then median), mirroring stixpy's + ``create_meta_pixels`` normalization: + ``flux[t,E] = Σ_dp counts / ((Σ_d livefrac[t,d]·timedel[t]) · (Σ_p area[p]) · dE[E])``. + Both are aligned to the product's own ``energies`` table (length = telemetered + channels). Reads ``cpd.data["counts"]`` directly (``get_data`` is not robust on all + files). Any failure computing the flux yields NaN flux (counts still returned). + + Returns + ------- + (counts, flux, energies) + """ + counts = cpd.data["counts"] + counts_q = counts if hasattr(counts, "unit") else np.asarray(counts, dtype=float) * u.ct # (nt,32,npix,nE) ct + nE = counts_q.shape[-1] + energies = cpd.energies + csum = counts_q[:, _IMAGING_DETECTORS, :, :].sum(axis=(1, 2)) # (nt, nE) ct over imaging det+pix + counts_spec = _align_to_energies(np.nanmedian(csum, axis=0), nE, energies, cpd) # Quantity ct + + flux_spec = np.full(len(energies), np.nan) * FLUX_UNIT + try: + timedel = cpd.data["timedel"].to(u.s) # (nt,) — units carried through the livetime rate + trig = np.asarray(getattr(cpd.data["triggers"], "value", cpd.data["triggers"]), dtype=float) # (nt,16) + from stixpy.calibration.visibility import STIX_INSTRUMENT + + subcol = np.asarray(STIX_INSTRUMENT.subcol_adc_mapping) # (32,) + livefrac = get_livetime_fraction(trig[:, subcol] / timedel[:, None])[0] # (nt, 32), dimensionless + livefrac = np.asarray(getattr(livefrac, "value", livefrac), dtype=float) + exp_t = (livefrac[:, _IMAGING_DETECTORS] * timedel[:, None]).sum(axis=1) # (nt,) s (Σ over dets) + pmask = _cpd_pixel_mask(cpd) # which of 12 pixels are telemetered + pa = _pixel_area_cm2() + area = (pa[pmask].sum() if pmask is not None else pa[: counts_q.shape[2]].sum()) * u.cm**2 # cm^2 + dE = (energies["e_high"] - energies["e_low"]).to(u.keV) # (ne_en,) keV + dE_c = dE if nE == len(energies) else dE[:nE] + with np.errstate(divide="ignore", invalid="ignore"): + # astropy derives the unit; .to(FLUX_UNIT) both converts and asserts it is correct + flux_t = (csum / (exp_t[:, None] * area * dE_c[None, :])).to(FLUX_UNIT) # (nt, nE) + flux_spec = _align_to_energies(np.nanmedian(flux_t, axis=0), nE, energies, cpd) + except Exception as e: + logger.warning(f"could not compute background flux: {e}") + + return counts_spec, flux_spec, energies + + +def _cpd_pixel_mask(cpd): + """Boolean (length 12) of telemetered pixels from the CPD ``pixel_masks`` (row 0), + or ``None`` when unavailable.""" + data = getattr(cpd, "data", None) + try: + if data is not None and "pixel_masks" in data.colnames: + return np.asarray(data["pixel_masks"][0], dtype=bool).ravel() + except Exception: + pass + return None + + +def rebin_spectrum_to_ql(counts_32, energies_32, ql_block): + """Rebin a science *counts* spectrum onto the QL energy bands of ``ql_block``. + + ``ql_block`` carries the target band edges (``e_low``/``e_high``) read from the + energy table for a given flare (never hardcoded, so a QL binning that changes + over time is honored). A science channel contributes to a band when its + ``[e_low, e_high]`` lies within the band; whole channels are summed (the edges + align). Summing preserves the input unit, so the result carries ``counts_32``'s unit. + """ + e_low = _edges_kev(energies_32["e_low"]) + e_high = _edges_kev(energies_32["e_high"]) + b_low = _edges_kev(ql_block["e_low"]) + b_high = _edges_kev(ql_block["e_high"]) + unit = getattr(counts_32, "unit", None) + counts = np.asarray(getattr(counts_32, "value", counts_32), dtype=float) + out = np.full(len(b_low), np.nan) + for b in range(len(b_low)): + chans = (e_low >= b_low[b]) & (e_high <= b_high[b]) + if np.any(chans): + out[b] = np.nansum(counts[chans]) + return out * unit if unit is not None else out + + +def rebin_flux_to_ql(flux_32, energies_32, ql_block): + """Rebin a science *flux* spectrum (per keV) onto the QL bands of ``ql_block``. + + Flux is a density, so bands combine as the dE-weighted mean + ``flux_band = Σ(flux_ch·dE_ch) / Σ dE_ch`` over each band's channels (edges align). + The dE weights cancel dimensionally, so the result carries ``flux_32``'s unit. + """ + e_low = _edges_kev(energies_32["e_low"]) + e_high = _edges_kev(energies_32["e_high"]) + dE = e_high - e_low + b_low = _edges_kev(ql_block["e_low"]) + b_high = _edges_kev(ql_block["e_high"]) + unit = getattr(flux_32, "unit", None) + flux = np.asarray(getattr(flux_32, "value", flux_32), dtype=float) + out = np.full(len(b_low), np.nan) + for b in range(len(b_low)): + chans = (e_low >= b_low[b]) & (e_high <= b_high[b]) + denom = np.nansum(dE[chans]) + if np.any(chans) and denom > 0: + out[b] = np.nansum(flux[chans] * dE[chans]) / denom + return out * unit if unit is not None else out + + +def _intern_energy_block(energy, energy_look_up, energies_src): + """Append ``energies_src`` (channel/e_low/e_high) to the ``energy`` table as a new + block if its binning is unseen; return ``(energy, index)``. Blocks are hash-deduped + and get an ``index`` one past the current maximum so they never collide with the + QL blocks already present.""" + e_sub = QTable() + e_sub["channel"] = energies_src["channel"] + e_sub["e_low"] = energies_src["e_low"] + e_sub["e_high"] = energies_src["e_high"] + e_hash = frozenset(pd.core.util.hashing.hash_array(e_sub.as_array())) + if e_hash in energy_look_up: + return energy, energy_look_up[e_hash] + idx = (int(np.max(energy["index"])) + 1) if (len(energy) > 0 and "index" in energy.colnames) else 0 + energy_look_up[e_hash] = idx + e_sub["index"] = Column(idx, description="energy edge table index", dtype=np.int8) + return vstack([energy, e_sub]), idx + + +def _ql_block_for(energy, eidx): + """The rows of the ``energy`` table belonging to block ``eidx`` (a flare's QL + binning), or ``None`` when the table is empty / has no such block.""" + if energy is None or len(energy) == 0 or "index" not in energy.colnames: + return None + sel = np.asarray(energy["index"]) == int(eidx) + return energy[sel] if np.any(sel) else None + + #: Result of :func:`find_background_file_for_time`. ``path`` is a `~pathlib.Path` #: (or ``None`` when nothing qualifies), ``rid`` the selected BSD request id #: (``-1`` when none), and ``valid_from``/``valid_to`` the `~astropy.time.Time` @@ -353,6 +586,19 @@ def _valid_to(chosen): class FlareListManager: + """Base class for the flare-list source managers. + + Holds the source flare list and the product class it feeds, and provides the shared + enrichment machinery that turns raw flare definitions into an enriched flare list: + `~stixcore.io.FlareListManager.FlareListManager._build_ql_month_timeline`, + `~stixcore.io.FlareListManager.FlareListManager.add_lc_bkg_columns` (quicklook peak / + background counts and fluxes) and + `~stixcore.io.FlareListManager.FlareListManager.add_background_file_column` (quiet-time + background spectrum). The active subclass is + `~stixcore.io.FlareListManager.SDCFlareListManager`, which mirrors the operational STIX + Data Center flare list. See :doc:`/products/flarelist`. + """ + @property def flarelist(self): return self._flarelist @@ -424,6 +670,12 @@ def _build_ql_month_timeline(self, *, start, end, fido_client, data_product, n_d rate = compute_ql_count_rate( counts, p.data["timedel"], p.data["triggers"], energy_delta, n_detectors=n_detectors ) + # area-normalize to a flux (ct/s/keV/cm^2): use the product's detector/pixel + # masks, falling back to the fixed open-detector area for the QL background monitor + area = collecting_area_cm2(p) + if area is None: + area = _bkg_monitor_area_cm2() + rate = rate / (area * u.cm**2) daily = QTable() daily["time"] = p.data["time"] @@ -454,13 +706,14 @@ def _build_ql_month_timeline(self, *, start, end, fido_client, data_product, n_d return build_month_timeline(daily_tables), energy, date_to_eidx def add_lc_bkg_columns(self, data, *, start, end, fido_client): - """Populate LC/BKG peak counts + rates, RCR and ``att_in`` on ``data`` from the + """Populate LC/BKG peak counts + flux, RCR and ``att_in`` on ``data`` from the real L1 QL lightcurve + background products, and return the energy QTable. ``data`` must already carry ``flare_id`` and the astropy ``Time`` columns ``start_UTC`` / ``end_UTC`` / ``peak_UTC``. Columns are added in place: - ``lc_peak``, ``lc_peak_rate``, ``lc_bgk_peak``, ``lc_bgk_peak_rate``, - ``rcr_at_peak``, ``rcr_max``, ``att_in``, ``energy_index``. + ``lc_peak``, ``lc_peak_flux``, ``lc_bkg_peak``, ``lc_bkg_peak_flux``, + ``rcr_at_peak``, ``rcr_max``, ``att_in``, ``energy_index``. ``*_flux`` are + livetime- and area-normalized (ct/s/keV/cm2). """ n = len(data) tol = CONFIG.getfloat("Processing", "flarelist_peak_max_dist_s", fallback=60.0) * u.s @@ -482,15 +735,18 @@ def add_lc_bkg_columns(self, data, *, start, end, fido_client): track_energy=False, ) - lc_peak = np.zeros((n, 5), dtype=np.int64) - lc_peak_rate = np.zeros((n, 5), dtype=np.float64) - lc_bgk_peak = np.zeros((n, 5), dtype=np.int64) - lc_bgk_peak_rate = np.zeros((n, 5), dtype=np.float64) + # unit-carrying accumulators: assigning a Quantity slice below is unit-checked, so + # a wrong-unit timeline value would raise rather than be silently stored + # raw counts kept as unsigned int32 (u.Quantity floats by default; explicit dtype keeps + # it uint32 -> FITS 'J' + unsigned TZERO, 4 B/elem exact). No-match rows stay 0, never NaN. + lc_peak = u.Quantity(np.zeros((n, 5), dtype=np.uint32), u.ct, dtype=np.uint32) + lc_peak_flux = np.zeros((n, 5), dtype=np.float32) * FLUX_UNIT + lc_bkg_peak = u.Quantity(np.zeros((n, 5), dtype=np.uint32), u.ct, dtype=np.uint32) + lc_bkg_peak_flux = np.zeros((n, 5), dtype=np.float32) * FLUX_UNIT rcr_at_peak = np.full(n, -1, dtype=np.int8) rcr_max = np.full(n, -1, dtype=np.int8) energy_index = np.zeros(n, dtype=np.int8) - rate_unit = u.ct / (u.s * u.keV) lc_has = len(lc_timeline) > 0 bkg_has = len(bkg_timeline) > 0 if not lc_has: @@ -504,8 +760,8 @@ def add_lc_bkg_columns(self, data, *, start, end, fido_client): if lc_has: j = nearest_bin_index(lc_timeline["time"], peak, tol) if j is not None: - lc_peak[i] = lc_timeline["counts"][j].to_value(u.ct) - lc_peak_rate[i] = lc_timeline["counts_rate"][j].to_value(rate_unit) + lc_peak[i] = lc_timeline["counts"][j] # Quantity ct -> unit-checked assignment + lc_peak_flux[i] = lc_timeline["counts_rate"][j].to(FLUX_UNIT) # assert flux unit rcr_at_peak[i] = int(lc_timeline["rcr"][j]) rcr_max[i] = max_rcr_in_window( lc_timeline["time"], @@ -520,28 +776,38 @@ def add_lc_bkg_columns(self, data, *, start, end, fido_client): if bkg_has: k = nearest_bin_index(bkg_timeline["time"], peak, tol) if k is not None: - lc_bgk_peak[i] = bkg_timeline["counts"][k].to_value(u.ct) - lc_bgk_peak_rate[i] = bkg_timeline["counts_rate"][k].to_value(rate_unit) + lc_bkg_peak[i] = bkg_timeline["counts"][k] # Quantity ct -> unit-checked assignment + lc_bkg_peak_flux[i] = bkg_timeline["counts_rate"][k].to(FLUX_UNIT) # assert flux unit else: logger.warning(f"flare {fid}: no BKG bin within {tol} of peak {peak.isot}") + # Drop orphan energy-binning blocks: a QL product with a different binning can add + # rows to `energy` that no flare references (date_to_eidx keeps the first binning + # seen per date). Keep only referenced blocks and renumber energy_index contiguously. + if len(energy) > 0 and "index" in energy.colnames: + used = sorted({int(x) for x in energy_index}) + remap = {old: new for new, old in enumerate(used)} + energy = energy[[k for k, e in enumerate(energy["index"]) if int(e) in remap]] + energy["index"] = np.array([remap[int(x)] for x in energy["index"]], dtype=energy["index"].dtype) + energy_index = np.array([remap[int(x)] for x in energy_index], dtype=energy_index.dtype) + data["lc_peak"] = Column( - lc_peak * u.ct, + lc_peak, # already a Quantity in ct description="raw counts at the L1 QL lightcurve bin nearest the flare peak (5 energy channels)", - dtype=np.int64, + dtype=np.uint32, ) - data["lc_peak_rate"] = Column( - lc_peak_rate * rate_unit, - description="livetime-corrected count rate at the peak lightcurve bin (5 energy channels)", + data["lc_peak_flux"] = Column( + lc_peak_flux, # already a Quantity in ct/s/keV/cm2 + description="livetime- and area-corrected flux at the peak lightcurve bin (ct/s/keV/cm2, 5 energy channels)", ) - data["lc_bgk_peak"] = Column( - lc_bgk_peak * u.ct, - description="raw background counts at the L1 QL background bin nearest the flare peak (5 energy channels)", - dtype=np.int64, + data["lc_bkg_peak"] = Column( + lc_bkg_peak, # already a Quantity in ct + description="raw counts from the STIX background detector at the L1 QL background bin nearest the flare peak (5 energy channels) — unmodulated by imaging subcollimators and not affected by the attenuator", + dtype=np.uint32, ) - data["lc_bgk_peak_rate"] = Column( - lc_bgk_peak_rate * rate_unit, - description="livetime-corrected background count rate at the peak bin (5 energy channels)", + data["lc_bkg_peak_flux"] = Column( + lc_bkg_peak_flux, # already a Quantity in ct/s/keV/cm2 + description="livetime- and area-corrected STIX background detector flux at the peak bin (ct/s/keV/cm2, 5 energy channels)", ) data["rcr_at_peak"] = Column( rcr_at_peak, description="rate control regime at the peak bin (>0 attenuator in)", dtype=np.int8 @@ -552,20 +818,41 @@ def add_lc_bkg_columns(self, data, *, start, end, fido_client): data["att_in"] = Column(rcr_max > 0, description="was attenuator in during flare (rcr_max > 0)") data["energy_index"] = Column(energy_index, description="energy band index", dtype=np.int8) - return energy + # these CSV-derived placeholders (added in get_data) are superseded here; drop if present + for _col in ("bkg_baseline", "bkg_quiet_period"): + if _col in data.colnames: + del data[_col] - def add_background_file_column(self, data, *, fido_client): - """Add ``bkg_file`` / ``bkg_rid`` columns: the best quiet-time background - CPD file for each flare peak. + return energy - ``data`` rows are assumed to be peak-time ascending (as produced by the - source flare list), so each per-time background search - (:func:`find_background_file_for_time`) is cached with its validity - interval and only re-run once a flare peak crosses out of that period. + def add_background_file_column(self, data, *, energy, fido_client): + """Add background-file and quiet-period background-spectrum columns per flare. + + Writes ``bkg_file`` / ``bkg_rid`` (the best quiet-time background CPD file for + each flare peak) and, extracted from that file, the median quiet-period + background spectrum: raw counts in the native science channels (``bkg_spec``) and + rebinned to the flare's QL bands (``bkg_spec_ql``), plus the livetime- and + area-normalized flux (ct/s/keV/cm2) in native channels (``bkg_spec_flux``) and + rebinned to the QL bands (``bkg_spec_flux_ql``); ``bkg_energy_index`` points at the + native binning appended to the ``energy`` table. + + ``data`` rows are assumed peak-time ascending, so the per-time background + search (:func:`find_background_file_for_time`) is cached with its validity + interval and only re-run when a flare peak leaves that period. The extracted + spectrum is likewise cached per background file id, so each CPD is opened only + once no matter how many flares share it. Requires the ``energy`` table (from + :meth:`add_lc_bkg_columns`) so the QL rebin follows the actual QL binning; + returns the (possibly extended, orphan-pruned) ``energy`` table. """ n = len(data) bkg_files = [""] * n bkg_rids = np.full(n, -1, dtype=np.int64) + spec_rows = [None] * n # native-binning counts spectrum per flare (Quantity ct, length varies) + flux_rows = [None] * n # native-binning flux spectrum per flare (Quantity, ct/s/keV/cm2) + bkg_spec_ql = np.full((n, 5), np.nan) * u.ct # native counts rebinned to QL bands + bkg_spec_flux_ql = np.full((n, 5), np.nan) * FLUX_UNIT # unit-checked on assignment below + bkg_energy_index = np.full(n, -1, dtype=np.int16) + bkg_solo_sun_distance = np.full(n, np.nan) * u.km # SOLO-Sun distance at the bkg CPD's time primer = "" baseurl = getattr(fido_client, "baseurl", None) @@ -574,23 +861,112 @@ def add_background_file_column(self, data, *, fido_client): primer = baseurl.replace(datapath, "") primer = primer[7:] if primer.startswith("file://") else primer + energy_look_up_32 = {} # native binning hash -> index in `energy` + spectrum_cache = {} # rid -> (counts, flux, energies, eidx, dsun_km) + rebin_cache = {} # (rid, energy_index) -> flux_ql (5,) selection = None searches = 0 + opens = 0 for i, row in enumerate(data): peak = row["peak_UTC"] if selection is None or not (selection.valid_from <= peak <= selection.valid_to): selection = find_background_file_for_time(peak, fido_client=fido_client) searches += 1 - if selection.path is not None: - bkg_files[i] = str(selection.path).replace(primer, "") - bkg_rids[i] = selection.rid + if selection.path is None: + continue + bkg_files[i] = str(selection.path).replace(primer, "") + bkg_rids[i] = selection.rid + + if selection.rid not in spectrum_cache: + try: + cpd = STIXPYProduct(selection.path) + c32, f32, e32 = background_spectrum_from_cpd(cpd) + energy, eidx32 = _intern_energy_block(energy, energy_look_up_32, e32) + meta = getattr(cpd, "meta", None) or {} + dsun = float(meta["DSUN_OBS"]) * u.m if "DSUN_OBS" in meta else np.nan * u.m + spectrum_cache[selection.rid] = (c32, f32, e32, eidx32, dsun.to(u.km)) + opens += 1 + except Exception as e: + logger.warning(f"could not extract background spectrum from {selection.path}: {e}") + spectrum_cache[selection.rid] = None + cached = spectrum_cache[selection.rid] + if cached is None: + continue + c32, f32, e32, eidx32, dsun = cached + spec_rows[i] = c32 + flux_rows[i] = f32 + bkg_energy_index[i] = eidx32 + bkg_solo_sun_distance[i] = dsun + + eidx_ql = int(data["energy_index"][i]) if "energy_index" in data.colnames else -1 + ql_block = _ql_block_for(energy, eidx_ql) + if ql_block is not None: + key = (selection.rid, eidx_ql) + if key not in rebin_cache: + rebin_cache[key] = ( + rebin_spectrum_to_ql(c32, e32, ql_block), + rebin_flux_to_ql(f32, e32, ql_block), + ) + bkg_spec_ql[i], bkg_spec_flux_ql[i] = rebin_cache[key] + + # Drop orphan energy blocks (keep those referenced by either the QL energy_index + # or the background bkg_energy_index) and renumber both index columns contiguously. + if len(energy) > 0 and "index" in energy.colnames: + ref_ql = {int(x) for x in data["energy_index"]} if "energy_index" in data.colnames else set() + ref_bkg = {int(x) for x in bkg_energy_index if x >= 0} + used = sorted(ref_ql | ref_bkg) + remap = {old: new for new, old in enumerate(used)} + energy = energy[[k for k, e in enumerate(energy["index"]) if int(e) in remap]] + energy["index"] = np.array([remap[int(x)] for x in energy["index"]], dtype=energy["index"].dtype) + if "energy_index" in data.colnames: + data["energy_index"] = np.array( + [remap.get(int(x), 0) for x in data["energy_index"]], dtype=data["energy_index"].dtype + ) + bkg_energy_index = np.array([remap[int(x)] if x >= 0 else -1 for x in bkg_energy_index], dtype=np.int16) + + # native-binning spectra can differ in length across files (telemetered channel + # count varies); build rectangular Quantity columns at the widest, padding with NaN. + # Assigning the Quantity rows into the Quantity arrays is unit-checked. + width = max((len(r) for r in spec_rows if r is not None), default=32) + bkg_spec = np.full((n, width), np.nan) * u.ct + bkg_spec_flux = np.full((n, width), np.nan) * FLUX_UNIT + for i in range(n): + if spec_rows[i] is not None: + bkg_spec[i, : len(spec_rows[i])] = spec_rows[i] + if flux_rows[i] is not None: + bkg_spec_flux[i, : len(flux_rows[i])] = flux_rows[i] data["bkg_file"] = Column(bkg_files, description="path to the quiet-time background CPD file") data["bkg_rid"] = Column( bkg_rids, description="BSD request id of the selected background file (-1 if none)", dtype=np.int64 ) - logger.info(f"background file search ran {searches}x for {n} flares") - return data + data["bkg_spec"] = Column( + bkg_spec, # already a Quantity in ct + description="median quiet-period background counts per science energy channel (30 imaging detectors)", + ) + data["bkg_spec_ql"] = Column( + bkg_spec_ql, # already a Quantity in ct + description="median quiet-period background counts rebinned to the QL lightcurve energy bands", + ) + data["bkg_spec_flux"] = Column( + bkg_spec_flux, # already a Quantity in ct/s/keV/cm2 + description="median quiet-period background flux per science channel (ct/s/keV/cm2, 30 imaging detectors)", + ) + data["bkg_spec_flux_ql"] = Column( + bkg_spec_flux_ql, # already a Quantity in ct/s/keV/cm2 + description="median quiet-period background flux rebinned to the QL lightcurve energy bands (ct/s/keV/cm2)", + ) + data["bkg_energy_index"] = Column( + bkg_energy_index, + description="energy table index of the native background binning (-1 if no background file)", + dtype=np.int16, + ) + data["bkg_solo_sun_distance"] = Column( + bkg_solo_sun_distance, # already a Quantity in km + description="SOLO-Sun distance at the background CPD's observation time (from DSUN_OBS; NaN if none)", + ) + logger.info(f"background file search ran {searches}x, opened {opens} CPD files for {n} flares") + return energy class SCFlareListManager(FlareListManager, metaclass=Singleton): @@ -812,7 +1188,7 @@ def get_data(self, *, start, end, fido_client): dtype=np.int64, ) - data["lc_bgk_peak"] = Column( + data["lc_bkg_peak"] = Column( ( np.vstack( ( @@ -912,22 +1288,25 @@ def update_list(self): @classmethod def read_flarelist(cls, file, update=False): - """Reads or creates the LUT of all BSD RIDs and the request reason comment. + """Read the local flare-list CSV mirror, optionally refreshing it from the STIX Data Center. - On creation or update an api endpoint from the STIX data center is used - to get the information and persists as a LUT locally. + When ``update`` is set (or the file does not yet exist) the operational flare list is + fetched from the STIX Data Center via ``stixdcpy.fetch_flare_list`` in ~monthly chunks + (the API is batched by month and throttled, so the loop sleeps between chunks) from + 2020-01-01 to now; an incremental update re-fetches the last ~60 days. The chunks are + concatenated, de-duplicated, sorted by ``peak_UTC`` and cached back to ``file``. Parameters ---------- file : Path - path the to LUT file. + Path to the local flare-list CSV mirror. update : bool, optional - should the LUT be updated at start up?, by default False + Refresh from the STIX Data Center before reading, by default False. Returns ------- - Table - the LUT od RIDs and request reasons. + `~pandas.DataFrame` + The full flare list. """ if update or not file.exists(): # the api is limited to batch sizes of a month. in order to get the full table we have @@ -980,6 +1359,30 @@ def filter_flare_function(col): return col["lc_peak"][0].value > CONFIG.getint("Processing", "flarelist_sdc_min_count", fallback=1000) def get_data(self, *, start, end, fido_client): + """Build the enriched SDC flare list for the ``[start, end)`` month. + + Slices the requested month from the local CSV mirror and builds the base columns + (``flare_id``, the UTC time columns, GOES class/flux and the source quiet-period + background ``bkg_baseline`` / ``bkg_quiet_period``). It then enriches each flare with + the quicklook peak/background counts and fluxes + (`~stixcore.io.FlareListManager.FlareListManager.add_lc_bkg_columns`) and the quiet-time + background spectrum + (`~stixcore.io.FlareListManager.FlareListManager.add_background_file_column`). The + source CSV's own at-peak lightcurve counts and attenuator flag are unreliable and are + replaced by these STIX-derived values. See :doc:`/products/flarelist`. + + Parameters + ---------- + start, end : `~datetime.datetime` + Half-open month boundaries; flares with ``start <= start_UTC < end`` are kept. + fido_client : `~stixpy.net.client.STIXClient` + Client used to resolve the quicklook and CPD files during enrichment. + + Returns + ------- + tuple + ``(data, control, energy)`` QTables, or ``(None, None, None)`` if the month is empty. + """ month_data = self.flarelist[ (self.flarelist["start_UTC"] >= start.isoformat()) & (self.flarelist["start_UTC"] < end.isoformat()) ] @@ -1087,8 +1490,9 @@ def get_data(self, *, start, end, fido_client): # unreliable) using one monthly timeline per product built once. energy = self.add_lc_bkg_columns(data, start=start, end=end, fido_client=fido_client) - # select the best quiet-time background data file for each flare peak - self.add_background_file_column(data, fido_client=fido_client) + # select the best quiet-time background data file for each flare peak and extract + # its median quiet-period background spectrum (adds the 32-ch binning to `energy`) + energy = self.add_background_file_column(data, energy=energy, fido_client=fido_client) data.add_index("flare_id") diff --git a/stixcore/io/tests/test_flarelistmanager.py b/stixcore/io/tests/test_flarelistmanager.py index 95b9dd93..a6e90d99 100644 --- a/stixcore/io/tests/test_flarelistmanager.py +++ b/stixcore/io/tests/test_flarelistmanager.py @@ -11,15 +11,20 @@ from stixcore.io.FlareListManager import ( BackgroundSelection, FlareListManager, + background_spectrum_from_cpd, build_month_timeline, + collecting_area_cm2, compute_ql_count_rate, find_background_file_for_time, max_rcr_in_window, nearest_bin_index, + rebin_flux_to_ql, + rebin_spectrum_to_ql, ) from stixcore.io.RidLutManager import RidLutManager, search_background_candidates RATE_UNIT = u.ct / (u.s * u.keV) +FLUX_UNIT = u.ct / (u.s * u.keV * u.cm**2) # --- helpers to build synthetic QL data --------------------------------------- @@ -228,9 +233,10 @@ def test_add_lc_bkg_columns(flare_data, monkeypatch): assert flare_data["lc_peak"].shape == (3, 5) assert flare_data["lc_peak"].unit == u.ct assert np.all(flare_data["lc_peak"].value == np.round(flare_data["lc_peak"].value)) - assert flare_data["lc_peak_rate"].unit.is_equivalent(RATE_UNIT) - assert flare_data["lc_bgk_peak"].shape == (3, 5) - assert flare_data["lc_bgk_peak"].unit == u.ct + assert flare_data["lc_peak_flux"].unit.is_equivalent(FLUX_UNIT) + assert flare_data["lc_bkg_peak_flux"].unit.is_equivalent(FLUX_UNIT) + assert flare_data["lc_bkg_peak"].shape == (3, 5) + assert flare_data["lc_bkg_peak"].unit == u.ct assert flare_data["att_in"].dtype == bool assert flare_data["energy_index"].dtype == np.int8 @@ -248,7 +254,7 @@ def test_add_lc_bkg_columns(flare_data, monkeypatch): # counts pulled from the real timeline for in-range flares assert np.all(flare_data["lc_peak"][0].to_value(u.ct) == 100) - assert np.all(flare_data["lc_bgk_peak"][0].to_value(u.ct) == 100) + assert np.all(flare_data["lc_bkg_peak"][0].to_value(u.ct) == 100) # returned energy table schema assert set(energy.colnames) == {"channel", "e_low", "e_high", "index"} @@ -264,13 +270,47 @@ def test_add_lc_bkg_columns_no_files(flare_data, monkeypatch): ) assert np.all(flare_data["lc_peak"].to_value(u.ct) == 0) - assert np.all(flare_data["lc_bgk_peak"].to_value(u.ct) == 0) + assert np.all(flare_data["lc_bkg_peak"].to_value(u.ct) == 0) assert np.all(flare_data["rcr_at_peak"] == -1) assert np.all(flare_data["rcr_max"] == -1) assert not np.any(flare_data["att_in"]) assert len(energy) == 0 +def _energies_alt(): + """A second, different QL energy binning (distinct edges -> distinct hash).""" + e = QTable() + e["channel"] = np.arange(5, dtype=np.uint8) + e["e_low"] = [4, 11, 16, 26, 51] * u.keV + e["e_high"] = [11, 16, 26, 51, 85] * u.keV + return e + + +def test_energy_table_prunes_orphan_binning(flare_data, monkeypatch): + from datetime import date + + # two LC products for the month with DIFFERENT binnings, both on the same date; + # date_to_eidx keeps the first (primary) binning, so the alternate block is an orphan. + n = 20 + rcr = np.zeros(n, dtype=np.ubyte) + rcr[5:8] = 1 + lc = FakeProduct(_ql_data("2024-06-15T12:00:00", n, rcr=rcr), _energies()) + lc_alt = FakeProduct(_ql_data("2024-06-15T12:00:00", n, rcr=rcr), _energies_alt()) + bkg = FakeProduct(_ql_data("2024-06-15T12:00:00", n, with_rcr=False), _energies()) + products = {"lc": lc, "lc_alt": lc_alt, "bkg": bkg} + monkeypatch.setattr("stixcore.io.FlareListManager.STIXPYProduct", lambda path: products[path]) + fido = FakeFido(lc_paths=["lc", "lc_alt"], bkg_paths=["bkg"]) + + energy = FlareListManager().add_lc_bkg_columns( + flare_data, start=date(2024, 6, 1), end=date(2024, 7, 1), fido_client=fido + ) + + # the orphan alternate block is dropped: only the single referenced 5-row block remains + assert len(energy) == 5 + assert {int(x) for x in energy["index"]} == {0} + assert {int(x) for x in flare_data["energy_index"]} == {0} + + # --- background candidate search (RID LUT) ------------------------------------ @@ -602,8 +642,287 @@ def fake_find(time, *, fido_client, **kwargs): data["peak_UTC"] = Time( ["2023-06-15T00:00:00", "2023-06-16T00:00:00", "2023-06-18T00:00:00"] # 3rd is beyond the 1st period ) - FlareListManager().add_background_file_column(data, fido_client=object()) + # STIXPYProduct isn't patched here; the missing "bkg.fits" is swallowed, leaving NaN spectra. + FlareListManager().add_background_file_column(data, energy=QTable(), fido_client=object()) assert len(calls) == 2 # 1st + 3rd flare trigger a search; 2nd reuses the cache assert list(data["bkg_rid"]) == [42, 42, 42] assert list(data["bkg_file"]) == ["bkg.fits"] * 3 + + +# --- background-spectrum extraction ------------------------------------------ + + +def _sci_energies_32(): + """The 32-channel science energy binning (channel/e_low/e_high in keV).""" + lows = [ + 0, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 18, + 20, + 22, + 25, + 28, + 32, + 36, + 40, + 45, + 50, + 56, + 63, + 70, + 76, + 84, + 100, + 120, + 150, + ] + highs = [ + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 20, + 22, + 25, + 28, + 32, + 36, + 40, + 45, + 50, + 56, + 63, + 70, + 76, + 84, + 100, + 120, + 150, + 1e6, + ] + e = QTable() + e["channel"] = np.arange(32, dtype=np.uint8) + e["e_low"] = lows * u.keV + e["e_high"] = highs * u.keV + return e + + +class FakeCPD: + """CPD stand-in: builds a data table with counts + timedel + zero triggers + (livefrac=1) + pixel_masks (first npix pixels present) so the flux path runs.""" + + def __init__(self, counts, energies, control=None, timedel_s=1.0, dsun_m=1.496e11): + nt, ndet, npix, nE = counts.shape + d = QTable() + d["counts"] = counts + d["timedel"] = np.full(nt, timedel_s) * u.s + d["triggers"] = np.zeros((nt, 16)) + pm = np.zeros((nt, 12), dtype=np.ubyte) + pm[:, :npix] = 1 + d["pixel_masks"] = pm + self.data = d + self.energies = energies + self.control = control + self.meta = {"DSUN_OBS": dsun_m} # SOLO-Sun distance (m), as in a CPD primary header + + +class FakeCtrlProduct: + """Minimal product exposing a control table with detector/pixel masks.""" + + def __init__(self, detector_mask, pixel_mask): + c = QTable() + c["detector_mask"] = [np.asarray(detector_mask, dtype=np.ubyte)] + c["pixel_mask"] = [np.asarray(pixel_mask, dtype=np.ubyte)] + self.control = c + + +def test_collecting_area_cm2(): + # 30 imaging detectors, all 12 pixels -> 30 * sum(12 pixel areas) + det = np.ones(32, dtype=np.ubyte) + det[[8, 9]] = 0 + from stixcore.io.FlareListManager import _pixel_area_cm2 + + a_full = collecting_area_cm2(FakeCtrlProduct(det, np.ones(12))) + assert np.isclose(a_full, 30 * _pixel_area_cm2().sum()) + # 8 large pixels only -> smaller + pm8 = np.zeros(12, dtype=np.ubyte) + pm8[:8] = 1 + a8 = collecting_area_cm2(FakeCtrlProduct(det, pm8)) + assert np.isclose(a8, 30 * _pixel_area_cm2()[:8].sum()) + assert a8 < a_full + # no masks -> None + assert collecting_area_cm2(FakeCPD(np.ones((1, 32, 12, 5)) * u.ct, _energies())) is None + + +def test_background_spectrum_from_cpd(): + from stixcore.io.FlareListManager import _pixel_area_cm2 + + nt = 5 + counts = np.ones((nt, 32, 12, 32), dtype=float) + for t in range(nt): + counts[t] *= t + 1 # time-varying so the median is exercised + counts[:, 8] = 9999 # CFL detector - must be excluded + counts[:, 9] = 9999 # BKG monitor - must be excluded + spec, flux, e32 = background_spectrum_from_cpd(FakeCPD(counts * u.ct, _sci_energies_32(), timedel_s=1.0)) + # counts: 30 imaging dets * 12 pixels * (t+1) -> [360..1800]; median = 1080 + assert spec.shape == (32,) + assert spec.unit == u.ct + assert np.allclose(spec.to_value(u.ct), 1080.0) + assert len(e32) == 32 + # flux = counts / (exp * area * dE); triggers=0 -> livefrac=1, timedel=1s + # exp = 30 dets * 1s = 30; area = sum(12 pixel areas) + dE = e32["e_high"].to_value(u.keV) - e32["e_low"].to_value(u.keV) + area = _pixel_area_cm2().sum() + expected = 1080.0 / (30.0 * area * dE) + assert flux.shape == (32,) + assert flux.unit.is_equivalent(FLUX_UNIT) + assert np.allclose(flux.to_value(FLUX_UNIT), expected, rtol=1e-6) + + +def test_background_spectrum_from_cpd_ragged_energies(): + # real archive CPDs telemeter a subset of channels (e.g. 20) with fewer pixels (8); + # counts nE must stay aligned to the energies table (no forced 32-channel expansion) + energies = _sci_energies_32()[:20] # 20-channel binning + counts = np.ones((4, 32, 8, 20), dtype=float) + counts[:, 8] = 5.0 # CFL - excluded + counts[:, 9] = 5.0 # BKG monitor - excluded + spec, flux, e = background_spectrum_from_cpd(FakeCPD(counts * u.ct, energies)) + assert spec.shape == (20,) # matches energies, not forced to 32 + assert len(e) == 20 + assert np.allclose(spec.to_value(u.ct), 30 * 8) # 30 imaging dets * 8 pixels * 1 count + assert flux.shape == (20,) + assert np.all(np.isfinite(flux.to_value(FLUX_UNIT))) + # rebin against a QL block works (lengths consistent) and preserves units + c5 = rebin_spectrum_to_ql(spec, e, _energies()) + assert c5.shape == (5,) + assert c5.unit == u.ct + assert c5.to_value(u.ct)[0] == 6 * (30 * 8) # 4-10 keV -> 6 sci channels + f5 = rebin_flux_to_ql(flux, e, _energies()) + assert f5.shape == (5,) + assert f5.unit.is_equivalent(FLUX_UNIT) + assert np.isfinite(f5.to_value(FLUX_UNIT)[0]) + + +def test_rebin_flux_to_ql(): + e32 = _sci_energies_32() + # constant flux density -> dE-weighted mean is the same constant in every band + flux = np.full(32, 3.0) + f5 = rebin_flux_to_ql(flux, e32, _energies()) + assert np.allclose(f5, 3.0) + # a shifted top band changes the channel set but a constant stays constant; + # use a ramp to show band membership matters + flux2 = np.arange(32, dtype=float) + a = rebin_flux_to_ql(flux2, e32, _energies()) + ql2 = _energies() + ql2["e_high"] = [10, 15, 25, 50, 100] * u.keV + b = rebin_flux_to_ql(flux2, e32, ql2) + assert a[4] != b[4] # 50-84 vs 50-100 include different channels + + +def test_rebin_spectrum_to_ql(): + e32 = _sci_energies_32() + counts32 = np.arange(32, dtype=float) # channel c contributes c counts + ql = _energies() # QL bands read from the (energy) table, not hardcoded + c5 = rebin_spectrum_to_ql(counts32, e32, ql) + assert c5[0] == sum(range(1, 7)) # 4-10 keV -> ch 1..6 + assert c5[1] == sum(range(7, 12)) # 10-15 -> ch 7..11 + assert c5[2] == sum(range(12, 17)) # 15-25 -> ch 12..16 + assert c5[3] == sum(range(17, 23)) # 25-50 -> ch 17..22 + assert c5[4] == sum(range(23, 28)) # 50-84 -> ch 23..27 + + # a shifted top band (50-100) must pull in channel 28 (84-100): proves it follows the table + ql2 = _energies() + ql2["e_high"] = [10, 15, 25, 50, 100] * u.keV + c5b = rebin_spectrum_to_ql(counts32, e32, ql2) + assert c5b[4] == sum(range(23, 29)) + assert c5b[4] != c5[4] + + +def test_add_background_file_column_extracts_and_caches(monkeypatch): + t0 = "2024-10-01T00:00:00" + T0 = Time(t0) + data = QTable() + data["flare_id"] = [1, 2, 3, 4] + data["peak_UTC"] = Time([t0, "2024-10-01T01:00:00", "2024-10-01T02:00:00", "2024-10-01T05:00:00"]) + data["energy_index"] = np.zeros(4, dtype=np.int8) # all reference the QL block (index 0) + + energy = _energies() + energy["index"] = np.zeros(5, dtype=np.int8) # QL 5-ch block, index 0 + + def fake_find(time, *, fido_client, **kwargs): + if time <= T0 + 1.5 * u.h: + return BackgroundSelection(path="cpdA", rid=100, valid_from=T0, valid_to=T0 + 1.5 * u.h) + if time <= T0 + 3.5 * u.h: + return BackgroundSelection(path="cpdB", rid=200, valid_from=T0 + 1.5 * u.h, valid_to=T0 + 3.5 * u.h) + return BackgroundSelection(path=None, rid=-1, valid_from=time, valid_to=time + 1 * u.h) + + monkeypatch.setattr(flm_mod, "find_background_file_for_time", fake_find) + + opens = [] + + dsun_by_path = {"cpdA": 0.5 * 1.495978707e11, "cpdB": 0.8 * 1.495978707e11} # m + + def fake_product(path): + opens.append(path) + return FakeCPD(np.ones((3, 32, 12, 32)) * u.ct, _sci_energies_32(), dsun_m=dsun_by_path[path]) + + monkeypatch.setattr(flm_mod, "STIXPYProduct", fake_product) + + energy = FlareListManager().add_background_file_column(data, energy=energy, fido_client=object()) + + # each CPD opened exactly once (cpdA shared by flares 0 & 1; cpdB flare 2; flare 3 has no file) + assert opens == ["cpdA", "cpdB"] + assert list(data["bkg_rid"]) == [100, 100, 200, -1] + # both CPDs share the same 32-ch binning -> one new block (index 1); QL block stays index 0 + assert {int(x) for x in energy["index"]} == {0, 1} + assert int(np.sum(np.asarray(energy["index"]) == 1)) == 32 + assert list(data["bkg_energy_index"]) == [1, 1, 1, -1] + # counts spectra filled for in-file flares, NaN for the no-file flare + assert data["bkg_spec"].shape == (4, 32) + assert not np.isnan(data["bkg_spec"][0].to_value(u.ct)).any() + assert np.isnan(data["bkg_spec"][3].to_value(u.ct)).all() + assert np.allclose(data["bkg_spec"][0].to_value(u.ct), 360.0) # 30 dets * 12 pix * 1 + # counts rebinned to the QL bands + assert data["bkg_spec_ql"].shape == (4, 5) + assert data["bkg_spec_ql"].unit == u.ct + assert not np.isnan(data["bkg_spec_ql"][0].to_value(u.ct)).any() + assert np.isnan(data["bkg_spec_ql"][3].to_value(u.ct)).all() + # each QL band sums whole native channels (360 ct each), so every band is a multiple of 360 + assert np.allclose(data["bkg_spec_ql"][0].to_value(u.ct) % 360.0, 0.0) + # flux columns (ct/s/keV/cm2), native + QL-rebinned + assert data["bkg_spec_flux"].shape == (4, 32) + assert data["bkg_spec_flux"].unit.is_equivalent(FLUX_UNIT) + assert not np.isnan(data["bkg_spec_flux"][0].to_value(FLUX_UNIT)).any() + assert np.isnan(data["bkg_spec_flux"][3].to_value(FLUX_UNIT)).all() + assert data["bkg_spec_flux_ql"].shape == (4, 5) + assert not np.isnan(data["bkg_spec_flux_ql"][0].to_value(FLUX_UNIT)).any() + assert np.isnan(data["bkg_spec_flux_ql"][3].to_value(FLUX_UNIT)).all() + # bkg_solo_sun_distance read from each CPD's DSUN_OBS header (km); NaN for the no-file flare + d = data["bkg_solo_sun_distance"].to_value(u.AU) + assert np.isclose(d[0], 0.5) + assert np.isclose(d[1], 0.5) + assert np.isclose(d[2], 0.8) + assert np.isnan(d[3]) diff --git a/stixcore/processing/FlareListL3.py b/stixcore/processing/FlareListL3.py index 76f5c070..20a5f3ec 100644 --- a/stixcore/processing/FlareListL3.py +++ b/stixcore/processing/FlareListL3.py @@ -54,19 +54,19 @@ def find_processing_months(self, phs: ProcessingHistoryStorage) -> list[date]: return fl_months def test_for_processing(self, month: date, phm: ProcessingHistoryStorage) -> TestForProcessingResult: - """_summary_ + """Decide whether the flare list for ``month`` needs (re)processing. Parameters ---------- - candidate : Path - a fits file candidate + month : date + the month to test phm : ProcessingHistoryStorage the processing history persistent handler Returns ------- TestForProcessingResult - what should happen with the candidate in the next processing step + what should happen with the month in the next processing step """ try: wp = phm.has_processed_fits_products( diff --git a/stixcore/processing/pipeline_daily.py b/stixcore/processing/pipeline_daily.py index 642ed876..680e9ec3 100644 --- a/stixcore/processing/pipeline_daily.py +++ b/stixcore/processing/pipeline_daily.py @@ -25,7 +25,7 @@ from stixcore.processing.pipeline import PipelineStatus from stixcore.processing.SingleStep import SingleProcessingStepResult from stixcore.products.level1.quicklookL1 import LightCurve -from stixcore.products.level3.flarelist import FlarelistSDC, FlarelistSDCLoc +from stixcore.products.level3.flarelist import FlarelistSDC, FlarelistSDCLocation from stixcore.products.lowlatency.quicklookLL import LightCurveL3 from stixcore.soop.manager import SOOPManager from stixcore.util.logging import STX_LOGGER_DATE_FORMAT, STX_LOGGER_FORMAT, get_logger @@ -248,10 +248,10 @@ def run_daily_pipeline(args): fits_in_dir, fits_out_dir, products_in_out=[ - (FlarelistSDC, FlarelistSDCLoc), - # (FlarelistSDCLoc, FlarelistSDCLocImg), - # (FlarelistSC, FlarelistSCLoc), - # (FlarelistSCLoc, FlarelistSCLocImg), + (FlarelistSDC, FlarelistSDCLocation), + # (FlarelistSDCLocation, FlarelistSDCLocationImage), + # (FlarelistSC, FlarelistSCLocation), + # (FlarelistSCLocation, FlarelistSCLocationImage), ], cadence=timedelta(seconds=1), ) @@ -274,7 +274,8 @@ def run_daily_pipeline(args): # TODO reactivate once flarelist processing is finalized fl_sdc_months = flarelist_sdc.find_processing_months(phs) - # fl_sdc_months = [] + fl_sdc_months = [] + # fl_sdc_months = [f for f in fl_sdc_months if f.year == 2024 and f.month == 10] # TODO reactivate once flarelist processing is finalized # fl_sc_months = flarelist_sc.find_processing_months(phs) @@ -284,8 +285,7 @@ def run_daily_pipeline(args): fl_to_fl_files = fl_to_fl.get_processing_files(phs) # fl_to_fl_files = fl_to_fl_files[2:-1] # fl_to_fl_files = [] - - fl_to_fl_files = [f for f in fl_to_fl_files if "/2024/" in str(f[2])] + fl_to_fl_files = [f for f in fl_to_fl_files if "/2024/10/" in str(f[2]) and "V04" in str(f[2])] # all processing files should be terminated before the next step as the different # processing steeps might create new candidates diff --git a/stixcore/products/level3/flarelist.py b/stixcore/products/level3/flarelist.py index e38f5cf2..21418ccc 100644 --- a/stixcore/products/level3/flarelist.py +++ b/stixcore/products/level3/flarelist.py @@ -1,6 +1,7 @@ from pathlib import Path from datetime import datetime from itertools import groupby +from collections import namedtuple import numpy as np from stixpy.calibration.visibility import ( @@ -28,7 +29,7 @@ from stixcore.ephemeris.manager import Spice from stixcore.products.level3.flarelistproduct import PeakPreviewImage from stixcore.products.level3.processing import stx_estimate_flare_location -from stixcore.products.product import CountDataMixin, GenericProduct, L2Mixin, read_qtable +from stixcore.products.product import CountDataMixin, GenericProduct, L2Mixin from stixcore.soop.manager import SOOPManager from stixcore.time import SCETime, SCETimeRange from stixcore.util.logging import get_logger @@ -41,11 +42,13 @@ "FlarePositionMixin", "FlareSOOPMixin", "FlareList", - "FlarelistSDCLocImg", + "FlarelistSDCLocation", + "FlarelistSDCLocationImage", "FlarePeakPreviewMixin", "FlarelistSC", - "FlarelistSCLoc", - "FlarelistSCLocImg", + "FlarelistSCLocation", + "FlarelistSCLocationImage", + "add_distance_normalized_flux", ] logger = get_logger(__name__) @@ -78,6 +81,106 @@ def make_stix_fitswcs_header(data, flare_position, *, scale, exposure, rotation_ return header +#: Which per-flare distance column each flux column is normalized by. LC peak fluxes use the +#: flare-time SOLO distance; the quiet-period background fluxes use the background CPD's own time. +_FLUX_DISTANCE_COLS = { + "lc_peak_flux": "solo_sun_distance", + "lc_bkg_peak_flux": "solo_sun_distance", + "bkg_spec_flux": "bkg_solo_sun_distance", + "bkg_spec_flux_ql": "bkg_solo_sun_distance", +} + + +def add_distance_normalized_flux(data, mapping=_FLUX_DISTANCE_COLS): + """Add ``_at_1au`` = `` * (r/1AU)**2`` for each present flux column, using the + per-flare distance in its mapped column (flux scales as 1/r**2). + + The existing flux columns are kept unchanged; the scale factor is dimensionless so the + ``*_at_1au`` columns keep the flux unit (ct/s/keV/cm2). NaN distance -> NaN; a flux column + (or its distance column) that is absent is skipped. + """ + for col, dist_col in mapping.items(): + if col not in data.colnames or dist_col not in data.colnames: + continue + factor = (data[dist_col] / (1 * u.AU)).decompose().value ** 2 # (N,), dimensionless + f = data[col] + data[col + "_at_1au"] = f * (factor[:, None] if f.ndim == 2 else factor) + data[ + col + "_at_1au" + ].info.description = f"{col} scaled to what would be seen at 1 AU (x (r_solo/1AU)**2, r_solo from {dist_col})" + + +#: Per-flare result of :meth:`FlarePositionMixin.add_flare_position`. One record per flare row; +#: skipped/failed flares use :func:`_empty_flare_position` (NaN geometry, ``peak_time`` placeholder). +FlarePositionResult = namedtuple( + "FlarePositionResult", + [ + "anc_path", + "cpd_path", + "status", + "message", + "flare_x", # HGS cartesian, km + "flare_y", + "flare_z", + "solo_time", # location time center + "duration", # location window length + "solo_x", # SOLO HGS cartesian, km + "solo_y", + "solo_z", + "rcr_at_peak", + "sidelobe", + "min_exposure", # CPD per-bin exposure ds over the flare window (u.ds) + "max_exposure", + ], +) + + +def _empty_flare_position(solo_time, **overrides): + """A :class:`FlarePositionResult` for a skipped/failed flare. + + NaN geometry and zero window, with ``solo_time`` kept as the per-row time placeholder (it + must stay a valid `~astropy.time.Time` for the downstream coordinate/Spice handling). + ``overrides`` set the few fields a given skip site knows, e.g. ``message`` / ``anc_path`` / + ``cpd_path``. + """ + base = dict( + anc_path="", + cpd_path="", + status=False, + message="", + flare_x=np.nan * u.km, + flare_y=np.nan * u.km, + flare_z=np.nan * u.km, + solo_time=solo_time, + duration=0 * u.s, + solo_x=np.nan * u.km, + solo_y=np.nan * u.km, + solo_z=np.nan * u.km, + rcr_at_peak=0, + sidelobe=np.nan, + min_exposure=np.nan * u.ds, + max_exposure=np.nan * u.ds, + ) + base.update(overrides) + return FlarePositionResult(**base) + + +def cpd_timedel_range(times, timedels, start, end): + """Min and max CPD time-step ``ds`` (``timedel``) for bins overlapping ``[start, end]``. + + Uses the same half-bin overlap test as the peak-window selection so partially covered + windows still contribute. Returns ``(min_exposure, max_exposure)`` as + `~astropy.units.Quantity` in deciseconds (``u.ds``, STIX's native ``timedel`` unit); + ``(NaN ds, NaN ds)`` if no bin overlaps. + """ + half = timedels / 2 + mask = (times + half >= start) & (times - half <= end) + sel = timedels[mask] + if len(sel) == 0: + return np.nan * u.ds, np.nan * u.ds + return sel.min().to(u.ds), sel.max().to(u.ds) + + class _SerializeMixin: """No-op chain terminator for on_serialize/on_deserialize. @@ -93,7 +196,19 @@ def on_deserialize(self, data, **kwargs): class FlarePositionMixin(_SerializeMixin): - """_summary_""" + """Mixin adding a STIX-derived flare location to a flare-list product. + + For every flare it selects a compressed-pixel-data (CPD) file, estimates the source + position by back-projection imaging (`~stixcore.products.level3.processing.stx_estimate_flare_location`), + and stores the location (Heliographic Stonyhurst / Helioprojective), the Solar Orbiter + position and distance, an imaging-quality metric and related timing columns. On + serialization the location columns are converted to ICRS (and back on deserialization) so + they survive the FITS round-trip. + + See :doc:`/products/flarelist` for the CPD-selection and imaging details. Used by + `~stixcore.products.level3.flarelist.FlarelistSDCLocation` (and the image product built on + it). + """ @classmethod def add_flare_position( @@ -109,11 +224,30 @@ def add_flare_position( keep_all_flares=True, month=None, ): - anc_ephemeris_paths = [] - cpd_paths = [] - position_statuses = [] - position_messages = [] - solo_cartesian_list = [] + """Estimate and add the flare location columns for every flare in ``data``. + + For each flare passing ``filter_function`` the daily ancillary ephemeris and the + science CPD file(s) covering ``[start, end]`` are looked up; the best CPD is scored and + selected, a constant-``rcr`` time window around the peak is chosen, and the location is + estimated by back-projection imaging. See :doc:`/products/flarelist` for the selection + and imaging parameters. + + Parameters + ---------- + data : `~astropy.table.QTable` + The flare list; location columns are added in place. + fido_client : `~stixpy.net.client.STIXClient` + Client used to search for the ephemeris and CPD files. + filter_function : callable, optional + ``row -> bool`` deciding which flares to process (default: all). + peak_time_colname, start_time_colname, end_time_colname, location_time_colname : str + Names of the time columns to read / write. + keep_all_flares : bool, optional + If False, flares that did not pass ``filter_function`` are removed from ``data``. + month : optional + Month being processed, used for logging only. + """ + position_results = [] to_remove = [] pass_filter = 0 @@ -126,10 +260,6 @@ def add_flare_position( day_asp_ephemeris_cache = dict() for i, row in enumerate(data): - _anc_path = "" - _cpd_path = "" - _status = False - _message = "" peak_time = row[peak_time_colname] start_time = row[start_time_colname] end_time = row[end_time_colname] @@ -151,93 +281,56 @@ def add_flare_position( if len(anc_res) < 1: logger.warning(f"No ephemeris data found for flare at time {start_time} : {end_time}") - _message = "no ephemeris data found" no_ephemeris += 1 - solo_cartesian_list.append( - ( - np.nan * u.km, - np.nan * u.km, - np.nan * u.km, - peak_time, - 0 * u.s, - np.nan * u.km, - np.nan * u.km, - np.nan * u.km, - 0, - np.nan, - ) - ) - anc_ephemeris_paths.append(_anc_path) - cpd_paths.append(_cpd_path) - position_statuses.append(_status) - position_messages.append(_message) + position_results.append(_empty_flare_position(peak_time, message="no ephemeris data found")) continue _anc_path = str(anc_res["path"][0]) - if start_time.datetime.hour < 2: - start_time = start_time - 2 * u.hour + # widen only the FIDO search window near midnight; keep the true flare start_time + search_start = start_time - 2 * u.hour if start_time.datetime.hour < 2 else start_time cpd_res = fido_client.search( - a.Time(start_time, end_time), a.Instrument.stix, a.stix.DataProduct.sci_xray_cpd + a.Time(search_start, end_time), a.Instrument.stix, a.stix.DataProduct.sci_xray_cpd ) if cpd_res: cpd_res.filter_for_latest_version() url_to_path(cpd_res) if len(cpd_res) < 1: - logger.warning(f"No CPD data found for flare at time {start_time} : {end_time}") - _message = "no CPD data found" + logger.warning(f"No CPD data found for flare at time {search_start} : {end_time}") no_cpd += 1 - solo_cartesian_list.append( - ( - np.nan * u.km, - np.nan * u.km, - np.nan * u.km, - peak_time, - 0 * u.s, - np.nan * u.km, - np.nan * u.km, - np.nan * u.km, - 0, - np.nan, - ) + position_results.append( + _empty_flare_position(peak_time, anc_path=_anc_path, message="no CPD data found") ) - anc_ephemeris_paths.append(_anc_path) - cpd_paths.append(_cpd_path) - position_statuses.append(_status) - position_messages.append(_message) continue if len(cpd_res) > 1: - logger.debug(f"Many CPD data found for flare at time {start_time} : {end_time}") - # select the best available CPD data file - cpd_res["inc_peak"] = np.logical_and( - peak_time >= cpd_res["Start Time"], peak_time <= cpd_res["End Time"] - ) - cpd_res["file_size"] = [path.stat().st_size for path in cpd_res["path"]] - cpd_res["exposure"] = 0.0 - cpd_res["duration"] = 0.0 - cpd_res["tbins"] = 0 - cpd_res["ebins"] = 0 - cpd_res["estart"] = 0.0 - cpd_res["eend"] = 0.0 - cpd_res["maxcount"] = 0 + logger.debug(f"Many CPD data found for flare at time {search_start} : {end_time}") + # select the best available CPD data file (full product read per candidate) + cpd_res["inc_peak"] = False # flare peak time inside the file + cpd_res["inc_flare"] = 0.0 # % of the flare duration covered by the file + cpd_res["min_dt"] = np.inf # min time resolution (ds) during the flare; inf if no overlap + cpd_res["ebins"] = 0 # number of energy bins (energy table) many_cpd += 1 - for i, path in enumerate(cpd_res["path"]): - header = fits.getheader(path) - header_data = fits.getheader(path, "DATA") - energies = read_qtable(path, "ENERGIES") - cpd_res["maxcount"][i] = header["DATAMAX"] - cpd_res["exposure"][i] = header["XPOSURE"] - cpd_res["tbins"][i] = header_data["NAXIS2"] - cpd_res["ebins"][i] = len(energies) - cpd_res["estart"][i] = energies["e_low"][0].value - cpd_res["eend"][i] = ( - energies["e_high"][-1].value if len(energies) < 31 else energies["e_high"][-2].value - ) - cpd_res["duration"][i] = header["OBT_END"] - header["OBT_BEG"] - + flare_dur_s = (end_time - start_time).sec # flare duration in seconds + for ci, path in enumerate(cpd_res["path"]): + cpd = STIXPYProduct(Path(path)) + # FIDO reports Start/End Time as strings; parse to Time for arithmetic + file_start = Time(cpd_res["Start Time"][ci]) + file_end = Time(cpd_res["End Time"][ci]) + cpd_res["inc_peak"][ci] = file_start <= peak_time <= file_end + # percentage of the flare [start_time, end_time] duration covered by the file + overlap_s = (min(file_end, end_time) - max(file_start, start_time)).sec + cpd_res["inc_flare"][ci] = 100.0 * max(0.0, overlap_s) / flare_dur_s if flare_dur_s > 0 else 0.0 + # shortest time resolution among bins overlapping the flare (shorter is better) + dt_min, _ = cpd_timedel_range(cpd.data["time"], cpd.data["timedel"], start_time, end_time) + cpd_res["min_dt"][ci] = dt_min.to_value(u.ds) if np.isfinite(dt_min.value) else np.inf + cpd_res["ebins"][ci] = len(cpd.energies) + + # best = peak inside, then most flare coverage, then shortest time resolution, + # then most energy bins. shorter min_dt is better, so sort on its negative. # TODO: add more criteria to select the best CPD file - cpd_res.sort(["inc_peak", "tbins", "duration"], reverse=True) + cpd_res["_neg_min_dt"] = -cpd_res["min_dt"] + cpd_res.sort(["inc_peak", "inc_flare", "_neg_min_dt", "ebins"], reverse=True) # cpd_res.pprint() best_cpd_idx = 0 else: @@ -247,6 +340,12 @@ def add_flare_position( try: stixpy_cpd = STIXPYProduct(Path(_cpd_path)) + + # CPD per-bin exposure (ds) over the full flare start..end window (partial overlap ok) + min_exposure, max_exposure = cpd_timedel_range( + stixpy_cpd.data["time"], stixpy_cpd.data["timedel"], start_time, end_time + ) + time_range = TimeRange(max(peak_time - 20 * u.s, start_time), min(peak_time + 20 * u.s, end_time)) overlaps = calculate_overlap(stixpy_cpd.time_range, time_range) if overlaps is None: @@ -296,66 +395,61 @@ def add_flare_position( center_hgs = flare_loc.transform_to( HeliographicStonyhurst(obstime=img_time_range.center) ).cartesian - solo_cartesian_list.append( - ( - center_hgs.x, - center_hgs.y, - center_hgs.z, - img_time_range.center, - img_time_range.seconds, - solo.x, - solo.y, - solo.z, - rcr_at_peak, - sidelobe, + position_results.append( + FlarePositionResult( + anc_path=_anc_path, + cpd_path=_cpd_path, + status=True, + message="OK", + flare_x=center_hgs.x, + flare_y=center_hgs.y, + flare_z=center_hgs.z, + solo_time=img_time_range.center, + duration=img_time_range.seconds, + solo_x=solo.x, + solo_y=solo.y, + solo_z=solo.z, + rcr_at_peak=rcr_at_peak, + sidelobe=sidelobe, + min_exposure=min_exposure, + max_exposure=max_exposure, ) ) - - _status = True - _message = "OK" except Exception as e: - _status = False - _message = f"Error: {type(e)}" logger.warning(f"Error calculating flare position for flare at time {start_time} : {end_time}: {e}") - solo_cartesian_list.append( - ( - np.nan * u.km, - np.nan * u.km, - np.nan * u.km, - peak_time, - 0 * u.s, - np.nan * u.km, - np.nan * u.km, - np.nan * u.km, - 0, - np.nan, + position_results.append( + _empty_flare_position( + peak_time, anc_path=_anc_path, cpd_path=_cpd_path, message=f"Error: {type(e)}" ) ) - anc_ephemeris_paths.append(_anc_path) - cpd_paths.append(_cpd_path) - position_statuses.append(_status) - position_messages.append(_message) else: to_remove.append(i) - solo_cartesian_list.append( - ( - np.nan * u.km, - np.nan * u.km, - np.nan * u.km, - peak_time, - 0 * u.s, - np.nan * u.km, - np.nan * u.km, - np.nan * u.km, - 0, - np.nan, - ) + position_results.append( + _empty_flare_position(peak_time, message="flare did not pass the filter function") ) - anc_ephemeris_paths.append(_anc_path) - cpd_paths.append(_cpd_path) - position_statuses.append(False) - position_messages.append("flare did not pass the filter function") + + # transpose the per-flare records in one pass (a namedtuple is a tuple), field order must + # match FlarePositionResult + ( + anc_ephemeris_paths, + cpd_paths, + position_statuses, + position_messages, + flare_x, + flare_y, + flare_z, + solo_times, + duration, + solo_x, + solo_y, + solo_z, + rcr_at_peak, + sidelobe, + min_exposure, + max_exposure, + ) = zip(*position_results) + solo_times = Time(solo_times) primer = fido_client.baseurl.replace(fido_client.datapath, "") primer = primer[7:] if primer.startswith("file://") else primer @@ -372,11 +466,6 @@ def add_flare_position( data["_position_message"] = position_messages data["_position_message"].info.description = "Message describing the status of the flare position calculation" - flare_x, flare_y, flare_z, solo_times, duration, solo_x, solo_y, solo_z, rcr_at_peak, sidelobe = zip( - *solo_cartesian_list - ) - solo_times = Time(solo_times) - hgs_coords = SkyCoord( u.Quantity(flare_x), u.Quantity(flare_y), @@ -402,6 +491,12 @@ def add_flare_position( data["solo_location_hgs"] = solo_coords data["solo_location_hgs"].info.description = "SOLO location in Heliographic Stonyhurst coordinates" + data["solo_sun_distance"] = solo_coords.cartesian.norm().to(u.km) + data["solo_sun_distance"].info.description = "distance of Solar Orbiter to Sun center" + + # add 1-AU-normalized twins of the flux columns (flux ~ 1/r^2); keeps the originals + add_distance_normalized_flux(data) + data["sidelobes_ratio"] = sidelobe data["sidelobes_ratio"].info.description = "Ratio of sidelobes in the STIX image used to assess imaging quality" @@ -421,6 +516,15 @@ def add_flare_position( data["location_duration"] = duration data["location_duration"].info.description = "duration of the flare location estimation time range" + data["min_exposure"] = u.Quantity(min_exposure) # unit u.ds (deciseconds) + data[ + "min_exposure" + ].info.description = "minimum CPD per-bin exposure ds (timedel) over the flare start..end window" + data["max_exposure"] = u.Quantity(max_exposure) + data[ + "max_exposure" + ].info.description = "maximum CPD per-bin exposure ds (timedel) over the flare start..end window" + ( time_shift, disc_size, @@ -508,12 +612,26 @@ def is_visible(cls, coord): class FlareSOOPMixin(_SerializeMixin): - """_summary_""" + """Mixin adding the Solar Orbiter observing-campaign (SOOP) columns to a flare list. + + For each flare it queries `~stixcore.soop.manager.SOOPManager` for the campaign active at + the flare peak and stores its encoded type, instance id and name (``soop_encoded_type``, + ``soop_id``, ``soop_type``). Used by `~stixcore.products.level3.flarelist.FlarelistSDC`. + """ @classmethod def add_soop( self, data, *, peak_time_colname="peak_UTC", start_time_colname="start_UTC", end_time_colname="end_UTC" ): + """Add the SOOP campaign columns for every flare in ``data`` (in place). + + Parameters + ---------- + data : `~astropy.table.QTable` + The flare list. + peak_time_colname, start_time_colname, end_time_colname : str + Names of the time columns; the campaign is looked up at ``peak_time_colname``. + """ soop_encoded_type = list() soop_id = list() soop_type = list() @@ -574,6 +692,9 @@ def add_peak_preview( keep_all_flares=True, month=None, ): + """Reconstruct and attach per-flare peak-preview CLEAN images (4-20 and 20-120 keV) for + each flare in ``data``, using its selected CPD file, writing one + `~stixcore.products.level3.flarelistproduct.PeakPreviewImage` product per flare.""" data["peak_preview_path"] = Column(" " * 500, dtype=str, description="TDB") data["preview_start_UTC"] = [Time(d, format="isot", scale="utc") for d in data[peak_time_colname]] data["preview_end_UTC"] = [Time(d, format="isot", scale="utc") for d in data[peak_time_colname]] @@ -783,12 +904,19 @@ def enhance_from_product(self, in_prod: GenericProduct): class FlarelistSDC(FlareList, FlareSOOPMixin): - """Flarelist product class for StixDataCenter flares. - - In L3 product format. + """Base SDC flare-list product (``NAME="sdc"``, ssid 2, level L3). + + Mirrors the operational STIX Data Center flare list and enriches every flare with the + quicklook peak / quiet-time background counts and fluxes (added by + `~stixcore.io.FlareListManager.SDCFlareListManager`) and the SOOP campaign (via + `~stixcore.products.level3.flarelist.FlareSOOPMixin`). It is the first level of the SDC + chain: `~stixcore.products.level3.flarelist.FlarelistSDC` -> + `~stixcore.products.level3.flarelist.FlarelistSDCLocation` -> + `~stixcore.products.level3.flarelist.FlarelistSDCLocationImage`. See + :doc:`/products/flarelist`. """ - PRODUCT_PROCESSING_VERSION = 3 + PRODUCT_PROCESSING_VERSION = 4 NAME = "sdc" def __init__(self, *, service_type=0, service_subtype=0, ssid=2, data, month, **kwargs): @@ -838,19 +966,23 @@ def is_datasource_for(cls, *, service_type, service_subtype, ssid, **kwargs): return kwargs["level"] == "L3" and service_type == 0 and service_subtype == 0 and ssid == 2 -class FlarelistSDCLoc(FlarelistSDC, FlarePositionMixin): - """Flarelist product class for StixDataCenter flares. +class FlarelistSDCLocation(FlarelistSDC, FlarePositionMixin): + """SDC flare list with a STIX-derived flare location (``NAME="sdcloc"``, ssid 3, level L3). - In ANC product format. + Extends `~stixcore.products.level3.flarelist.FlarelistSDC` with the flare position + (and SOLO position/distance, imaging-quality metric and 1-AU-normalized fluxes) via + `~stixcore.products.level3.flarelist.FlarePositionMixin`. Only flares above + ``[Processing] flarelist_sdc_min_count`` peak counts are located. See + :doc:`/products/flarelist`. """ - PRODUCT_PROCESSING_VERSION = 3 + PRODUCT_PROCESSING_VERSION = 4 NAME = "sdcloc" def __init__(self, *, service_type=0, service_subtype=0, ssid=3, data, month, **kwargs): super().__init__(service_type=0, service_subtype=0, ssid=3, data=data, month=month, **kwargs) - self.name = FlarelistSDCLoc.NAME + self.name = FlarelistSDCLocation.NAME self.ssid = 3 self.location_time_colname = "location_time_UTC" @@ -879,19 +1011,22 @@ def is_datasource_for(cls, *, service_type, service_subtype, ssid, **kwargs): return kwargs["level"] == "L3" and service_type == 0 and service_subtype == 0 and ssid == 3 -class FlarelistSDCLocImg(FlarelistSDCLoc, FlarePeakPreviewMixin): - """Flarelist product class for StixDataCenter flares. +class FlarelistSDCLocationImage(FlarelistSDCLocation, FlarePeakPreviewMixin): + """Located SDC flare list plus peak-preview images (``NAME="sdclocimg"``, ssid 4, level L3). - In ANC product format. + Extends `~stixcore.products.level3.flarelist.FlarelistSDCLocation` with per-flare + peak-preview CLEAN images (`~stixcore.products.level3.flarelistproduct.PeakPreviewImage`), + reconstructed on the same CPD file used for the flare location, via + `~stixcore.products.level3.flarelist.FlarePeakPreviewMixin`. See :doc:`/products/flarelist`. """ - PRODUCT_PROCESSING_VERSION = 2 + PRODUCT_PROCESSING_VERSION = 4 NAME = "sdclocimg" def __init__(self, *, service_type=0, service_subtype=0, ssid=4, data, month, **kwargs): super().__init__(service_type=0, service_subtype=0, ssid=4, data=data, month=month, **kwargs) - self.name = FlarelistSDCLocImg.NAME + self.name = FlarelistSDCLocationImage.NAME self.ssid = 4 def enhance_from_product(self, in_prod: GenericProduct): @@ -976,7 +1111,7 @@ def is_datasource_for(cls, *, service_type, service_subtype, ssid, **kwargs): return kwargs["level"] == "L3" and service_type == 0 and service_subtype == 0 and ssid == 6 -class FlarelistSCLoc(FlarelistSC, FlarePositionMixin): +class FlarelistSCLocation(FlarelistSC, FlarePositionMixin): """Flarelist product class for STIXCore flares. In L3 product format. @@ -988,7 +1123,7 @@ class FlarelistSCLoc(FlarelistSC, FlarePositionMixin): def __init__(self, *, service_type=0, service_subtype=0, ssid=7, data, month, **kwargs): super().__init__(service_type=0, service_subtype=0, ssid=7, data=data, month=month, **kwargs) - self.name = FlarelistSCLoc.NAME + self.name = FlarelistSCLocation.NAME self.ssid = 7 self.peak_time_colname = "peak_UTC" @@ -1017,7 +1152,7 @@ def is_datasource_for(cls, *, service_type, service_subtype, ssid, **kwargs): return kwargs["level"] == "L3" and service_type == 0 and service_subtype == 0 and ssid == 7 -class FlarelistSCLocImg(FlarelistSCLoc, FlarePeakPreviewMixin): +class FlarelistSCLocationImage(FlarelistSCLocation, FlarePeakPreviewMixin): """Flarelist product class for StixCore flares. In ANC product format. @@ -1029,7 +1164,7 @@ class FlarelistSCLocImg(FlarelistSCLoc, FlarePeakPreviewMixin): def __init__(self, *, service_type=0, service_subtype=0, ssid=8, data, month, **kwargs): super().__init__(service_type=0, service_subtype=0, ssid=8, data=data, month=month, **kwargs) - self.name = FlarelistSCLocImg.NAME + self.name = FlarelistSCLocationImage.NAME self.ssid = 8 def enhance_from_product(self, in_prod: GenericProduct): diff --git a/stixcore/products/level3/flarelistproduct.py b/stixcore/products/level3/flarelistproduct.py index 032864e1..0729ae50 100644 --- a/stixcore/products/level3/flarelistproduct.py +++ b/stixcore/products/level3/flarelistproduct.py @@ -19,16 +19,37 @@ class FlareListProduct(GenericProduct, L3Mixin): @classmethod def from_timerange(cls, timerange: SCETimeRange, *, flarelistparent: str = ""): + """Create a product for the given time range and parent flare list. + + Parameters + ---------- + timerange : `~stixcore.time.datetime.SCETimeRange` + The time range the product covers. + flarelistparent : str, optional + Identifier of the parent flare list. + + Notes + ----- + Not yet implemented (stub). + """ pass class PeakPreviewImage(FlareListProduct): + """Per-flare peak-preview image product (``Name="peakpreviewimg"``, ssid 5, level L3). + + Holds the CLEAN back-projection `~sunpy.map.Map` images reconstructed around a flare peak + (see `~stixcore.products.level3.flarelist.FlarePeakPreviewMixin`), together with the parent + flare-list product(s) they were derived from. One instance is written per flare. + """ + PRODUCT_PROCESSING_VERSION = 1 Level = "L3" Type = "sci" Name = "peakpreviewimg" def __init__(self, control, data, energy, maps, parents, *, product_name_suffix="", **kwargs): + """Build the product from its tables, reconstructed maps and parent product(s).""" super().__init__(service_type=0, service_subtype=0, ssid=5, control=control, data=data, energy=energy, **kwargs) self.name = f"{PeakPreviewImage.Name}-{product_name_suffix}" self.level = PeakPreviewImage.Level @@ -41,14 +62,17 @@ def __init__(self, control, data, energy, maps, parents, *, product_name_suffix= @property def parent(self): + """The parent flare-list product(s) this image was derived from, as a 1-d array.""" return np.atleast_1d(self.parents) @property def utc_timerange(self): + """The preview time range in UTC (`~sunpy.time.TimeRange`).""" return TimeRange(self.data["preview_start_UTC"][0], self.data["preview_end_UTC"][0]) @property def scet_timerange(self): + """The preview time range in spacecraft elapsed time (approximated via Spice).""" tr = self.utc_timerange logger.warning( "scet_timerange will be approximated using Spice. Better to work with utc_timerange property to avoid automatic time conversion" @@ -58,8 +82,10 @@ def scet_timerange(self): return SCETimeRange(start=start, end=end) def split_to_files(self): + """Yield the product(s) to write; a peak-preview image is always a single file.""" return [self] @classmethod def is_datasource_for(cls, *, service_type, service_subtype, ssid, **kwargs): + """Whether this class handles the given L3 / ssid-5 product identifiers.""" return kwargs["level"] == PeakPreviewImage.Level and service_type == 0 and service_subtype == 0 and ssid == 5 diff --git a/stixcore/products/level3/processing.py b/stixcore/products/level3/processing.py index 73106521..722d8282 100644 --- a/stixcore/products/level3/processing.py +++ b/stixcore/products/level3/processing.py @@ -25,6 +25,15 @@ from astropy.coordinates.representation import CartesianRepresentation from astropy.time import Time +__all__ = [ + "get_rsun_obs", + "get_distance_off_limb", + "generate_blank_map", + "is_visible", + "stx_estimate_flare_location", + "calculate_sidelobes_ratio", +] + def get_rsun_obs(observer): """ diff --git a/stixcore/products/tests/test_flarelist.py b/stixcore/products/tests/test_flarelist.py index 07522646..b64066fd 100644 --- a/stixcore/products/tests/test_flarelist.py +++ b/stixcore/products/tests/test_flarelist.py @@ -14,8 +14,12 @@ from stixcore.io.product_processors.fits.processors import FitsL3Processor from stixcore.products.level3.flarelist import ( - FlarelistSDCLoc, + FlarelistSDCLocation, + FlarePositionResult, + _empty_flare_position, + add_distance_normalized_flux, calculate_overlap, + cpd_timedel_range, longest_constant_sequence, ) from stixcore.products.product import Product @@ -54,7 +58,7 @@ def flare_data(): @pytest.fixture def written_fits(flare_data, tmp_path): - prod = FlarelistSDCLoc( + prod = FlarelistSDCLocation( data=flare_data, month=date(2022, 1, 1), control=QTable(), @@ -86,7 +90,7 @@ def test_flarelist_sdcloc_location_roundtrip(written_fits): # read back via Product factory — calls on_deserialize internally recovered = Product(fits_path) - assert isinstance(recovered, FlarelistSDCLoc) + assert isinstance(recovered, FlarelistSDCLocation) assert_quantity_allclose(recovered.data["location_hgs"].lon, orig_hgs_lon, atol=1e-6 * u.deg, equal_nan=True) assert_quantity_allclose(recovered.data["location_hgs"].lat, orig_hgs_lat, atol=1e-6 * u.deg, equal_nan=True) @@ -189,3 +193,98 @@ def test_overlap_identical(): assert result is not None assert result.start == r1.start assert result.end == r1.end + + +# --- add_distance_normalized_flux --- + +FLUX_UNIT = u.ct / (u.s * u.keV * u.cm**2) + + +def test_add_distance_normalized_flux(): + data = QTable() + # flare-time distance (for LC) and background-time distance (for bkg) differ per row + data["solo_sun_distance"] = [0.5, 1.0, np.nan] * u.AU + data["bkg_solo_sun_distance"] = [0.8, 1.0, 0.5] * u.AU + data["lc_peak_flux"] = np.array([[100.0, 10, 1, 5, 2]] * 3) * FLUX_UNIT + data["bkg_spec_flux"] = np.array([[4.0, 3, 2, 1, 0.5]] * 3) * FLUX_UNIT + # lc_bkg_peak_flux / bkg_spec_flux_ql intentionally absent -> must be skipped, no error + + add_distance_normalized_flux(data) + + # new columns added, originals unchanged + assert "lc_peak_flux_at_1au" in data.colnames + assert "bkg_spec_flux_at_1au" in data.colnames + assert "lc_bkg_peak_flux_at_1au" not in data.colnames # source column was absent + assert np.allclose(data["lc_peak_flux"].to_value(FLUX_UNIT), [[100, 10, 1, 5, 2]] * 3) + + # LC uses solo_sun_distance: 0.5 AU -> x0.25, 1 AU -> x1, NaN -> NaN + lc = data["lc_peak_flux_at_1au"].to_value(FLUX_UNIT) + assert np.allclose(lc[0], np.array([100, 10, 1, 5, 2]) * 0.25) + assert np.allclose(lc[1], [100, 10, 1, 5, 2]) + assert np.all(np.isnan(lc[2])) + # bkg uses bkg_solo_sun_distance: 0.8 AU -> x0.64, 0.5 AU (row 2) -> x0.25 + bk = data["bkg_spec_flux_at_1au"].to_value(FLUX_UNIT) + assert np.allclose(bk[0], np.array([4, 3, 2, 1, 0.5]) * 0.64) + assert np.allclose(bk[2], np.array([4, 3, 2, 1, 0.5]) * 0.25) + # unit preserved + assert data["lc_peak_flux_at_1au"].unit.is_equivalent(FLUX_UNIT) + + +# --- cpd_timedel_range --- + + +def test_cpd_timedel_range(): + # four bins at t = 0,10,20,30 s with widths 4,4,10,20 ds (0.4,0.4,1.0,2.0 s) + times = np.array([0, 10, 20, 30]) * u.s + timedels = np.array([4, 4, 10, 20]) * u.ds + + # window fully inside the file -> bins 1 and 2 overlap [5, 25] s + ds_min, ds_max = cpd_timedel_range(times, timedels, 5 * u.s, 25 * u.s) + assert ds_min.unit == u.ds + assert ds_max.unit == u.ds + assert ds_min == 4 * u.ds + assert ds_max == 10 * u.ds + + # window extends beyond the file (not fully covered) -> all bins overlap + ds_min, ds_max = cpd_timedel_range(times, timedels, -100 * u.s, 100 * u.s) + assert ds_min == 4 * u.ds + assert ds_max == 20 * u.ds + + # no overlap -> (NaN ds, NaN ds) + ds_min, ds_max = cpd_timedel_range(times, timedels, 100 * u.s, 200 * u.s) + assert ds_min.unit == u.ds + assert ds_max.unit == u.ds + assert np.isnan(ds_min.value) + assert np.isnan(ds_max.value) + + +# --- _empty_flare_position --- + + +def test_empty_flare_position_defaults(): + peak = Time("2022-01-01T12:00:00") + r = _empty_flare_position(peak) + + assert isinstance(r, FlarePositionResult) + assert r.status is False + assert r.message == "" + assert r.anc_path == "" + assert r.cpd_path == "" + assert r.rcr_at_peak == 0 + assert r.solo_time == peak # per-row placeholder passed through + # NaN geometry + for f in (r.flare_x, r.flare_y, r.flare_z, r.solo_x, r.solo_y, r.solo_z): + assert f.unit == u.km + assert np.isnan(f.value) + assert np.isnan(r.sidelobe) + # new ds columns default to NaN deciseconds + assert r.min_exposure.unit == u.ds + assert np.isnan(r.min_exposure.value) + assert r.max_exposure.unit == u.ds + assert np.isnan(r.max_exposure.value) + + # overrides applied + r2 = _empty_flare_position(peak, message="no CPD data found", anc_path="/some/anc.fits") + assert r2.message == "no CPD data found" + assert r2.anc_path == "/some/anc.fits" + assert r2.status is False # untouched