From 2a72a3672ecaae1e70b0ea3faa4b936ab138e660 Mon Sep 17 00:00:00 2001 From: Cail Daley Date: Sat, 10 Oct 2026 06:44:18 -0400 Subject: [PATCH 1/4] make_cat: N_EPOCH counts only PSF-validated epochs The tile SExtractor step counts every exposure CCD whose footprint holds an object, before any PSF model exists. psfex_interp then drops CCDs that fail PSF validation (STAR_THRESH, CHI2_THRESH), and ngmix fits only epochs with a PSF, so the final N_EPOCH over-counted by one for 0.6-1.3% of objects on real campaigns. make_cat now saves the SExtractor count as N_EPOCH_OVERLAP and writes N_EPOCH from the galaxy_psf catalogue: the number of epochs it holds for the object (0 for "empty"). The runner always runs the psf pass for this column; the per-epoch HSM/EXP_ID/CCD slot families still follow SAVE_PSF_DATA. The default slot count is max(N_EPOCH_OVERLAP) + 1, an upper bound on every object's PSF epoch count. The upstream SExtractor N_EPOCH, used by vignetmaker and psfex_interp/mccd for array sizing, is unchanged. The final catalogue param files list N_EPOCH_OVERLAP. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01DfF1SjWsjHQzZYjL8bbnLo --- example/cfis_image_sims/final_cat.param | 4 +- .../modules/make_cat_package/__init__.py | 8 +- .../modules/make_cat_package/make_cat.py | 67 ++++++-- src/shapepipe/modules/make_cat_runner.py | 12 +- tests/module/test_make_cat.py | 148 +++++++++++++++--- workflow/config/cfis/final_cat.param | 4 +- .../config/cfis_image_sims/final_cat.param | 4 +- 7 files changed, 200 insertions(+), 47 deletions(-) diff --git a/example/cfis_image_sims/final_cat.param b/example/cfis_image_sims/final_cat.param index 5f96d959e..e94994122 100644 --- a/example/cfis_image_sims/final_cat.param +++ b/example/cfis_image_sims/final_cat.param @@ -31,8 +31,10 @@ NGMIX_MCAL_TYPES_FAIL NGMIX_G1_PSF_ORIG_NOSHEAR NGMIX_G2_PSF_ORIG_NOSHEAR -# Number of epochs (exposures) +# Number of epochs: N_EPOCH with a validated PSF model, N_EPOCH_OVERLAP +# exposure CCDs covering the object, NGMIX_N_EPOCH fitted by ngmix N_EPOCH +N_EPOCH_OVERLAP NGMIX_N_EPOCH ## Shape measurement outputs diff --git a/src/shapepipe/modules/make_cat_package/__init__.py b/src/shapepipe/modules/make_cat_package/__init__.py index cf1241168..435000c1c 100644 --- a/src/shapepipe/modules/make_cat_package/__init__.py +++ b/src/shapepipe/modules/make_cat_package/__init__.py @@ -21,7 +21,10 @@ previous module runs. This gathers all relevant information on the measured galaxies for weak-lensing post-processing. This includes galaxy detection and basic measurement parameters, the PSF model at galaxy positions, and the -shape measurement. Every detected object is kept: the catalogue carries no +shape measurement. ``N_EPOCH`` counts the object's epochs with a validated +PSF model, the epochs ngmix can fit; ``N_EPOCH_OVERLAP`` counts the exposure +CCDs whose footprint holds the object, including CCDs whose PSF model failed +validation. Every detected object is kept: the catalogue carries no star/galaxy classification, which is done downstream. Each object is tagged with its source tile via the ``TILE_ID`` column and carries the survey-wide object ID ``TILE_UNIQUE_ID = tile_id * 10**6 + NUMBER`` @@ -43,7 +46,8 @@ (``HSM_*_PSF_n``, ``EXP_ID_n``, ``CCD_n``) when ``SAVE_PSF_DATA`` is ``True``; unfilled slots hold the family's sentinel, and an object with more epochs than slots raises an error. A fixed count gives every tile - the same schema. Default is the tile's maximum ``N_EPOCH`` plus one + the same schema. Default is the tile's maximum ``N_EPOCH_OVERLAP`` plus + one """ diff --git a/src/shapepipe/modules/make_cat_package/make_cat.py b/src/shapepipe/modules/make_cat_package/make_cat.py index dce52d590..eb004beb4 100644 --- a/src/shapepipe/modules/make_cat_package/make_cat.py +++ b/src/shapepipe/modules/make_cat_package/make_cat.py @@ -10,6 +10,7 @@ import re import numpy as np +from numpy.lib.recfunctions import rename_fields from astropy import coordinates as coords from astropy import units as u from astropy.wcs import WCS @@ -106,6 +107,12 @@ def save_sextractor_data(final_cat_file, sexcat_path, remove_vignet=True): of the tile and ``NUMBER``). The tile is read from the catalogue's file name, e.g. ``sexcat-301-279.fits``. + The SExtractor catalogue's ``N_EPOCH`` counts the exposure CCDs whose + footprint holds the object, before any PSF model exists; it is saved as + ``N_EPOCH_OVERLAP``. The final catalogue's ``N_EPOCH`` counts only the + epochs with a validated PSF and is written by + :meth:`SaveCatalogue.process` in ``psf`` mode. + Parameters ---------- final_cat_file : file_io.FITSCatalogue @@ -124,7 +131,9 @@ def save_sextractor_data(final_cat_file, sexcat_path, remove_vignet=True): """ sexcat_file = file_io.FITSCatalogue(sexcat_path, SEx_catalogue=True) sexcat_file.open() - data = np.copy(sexcat_file.get_data()) + data = rename_fields( + np.copy(sexcat_file.get_data()), {"N_EPOCH": "N_EPOCH_OVERLAP"} + ) if remove_vignet: data = remove_field_name(data, "VIGNET") cat_size = len(data) @@ -250,6 +259,7 @@ def process( cat_path=None, moments=False, n_epoch_slots=None, + epoch_slots=True, ): """Process Catalogue. @@ -263,7 +273,10 @@ def process( Option to run ``ngmix`` mode with moments n_epoch_slots : int, optional Number of per-epoch slots in ``psf`` mode; if ``None``, the - tile's ``max(N_EPOCH) + 1`` + tile's ``max(N_EPOCH_OVERLAP) + 1`` + epoch_slots : bool, optional + In ``psf`` mode, write the per-epoch column families as well as + ``N_EPOCH``; default ``True`` Returns -------- @@ -280,7 +293,7 @@ def process( if mode == "ngmix": err_msg = self._save_ngmix_data(cat_path, moments) elif mode == "psf": - self._save_psf_data(cat_path, n_epoch_slots) + self._save_psf_data(cat_path, n_epoch_slots, epoch_slots) else: err_msg = ( f"Invalid process mode ({mode}) for " @@ -589,14 +602,24 @@ def _save_ngmix_data(self, ngmix_cat_path, moments=False): return None - def _save_psf_data(self, galaxy_psf_path, n_epoch_slots=None): + def _save_psf_data( + self, galaxy_psf_path, n_epoch_slots=None, epoch_slots=True + ): """Save PSF data. - Save the PSF catalogue into the final one, as per-epoch column + Save the PSF catalogue into the final one: the per-object epoch + count ``N_EPOCH`` and, if ``epoch_slots``, the per-epoch column families ``HSM_*_PSF_n`` (``psf_shape_cols``), ``EXP_ID_n`` and - ``CCD_n``. Slot ``n`` of every - family refers to the same epoch; slots no epoch fills keep the - family's sentinel. + ``CCD_n``. Slot ``n`` of every family refers to the same epoch; + slots no epoch fills keep the family's sentinel. + + ``N_EPOCH`` counts the object's entries in the PSF catalogue, i.e. + the epochs whose CCD passed PSF-model validation and so have an + interpolated PSF; ngmix fits no other epoch. It is at most + ``N_EPOCH_OVERLAP``, the count of exposure CCDs whose footprint holds + the object, and 0 for an object the PSF catalogue marks + ``"empty"``. An epoch whose PSF shape fit failed + (``HSM_FLAG_PSF != 0``) still counts: its PSF exists. @sc [label:schema] psf-epoch-slot-columns The per-epoch families are ``psf_shape_cols`` plus ``EXP_ID``/``CCD``, @@ -610,7 +633,10 @@ def _save_psf_data(self, galaxy_psf_path, n_epoch_slots=None): Path to the PSF catalogue to save n_epoch_slots : int, optional Number of slots written per family; if ``None``, the tile's - ``max(N_EPOCH) + 1`` + ``max(N_EPOCH_OVERLAP) + 1``, which bounds every object's PSF + epoch count + epoch_slots : bool, optional + Write the per-epoch column families; default ``True`` Raises ------ @@ -620,12 +646,17 @@ def _save_psf_data(self, galaxy_psf_path, n_epoch_slots=None): """ galaxy_psf_cat = SqliteDict(galaxy_psf_path) - n_epoch = self._final_cat_file.get_data()["N_EPOCH"] - if n_epoch_slots is None: - n_slots = np.max(n_epoch) + 1 + n_obj = len(self._obj_id) + n_epoch = np.zeros(n_obj, dtype="int32") + + if not epoch_slots: + n_slots = 0 + elif n_epoch_slots is None: + n_slots = ( + np.max(self._final_cat_file.get_data()["N_EPOCH_OVERLAP"]) + 1 + ) else: n_slots = n_epoch_slots - n_obj = len(self._obj_id) # Per-epoch PSF shape columns copied from the producer's SHAPES dict: # (column, empty-slot fill, dtype). Fills are out of physical range @@ -657,11 +688,15 @@ def _save_psf_data(self, galaxy_psf_path, n_epoch_slots=None): if obj_epochs == "empty": continue + n_epoch[idx] = len(obj_epochs) + if not epoch_slots: + continue + if len(obj_epochs) > n_slots: galaxy_psf_cat.close() raise ValueError( - f"Object {id_tmp} has {len(obj_epochs)} PSF epochs" - + f" (N_EPOCH={n_epoch[idx]}), more than the {n_slots}" + f"Object {id_tmp} has N_EPOCH={n_epoch[idx]} PSF epochs," + + f" more than the {n_slots}" + f" per-epoch slots (N_EPOCH_SLOTS={n_epoch_slots})" ) @@ -689,3 +724,5 @@ def _save_psf_data(self, galaxy_psf_path, n_epoch_slots=None): ) galaxy_psf_cat.close() + + self._output_dict = {"N_EPOCH": n_epoch, **self._output_dict} diff --git a/src/shapepipe/modules/make_cat_runner.py b/src/shapepipe/modules/make_cat_runner.py index 468e7e4e1..5666b9dd3 100644 --- a/src/shapepipe/modules/make_cat_runner.py +++ b/src/shapepipe/modules/make_cat_runner.py @@ -103,10 +103,14 @@ def make_cat_runner( #raise ValueError(err_msg) w_log.info(err_msg) - if save_psf: - err_msg = sc_inst.process( - "psf", galaxy_psf_path, n_epoch_slots=n_epoch_slots - ) + # N_EPOCH (epochs with a validated PSF) always; the per-epoch PSF + # columns only when SAVE_PSF_DATA is set. + sc_inst.process( + "psf", + galaxy_psf_path, + n_epoch_slots=n_epoch_slots, + epoch_slots=save_psf, + ) # Optional per-band external healsparse mask lookup (UNIONS-WL/spherex#38): # add one MASK_ column per band, queried at each object's world diff --git a/tests/module/test_make_cat.py b/tests/module/test_make_cat.py index e29109d19..d453b59cd 100644 --- a/tests/module/test_make_cat.py +++ b/tests/module/test_make_cat.py @@ -397,23 +397,29 @@ def _write_galaxy_psf_cat(path, per_obj): class _FinalCatStub: - """Stand-in for the FITSCatalogue ``_save_psf_data`` reads N_EPOCH from.""" + """Stand-in for the FITSCatalogue ``_save_psf_data`` reads N_EPOCH_OVERLAP from.""" - def __init__(self, n_epoch): - self._n_epoch = np.asarray(n_epoch) + def __init__(self, n_overlap): + self._n_overlap = np.asarray(n_overlap) def get_data(self): - return {"N_EPOCH": self._n_epoch} + return {"N_EPOCH_OVERLAP": self._n_overlap} -def _run_save_psf(galaxy_psf_path, obj_id, n_epoch, n_epoch_slots=None): +def _run_save_psf( + galaxy_psf_path, obj_id, n_overlap, n_epoch_slots=None, epoch_slots=True +): """Drive ``_save_psf_data`` and return its populated output dict.""" inst = object.__new__(SaveCatalogue) inst._obj_id = np.asarray(obj_id) inst._output_dict = {} - inst._final_cat_file = _FinalCatStub(n_epoch) + inst._final_cat_file = _FinalCatStub(n_overlap) - inst._save_psf_data(str(galaxy_psf_path), n_epoch_slots=n_epoch_slots) + inst._save_psf_data( + str(galaxy_psf_path), + n_epoch_slots=n_epoch_slots, + epoch_slots=epoch_slots, + ) return inst._output_dict @@ -439,7 +445,7 @@ def test_save_psf_data_exp_id_ccd_align_with_hsm_psf_slots(tmp_path): } _write_galaxy_psf_cat(galaxy_psf_path, per_obj) - out = _run_save_psf(galaxy_psf_path, [101], n_epoch=[2]) + out = _run_save_psf(galaxy_psf_path, [101], n_overlap=[2]) npt.assert_allclose(out["HSM_G1_PSF_1"], [0.03]) npt.assert_allclose(out["HSM_G1_PSF_2"], [0.01]) @@ -468,7 +474,7 @@ def test_save_psf_data_identity_survives_failed_hsm_fit(tmp_path): } _write_galaxy_psf_cat(galaxy_psf_path, per_obj) - out = _run_save_psf(galaxy_psf_path, [202], n_epoch=[2]) + out = _run_save_psf(galaxy_psf_path, [202], n_overlap=[2]) npt.assert_allclose(out["HSM_G1_PSF_1"], [0.05]) # Sentinel: the epoch-2 HSM fit failed, so the pre-fill value stands. @@ -483,7 +489,7 @@ def test_save_psf_data_identity_survives_failed_hsm_fit(tmp_path): def test_save_psf_data_fills_sentinel_for_absent_epochs(tmp_path): """Unused epoch slots and "empty" objects keep the -1 sentinel. - ``max_epoch`` (from the largest ``N_EPOCH`` across the catalogue) can + ``max_epoch`` (from the largest ``N_EPOCH_OVERLAP`` across the catalogue) can exceed a given object's own epoch count, and an object the PSF catalogue reports no epochs at all for is marked ``"empty"``; both cases must leave EXP_ID_n/CCD_n at -1, an exposure ID / CCD number no @@ -496,8 +502,8 @@ def test_save_psf_data_fills_sentinel_for_absent_epochs(tmp_path): } _write_galaxy_psf_cat(galaxy_psf_path, per_obj) - # max(N_EPOCH) = 2 -> 3 slot columns, though obj 101 only fills slot 1. - out = _run_save_psf(galaxy_psf_path, [101, 303], n_epoch=[1, 2]) + # max(N_EPOCH_OVERLAP) = 2 -> 3 slot columns, though obj 101 only fills slot 1. + out = _run_save_psf(galaxy_psf_path, [101, 303], n_overlap=[1, 2]) assert out["EXP_ID_1"][0] == 2113864 assert out["CCD_1"][0] == 7 @@ -507,6 +513,52 @@ def test_save_psf_data_fills_sentinel_for_absent_epochs(tmp_path): assert out[col][1] == -1, col +def test_save_psf_data_n_epoch_counts_only_psf_validated_epochs(tmp_path): + """N_EPOCH counts the epochs with an interpolated PSF, not the overlaps. + + Object 101 overlaps three CCDs but one failed PSF-model validation, so + the PSF catalogue holds two epochs for it and ngmix sees two; object 202 + has an epoch whose PSF shape fit failed, which still counts (its PSF + exists); object 303 overlaps a CCD with no validated PSF at all and is + ``"empty"``. + """ + galaxy_psf_path = tmp_path / "galaxy_psf.sqlite" + per_obj = { + 101: { + "2603237-12": _psf_epoch(0.01, 0.02, 0.5), + "2603241-12": _psf_epoch(0.03, 0.04, 0.6), + }, + 202: { + "2603237-13": _psf_epoch(0.05, 0.06, 0.7), + "2603241-13": _psf_epoch(-10.0, -10.0, 0.0, flag=5), + }, + 303: "empty", + } + _write_galaxy_psf_cat(galaxy_psf_path, per_obj) + + out = _run_save_psf(galaxy_psf_path, [101, 202, 303], n_overlap=[3, 2, 1]) + + npt.assert_array_equal(out["N_EPOCH"], [2, 2, 0]) + assert out["EXP_ID_3"][0] == -1 + + +def test_save_psf_data_without_epoch_slots_writes_only_n_epoch(tmp_path): + """With per-epoch slots off, ``N_EPOCH`` is the only column written.""" + galaxy_psf_path = tmp_path / "galaxy_psf.sqlite" + per_obj = { + 101: {"2603237-12": _psf_epoch(0.01, 0.02, 0.5)}, + 303: "empty", + } + _write_galaxy_psf_cat(galaxy_psf_path, per_obj) + + out = _run_save_psf( + galaxy_psf_path, [101, 303], n_overlap=[2, 1], epoch_slots=False + ) + + assert list(out) == ["N_EPOCH"] + npt.assert_array_equal(out["N_EPOCH"], [1, 0]) + + # --- make_cat_runner: end-to-end catalogue assembly --- @@ -548,6 +600,7 @@ def test_make_cat_runner_ships_every_detection_unclassified(tmp_path): tile_sexcat_path = tmp_path / "tile_sexcat-350-100.fits" _write_sex_like_cat(tile_sexcat_path, _numbered_data(obj_ids)) galaxy_psf_path = tmp_path / "galaxy_psf-350-100.sqlite" + _write_galaxy_psf_cat(galaxy_psf_path, dict.fromkeys(obj_ids, "empty")) ngmix_path = tmp_path / "ngmix-350-100.fits" _write_ngmix_cat(ngmix_path, obj_ids) @@ -574,6 +627,55 @@ def test_make_cat_runner_ships_every_detection_unclassified(tmp_path): assert not [name for name in data.dtype.names if "SPREAD" in name] +def test_make_cat_runner_n_epoch_excludes_unvalidated_psf_epochs(tmp_path): + """The final N_EPOCH counts PSF-validated epochs; N_EPOCH_OVERLAP keeps the geometry. + + The SExtractor ``N_EPOCH`` counts every exposure CCD covering an object. + Object 1 lies on two CCDs of which one failed PSF validation, object 3 + on one such CCD only. Without SAVE_PSF_DATA the runner still writes + ``N_EPOCH`` from the PSF catalogue, and no per-epoch slot column. + """ + obj_ids = [1, 2, 3] + sexcat = np.array( + list(zip(obj_ids, [2, 3, 1])), + dtype=[("NUMBER", "i8"), ("N_EPOCH", "i8")], + ) + tile_sexcat_path = tmp_path / "tile_sexcat-350-100.fits" + _write_sex_like_cat(tile_sexcat_path, sexcat) + galaxy_psf_path = tmp_path / "galaxy_psf-350-100.sqlite" + _write_galaxy_psf_cat( + galaxy_psf_path, + { + 1: {"2603241-12": _psf_epoch(0.01, 0.02, 0.5)}, + 2: { + "2603237-12": _psf_epoch(0.05, 0.06, 0.7), + "2603241-12": _psf_epoch(0.07, 0.08, 0.9), + "2603246-12": _psf_epoch(0.03, 0.04, 0.6), + }, + 3: "empty", + }, + ) + ngmix_path = tmp_path / "ngmix-350-100.fits" + _write_ngmix_cat(ngmix_path, obj_ids) + + config = CustomParser() + config.read_dict({"MAKE_CAT_RUNNER": {"SHAPE_MEASUREMENT_TYPE": "ngmix"}}) + assert make_cat_runner( + [str(tile_sexcat_path), str(galaxy_psf_path), str(ngmix_path)], + {"output": str(tmp_path)}, + "-350-100", + config, + "MAKE_CAT_RUNNER", + _NullLogger(), + ) == (None, None) + + with fits.open(make_cat.get_output_name(str(tmp_path), "-350-100")) as hdul: + data = hdul[1].data + npt.assert_array_equal(data["N_EPOCH"], [1, 3, 0]) + npt.assert_array_equal(data["N_EPOCH_OVERLAP"], [2, 3, 1]) + assert "EXP_ID_1" not in data.columns.names + + def test_make_cat_runner_work_dir_publishes_same_catalogue( tmp_path, monkeypatch ): @@ -759,8 +861,8 @@ def test_galaxy_cut_admits_only_measured_objects(tmp_path): class _ProcessCatStub(_FinalCatStub): """FITSCatalogue stand-in for ``SaveCatalogue.process``; records add_cols.""" - def __init__(self, obj_id, n_epoch): - super().__init__(n_epoch) + def __init__(self, obj_id, n_overlap): + super().__init__(n_overlap) self._number = np.asarray(obj_id) self.cols = {} @@ -771,7 +873,7 @@ def close(self): pass def get_data(self): - return {"NUMBER": self._number, "N_EPOCH": self._n_epoch} + return {"NUMBER": self._number, "N_EPOCH_OVERLAP": self._n_overlap} def add_cols(self, columns): self.cols.update(columns) @@ -808,7 +910,7 @@ def test_save_psf_data_fixed_slots_pad_every_family(tmp_path): # Driven through ``process``, the entry point the runner calls. n_slots = 7 - cat = _ProcessCatStub([101, 202, 303], n_epoch=[1, 2, 0]) + cat = _ProcessCatStub([101, 202, 303], n_overlap=[1, 2, 0]) sc = SaveCatalogue(cat, 3, _NullLogger()) assert sc.process("psf", str(galaxy_psf_path), n_epoch_slots=n_slots) is None out = cat.cols @@ -841,7 +943,7 @@ def test_save_psf_data_more_epochs_than_slots_raises(tmp_path): with pytest.raises(ValueError) as excinfo: _run_save_psf( - galaxy_psf_path, [101, 404], n_epoch=[1, 3], n_epoch_slots=2 + galaxy_psf_path, [101, 404], n_overlap=[1, 3], n_epoch_slots=2 ) msg = str(excinfo.value) assert "404" in msg @@ -860,15 +962,15 @@ def test_save_psf_data_exactly_slots_epochs_fits(tmp_path): } _write_galaxy_psf_cat(galaxy_psf_path, per_obj) - out = _run_save_psf(galaxy_psf_path, [404], n_epoch=[2], n_epoch_slots=2) + out = _run_save_psf(galaxy_psf_path, [404], n_overlap=[2], n_epoch_slots=2) assert _slot_numbers(out, "EXP_ID") == [1, 2] assert out["EXP_ID_2"][0] == 2229900 npt.assert_allclose(out["HSM_G1_PSF_2"], [0.03]) -def test_save_psf_data_unset_slots_uses_tile_max_n_epoch_plus_one(tmp_path): - """Without N_EPOCH_SLOTS the slot count is the tile's max(N_EPOCH) + 1.""" +def test_save_psf_data_unset_slots_uses_tile_max_n_overlap_plus_one(tmp_path): + """Without N_EPOCH_SLOTS the slot count is the tile's max(N_EPOCH_OVERLAP) + 1.""" galaxy_psf_path = tmp_path / "galaxy_psf.sqlite" per_obj = { 101: {"2113864-7": _psf_epoch(0.01, 0.02, 0.5)}, @@ -880,7 +982,7 @@ def test_save_psf_data_unset_slots_uses_tile_max_n_epoch_plus_one(tmp_path): } _write_galaxy_psf_cat(galaxy_psf_path, per_obj) - out = _run_save_psf(galaxy_psf_path, [101, 202], n_epoch=[1, 3]) + out = _run_save_psf(galaxy_psf_path, [101, 202], n_overlap=[1, 3]) for family in _PSF_SLOT_SENTINELS: assert _slot_numbers(out, family) == [1, 2, 3, 4], family @@ -908,7 +1010,7 @@ def test_save_psf_data_fixed_slots_keep_epoch_alignment(tmp_path): n_slots = 6 out = _run_save_psf( - galaxy_psf_path, [505], n_epoch=[3], n_epoch_slots=n_slots + galaxy_psf_path, [505], n_overlap=[3], n_epoch_slots=n_slots ) for n, (key, g1, g2, t, flag) in enumerate(epochs, start=1): @@ -944,7 +1046,7 @@ def test_save_psf_data_carries_fourth_moments_per_epoch(tmp_path): } _write_galaxy_psf_cat(galaxy_psf_path, per_obj) - out = _run_save_psf(galaxy_psf_path, [101], n_epoch=[2]) + out = _run_save_psf(galaxy_psf_path, [101], n_overlap=[2]) npt.assert_allclose(out["HSM_M4_1_PSF_1"], [0.11]) npt.assert_allclose(out["HSM_M4_2_PSF_1"], [-0.22]) diff --git a/workflow/config/cfis/final_cat.param b/workflow/config/cfis/final_cat.param index bb1b2efe2..5b8ad62fe 100644 --- a/workflow/config/cfis/final_cat.param +++ b/workflow/config/cfis/final_cat.param @@ -51,8 +51,10 @@ NGMIX_MCAL_FLAGS NGMIX_G1_PSF_ORIG_NOSHEAR NGMIX_G2_PSF_ORIG_NOSHEAR -# Number of epochs (exposures) +# Number of epochs: N_EPOCH with a validated PSF model, N_EPOCH_OVERLAP +# exposure CCDs covering the object, NGMIX_N_EPOCH fitted by ngmix N_EPOCH +N_EPOCH_OVERLAP NGMIX_N_EPOCH # Per-epoch identity and PSF shape, slot n = 1..12 (config_tile_Mc.ini's diff --git a/workflow/config/cfis_image_sims/final_cat.param b/workflow/config/cfis_image_sims/final_cat.param index 9db095b66..3357cde0f 100644 --- a/workflow/config/cfis_image_sims/final_cat.param +++ b/workflow/config/cfis_image_sims/final_cat.param @@ -35,8 +35,10 @@ NGMIX_NEIGHBOUR_FLAG NGMIX_G1_PSF_ORIG_NOSHEAR NGMIX_G2_PSF_ORIG_NOSHEAR -# Number of epochs (exposures) +# Number of epochs: N_EPOCH with a validated PSF model, N_EPOCH_OVERLAP +# exposure CCDs covering the object, NGMIX_N_EPOCH fitted by ngmix N_EPOCH +N_EPOCH_OVERLAP NGMIX_N_EPOCH ## Shape measurement outputs From 8709707bdba5b9d9da6e3640b4057840ace6354f Mon Sep 17 00:00:00 2001 From: Cail Daley Date: Sat, 10 Oct 2026 06:44:19 -0400 Subject: [PATCH 2/4] Describe N_EPOCH / N_EPOCH_OVERLAP in the decision record and ngmix_range Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01DfF1SjWsjHQzZYjL8bbnLo --- astra.yaml | 6 +++++- workflow/scripts/ngmix_range.py | 7 ++++--- 2 files changed, 9 insertions(+), 4 deletions(-) diff --git a/astra.yaml b/astra.yaml index b58cdafed..d382fd5cc 100644 --- a/astra.yaml +++ b/astra.yaml @@ -679,7 +679,8 @@ analyses: admits its upper endpoint, column 2080. [HARDCODED] The comparisons are strict in make_post_process; CCD_SIZE sets only the bounds. A WCS inversion failure skips that CCD, lowering N_EPOCH, which caps - how many exposures enter each galaxy's multi-epoch fit. + how many exposures enter each galaxy's multi-epoch fit. The final + catalogue keeps this geometric count as N_EPOCH_OVERLAP. Values: SEXTRACTOR_RUNNER.CCD_SIZE = 33,2080,1,4612; SEXTRACTOR_RUNNER.MAKE_POST_PROCESS = True. @@ -1119,6 +1120,9 @@ analyses: 22-star floor to that accepted count. In the science path the CCD's epoch is dropped for every object on it, and an object left with no epoch has no shape. + The final catalogue's N_EPOCH counts the epochs that pass this + gate, N_EPOCH_OVERLAP the CCDs covering the object, so their + difference is the number of the object's CCDs the gate rejected. The pipeline has no minimum-epoch floor: NGMIX_N_EPOCH records what survived and epoch-count cuts happen downstream. Guinot+22 discards the CCD from PSF estimation rather than gating at interpolation. diff --git a/workflow/scripts/ngmix_range.py b/workflow/scripts/ngmix_range.py index 6f2cbbf03..746c5b66f 100644 --- a/workflow/scripts/ngmix_range.py +++ b/workflow/scripts/ngmix_range.py @@ -193,14 +193,15 @@ def object_epochs(run_dir: Path): discovered by name, never assumed — each with ``n_obj`` rows in ``LDAC_OBJECTS`` row order and ``CCD_N < 0`` where the object misses that exposure. Summing - ``CCD_N >= 0`` across them reproduces the final catalogue's ``N_EPOCH`` - column exactly — checked row by row against 186.307's + ``CCD_N >= 0`` across them reproduces the final catalogue's + ``N_EPOCH_OVERLAP`` column exactly — checked row by row against 186.307's ``run_sp_tile_Mc/.../final_cat-186-307.fits``, all 35,298 of them, 7 extensions, 116,727 pairs, mean 3.31. The post-process is upstream of the whole tile_shape group, so the extensions always exist by the time ngmix runs; their absence is a broken tile, not a case to accommodate. - N_EPOCH is a GEOMETRIC count and mildly over-states the work, because ngmix + This is a GEOMETRIC count and mildly over-states the work, because ngmix + fits only epochs with a validated PSF (the final catalogue's N_EPOCH) and drops epochs it cannot fit. The same catalogue's NGMIX_N_EPOCH is lower for 2,203 of the 35,298 objects and never higher: 113,947 pairs against 116,727, 2.4%. That gap is the whole of the ~3.4% by which this cost model From 494d79afe82c3adde68b5b1d530b99b0f8f11a5a Mon Sep 17 00:00:00 2001 From: Cail Daley Date: Sat, 10 Oct 2026 07:00:23 -0400 Subject: [PATCH 3/4] make_cat: name the file on a missing galaxy-PSF entry; CONTRACTS and decision record cover N_EPOCH_OVERLAP and the mccd path Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01DfF1SjWsjHQzZYjL8bbnLo --- astra.yaml | 8 +++++--- src/shapepipe/modules/make_cat_package/make_cat.py | 10 +++++++++- workflow/CONTRACTS | 2 +- 3 files changed, 15 insertions(+), 5 deletions(-) diff --git a/astra.yaml b/astra.yaml index 1f48383a0..2352e6eb3 100644 --- a/astra.yaml +++ b/astra.yaml @@ -1120,9 +1120,11 @@ analyses: 22-star floor to that accepted count. In the science path the CCD's epoch is dropped for every object on it, and an object left with no epoch has no shape. - The final catalogue's N_EPOCH counts the epochs that pass this - gate, N_EPOCH_OVERLAP the CCDs covering the object, so their - difference is the number of the object's CCDs the gate rejected. + The final catalogue's N_EPOCH counts the epochs with an + interpolated PSF (for psfex, those passing this gate; for mccd, + those with a model), N_EPOCH_OVERLAP the CCDs covering the object, + so their difference is the number of the object's CCDs without a + usable PSF. The pipeline has no minimum-epoch floor: NGMIX_N_EPOCH records what survived and epoch-count cuts happen downstream. Guinot+22 discards the CCD from PSF estimation rather than gating at interpolation. diff --git a/src/shapepipe/modules/make_cat_package/make_cat.py b/src/shapepipe/modules/make_cat_package/make_cat.py index ad2209ab7..45f939f51 100644 --- a/src/shapepipe/modules/make_cat_package/make_cat.py +++ b/src/shapepipe/modules/make_cat_package/make_cat.py @@ -639,7 +639,15 @@ def _save_psf_data( for idx, id_tmp in enumerate(self._obj_id): - obj_epochs = galaxy_psf_cat[str(id_tmp)] + try: + obj_epochs = galaxy_psf_cat[str(id_tmp)] + except KeyError: + galaxy_psf_cat.close() + raise KeyError( + f"Object {id_tmp} is missing from {galaxy_psf_path}; " + "the galaxy PSF catalogue and the SExtractor catalogue " + "do not match" + ) from None if obj_epochs == "empty": continue diff --git a/workflow/CONTRACTS b/workflow/CONTRACTS index 9538e78cf..b2a6c8a7d 100644 --- a/workflow/CONTRACTS +++ b/workflow/CONTRACTS @@ -23,7 +23,7 @@ column list, in order: `final_cat_merge` writes those columns and no others, and the reader raises on a listed column a tile lacks. So a name belongs there only if `make_cat` writes it on EVERY tile. A per-epoch family (`EXP_ID_n`, `CCD_n`, `HSM_*_PSF_n`) qualifies only with a fixed slot count, since -`make_cat` otherwise sizes it from the tile's own maximum `N_EPOCH`. Enforced +`make_cat` otherwise sizes it from the tile's own maximum `N_EPOCH_OVERLAP`. Enforced by tests/unit/test_final_cat_merge_invariants.py (the merge) and tests/module/test_psf_grammar_properties.py (the shipped names). From 285af672e00541aac03902007337d8a1dba81aed Mon Sep 17 00:00:00 2001 From: Cail Daley Date: Sat, 10 Oct 2026 10:19:03 -0400 Subject: [PATCH 4/4] Keep only PSF-validated N_EPOCH in the final catalogue Use the geometric sexcat count only as the internal slot-sizing upper bound. Record the single-column rationale and assert the overlap column is absent. Co-Authored-By: GPT-6.1 Sol via Claude Opus 5.5 --- astra.yaml | 12 ++++++----- example/cfis_image_sims/final_cat.param | 5 ++--- .../modules/make_cat_package/__init__.py | 7 +++---- .../modules/make_cat_package/make_cat.py | 14 ++++++------- tests/module/test_make_cat.py | 20 +++++++++---------- workflow/CONTRACTS | 2 +- workflow/config/cfis/config_tile_Mc.ini | 2 +- workflow/config/cfis/final_cat.param | 5 ++--- .../config/cfis_image_sims/final_cat.param | 5 ++--- workflow/scripts/ngmix_range.py | 4 ++-- 10 files changed, 37 insertions(+), 39 deletions(-) diff --git a/astra.yaml b/astra.yaml index 2352e6eb3..e0251df06 100644 --- a/astra.yaml +++ b/astra.yaml @@ -679,8 +679,9 @@ analyses: admits its upper endpoint, column 2080. [HARDCODED] The comparisons are strict in make_post_process; CCD_SIZE sets only the bounds. A WCS inversion failure skips that CCD, lowering N_EPOCH, which caps - how many exposures enter each galaxy's multi-epoch fit. The final - catalogue keeps this geometric count as N_EPOCH_OVERLAP. + how many exposures enter each galaxy's multi-epoch fit. This geometric + count remains internal; the final catalogue's N_EPOCH counts only + epochs with a validated PSF. Values: SEXTRACTOR_RUNNER.CCD_SIZE = 33,2080,1,4612; SEXTRACTOR_RUNNER.MAKE_POST_PROCESS = True. @@ -1122,9 +1123,10 @@ analyses: epoch has no shape. The final catalogue's N_EPOCH counts the epochs with an interpolated PSF (for psfex, those passing this gate; for mccd, - those with a model), N_EPOCH_OVERLAP the CCDs covering the object, - so their difference is the number of the object's CCDs without a - usable PSF. + those with a model). Only this count is written as N_EPOCH: one column + avoids downstream cuts on the geometric count by mistake. + PSF-gate rejections are studied per CCD in validation output and + the exposure maps (#887), rather than per object. The pipeline has no minimum-epoch floor: NGMIX_N_EPOCH records what survived and epoch-count cuts happen downstream. Guinot+22 discards the CCD from PSF estimation rather than gating at interpolation. diff --git a/example/cfis_image_sims/final_cat.param b/example/cfis_image_sims/final_cat.param index e94994122..3ac278819 100644 --- a/example/cfis_image_sims/final_cat.param +++ b/example/cfis_image_sims/final_cat.param @@ -31,10 +31,9 @@ NGMIX_MCAL_TYPES_FAIL NGMIX_G1_PSF_ORIG_NOSHEAR NGMIX_G2_PSF_ORIG_NOSHEAR -# Number of epochs: N_EPOCH with a validated PSF model, N_EPOCH_OVERLAP -# exposure CCDs covering the object, NGMIX_N_EPOCH fitted by ngmix +# Number of epochs: N_EPOCH with a validated PSF model, +# NGMIX_N_EPOCH fitted by ngmix N_EPOCH -N_EPOCH_OVERLAP NGMIX_N_EPOCH ## Shape measurement outputs diff --git a/src/shapepipe/modules/make_cat_package/__init__.py b/src/shapepipe/modules/make_cat_package/__init__.py index 435000c1c..dc9b93bdc 100644 --- a/src/shapepipe/modules/make_cat_package/__init__.py +++ b/src/shapepipe/modules/make_cat_package/__init__.py @@ -22,9 +22,8 @@ galaxies for weak-lensing post-processing. This includes galaxy detection and basic measurement parameters, the PSF model at galaxy positions, and the shape measurement. ``N_EPOCH`` counts the object's epochs with a validated -PSF model, the epochs ngmix can fit; ``N_EPOCH_OVERLAP`` counts the exposure -CCDs whose footprint holds the object, including CCDs whose PSF model failed -validation. Every detected object is kept: the catalogue carries no +PSF model, the epochs ngmix can fit. +The geometric count of covering CCDs is used internally, not written out. Every detected object is kept: the catalogue carries no star/galaxy classification, which is done downstream. Each object is tagged with its source tile via the ``TILE_ID`` column and carries the survey-wide object ID ``TILE_UNIQUE_ID = tile_id * 10**6 + NUMBER`` @@ -46,7 +45,7 @@ (``HSM_*_PSF_n``, ``EXP_ID_n``, ``CCD_n``) when ``SAVE_PSF_DATA`` is ``True``; unfilled slots hold the family's sentinel, and an object with more epochs than slots raises an error. A fixed count gives every tile - the same schema. Default is the tile's maximum ``N_EPOCH_OVERLAP`` plus + the same schema. Default is the sexcat's maximum geometric ``N_EPOCH`` plus one """ diff --git a/src/shapepipe/modules/make_cat_package/make_cat.py b/src/shapepipe/modules/make_cat_package/make_cat.py index 45f939f51..e6640d006 100644 --- a/src/shapepipe/modules/make_cat_package/make_cat.py +++ b/src/shapepipe/modules/make_cat_package/make_cat.py @@ -82,8 +82,8 @@ def read_sextractor_data(sexcat_path, remove_vignet=True): ``sexcat-301-279.fits``. The SExtractor catalogue's ``N_EPOCH`` counts the exposure CCDs whose - footprint holds the object, before any PSF model exists; it is kept as - ``N_EPOCH_OVERLAP``. The final catalogue's ``N_EPOCH`` counts only the + footprint holds the object, before any PSF model exists; it is used + internally for slot sizing and overwritten before output. The final catalogue's ``N_EPOCH`` counts only the epochs with a validated PSF and is written by :meth:`SaveCatalogue.process` in ``psf`` mode. @@ -107,7 +107,7 @@ def read_sextractor_data(sexcat_path, remove_vignet=True): sexcat_file.close() columns = { - "N_EPOCH_OVERLAP" if name == "N_EPOCH" else name: data[name] + name: data[name] for name in data.dtype.names if not (remove_vignet and name == "VIGNET") } @@ -234,7 +234,7 @@ def process( Option to run ``ngmix`` mode with moments n_epoch_slots : int, optional Number of per-epoch slots in ``psf`` mode; if ``None``, the - tile's ``max(N_EPOCH_OVERLAP) + 1`` + sexcat's geometric ``max(N_EPOCH) + 1`` epoch_slots : bool, optional In ``psf`` mode, write the per-epoch column families as well as ``N_EPOCH``; default ``True`` @@ -573,7 +573,7 @@ def _save_psf_data( ``N_EPOCH`` counts the object's entries in the PSF catalogue, i.e. the epochs whose CCD passed PSF-model validation and so have an interpolated PSF; ngmix fits no other epoch. It is at most - ``N_EPOCH_OVERLAP``, the count of exposure CCDs whose footprint holds + the sexcat's geometric ``N_EPOCH``, the count of CCDs whose footprint holds the object, and 0 for an object the PSF catalogue marks ``"empty"``. An epoch whose PSF shape fit failed (``HSM_FLAG_PSF != 0``) still counts: its PSF exists. @@ -590,7 +590,7 @@ def _save_psf_data( Path to the PSF catalogue to save n_epoch_slots : int, optional Number of slots written per family; if ``None``, the tile's - ``max(N_EPOCH_OVERLAP) + 1``, which bounds every object's PSF + geometric ``max(N_EPOCH) + 1`` from the sexcat, which bounds every object's PSF epoch count epoch_slots : bool, optional Write the per-epoch column families; default ``True`` @@ -609,7 +609,7 @@ def _save_psf_data( if not epoch_slots: n_slots = 0 elif n_epoch_slots is None: - n_slots = np.max(self._final_cat["N_EPOCH_OVERLAP"]) + 1 + n_slots = np.max(self._final_cat["N_EPOCH"]) + 1 else: n_slots = n_epoch_slots diff --git a/tests/module/test_make_cat.py b/tests/module/test_make_cat.py index b53daf2c6..6ef58b9e7 100644 --- a/tests/module/test_make_cat.py +++ b/tests/module/test_make_cat.py @@ -403,7 +403,7 @@ def _run_save_psf( inst = object.__new__(SaveCatalogue) inst._obj_id = np.asarray(obj_id) inst._output_dict = {} - inst._final_cat = {"N_EPOCH_OVERLAP": np.asarray(n_overlap)} + inst._final_cat = {"N_EPOCH": np.asarray(n_overlap)} inst._save_psf_data( str(galaxy_psf_path), @@ -479,7 +479,7 @@ def test_save_psf_data_identity_survives_failed_hsm_fit(tmp_path): def test_save_psf_data_fills_sentinel_for_absent_epochs(tmp_path): """Unused epoch slots and "empty" objects keep the -1 sentinel. - ``max_epoch`` (from the largest ``N_EPOCH_OVERLAP`` across the catalogue) can + ``max_epoch`` (from the largest geometric sexcat ``N_EPOCH``) can exceed a given object's own epoch count, and an object the PSF catalogue reports no epochs at all for is marked ``"empty"``; both cases must leave EXP_ID_n/CCD_n at -1, an exposure ID / CCD number no @@ -492,7 +492,7 @@ def test_save_psf_data_fills_sentinel_for_absent_epochs(tmp_path): } _write_galaxy_psf_cat(galaxy_psf_path, per_obj) - # max(N_EPOCH_OVERLAP) = 2 -> 3 slot columns, though obj 101 only fills slot 1. + # Geometric max(N_EPOCH) = 2 -> 3 slots; obj 101 only fills slot 1. out = _run_save_psf(galaxy_psf_path, [101, 303], n_overlap=[1, 2]) assert out["EXP_ID_1"][0] == 2113864 @@ -607,7 +607,7 @@ def test_make_cat_runner_ships_every_detection_unclassified(tmp_path): def test_make_cat_runner_n_epoch_excludes_unvalidated_psf_epochs(tmp_path): - """The final N_EPOCH counts PSF-validated epochs; N_EPOCH_OVERLAP keeps the geometry. + """The final N_EPOCH counts only PSF-validated epochs. The SExtractor ``N_EPOCH`` counts every exposure CCD covering an object. Object 1 lies on two CCDs of which one failed PSF validation, object 3 @@ -650,7 +650,7 @@ def test_make_cat_runner_n_epoch_excludes_unvalidated_psf_epochs(tmp_path): with h5py.File(tmp_path / "final_cat-350-100.hdf5", "r") as cat: npt.assert_array_equal(cat["N_EPOCH"][()], [1, 3, 0]) - npt.assert_array_equal(cat["N_EPOCH_OVERLAP"][()], [2, 3, 1]) + assert "N_EPOCH_OVERLAP" not in cat assert "EXP_ID_1" not in cat @@ -721,12 +721,12 @@ def test_make_cat_runner_writes_one_hdf5_dataset_per_column(tmp_path): with h5py.File(out_dir / "final_cat-350-100.hdf5", "r") as cat: names = list(cat) assert names[:7] == [ - "NUMBER", "N_EPOCH_OVERLAP", "XWIN_WORLD", "YWIN_WORLD", + "NUMBER", "N_EPOCH", "XWIN_WORLD", "YWIN_WORLD", "FLUX_APER", "TILE_ID", "TILE_UNIQUE_ID", ] assert ( - names.index("NGMIX_G1_NOSHEAR") - < names.index("N_EPOCH") + names.index("N_EPOCH") + < names.index("NGMIX_G1_NOSHEAR") < names.index("HSM_G1_PSF_1") ) assert names[-1] == "MASK_r" @@ -871,7 +871,7 @@ def test_save_psf_data_fixed_slots_pad_every_family(tmp_path): n_slots = 7 out = { "NUMBER": np.array([101, 202, 303]), - "N_EPOCH_OVERLAP": np.array([1, 2, 0]), + "N_EPOCH": np.array([1, 2, 0]), } sc = SaveCatalogue(out, 3, _NullLogger()) assert sc.process("psf", str(galaxy_psf_path), n_epoch_slots=n_slots) is None @@ -931,7 +931,7 @@ def test_save_psf_data_exactly_slots_epochs_fits(tmp_path): def test_save_psf_data_unset_slots_uses_tile_max_n_overlap_plus_one(tmp_path): - """Without N_EPOCH_SLOTS the slot count is the tile's max(N_EPOCH_OVERLAP) + 1.""" + """Without N_EPOCH_SLOTS use the sexcat's geometric max(N_EPOCH) + 1.""" galaxy_psf_path = tmp_path / "galaxy_psf.sqlite" per_obj = { 101: {"2113864-7": _psf_epoch(0.01, 0.02, 0.5)}, diff --git a/workflow/CONTRACTS b/workflow/CONTRACTS index b2a6c8a7d..8dbaba1ef 100644 --- a/workflow/CONTRACTS +++ b/workflow/CONTRACTS @@ -23,7 +23,7 @@ column list, in order: `final_cat_merge` writes those columns and no others, and the reader raises on a listed column a tile lacks. So a name belongs there only if `make_cat` writes it on EVERY tile. A per-epoch family (`EXP_ID_n`, `CCD_n`, `HSM_*_PSF_n`) qualifies only with a fixed slot count, since -`make_cat` otherwise sizes it from the tile's own maximum `N_EPOCH_OVERLAP`. Enforced +`make_cat` otherwise sizes it from the sexcat's maximum geometric `N_EPOCH`. Enforced by tests/unit/test_final_cat_merge_invariants.py (the merge) and tests/module/test_psf_grammar_properties.py (the shipped names). diff --git a/workflow/config/cfis/config_tile_Mc.ini b/workflow/config/cfis/config_tile_Mc.ini index 648f1c418..695654523 100644 --- a/workflow/config/cfis/config_tile_Mc.ini +++ b/workflow/config/cfis/config_tile_Mc.ini @@ -82,7 +82,7 @@ SHAPE_MEASUREMENT_TYPE = ngmix # Save per-epoch PSF shapes and epoch identity (HSM_*_PSF_n, EXP_ID_n, # CCD_n). The campaign merge needs one column schema across all tiles, so -# the slot count is fixed rather than per-tile max(N_EPOCH_OVERLAP)+1. It must +# the slot count is fixed rather than the sexcat's geometric max(N_EPOCH)+1. It must # exceed the survey's maximum epochs per object with margin (smk-g7: max 6 # over 8 inspected tiles); an object with more epochs than slots is an error. SAVE_PSF_DATA = True diff --git a/workflow/config/cfis/final_cat.param b/workflow/config/cfis/final_cat.param index 5b8ad62fe..b1958794d 100644 --- a/workflow/config/cfis/final_cat.param +++ b/workflow/config/cfis/final_cat.param @@ -51,10 +51,9 @@ NGMIX_MCAL_FLAGS NGMIX_G1_PSF_ORIG_NOSHEAR NGMIX_G2_PSF_ORIG_NOSHEAR -# Number of epochs: N_EPOCH with a validated PSF model, N_EPOCH_OVERLAP -# exposure CCDs covering the object, NGMIX_N_EPOCH fitted by ngmix +# Number of epochs: N_EPOCH with a validated PSF model, +# NGMIX_N_EPOCH fitted by ngmix N_EPOCH -N_EPOCH_OVERLAP NGMIX_N_EPOCH # Per-epoch identity and PSF shape, slot n = 1..12 (config_tile_Mc.ini's diff --git a/workflow/config/cfis_image_sims/final_cat.param b/workflow/config/cfis_image_sims/final_cat.param index 3357cde0f..8b5f34e2a 100644 --- a/workflow/config/cfis_image_sims/final_cat.param +++ b/workflow/config/cfis_image_sims/final_cat.param @@ -35,10 +35,9 @@ NGMIX_NEIGHBOUR_FLAG NGMIX_G1_PSF_ORIG_NOSHEAR NGMIX_G2_PSF_ORIG_NOSHEAR -# Number of epochs: N_EPOCH with a validated PSF model, N_EPOCH_OVERLAP -# exposure CCDs covering the object, NGMIX_N_EPOCH fitted by ngmix +# Number of epochs: N_EPOCH with a validated PSF model, +# NGMIX_N_EPOCH fitted by ngmix N_EPOCH -N_EPOCH_OVERLAP NGMIX_N_EPOCH ## Shape measurement outputs diff --git a/workflow/scripts/ngmix_range.py b/workflow/scripts/ngmix_range.py index 746c5b66f..3bb816f5b 100644 --- a/workflow/scripts/ngmix_range.py +++ b/workflow/scripts/ngmix_range.py @@ -193,8 +193,8 @@ def object_epochs(run_dir: Path): discovered by name, never assumed — each with ``n_obj`` rows in ``LDAC_OBJECTS`` row order and ``CCD_N < 0`` where the object misses that exposure. Summing - ``CCD_N >= 0`` across them reproduces the final catalogue's - ``N_EPOCH_OVERLAP`` column exactly — checked row by row against 186.307's + ``CCD_N >= 0`` across them reproduces the sexcat's geometric + ``N_EPOCH`` — checked row by row against the pre-change catalogue for 186.307, ``run_sp_tile_Mc/.../final_cat-186-307.fits``, all 35,298 of them, 7 extensions, 116,727 pairs, mean 3.31. The post-process is upstream of the whole tile_shape group, so the extensions always exist by the time ngmix runs;