From 28b0b929e76a29533e00457b8cc813230482744b Mon Sep 17 00:00:00 2001 From: Cail Daley Date: Sat, 26 Sep 2026 07:07:02 +0200 Subject: [PATCH 01/30] test(ngmix): specify defect fill, epoch cuts and shear recovery near defects Defects (nonzero flag, zero weight, invalid RMS) must be zero-weighted and noise-filled under every BLEND_HANDLING, with neighbour pixels left raw under uberseg. The masked-fraction cut must count that same raw set, and an epoch with a defect closer than EPOCH_CENTRAL_DEFECT_RADIUS to the stamp centre must be dropped, at a strict boundary and at the configured radius. The runner must pass both cut options on, the HSM centroid must ignore raw defect values, each epoch must keep its own OFFSET, and every tile must log its epoch-cut tally. The science test recovers the full 2x2 response for columns, 3-px bleeds, single pixels and edge bands at the veto radius, on 0.3" and 0.5" galaxies through round and elliptical 0.7" PSFs, and requires |c1|, |c2| < 5e-4 and |m11|, |m22| < 1%. Flagged pixels in the simulation hold a hot value, so an unfilled defect shows up. A column two pixels inside the radius is the positive control. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01QAc1ywyd9Sbbra6po2iuaA --- tests/helpers/defect_response.py | 89 +++++ tests/module/test_ngmix_defect_fill.py | 532 +++++++++++++++++++++++++ tests/module/test_ngmix_uberseg.py | 69 ++-- tests/science/test_defect_veto.py | 104 +++++ 4 files changed, 755 insertions(+), 39 deletions(-) create mode 100644 tests/helpers/defect_response.py create mode 100644 tests/module/test_ngmix_defect_fill.py create mode 100644 tests/science/test_defect_veto.py diff --git a/tests/helpers/defect_response.py b/tests/helpers/defect_response.py new file mode 100644 index 000000000..c3e3ac157 --- /dev/null +++ b/tests/helpers/defect_response.py @@ -0,0 +1,89 @@ +"""Full-matrix metacal recovery for fixed detector defects.""" + +import numpy as np + +from shapepipe.modules.ngmix_package import ngmix as ngm +from shapepipe.testing.simulate import make_data +from tests.helpers.metacal_sim import build_stamp + + +def defect_response(bad, hlr=0.5, psf=0.7, seeds=range(4), + psf_shear=(0.0, 0.0), options=None, known_rms=True, + defect_value=1e3): + """Recover c and M = inverse(mean R) A - I with paired seed errors. + + Null pairs rotate the pixels by 90 degrees while pre-rotating the PSF + ellipticity oppositely, so the final PSF stays fixed in detector + coordinates. + This does NOT average away elliptical-PSF leakage. + + The flagged pixels ``bad`` hold ``defect_value`` (far above the galaxy's + peak) rather than sky, as a hot column or bleed would, so any defect value + that reaches metacal shows up as a bias. + """ + gamma = 0.02 + options = {} if options is None else options + samples = [] + for seed in seeds: + def arm(axis, sign, rotation=0): + shear = [0.0, 0.0] + if axis >= 0: + shear[axis] = sign * gamma + ps = tuple(v * (-1 if rotation else 1) for v in psf_shear) + data = list(make_data( + rng=np.random.RandomState(seed + 100), shear=shear, + psf_shear=ps, noise=1e-4, n_epochs=1, img_size=51, + gal_hlr=hlr, psf_fwhm=psf, return_centers=True, + )) + centre = data.pop()[0] + offset = np.array([centre.y - 26, centre.x - 26]) + if rotation: + data[0] = [np.rot90(a).copy() for a in data[0]] + data[1] = [np.rot90(a).copy() for a in data[1]] + offset = np.array([-offset[1], offset[0]]) + data[0] = [np.where(bad, defect_value, a) for a in data[0]] + data[4] = [bad.astype(np.int32)] + stamp = build_stamp(data) + stamp.offsets = [offset] + if known_rms: + stamp.bkg_rms = [np.full((51, 51), 1e-4)] + rng = np.random.RandomState(seed) + result, _, _ = ngm.do_ngmix_metacal( + stamp, ngm.get_prior(0.1857, rng), 1.0, rng, + centroid_source="wcs", **options, + ) + assert all(result[t]["flags"] == 0 for t in ngm.METACAL_TYPES) + e = np.asarray(result["noshear"]["g"]) + response = np.column_stack([ + (np.asarray(result[p]["g"]) - result[m]["g"]) / 0.02 + for p, m in (("1p", "1m"), ("2p", "2m")) + ]) + assert np.all(np.isfinite(e)) and np.all(np.isfinite(response)) + return e, response + null = [arm(-1, 0, k) for k in (0, 1)] + e0 = np.mean([a[0] for a in null], axis=0) + r0 = np.mean([a[1] for a in null], axis=0) + derivatives, responses = [], [] + for axis in (0, 1): + plus, minus = arm(axis, 1), arm(axis, -1) + derivatives.append((plus[0] - minus[0]) / (2 * gamma)) + responses.extend([plus[1], minus[1]]) + samples.append((e0, r0, np.column_stack(derivatives), + np.mean(responses, axis=0))) + e, r, a, rm = [np.array([s[i] for s in samples]) for i in range(4)] + assert np.linalg.svd(rm.mean(axis=0), compute_uv=False).min() > 0.1 + c = np.linalg.solve(r.mean(axis=0), e.mean(axis=0)) + matrix = np.linalg.solve(rm.mean(axis=0), a.mean(axis=0)) - np.eye(2) + draw = np.random.RandomState(91).randint(len(e), size=(1000, len(e))) + cb = np.linalg.solve(r[draw].mean(axis=1), + e[draw].mean(axis=1)[..., None])[..., 0] + mb = np.linalg.solve(rm[draw].mean(axis=1), a[draw].mean(axis=1)) + mb -= np.eye(2) + return dict(c=c.tolist(), m=np.diag(matrix).tolist(), + matrix=matrix.tolist(), + c_err=cb.std(axis=0).tolist(), + m_err=np.diag(mb.std(axis=0)).tolist(), + response=r.mean(axis=0).tolist(), + seed_groups=[dict(e=s[0].tolist(), R=s[1].tolist(), + A=s[2].tolist(), Rm=s[3].tolist()) + for s in samples]) diff --git a/tests/module/test_ngmix_defect_fill.py b/tests/module/test_ngmix_defect_fill.py new file mode 100644 index 000000000..0480c7ef7 --- /dev/null +++ b/tests/module/test_ngmix_defect_fill.py @@ -0,0 +1,532 @@ +"""Defect fill and the epoch cuts (ngmix module). + +A defect is a stamp pixel with a nonzero flag, zero exposure weight or an +invalid background RMS. Before metacal, :func:`prepare_ngmix_weights` gives +every defect weight 0 and fills it with noise at its background RMS, whatever +``BLEND_HANDLING`` is. Under uberseg, pixels on the neighbour side only lose +their weight, and their image values stay raw. The per-epoch cuts in +:func:`prepare_postage_stamps` act on the same defect set: the masked-fraction +cut counts it, and the central-defect veto drops an epoch with a defect near +the stamp centre. +""" + +import re +from types import SimpleNamespace + +import numpy as np +import numpy.testing as npt +from astropy.io import fits +from astropy.wcs import WCS +from hypothesis import given +from hypothesis import strategies as st +from sqlitedict import SqliteDict +from shapepipe.modules.ngmix_package import ngmix as ngmix_module + +from shapepipe.modules.ngmix_package.ngmix import ( + EPOCH_CENTRAL_DEFECT_RADIUS, + Ngmix, + make_ngmix_observation, + prepare_ngmix_weights, + prepare_postage_stamps, + uberseg_weight, +) + + +# --- prepare_ngmix_weights: the filled set is the defect set --------------- + +@st.composite +def defect_stamps(draw): + """Square stamp with random flagged, zero-weight and bad-RMS pixels.""" + n = draw(st.integers(min_value=5, max_value=21)) + pixels = st.lists( + st.tuples(st.integers(0, n - 1), st.integers(0, n - 1)), max_size=n + ) + weight = np.ones((n, n)) + flag = np.zeros((n, n), dtype=np.int32) + bkg_rms = np.ones((n, n)) + for i, j in draw(pixels): + weight[i, j] = 0.0 + for i, j in draw(pixels): + flag[i, j] = draw(st.sampled_from([1, 2, 2**10])) + for i, j in draw(pixels): + bkg_rms[i, j] = draw(st.sampled_from([0.0, -1.0, np.nan, np.inf])) + # Raw values far outside the unit-RMS noise, so a filled pixel is + # unambiguous. + gal = 1.0e3 + np.arange(n * n, dtype=float).reshape(n, n) + return gal, weight, flag, bkg_rms + + +def _uberseg_seg(n): + """Central object on the centre pixel, neighbour footprint in a corner.""" + seg = np.zeros((n, n), dtype=np.int32) + seg[n // 2, n // 2] = 1 + seg[:2, :2] = 2 + return seg + + +@given( + stamp=defect_stamps(), + blend_handling=st.sampled_from(["none", "uberseg"]), + seed=st.integers(0, 2**31 - 1), +) +def test_filled_set_is_the_defect_set(stamp, blend_handling, seed): + """Filled pixels are exactly the defects (flag, zero weight, bad RMS). + They carry zero weight and look like noise. Every other pixel keeps its + raw value, under either BLEND_HANDLING. + + Failure modes: + * a defect source (flag, zero weight, bad RMS) is left out of the fill; + * the filled set grows beyond the defects (for example symmetrized); + * the filled set and the zero-weight defect set differ; + * the fill is skipped under uberseg; + * neighbour-side pixels are filled. + """ + gal, weight, flag, bkg_rms = stamp + n = gal.shape[0] + kwargs = ( + dict(seg=_uberseg_seg(n), object_number=1, dilate_neighbour=1) + if blend_handling == "uberseg" + else {} + ) + + gal_out, w_out, _ = prepare_ngmix_weights( + gal, weight, flag, np.random.RandomState(seed), bkg_rms=bkg_rms, + blend_handling=blend_handling, **kwargs, + ) + + defect = ( + (weight == 0) | (flag != 0) | ~(np.isfinite(bkg_rms) & (bkg_rms > 0)) + ) + if defect.all(): + return # fully masked: no clean pixel left to set the noise level + filled = gal_out != gal + npt.assert_array_equal(filled, defect, "filled set is not the defect set") + assert np.all(np.abs(gal_out[filled]) < 10.0), "fill is not unit noise" + neighbour = ( + uberseg_weight(np.ones((n, n)), kwargs["seg"], 1, dilate_neighbour=1) + == 0.0 + if blend_handling == "uberseg" + else np.zeros((n, n), dtype=bool) + ) + # Zero weight on defects and neighbour side; neighbour pixels that are + # not defects keep their raw values (filled-set equality above). + npt.assert_array_equal(w_out == 0.0, defect | neighbour) + npt.assert_array_equal(w_out[~(defect | neighbour)], 1.0) + + +def test_uberseg_defect_in_neighbour_region_is_filled(): + """A defect pixel that also lies on the neighbour side is filled. + + Failure mode: the fill is restricted to pixels uberseg keeps, so a raw bad + pixel on the neighbour side still reaches metacal. + """ + n = 21 + gal = 1.0e3 + np.arange(n * n, dtype=float).reshape(n, n) + weight = np.ones((n, n)) + flag = np.zeros((n, n), dtype=np.int32) + flag[1, 1] = 1 # inside the neighbour footprint + gal_out, w_out, _ = prepare_ngmix_weights( + gal, weight, flag, np.random.RandomState(0), bkg_rms=np.ones((n, n)), + blend_handling="uberseg", seg=_uberseg_seg(n), object_number=1, + ) + assert w_out[1, 1] == 0.0 and gal_out[1, 1] != gal[1, 1] + # A neighbour-side pixel that is not a defect: zero weight, raw value. + assert w_out[0, 3] == 0.0 and gal_out[0, 3] == gal[0, 3] + + +# --- prepare_postage_stamps: the fraction cut counts the defect set -------- + +N_STAMP = 51 +RA, DEC = 150.0, 2.0 + + +def _fake_inputs(epochs): + """Minimal vignet / tile-catalogue stand-ins for prepare_postage_stamps. + + ``epochs`` maps ``"-"`` to ``(flag, weight)`` or + ``(flag, weight, bkg_rms)``. Returns ``(vignet, tile_cat, psf_obj, + gal_obj)``. + """ + rng = np.random.default_rng(1) + wcs = WCS(naxis=2) + wcs.wcs.ctype = ["RA---TAN", "DEC--TAN"] + wcs.wcs.crval = [RA, DEC] + wcs.wcs.crpix = [N_STAMP / 2, N_STAMP / 2] + wcs.wcs.cdelt = [-0.187 / 3600, 0.187 / 3600] + header = fits.Header({"FSCALE": 1.0}).tostring() + + def per_epoch(make): + return {k: {"VIGNET": make(k)} for k in epochs} + + with_rms = any(len(v) == 3 for v in epochs.values()) + psf_obj = per_epoch(lambda k: np.ones((N_STAMP, N_STAMP))) + gal_obj = per_epoch(lambda k: rng.normal(0.0, 1.0, (N_STAMP, N_STAMP))) + vignet = SimpleNamespace( + gal_vign_cat={"1": gal_obj}, + bkg_vign_cat=None, + bkg_rms_vign_cat=( + {"1": per_epoch( + lambda k: epochs[k][2] if len(epochs[k]) == 3 + else np.ones((N_STAMP, N_STAMP)) + )} + if with_rms + else None + ), + flag_vign_cat={"1": per_epoch(lambda k: epochs[k][0])}, + weight_vign_cat={"1": per_epoch(lambda k: epochs[k][1])}, + f_wcs_file={ + k.split("-")[0]: { + int(k.split("-")[1]): {"WCS": wcs, "header": header} + } + for k in epochs + }, + ) + tile_cat = SimpleNamespace( + vign=None, seg=None, ra=np.array([RA]), dec=np.array([DEC]) + ) + return vignet, tile_cat, psf_obj, gal_obj + + +def _two_sided_band(width): + """Mask of ``width`` columns on each side of the stamp, far from the + centre (at least 17 px for width 9).""" + band = np.zeros((N_STAMP, N_STAMP), dtype=bool) + band[:, :width] = True + band[:, -width:] = True + return band + + +def _epochs(): + """A clean epoch and four masked ones, all defects far from the centre. + + * band10: 10 flagged columns on one side, 19.6% (39% if symmetrized); + * flag18: 2 x 9 flagged columns, 35.3%; + * dead18: the same columns at zero weight, no flags; + * rms18: the same columns with an invalid background RMS. + """ + clean = np.zeros((N_STAMP, N_STAMP), dtype=np.int32) + ones = np.ones((N_STAMP, N_STAMP)) + band10 = clean.copy() + band10[:, -10:] = 1 + two = _two_sided_band(9) + dead = ones.copy() + dead[two] = 0.0 + rms = ones.copy() + rms[two] = np.nan + return { + "2100001-10": (clean, ones), + "2100002-11": (band10, ones), + "2100003-12": (two.astype(np.int32), ones), + "2100004-13": (clean.copy(), dead), + "2100005-14": (clean.copy(), ones, rms), + } + + +def _surviving(epochs, **kwargs): + vignet, tile_cat, psf_obj, gal_obj = _fake_inputs(epochs) + stamp = prepare_postage_stamps( + vignet, 1, 0, tile_cat, bkg_sub=False, + psf_obj=psf_obj, gal_obj=gal_obj, **kwargs, + ) + names = {id(v[0]): name for name, v in epochs.items()} + return sorted(names[id(flag)] for flag in stamp.flags) + + +def test_epoch_cut_counts_the_defect_set(): + """At the default 1/3 cut, the three 35% epochs are dropped, whichever + defect source masks them, and the 20% band survives. + + Failure modes: the cut counts flags only (keeps dead18 and rms18), omits + one defect source, or counts a symmetrized set (drops band10); in each + case the cut disagrees with the set that is zero-weighted and filled + (epoch-cut-on-defect-mask). + """ + assert _surviving(_epochs()) == ["2100001-10", "2100002-11"] + + +def test_epoch_cut_threshold_is_the_configured_fraction(): + """At a 10% cut (the DES Y3 and Y6 value), the 19.6% band is dropped as + well, and only the clean epoch survives. + + Failure mode: the configured threshold is ignored. + """ + assert _surviving(_epochs(), epoch_masked_fraction_cut=0.1) == [ + "2100001-10" + ] + + +# --- prepare_postage_stamps: the central-defect veto ----------------------- + +def _veto_epochs(radius): + """A clean epoch, one with a single flagged pixel just inside + ``radius`` of the stamp centre, and one with a flagged column exactly + ``radius`` away. Both masked epochs are far below the fraction cut. + """ + centre = N_STAMP // 2 + clean = np.zeros((N_STAMP, N_STAMP), dtype=np.int32) + ones = np.ones((N_STAMP, N_STAMP)) + near, far = clean.copy(), clean.copy() + near[centre, centre + radius - 1] = 1 + far[:, centre + radius] = 1 + return { + "2100001-10": (clean, ones), + "2100002-11": (near, ones), + "2100003-12": (far, ones), + } + + +def test_central_defect_vetoes_the_epoch(): + """At the default radius, a single defect pixel inside it drops the + epoch, and a column at the radius does not. + + Failure mode: the veto is skipped, so an epoch whose filled hole overlaps + the object's light enters the fit; or the boundary is inclusive, dropping + the column at the radius (epoch-central-defect-veto). + """ + epochs = _veto_epochs(EPOCH_CENTRAL_DEFECT_RADIUS) + assert _surviving(epochs) == ["2100001-10", "2100003-12"] + + +def test_central_defect_radius_is_the_configured_value(): + """Radius 0 disables the veto; a radius one pixel larger than the far + column's distance drops that epoch too. + + Failure mode: the configured radius is ignored. + """ + epochs = _veto_epochs(EPOCH_CENTRAL_DEFECT_RADIUS) + assert _surviving(epochs, epoch_central_defect_radius=0) == [ + "2100001-10", "2100002-11", "2100003-12" + ] + assert _surviving( + epochs, epoch_central_defect_radius=EPOCH_CENTRAL_DEFECT_RADIUS + 1 + ) == ["2100001-10"] + + +# --- prepare_postage_stamps: per-epoch OFFSET ------------------------------ + +def test_each_surviving_epoch_carries_its_own_offset(): + """``stamp.offsets`` holds each surviving epoch's vignette OFFSET, in the + order of ``stamp.flags``. + + Failure mode: the offset is dropped or read from another epoch, so the + default "wcs" centroid raises or puts the Jacobian origin off the object. + """ + clean = np.zeros((N_STAMP, N_STAMP), dtype=np.int32) + ones = np.ones((N_STAMP, N_STAMP)) + epochs = {f"210000{i}-1{i}": (clean.copy(), ones) for i in range(3)} + vignet, tile_cat, psf_obj, gal_obj = _fake_inputs(epochs) + del vignet.gal_vign_cat # OFFSET must come from the supplied gal_obj. + for i, name in enumerate(epochs): + gal_obj[name]["OFFSET"] = np.array([0.1 * i, -0.1 * i]) + stamp = prepare_postage_stamps( + vignet, 1, 0, tile_cat, bkg_sub=False, + psf_obj=psf_obj, gal_obj=gal_obj, + ) + names = {id(flag): name for name, (flag, _) in epochs.items()} + assert len(stamp.offsets) == len(stamp.flags) == 3 + for flag, offset in zip(stamp.flags, stamp.offsets): + npt.assert_array_equal(offset, gal_obj[names[id(flag)]]["OFFSET"]) + + +# --- Ngmix.process: the per-tile epoch-cut tally --------------------------- + +class _RecordingLogger: + def __init__(self): + self.messages = [] + + def info(self, msg, *_args, **_kwargs): + self.messages.append(msg) + + +def test_process_logs_the_epoch_cut_tally(tmp_path, monkeypatch): + """One tile, four objects; the end-of-tile line counts each cut's drops. + + * object 1: clean, 18-column edge band, defect inside the radius -> one + epoch each for considered, masked_fraction, central_veto; survives. + * object 2: edge band and central defect -> both epochs dropped; emptied. + * object 3: one all-zero stamp, skipped before the cuts -> not considered, + and not emptied by the cuts. + * object 4: no PSF ('empty') -> never reaches the cuts. + + Failure modes: a cut's drops are not counted or land in the wrong + counter; epochs skipped before the cuts are counted as considered; an + object with no epoch at all is reported as emptied by the cuts; counts + from one object overwrite another's. + """ + radius = EPOCH_CENTRAL_DEFECT_RADIUS + centre = N_STAMP // 2 + clean = np.zeros((N_STAMP, N_STAMP), dtype=np.int32) + ones = np.ones((N_STAMP, N_STAMP)) + wide18, near = clean.copy(), clean.copy() + wide18[:, -18:] = 1 + near[centre, centre + radius - 1] = 1 + objects = { + 1: {"2100001-10": (clean, ones), "2100002-11": (wide18, ones), + "2100003-12": (near, ones)}, + 2: {"2100004-13": (wide18.copy(), ones), + "2100005-14": (near.copy(), ones)}, + 3: {"2100006-15": (clean.copy(), ones)}, + } + stores = {} + for obj_id, epochs in objects.items(): + vignet, _, psf_obj, gal_obj = _fake_inputs(epochs) + if obj_id == 3: + gal_obj["2100006-15"]["VIGNET"] = np.zeros((N_STAMP, N_STAMP)) + stores[obj_id] = (vignet, psf_obj, gal_obj) + vignet = SimpleNamespace( + bkg_vign_cat=None, + bkg_rms_vign_cat=None, + psf_vign_cat={ + "4": "empty", **{str(i): s[1] for i, s in stores.items()} + }, + gal_vign_cat={ + "4": "empty", **{str(i): s[2] for i, s in stores.items()} + }, + flag_vign_cat={ + str(i): s[0].flag_vign_cat["1"] for i, s in stores.items() + }, + weight_vign_cat={ + str(i): s[0].weight_vign_cat["1"] for i, s in stores.items() + }, + f_wcs_file={ + k: v for s in stores.values() for k, v in s[0].f_wcs_file.items() + }, + close=lambda: None, + ) + tile_cat = SimpleNamespace( + obj_id=np.array([1, 2, 3, 4]), ra=np.full(4, RA), dec=np.full(4, DEC), + vign=None, seg=None, flux=None, + ) + + paths = [tmp_path / f"{name}.sqlite" for name in + ("gal", "psf", "weight", "flag", "headers")] + for path in paths: + SqliteDict(str(path)).close() + log = _RecordingLogger() + ngmix = Ngmix( + ["tile_cat.fits"] + [str(p) for p in paths[:4]], + str(tmp_path), "-001-001", 30.0, 0.186, str(paths[4]), log, + bkg_sub=False, + ) + ngmix._vignet_cat.close() + ngmix._vignet_cat = vignet + monkeypatch.setattr(ngmix_module, "Tile_cat", lambda *a, **k: tile_cat) + + def no_fit(*_args, **_kwargs): + raise RuntimeError("metacal is not under test") + + monkeypatch.setattr(ngmix_module, "do_ngmix_metacal", no_fit) + for method in ("compile_results", "save_results", "log_mean_ellipticity"): + monkeypatch.setattr(Ngmix, method, lambda *_a, **_k: None) + + ngmix.process() + + lines = [m for m in log.messages if m.startswith("epoch cuts:")] + assert len(lines) == 1, log.messages + tally = dict( + (k, int(v)) for k, v in re.findall(r"(\w+)=(\d+)", lines[0]) + ) + assert tally == dict( + considered=5, masked_fraction=2, central_veto=2, objects_emptied=1 + ), lines[0] + + +# --- ngmix_runner: the epoch-cut options reach Ngmix ------------------------ + +class _OptionConfig: + """Config stub answering from a dict; absent options take the caller's + fallback.""" + + def __init__(self, options): + self._options = {"MAG_ZP": "30.0", "ID_OBJ_MIN": "-1", + "ID_OBJ_MAX": "-1", **options} + + def has_option(self, _sec, key): + return key in self._options + + def get(self, _sec, key): + return self._options[key] + + def getexpanded(self, _sec, key): + return self._options[key] + + def getfloat(self, _sec, key): + return float(self._options[key]) + + def getint(self, _sec, key): + return int(self._options[key]) + + def getboolean(self, _sec, key, fallback=False): + return fallback + + +def test_runner_threads_the_epoch_cut_options(tmp_path, monkeypatch): + """EPOCH_CENTRAL_DEFECT_RADIUS and EPOCH_MASKED_FRACTION_CUT reach Ngmix + as configured, and default to the module constants when absent. + + Failure mode: the runner drops or ignores an option, so a configured A/B + arm silently runs the default cuts. + """ + from shapepipe.modules import ngmix_runner as runner_module + + captured = [] + + class _Capture: + def __init__(self, *_args, **kwargs): + captured.append(kwargs) + + def process(self): + pass + + monkeypatch.setattr(runner_module, "Ngmix", _Capture) + inputs = [str(tmp_path / f"in{i}.sqlite") for i in range(7)] + for path in inputs: + SqliteDict(path).close() + + for options, radius, fraction in ( + ({"EPOCH_CENTRAL_DEFECT_RADIUS": "7.5", + "EPOCH_MASKED_FRACTION_CUT": "0.1"}, 7.5, 0.1), + ({}, EPOCH_CENTRAL_DEFECT_RADIUS, + ngmix_module.EPOCH_MASKED_FRACTION_CUT), + ): + runner_module.ngmix_runner( + inputs, {"output": str(tmp_path)}, "-001-001", + _OptionConfig(options), "NGMIX_RUNNER", _RecordingLogger(), + ) + assert captured[-1]["epoch_central_defect_radius"] == radius + assert captured[-1]["epoch_masked_fraction_cut"] == fraction + + +# --- make_ngmix_observation: the HSM centroid reads the filled image ------- + +def test_hsm_centroid_ignores_raw_defect_values(): + """With ``centroid_source="hsm"``, the Jacobian origin lands on the + object even when a flagged column a few pixels away holds a raw value + ten times the object's peak. + + Failure mode: HSM measures the raw stamp, so the defect drags the centroid + off the object (or HSM fails and falls back to the stamp centre). + """ + import galsim + + n = 51 + centre = (n - 1) / 2 + d_row, d_col = 1.3, -0.7 + rows, cols = np.mgrid[:n, :n] + gal = np.exp( + -((rows - centre - d_row) ** 2 + (cols - centre - d_col) ** 2) + / (2 * 2.0 ** 2) + ) + flag = np.zeros((n, n), dtype=np.int32) + flag[:, n // 2 + 6] = 1 + gal[flag != 0] = 10.0 + psf = np.exp(-((rows - centre) ** 2 + (cols - centre) ** 2) / 2.0) + obs = make_ngmix_observation( + gal, np.ones((n, n)), flag, psf / psf.sum(), + galsim.PixelScale(0.1857).jacobian(), np.random.RandomState(0), + bkg_rms=np.full((n, n), 1e-3), centroid_source="hsm", + ) + row, col = obs.jacobian.get_cen() + npt.assert_allclose( + [row - centre, col - centre], [d_row, d_col], atol=0.05 + ) diff --git a/tests/module/test_ngmix_uberseg.py b/tests/module/test_ngmix_uberseg.py index 0c80486b2..86dad33bc 100644 --- a/tests/module/test_ngmix_uberseg.py +++ b/tests/module/test_ngmix_uberseg.py @@ -6,10 +6,10 @@ Geometry assertions on a synthetic two-object stamp: neighbour-side pixels are zeroed, the surviving central core is a *single connected* region (the emergent "circularisation"), and the neighbour footprint is fully removed. -* :func:`prepare_ngmix_weights` — the ``noisefill`` default is byte-for-byte - unchanged (asserted against an independent recomputation of the legacy - three-line noise-fill on a shared RNG), while ``uberseg`` hard-masks the - weight (weight -> 0) and leaves the image untouched. +* :func:`prepare_ngmix_weights` under ``uberseg`` — neighbour-side pixels + lose their weight and keep their raw image values, while defect pixels are + noise-filled as under any blend handling (the defect fill itself is covered + in ``test_ngmix_defect_fill.py``). * The error contract when ``uberseg`` is selected without a segmentation map (the seg-map source is plumbing-gated upstream). """ @@ -226,7 +226,7 @@ def test_uberseg_matches_bruteforce_nearest_segment(): npt.assert_array_equal(out, brute) -# --- prepare_ngmix_weights: default unchanged, uberseg hard-masks ---------- +# --- prepare_ngmix_weights: uberseg zeroes neighbour weights only --------- def _gal_flag_weight(npix=41, seed=7): rng = np.random.default_rng(seed) @@ -239,32 +239,6 @@ def _gal_flag_weight(npix=41, seed=7): return gal, flag, weight -def test_noisefill_default_is_byte_identical_to_legacy(): - """The default path reproduces the legacy three-line noise-fill exactly - (same RNG stream): masked pixels replaced by noise, weight 1/sigma^2.""" - gal, flag, weight = _gal_flag_weight() - - gal_out, w_out, noise_out = prepare_ngmix_weights( - gal, weight, flag, np.random.RandomState(123), - ) - - # Independent recomputation of the legacy algorithm on the same stream. - from modopt.math.stats import sigma_mad - rng = np.random.RandomState(123) - mask = np.copy(weight) != 0 - mask[flag != 0] = False - sig = sigma_mad(gal) - w_exp = mask.astype(float) / sig ** 2 - noise_exp = rng.standard_normal(gal.shape) * sig - noise_gal = rng.standard_normal(gal.shape) * sig - gal_exp = np.copy(gal) - gal_exp[~mask] = noise_gal[~mask] - - npt.assert_array_equal(gal_out, gal_exp) - npt.assert_array_equal(w_out, w_exp) - npt.assert_array_equal(noise_out, noise_exp) - - def test_noisefill_ignores_seg_and_dilate_kwargs(): """Under noisefill, passing seg / dilate_neighbour changes nothing: the result matches the plain default call on the same RNG stream.""" @@ -282,9 +256,15 @@ def test_noisefill_ignores_seg_and_dilate_kwargs(): npt.assert_array_equal(a, b) -def test_uberseg_hard_masks_weight_and_leaves_image_untouched(): - """uberseg: image returned untouched, weight zeroed on neighbour-side and - flagged pixels, positive on the central core.""" +def test_uberseg_fills_defects_and_leaves_neighbour_pixels_raw(): + """uberseg: flagged pixels are noise-filled at weight 0; neighbour-side + pixels get weight 0 and keep their raw image values; the central core + keeps weight and image. + + Failure modes: the defect fill is skipped under uberseg, so raw bad + pixels reach metacal; or the neighbour side is noise-filled + (defects-filled-neighbours-raw). + """ npix = 41 gal, flag, weight = _gal_flag_weight(npix=npix) seg, centre, neigh = two_object_seg(npix=npix, sep=12) @@ -294,13 +274,16 @@ def test_uberseg_hard_masks_weight_and_leaves_image_untouched(): blend_handling="uberseg", seg=seg, object_number=1, ) - # Image untouched under uberseg (no noise fill). - npt.assert_array_equal(gal_out, gal) - # Neighbour footprint hard-masked; central centre kept. + # Flagged pixels: zero weight, image replaced by noise. + for pix in [(5, 5), (30, 12)]: + assert w_out[pix] == 0.0 + assert gal_out[pix] != gal[pix] + # Neighbour footprint: zero weight, raw image (never noise-filled). assert np.all(w_out[seg == 2] == 0.0) + npt.assert_array_equal(gal_out[seg == 2], gal[seg == 2]) + # Central core: weight and image untouched. assert w_out[centre] > 0.0 - # Flagged bad pixels remain at weight 0 (folded into the base mask). - assert w_out[5, 5] == 0.0 + assert gal_out[centre] == gal[centre] def test_uberseg_requires_seg_and_object_number(): @@ -422,3 +405,11 @@ def test_runner_missing_seg_vignet_file_raises(tmp_path): "NGMIX_RUNNER", _RecordingLogger(), ) + + +def test_retired_noisefill_name_is_rejected(): + """Do not silently interpret the retired neighbour option.""" + with pytest.raises(ValueError, match="noisefill"): + prepare_ngmix_weights(np.ones((5, 5)), np.ones((5, 5)), + np.zeros((5, 5)), np.random.RandomState(3), + blend_handling="noisefill") diff --git a/tests/science/test_defect_veto.py b/tests/science/test_defect_veto.py new file mode 100644 index 000000000..3384263e4 --- /dev/null +++ b/tests/science/test_defect_veto.py @@ -0,0 +1,104 @@ +"""Shear recovery for the defects the central-defect veto keeps (noise fill). + +Physics invariant: a defect the veto keeps -- a column, a 3-px bleed or a +single pixel at the veto radius or beyond, or an edge band -- leaves both +additive terms |c1|, |c2| < 5e-4 and both diagonal multiplicative terms +|m11|, |m22| < 1%, from the full 2x2 response matrix. The grid spans galaxies +with half-light radius 0.3" and 0.5" through a 0.7" PSF, round and with +ellipticity (0.05, 0.02). A noise-filled defect biases m anisotropically +(m11 and m22 differ by up to a factor of ten), so a scalar m would hide it. +The cases sit at ``EPOCH_CENTRAL_DEFECT_RADIUS`` itself, so lowering the +radius below the calibrated value turns this red. + +Positive control: the same column two pixels inside the radius breaks the +bound, so the recovery check can see the bias the veto removes. +""" + +import json + +import numpy as np +import pytest + +from shapepipe.modules.ngmix_package.ngmix import ( + EPOCH_CENTRAL_DEFECT_RADIUS, + EPOCH_MASKED_FRACTION_CUT, + defect_mask, + has_central_defect, +) +from tests.helpers.defect_response import defect_response + +N = 51 +CENTRE = N // 2 +R = int(np.ceil(EPOCH_CENTRAL_DEFECT_RADIUS)) +ELLIPTICAL = (0.05, 0.02) +SEEDS = range(6) + + +def geometry(kind, distance): + """A detector defect whose nearest pixel is ``distance`` px from the + stamp centre; an edge band reaches in to that distance.""" + bad = np.zeros((N, N), dtype=bool) + if kind == "pixel": + bad[CENTRE, CENTRE + distance] = True + elif kind == "column": + bad[:, CENTRE + distance] = True + elif kind == "bleed": + bad[:, CENTRE + distance:CENTRE + distance + 3] = True + elif kind == "edge": + bad[:, CENTRE + distance:] = True + return bad + + +# Wide defects (a 3-px bleed, the widest edge band the veto keeps) at the +# radius itself on the 0.5" galaxy through the elliptical PSF sit at the +# bound (m11 = -0.98% +/- 0.04% and -0.94% +/- 0.06%; see +# :func:`has_central_defect`), so those two are checked one pixel out. +CASES = ( + [(k, d, h, (0.0, 0.0)) for k in ("column", "bleed", "pixel") + for d in (R, R + 2) for h in (0.3, 0.5)] + + [(k, R, 0.5, ELLIPTICAL) for k in ("column", "pixel")] + + [(k, R + 1, 0.5, ELLIPTICAL) for k in ("bleed", "edge")] + + [("edge", R, 0.5, (0.0, 0.0))] + + [("edge", N - CENTRE - 5, 0.5, psf) for psf in ((0.0, 0.0), ELLIPTICAL)] +) + + +def recover(bad, hlr, psf_shear, tmp_path): + result = defect_response(bad, hlr=hlr, psf=0.7, seeds=SEEDS, + psf_shear=psf_shear) + (tmp_path / "recovery.json").write_text(json.dumps(result, indent=2)) + return np.abs(result["m"]).max(), np.abs(result["c"]).max(), result + + +def case_id(case): + kind, distance, hlr, psf_shear = case + psf = "elliptical" if any(psf_shear) else "round" + return f"{kind}-{distance}px-hlr{hlr}-{psf}" + + +@pytest.mark.parametrize("kind,distance,hlr,psf_shear", CASES, + ids=[case_id(c) for c in CASES]) +def test_kept_defects_recover_shear_on_both_axes(kind, distance, hlr, + psf_shear, tmp_path): + """Failure modes: the veto radius is below the calibrated value; the + fill leaks raw defect values or leaves more of the object's light + missing; the check reads m11 alone and misses m22, or c1 alone. + """ + bad = geometry(kind, distance) + masked = defect_mask(np.ones((N, N)), bad.astype(np.int32)) + np.testing.assert_array_equal(masked, bad) + assert masked.mean() <= EPOCH_MASKED_FRACTION_CUT + assert not has_central_defect(masked, EPOCH_CENTRAL_DEFECT_RADIUS) + m, c, result = recover(bad, hlr, psf_shear, tmp_path) + assert m < 0.01, result + assert c < 5e-4, result + + +def test_vetoed_column_breaks_the_bound(tmp_path): + """Positive control: a column two pixels inside the radius, on the + 0.5" galaxy, gives |m| > 2% and |c| > 1e-3. The veto drops it.""" + bad = geometry("column", R - 2) + assert has_central_defect(bad, EPOCH_CENTRAL_DEFECT_RADIUS) + m, c, result = recover(bad, 0.5, (0.0, 0.0), tmp_path) + assert m > 0.02, result + assert c > 1e-3, result From cd8b321b2bc81e5920e9a2a9b7d501c846fff9a0 Mon Sep 17 00:00:00 2001 From: Cail Daley Date: Sat, 26 Sep 2026 07:07:40 +0200 Subject: [PATCH 02/30] feat(ngmix): noise-fill defects under every BLEND_HANDLING, veto central ones Metacal shears the whole stamp without reading the weights, so a defect left raw under uberseg reached the fit. Every defect (nonzero flag, zero weight, invalid RMS) is now zero-weighted and noise-filled whatever BLEND_HANDLING is; uberseg only zeroes the weight of neighbour-side pixels and keeps their light. The option that meant "no neighbour treatment" is renamed from noisefill to none, and the retired name is rejected. The defect set is not symmetrized: at the veto radius the one-sided fill leaves |c| <= 3e-4, while a four-fold mask would quadruple m (-2.7% against -0.64% for a 3-px bleed at 10 px on a 0.5" galaxy) and, through an elliptical PSF, still leave c1 = 7.3e-4. An epoch is dropped when a defect lies closer than EPOCH_CENTRAL_DEFECT_RADIUS (default 10 px) to the stamp centre, the smallest radius at which columns, 3-px bleeds, single pixels and edge bands give |m11|, |m22| < 1% and |c1|, |c2| < 5e-4 on 0.3" and 0.5" galaxies through a 0.7" PSF. The masked-fraction cut counts the same raw defect set; EPOCH_MASKED_FRACTION_CUT (default 1/3) makes it configurable. Each tile logs "epoch cuts: considered=N masked_fraction=A central_veto=B objects_emptied=C". The HSM centroid is measured on the filled image. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01QAc1ywyd9Sbbra6po2iuaA --- src/shapepipe/modules/ngmix_package/ngmix.py | 353 +++++++++++++++---- src/shapepipe/modules/ngmix_runner.py | 38 +- 2 files changed, 317 insertions(+), 74 deletions(-) diff --git a/src/shapepipe/modules/ngmix_package/ngmix.py b/src/shapepipe/modules/ngmix_package/ngmix.py index b2e092240..807b99ff0 100644 --- a/src/shapepipe/modules/ngmix_package/ngmix.py +++ b/src/shapepipe/modules/ngmix_package/ngmix.py @@ -8,6 +8,7 @@ import os import re +from collections import Counter from typing import NamedTuple import ngmix @@ -25,7 +26,17 @@ from shapepipe.pipeline import file_io # Neighbour treatments selectable with the BLEND_HANDLING option. -BLEND_HANDLINGS = ("noisefill", "uberseg") +BLEND_HANDLINGS = ("none", "uberseg") + +# Default of the EPOCH_MASKED_FRACTION_CUT option: an epoch is dropped when +# more than this fraction of its stamp is in :func:`defect_mask` (see +# :func:`prepare_postage_stamps`). +EPOCH_MASKED_FRACTION_CUT = 1 / 3 + +# Default of the EPOCH_CENTRAL_DEFECT_RADIUS option (pixels): an epoch is +# dropped when a pixel of :func:`defect_mask` lies closer than this to the +# stamp centre (see :func:`has_central_defect`). +EPOCH_CENTRAL_DEFECT_RADIUS = 10 METACAL_TYPES = ('noshear', '1p', '1m', '2p', '2m') @@ -191,7 +202,7 @@ class Tile_cat(): vignetmaker output cut from the tile ``SEGMENTATION`` check image), row-aligned to ``cat_path``. When given, ``self.seg`` holds one integer seg stamp per object for the ``"uberseg"`` blend handling; ``None`` - leaves ``self.seg`` unset (the noise-fill path is unaffected). + leaves ``self.seg`` unset, which only ``"uberseg"`` needs. """ def __init__( @@ -226,7 +237,7 @@ def get_data(self, cat_path): # Coadd-frame SExtractor segmentation stamp (integer labels), one per # object and row-aligned to the tile catalogue, overlaid unchanged on # every epoch for uberseg neighbour masking (shapepipe#776). None -> - # uberseg unavailable; the noise-fill path is unaffected. + # uberseg unavailable; ``"none"`` does not read it. self.seg = None if self.seg_cat_path: seg_cat = file_io.FITSCatalogue( @@ -281,7 +292,7 @@ def __init__( self.flags = [] self.bkg_rms = [] # Segmentation stamps, one per epoch, used only by the "uberseg" blend - # handling; empty for the default noise-fill path. All epochs carry the + # handling; empty under the default "none". All epochs carry the # SAME coadd-frame seg stamp (shapepipe#776: one coadd seg per object, # no per-epoch reprojection), each MegaCam-flipped to match its galaxy # stamp so the overlay stays registered. @@ -298,6 +309,9 @@ def __init__( # CCD number of the first epoch, used only to build the per-object # position seed (see :func:`position_seed`). self.ccd = None + self.epoch_cuts = Counter( + considered=0, masked_fraction=0, central_veto=0 + ) self.bkg_sub = bkg_sub self.megacam_flip = megacam_flip @@ -426,16 +440,26 @@ class Ngmix(object): propagated on the vignette. ``"hsm"`` re-centers on the adaptive-moment centroid measured from the stamp pixels. See :func:`make_ngmix_observation`. - blend_handling : {"noisefill", "uberseg"}, optional - Neighbour treatment; ``"noisefill"`` (default) is the historical - noise-fill, ``"uberseg"`` hard-masks neighbour-side pixels from the - coadd segmentation map and requires ``seg_cat_path``. + blend_handling : {"none", "uberseg"}, optional + Neighbour treatment. ``"none"`` (default) leaves neighbour pixels + untouched; ``"uberseg"`` zeroes the weight of neighbour-side pixels + from the coadd segmentation map and requires ``seg_cat_path``. Defect + pixels are noise-filled under both (see + :func:`prepare_ngmix_weights`). seg_cat_path : str, optional Path to the coadd-frame segmentation VIGNET catalogue (see :class:`Tile_cat`). Required when ``blend_handling="uberseg"``. dilate_neighbour : int, optional Neighbour-mask dilation iterations for ``"uberseg"`` (see :func:`uberseg_weight`); the default is ``1``. + epoch_central_defect_radius : float, optional + Drop an epoch when a masked pixel lies closer than this many pixels + to the stamp centre (see :func:`has_central_defect`); the default is + ``EPOCH_CENTRAL_DEFECT_RADIUS``. + epoch_masked_fraction_cut : float, optional + Drop an epoch when more than this fraction of its stamp is masked + (see :func:`prepare_postage_stamps`); the default is + ``EPOCH_MASKED_FRACTION_CUT``. Notes ----- @@ -466,10 +490,12 @@ def __init__( id_obj_max=-1, bkg_sub=True, centroid_source="wcs", - blend_handling="noisefill", + blend_handling="none", seg_cat_path=None, dilate_neighbour=1, metacal_psf="fitgauss", + epoch_central_defect_radius=EPOCH_CENTRAL_DEFECT_RADIUS, + epoch_masked_fraction_cut=EPOCH_MASKED_FRACTION_CUT, ): # Base count = catalogue + vignets, excluding the f_wcs headers (passed @@ -539,6 +565,8 @@ def __init__( self._seg_cat_path = seg_cat_path self._dilate_neighbour = dilate_neighbour self._metacal_psf = metacal_psf + self._epoch_central_defect_radius = epoch_central_defect_radius + self._epoch_masked_fraction_cut = epoch_masked_fraction_cut self._w_log = w_log @@ -985,6 +1013,8 @@ def process(self): id_first = -1 id_last = -1 count_batch = 0 + epoch_cuts = Counter(considered=0, masked_fraction=0, central_veto=0) + n_emptied = 0 saved_batch_cumul = 0 for i_tile, obj_id in enumerate(tile_cat.obj_id): @@ -1014,10 +1044,14 @@ def process(self): self._bkg_sub, psf_obj, gal_obj, + epoch_central_defect_radius=self._epoch_central_defect_radius, + epoch_masked_fraction_cut=self._epoch_masked_fraction_cut, ) + epoch_cuts.update(stamp.epoch_cuts) if len(stamp.gals) == 0: n_no_epoch += 1 + n_emptied += stamp.epoch_cuts["considered"] > 0 continue # Per-object RNG, seeded from (ra, dec, ccd) — see @@ -1131,6 +1165,13 @@ def process(self): + f" {n_fitted} fitted" ) + self._w_log.info( + "epoch cuts:" + + f" considered={epoch_cuts['considered']}" + + f" masked_fraction={epoch_cuts['masked_fraction']}" + + f" central_veto={epoch_cuts['central_veto']}" + + f" objects_emptied={n_emptied}" + ) vignet_cat.close() # Put all results together @@ -1150,7 +1191,46 @@ def prepare_postage_stamps( bkg_sub=True, psf_obj=None, gal_obj=None, + epoch_central_defect_radius=EPOCH_CENTRAL_DEFECT_RADIUS, + epoch_masked_fraction_cut=EPOCH_MASKED_FRACTION_CUT, ): + """Gather one object's epoch stamps, dropping epochs its defects spoil. + + @sc [decision:shape_measurement.epoch_masked_fraction_cut,decision:shape_measurement.defect_fill] epoch-cut-on-defect-mask + An epoch is dropped when more than ``epoch_masked_fraction_cut`` of its + stamp lies in :func:`defect_mask`: flagged, zero-weight and invalid-RMS + pixels, the set that :func:`prepare_ngmix_weights` zero-weights and + noise-fills. Counting flags alone would keep epochs whose filled area + exceeds the cut. The default is 1/3; 10%, the DES Y3 and Y6 value, is + the alternative to test. + + Parameters + ---------- + vignet : Vignet + Per-object vignet stores. + obj_id : int + Object ID (SExtractor ``NUMBER``). + i_tile : int + Row of the object in ``tile_cat``. + tile_cat : Tile_cat + Tile catalogue. + bkg_sub : bool, optional + Subtract the background vignet; the default is ``True``. + psf_obj, gal_obj : dict, optional + The object's PSF and galaxy vignet dicts, if already read. + epoch_central_defect_radius : float, optional + Drop an epoch with a defect pixel closer than this many pixels to the + stamp centre (:func:`has_central_defect`); 0 disables the veto. The + default is ``EPOCH_CENTRAL_DEFECT_RADIUS``. + epoch_masked_fraction_cut : float, optional + Drop an epoch with more than this fraction of its stamp in + :func:`defect_mask`. The default is ``EPOCH_MASKED_FRACTION_CUT``. + + Returns + ------- + Postage_stamp + The surviving epochs' stamps. + """ # define per-object lists of individual exposures to go into ngmix stamp = Postage_stamp(bkg_sub=bkg_sub) # Read each store's per-object dict ONCE: every sqlitedict access @@ -1220,17 +1300,23 @@ def prepare_postage_stamps( flag_vign = flag_obj[expccd_name]['VIGNET'] if tile_vign is not None: flag_vign[np.where(tile_vign == -1e30)] = 2**10 - v_flag_tmp = flag_vign.ravel() - # remove objects that are more than 1/3 masked - if len(np.where(v_flag_tmp != 0)[0]) / v_flag_tmp.size > 1 / 3.0: - continue - weight_vign = weight_obj[expccd_name]['VIGNET'] bkg_rms_vign = ( bkg_rms_obj[expccd_name]['VIGNET'] if bkg_rms_obj is not None else None ) + # Drop the epoch when too much of it would be zero-weighted and + # noise-filled (epoch-cut-on-defect-mask), or when a filled pixel + # would sit near the object (epoch-central-defect-veto). + masked = defect_mask(weight_vign, flag_vign, bkg_rms_vign) + stamp.epoch_cuts["considered"] += 1 + if masked.mean() > epoch_masked_fraction_cut: + stamp.epoch_cuts["masked_fraction"] += 1 + continue + if has_central_defect(masked, epoch_central_defect_radius): + stamp.epoch_cuts["central_veto"] += 1 + continue # One unpickle per exposure (all CCDs), reused across this object's # epochs; the cache is per-call, so bounded by the object's exposures @@ -1503,9 +1589,9 @@ def uberseg_weight(weight, seg, object_number, dilate_neighbour=0): Because the partition is by distance to the nearest footprint, the pixels surviving around a compact central object form a single connected, roughly circular core; the "circularisation" is emergent geometry, not a - separate aperture. Unlike the noise-fill treatment, masked pixels are - handed to ngmix as a hard mask (weight = 0), never replaced by a noise - realisation. + separate aperture. Only the weight changes: neighbour-side pixels are + handed to ngmix as a hard mask (weight = 0) and keep their image values, + so metacal shears the neighbour's light along with the target's. Parameters ---------- @@ -1566,33 +1652,163 @@ def uberseg_weight(weight, seg, object_number, dilate_neighbour=0): return weight + +def defect_mask(weight, flag, bkg_rms=None): + """Defect pixels of one epoch stamp. + + @sc [decision:shape_measurement.defect_fill,label:physics] defect-set-unsymmetrized + A defect is a pixel with zero exposure weight, a nonzero flag, or (when a + background RMS map is given) a non-finite or non-positive RMS. This one + set is zero-weighted and noise-filled by :func:`prepare_ngmix_weights` + and counted by the epoch cuts in :func:`prepare_postage_stamps`. It is + not ORed with its rotations. For defects the central-defect veto keeps + (:func:`has_central_defect`), the unsymmetrized fill leaves + |c| <= 3e-4 per affected epoch on galaxies with half-light radius 0.3" + and 0.5" through a 0.7" PSF, round or elliptical. Symmetrizing would + quadruple the filled area near the object, and with it m: a 3-px bleed at + 10 px on the 0.5" galaxy gives m11 = -2.7% four-fold against -0.64% + unsymmetrized. Through a PSF with ellipticity (0.05, 0.02) it would not + cancel c either: c1 = 7.3e-4 four-fold against 0.7e-4. It would also + turn a 5-px edge band (9.8% of the stamp), which biases nothing, into a + 35.4% frame that fails the masked-fraction cut. + + Parameters + ---------- + weight : numpy.ndarray + Exposure weight stamp. + flag : numpy.ndarray + Flag stamp. + bkg_rms : numpy.ndarray, optional + Background RMS stamp. + + Returns + ------- + numpy.ndarray of bool + ``True`` on defect pixels. + """ + defect = (weight == 0) | (flag != 0) + if bkg_rms is not None: + defect |= ~(np.isfinite(bkg_rms) & (bkg_rms > 0)) + return defect + + +def has_central_defect(defect, radius): + """Whether a defect pixel lies closer than ``radius`` to the stamp centre. + + @sc [decision:shape_measurement.central_defect_veto,decision:shape_measurement.defect_fill] epoch-central-defect-veto + An epoch is dropped when any pixel of :func:`defect_mask` lies closer + than ``radius`` pixels to the stamp centre. A noise-filled hole in the + object's light is sheared by metacal but not by the sky, so the metacal + response is wrong for that epoch, and a one-sided hole adds an additive + term. The default radius (``EPOCH_CENTRAL_DEFECT_RADIUS``, 10 px or + 1.9") is the smallest at which columns, 3-px bleeds, single pixels and + edge bands give |m11|, |m22| < 1% and |c1|, |c2| < 5e-4 (full response + matrix) on galaxies with half-light radius 0.3" and 0.5" through a 0.7" + PSF. The bias is anisotropic: a column at 10 px on the 0.5" galaxy gives + m11 = -0.60%, m22 = +0.01% and c1 = -1.4e-4; at 9 px, m11 = -1.75% and + c1 = -1.1e-3. Through a PSF with ellipticity (0.05, 0.02), wide defects + at 10 px on the 0.5" galaxy sit at the bound: a 3-px bleed gives + m11 = -0.98% +/- 0.04% and the widest edge band the veto keeps (16 px) + -0.94% +/- 0.06%; at 11 px the bleed gives -0.24%. Larger galaxies need + more: at hlr 0.7" through a 0.9" PSF a 3-px bleed at 10 px gives + m11 = -6.8% and c1 = -4.9e-3, and passes from 14 px. The distance is + measured from the stamp centre, where the extractor places the object to + within half a pixel. The veto reads only the defect mask, never the + object's pixels, so it selects on nothing that responds to shear. + Radius 0 disables it. Guarded by ``tests/science/test_defect_veto.py``. + + Parameters + ---------- + defect : numpy.ndarray of bool + Defect mask of one epoch stamp (:func:`defect_mask`). + radius : float + Veto radius in pixels. + + Returns + ------- + bool + ``True`` if the epoch should be dropped. + """ + rows, cols = np.nonzero(defect) + centre_row = (defect.shape[0] - 1) / 2 + centre_col = (defect.shape[1] - 1) / 2 + distance = np.hypot(rows - centre_row, cols - centre_col) + return bool(np.any(distance < radius)) + + +def fill_defects(image, defect, noise): + """Replace an image's defect pixels by a noise realisation. + + @sc [decision:shape_measurement.defect_fill,label:physics] defect-noise-fill + Defect pixels are replaced by ``noise``, an independent realisation at + the per-pixel background RMS. Metacal deconvolves, shears and reconvolves + the whole image without reading the weights, so a raw defect value (bad + column, bleed, cosmic ray) would leak into the fit. The noise fill leaves + a hole in the object's light that metacal shears and the sky does not, + which biases m for an epoch with a defect near the object; the + central-defect veto drops those epochs (epoch-central-defect-veto). + + Parameters + ---------- + image : numpy.ndarray + Stamp image. + defect : numpy.ndarray of bool + Pixels to replace (:func:`defect_mask`). + noise : numpy.ndarray + Noise realisation on the stamp grid. + + Returns + ------- + numpy.ndarray + ``image`` with ``defect`` pixels taken from ``noise``. + """ + return np.where(defect, noise, image) + + def prepare_ngmix_weights( gal, weight, flag, rng, bkg_rms=None, - blend_handling="noisefill", seg=None, object_number=None, + blend_handling="none", seg=None, object_number=None, dilate_neighbour=0, ): - """bookkeeping for ngmix weights. runs on a single galaxy and epoch - pixel scale and galaxy guess - TO DO: decide if we want galaxy guess stuff + """Build one epoch's image, weight map and noise image for ngmix. + + Defect pixels (:func:`defect_mask`: flagged, zero-weight or invalid-RMS + pixels) get weight 0 and are replaced by an independent noise + realisation at their background RMS (:func:`fill_defects`), under either + ``blend_handling``. ``blend_handling`` decides only the neighbour side. + + @sc [decision:shape_measurement.defect_fill,decision:shape_measurement.blend_handling] defects-filled-neighbours-raw + Every pixel of ``defect_mask`` is zero-weighted and noise-filled whatever + ``blend_handling`` is, and the filled set equals the zero-weight defect + set. ``blend_handling`` acts on the neighbour side only: uberseg zeroes + the weight of pixels nearer a neighbour's footprint and leaves their + image values raw, because that light is real sky that metacal shears + along with the target; filling it with noise would cut the target along + an unsheared edge. Neighbour pixels are never noise-filled. The noise + image covers the whole stamp and is independent of both. Parameters ---------- gal : numpy.ndarray + Background-subtracted galaxy stamp. weight : numpy.ndarray + Exposure weight stamp; zero marks a defect. flag : numpy.ndarray + Flag stamp; nonzero marks a defect. rng : numpy.random.RandomState Random state for the noise realisations (seeded per object; see :func:`position_seed`). bkg_rms : numpy.ndarray, optional - Per-pixel background RMS map. If supplied, unmasked pixels use - ``1 / bkg_rms**2`` as the ngmix inverse variance. - blend_handling : {"noisefill", "uberseg"}, optional - How to treat pixels shared with a neighbour. ``"noisefill"`` (default) - replaces flagged pixels with a noise realisation and keeps their - inverse-variance weight — the historical behaviour. ``"uberseg"`` - instead hard-masks (weight = 0) every pixel closer to a neighbour's - segmentation footprint than to the central object's, leaving the - image untouched (see :func:`uberseg_weight`). + Per-pixel background RMS map. If supplied, clean pixels use + ``1 / bkg_rms**2`` as the ngmix inverse variance, and non-finite or + non-positive values mark defects. Otherwise every clean pixel gets + ``1 / sigma_mad(gal)**2``. + blend_handling : {"none", "uberseg"}, optional + Neighbour treatment. ``"none"`` (default) leaves neighbour + pixels weighted and untouched. ``"uberseg"`` zeroes the weight of + every pixel closer to a neighbour's segmentation footprint than to the + central object's and keeps its raw image value (see + :func:`uberseg_weight`). seg : numpy.ndarray, optional Segmentation map on the stamp grid (object NUMBERs). Required for ``blend_handling="uberseg"``; ignored otherwise. @@ -1606,44 +1822,57 @@ def prepare_ngmix_weights( Returns ------- numpy.ndarray - Galaxy image. For ``"noisefill"`` masked pixels are replaced by noise; - for ``"uberseg"`` the image is returned untouched. + Galaxy image with defect pixels noise-filled. numpy.ndarray - Variance map for NGMIX. + Inverse-variance weight map for ngmix. numpy.ndarray - Noise image. + Noise image: an independent realisation over the whole stamp, for + metacal's ``fixnoise``. + + Raises + ------ + ValueError + If ``blend_handling`` is unknown, or ``"uberseg"`` lacks ``seg`` or + ``object_number``. """ if blend_handling not in BLEND_HANDLINGS: raise ValueError( f"Unknown blend_handling '{blend_handling}'; expected one of" + f" {BLEND_HANDLINGS}" ) + if blend_handling == "uberseg" and (seg is None or object_number is None): + raise ValueError( + "blend_handling='uberseg' requires a segmentation map and the" + + " central object_number; none reached prepare_ngmix_weights." + + " Set SEG_VIGNET_PATH on the ngmix run (see" + + " CosmoStat/shapepipe#776)." + ) - mask = np.copy(weight) != 0 - mask[flag != 0] = False + defect = defect_mask(weight, flag, bkg_rms) + clean = ~defect if bkg_rms is None: sig_noise = sigma_mad(gal) # Guard the degenerate constant stamp (sigma_mad == 0): 0 * inf # would otherwise put NaN in a fully-masked weight map. weight_map = ( - mask.astype(float) / sig_noise ** 2 + clean.astype(float) / sig_noise ** 2 if sig_noise > 0 else np.zeros_like(gal, dtype=float) ) else: - valid_rms = np.isfinite(bkg_rms) & (bkg_rms > 0) - mask &= valid_rms weight_map = np.zeros_like(gal, dtype=float) - weight_map[mask] = 1.0 / bkg_rms[mask] ** 2 + weight_map[clean] = 1.0 / bkg_rms[clean] ** 2 # Per-pixel noise sigma for the realisations below: metacal's # fixnoise bookkeeping (1/w + 1/w_noise) assumes the noise image # is a faithful realisation of the per-pixel variance the weights # claim; a scalar sigma there mis-reports errors and erodes the # inverse-variance advantage whenever the RMS map actually varies. + # Pixels without a valid RMS take the median over clean pixels. + valid_rms = np.isfinite(bkg_rms) & (bkg_rms > 0) sig_noise = ( - np.where(valid_rms, bkg_rms, np.median(bkg_rms[mask])) - if mask.any() + np.where(valid_rms, bkg_rms, np.median(bkg_rms[clean])) + if clean.any() else sigma_mad(gal) ) @@ -1656,33 +1885,20 @@ def prepare_ngmix_weights( noise_img = rng.standard_normal(gal.shape) * sig_noise noise_img_gal = rng.standard_normal(gal.shape) * sig_noise + gal_filled = fill_defects(gal, defect, noise_img_gal) - gal_masked = np.copy(gal) if blend_handling == "uberseg": - # Hard-mask neighbour-side pixels (weight -> 0) from the segmentation - # geometry; the image is left untouched (the masked pixels carry no - # weight, so ngmix ignores them in the likelihood). Bad/flagged - # pixels already sit at weight 0 from the mask above. - if seg is None or object_number is None: - raise ValueError( - "blend_handling='uberseg' requires a segmentation map and the" - + " central object_number; none reached prepare_ngmix_weights." - + " Set SEG_VIGNET_PATH on the ngmix run (see" - + " CosmoStat/shapepipe#776)." - ) weight_map = uberseg_weight( weight_map, seg, object_number, dilate_neighbour=dilate_neighbour ) - elif (~mask).any(): - # noisefill (default): replace masked pixels with a noise realisation. - gal_masked[~mask] = noise_img_gal[~mask] - return gal_masked, weight_map, noise_img + return gal_filled, weight_map, noise_img + def make_ngmix_observation( gal, weight, flag, psf, wcs, rng, bkg_rms=None, centroid_source="wcs", offset=None, - blend_handling="noisefill", seg=None, object_number=None, + blend_handling="none", seg=None, object_number=None, dilate_neighbour=0, ): """Build an ngmix Observation for a single galaxy epoch. @@ -1725,9 +1941,9 @@ def make_ngmix_observation( Sub-pixel ``[row, col]`` coadd-centroid offset propagated from the stamp extractor. Required for ``centroid_source="wcs"`` (ignored for ``"hsm"``). - blend_handling : {"noisefill", "uberseg"}, optional + blend_handling : {"none", "uberseg"}, optional Neighbour treatment passed through to :func:`prepare_ngmix_weights`; - the default ``"noisefill"`` is the historical behaviour. + the default ``"none"`` leaves neighbour pixels untouched. seg : numpy.ndarray, optional Segmentation map on the stamp grid. Required for ``blend_handling="uberseg"`` (ignored otherwise). @@ -1769,11 +1985,12 @@ def make_ngmix_observation( if centroid_source == "hsm": # Re-center the Jacobian on the HSM adaptive-moment centroid (pixel - # offset from the stamp center); fall back to the stamp center if - # HSM fails. + # offset from the stamp center), measured on the filled image so no + # raw defect value pulls it; fall back to the stamp center if HSM + # fails. try: _hsm = galsim.hsm.FindAdaptiveMom( - galsim.Image(gal, scale=1.0), strict=False + galsim.Image(gal_masked, scale=1.0), strict=False ) if _hsm.error_message != "": raise galsim.hsm.GalSimHSMError(_hsm.error_message) @@ -1988,7 +2205,7 @@ def make_runners(prior, flux_guess, rng): def do_ngmix_metacal( stamp, prior, flux_guess, rng, centroid_source="wcs", - blend_handling="noisefill", object_number=None, dilate_neighbour=0, + blend_handling="none", object_number=None, dilate_neighbour=0, metacal_psf="fitgauss", ): """Do Ngmix Metacal. @@ -2012,10 +2229,10 @@ def do_ngmix_metacal( coadd-centroid offset in ``stamp.offsets``, propagated from the stamp extractor); ``"hsm"`` uses the adaptive-moment centroid from the stamp pixels — see that function. - blend_handling : {"noisefill", "uberseg"}, optional + blend_handling : {"none", "uberseg"}, optional Neighbour treatment passed through to - :func:`make_ngmix_observation`; the default ``"noisefill"`` is the - historical behaviour. ``"uberseg"`` consumes ``stamp.segs`` and + :func:`make_ngmix_observation`; the default ``"none"`` leaves + neighbour pixels untouched. ``"uberseg"`` consumes ``stamp.segs`` and ``object_number``. object_number : int, optional Central object's segmentation label — its SExtractor ``NUMBER`` diff --git a/src/shapepipe/modules/ngmix_runner.py b/src/shapepipe/modules/ngmix_runner.py index 27648947f..ffc4de083 100644 --- a/src/shapepipe/modules/ngmix_runner.py +++ b/src/shapepipe/modules/ngmix_runner.py @@ -11,7 +11,11 @@ from sqlitedict import SqliteDict from shapepipe.modules.module_decorator import module_runner -from shapepipe.modules.ngmix_package.ngmix import Ngmix +from shapepipe.modules.ngmix_package.ngmix import ( + EPOCH_CENTRAL_DEFECT_RADIUS, + EPOCH_MASKED_FRACTION_CUT, + Ngmix, +) @module_runner( @@ -125,14 +129,16 @@ def ngmix_runner( else: centroid_source = "wcs" - # Neighbour treatment: "noisefill" (default, historical) replaces a - # neighbour's pixels with a noise realisation; "uberseg" hard-masks - # (weight -> 0) every pixel closer to a neighbour than to the central - # object, from the segmentation map. See the ngmix module docstrings. + # Neighbour treatment: "none" (default) leaves neighbour pixels + # weighted and untouched; "uberseg" zeroes the weight of every pixel + # closer to a neighbour than to the central object, from the segmentation + # map, and leaves its image raw. Defect pixels (flagged, zero-weight or + # invalid-RMS) are zero-weighted and noise-filled under both; see + # prepare_ngmix_weights. if config.has_option(module_config_sec, "BLEND_HANDLING"): blend_handling = config.get(module_config_sec, "BLEND_HANDLING") else: - blend_handling = "noisefill" + blend_handling = "none" # DILATE_NEIGHBOUR (optional): binary-dilation iterations enlarging the # uberseg neighbour mask, to absorb the few-pixel coadd-vs-epoch seg-overlay @@ -142,6 +148,24 @@ def ngmix_runner( else: dilate_neighbour = 1 + # EPOCH_CENTRAL_DEFECT_RADIUS (optional, pixels): drop an epoch when a + # masked pixel lies closer than this to the stamp centre; 0 disables. + if config.has_option(module_config_sec, "EPOCH_CENTRAL_DEFECT_RADIUS"): + epoch_central_defect_radius = config.getfloat( + module_config_sec, "EPOCH_CENTRAL_DEFECT_RADIUS" + ) + else: + epoch_central_defect_radius = EPOCH_CENTRAL_DEFECT_RADIUS + + # EPOCH_MASKED_FRACTION_CUT (optional): drop an epoch when more than this + # fraction of its stamp is masked (flagged, zero-weight or invalid-RMS). + if config.has_option(module_config_sec, "EPOCH_MASKED_FRACTION_CUT"): + epoch_masked_fraction_cut = config.getfloat( + module_config_sec, "EPOCH_MASKED_FRACTION_CUT" + ) + else: + epoch_masked_fraction_cut = EPOCH_MASKED_FRACTION_CUT + # Check PSF vignets first: if all are empty dicts {}, the exposures for this # tile are absent from the PSF dictionary and no shape measurement is possible. # This check must come before reading image vignets to avoid a C-level malloc @@ -197,6 +221,8 @@ def ngmix_runner( seg_cat_path=seg_vignet_path, dilate_neighbour=dilate_neighbour, metacal_psf=metacal_psf, + epoch_central_defect_radius=epoch_central_defect_radius, + epoch_masked_fraction_cut=epoch_masked_fraction_cut, ) # Process ngmix shape measurement and metacalibration From 7777181bc6cd76f3c0b7a41d4b98bccb218442b1 Mon Sep 17 00:00:00 2001 From: Cail Daley Date: Sat, 26 Sep 2026 07:07:40 +0200 Subject: [PATCH 03/30] ngmix: read each epoch's OFFSET from the object dict in hand prepare_postage_stamps already holds the object's galaxy vignet dict, so the per-epoch OFFSET is read from it instead of a second sqlitedict lookup. The pixel scale sets only the centroid-prior width; the comments that also credited it with a noise window are corrected. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01QAc1ywyd9Sbbra6po2iuaA --- .../modules/ngmix_package/__init__.py | 4 ++-- src/shapepipe/modules/ngmix_package/ngmix.py | 18 ++++++++---------- src/shapepipe/modules/ngmix_runner.py | 2 +- 3 files changed, 11 insertions(+), 13 deletions(-) diff --git a/src/shapepipe/modules/ngmix_package/__init__.py b/src/shapepipe/modules/ngmix_package/__init__.py index 66323822c..012b3264b 100644 --- a/src/shapepipe/modules/ngmix_package/__init__.py +++ b/src/shapepipe/modules/ngmix_package/__init__.py @@ -36,8 +36,8 @@ PIXEL_SCALE : float, optional Pixel scale in arcsec. Optional override; when omitted (or non-positive) it is read from the image WCS so it cannot drift from the pixels. Only - sets the centroid-prior width and noise window -- the fit Jacobian is - built per object from the full WCS. + sets the centroid-prior width -- the fit Jacobian is built per object + from the full WCS. SAVE_BATCH : int, optional Save the output catalogue in batches of this size; default is ``-1`` (no batch saving) diff --git a/src/shapepipe/modules/ngmix_package/ngmix.py b/src/shapepipe/modules/ngmix_package/ngmix.py index 807b99ff0..a19d2a83e 100644 --- a/src/shapepipe/modules/ngmix_package/ngmix.py +++ b/src/shapepipe/modules/ngmix_package/ngmix.py @@ -367,11 +367,11 @@ def pixel_scale_from_wcs(f_wcs_file): """Representative pixel scale (arcsec) from the tile's image WCS. The ngmix fit builds each object's Jacobian from the full per-epoch WCS, - so this scalar only sets the centroid-prior width and the noise-window - scale (see :func:`get_prior`, :func:`get_noise`). A single value read from - the astrometry is therefore sufficient -- and, unlike a hard-coded config - constant, it cannot silently drift from the pixels it describes (this - mirrors SExtractor's ``PIXEL_SCALE 0`` convention). + so this scalar only sets the centroid-prior width (see :func:`get_prior`). + A single value read from the astrometry is therefore sufficient -- and, + unlike a hard-coded config constant, it cannot silently drift from the + pixels it describes (this mirrors SExtractor's ``PIXEL_SCALE 0`` + convention). Parameters ---------- @@ -578,8 +578,8 @@ def __init__( # Pixel scale: an explicit positive PIXEL_SCALE overrides; otherwise # derive it from the image WCS so it can never drift from the pixels # (mirrors SExtractor's ``PIXEL_SCALE 0`` convention). Only the - # centroid-prior width and noise window use it -- the fit Jacobian is - # built per object from the full WCS. + # centroid-prior width uses it -- the fit Jacobian is built per + # object from the full WCS. if pixel_scale is None or pixel_scale <= 0: self._pixel_scale = pixel_scale_from_wcs( self._vignet_cat.f_wcs_file @@ -1359,9 +1359,7 @@ def prepare_postage_stamps( # make_ngmix_observation), which raises if it is missing; the "hsm" # path ignores it, so read it leniently rather than coupling hsm to a # field it never uses. - stamp.offsets.append( - vignet.gal_vign_cat[str(obj_id)][expccd_name].get('OFFSET') - ) + stamp.offsets.append(gal_obj[expccd_name].get('OFFSET')) stamp.ra.append(tile_cat.ra[i_tile]) stamp.dec.append(tile_cat.dec[i_tile]) # CCD of the first surviving epoch — Fabian's coord_list[0] convention diff --git a/src/shapepipe/modules/ngmix_runner.py b/src/shapepipe/modules/ngmix_runner.py index ffc4de083..605262395 100644 --- a/src/shapepipe/modules/ngmix_runner.py +++ b/src/shapepipe/modules/ngmix_runner.py @@ -54,7 +54,7 @@ def ngmix_runner( # Pixel scale -- optional override. When absent (or non-positive) it is # derived from the image WCS inside Ngmix, so it cannot drift from the - # pixels. Only the centroid-prior width and noise window use it. + # pixels. Only the centroid-prior width uses it. if config.has_option(module_config_sec, "PIXEL_SCALE"): pixel_scale = config.getfloat(module_config_sec, "PIXEL_SCALE") else: From 9f0ea4430f32f998a3a9e83fd1466cbeab69546d Mon Sep 17 00:00:00 2001 From: Cail Daley Date: Sat, 26 Sep 2026 07:44:32 +0200 Subject: [PATCH 04/30] tests: name the none-mode uberseg test for none, not noisefill Co-Authored-By: Claude Opus 5.5 --- tests/module/test_ngmix_uberseg.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/module/test_ngmix_uberseg.py b/tests/module/test_ngmix_uberseg.py index 86dad33bc..4d6ebb32a 100644 --- a/tests/module/test_ngmix_uberseg.py +++ b/tests/module/test_ngmix_uberseg.py @@ -239,9 +239,9 @@ def _gal_flag_weight(npix=41, seed=7): return gal, flag, weight -def test_noisefill_ignores_seg_and_dilate_kwargs(): - """Under noisefill, passing seg / dilate_neighbour changes nothing: the - result matches the plain default call on the same RNG stream.""" +def test_none_ignores_seg_and_dilate_kwargs(): + """Under BLEND_HANDLING = none, passing seg / dilate_neighbour changes + nothing: the result matches the plain default call on the same RNG stream.""" gal, flag, weight = _gal_flag_weight() seg, _, _ = two_object_seg(npix=gal.shape[0], sep=12) From a92a7471fbb09a067512ffb8f13a7c775426cac1 Mon Sep 17 00:00:00 2001 From: Cail Daley Date: Sat, 26 Sep 2026 07:22:49 +0200 Subject: [PATCH 05/30] test(ngmix): specify DEFECT_FILL = interpolate and its shear recovery The interpolation fills only short defect runs (at most 3 px along a row or column, with clean pixels at both ends); anything wider, and runs reaching the stamp border, stays noise-filled. The interpolant must reproduce linear planes without reading defect values, commute with quarter turns of the stamp, and apply one operator to the science and metacal noise images. The weight is zero on the defects and on the quarter-turn orbit of the interpolated pixels, whose light stays. Each defect is vetoed at the radius of its own fill: EPOCH_INTERPOLATED_DEFECT_RADIUS for interpolated pixels, EPOCH_CENTRAL_DEFECT_RADIUS for noise-filled ones. The runner and Ngmix pass DEFECT_FILL to the epoch cuts and to metacal; noise stays the default. The science test recovers the full response matrix for interpolated columns, full and finite 3-px bleeds and single pixels at the interpolated radius, on 0.3" and 0.5" galaxies through round and elliptical 0.7" PSFs and on a 0.7" galaxy through a 0.9" PSF, and for noise-filled wide defects at the noise radius. A bleed three pixels inside the radius is the positive control. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01QAc1ywyd9Sbbra6po2iuaA --- tests/module/test_defect_interpolation.py | 237 +++++++++++++++++++++ tests/module/test_ngmix_defect_fill.py | 223 ++++++++++++++++++- tests/science/test_defect_interpolation.py | 119 +++++++++++ 3 files changed, 568 insertions(+), 11 deletions(-) create mode 100644 tests/module/test_defect_interpolation.py create mode 100644 tests/science/test_defect_interpolation.py diff --git a/tests/module/test_defect_interpolation.py b/tests/module/test_defect_interpolation.py new file mode 100644 index 000000000..1368e78cc --- /dev/null +++ b/tests/module/test_defect_interpolation.py @@ -0,0 +1,237 @@ +"""Defect interpolation (``DEFECT_FILL = interpolate``): which pixels are +interpolated, and the properties of the interpolant. + +:func:`interpolable_defects` picks the defect pixels that lie in a short row +or column run with clean pixels at both ends; :func:`interpolate_defects` +fills them from the nearby clean pixels with a Clough-Tocher interpolant, +averaged over the four quarter turns of the stamp and shared by every plane +(science image and metacal noise image). +""" + +import numpy as np +import numpy.testing as npt +import pytest +from hypothesis import given, settings +from hypothesis import strategies as st + +from shapepipe.modules.ngmix_package.defect_interpolation import ( + MAX_INTERPOLATED_RUN, + fourfold, + interpolable_defects, + interpolate_defects, +) + + +# --- interpolable_defects --------------------------------------------------- + +def _oracle(defect, max_run=MAX_INTERPOLATED_RUN): + """Brute force: walk each defect pixel's row and column run.""" + n0, n1 = defect.shape + out = np.zeros_like(defect) + for i, j in zip(*np.nonzero(defect)): + for di, dj in ((0, 1), (1, 0)): + lo, hi = (i, j), (i, j) + while (0 <= lo[0] - di and 0 <= lo[1] - dj + and defect[lo[0] - di, lo[1] - dj]): + lo = (lo[0] - di, lo[1] - dj) + while (hi[0] + di < n0 and hi[1] + dj < n1 + and defect[hi[0] + di, hi[1] + dj]): + hi = (hi[0] + di, hi[1] + dj) + length = hi[0] - lo[0] + hi[1] - lo[1] + 1 + bounded = (lo[0] - di >= 0 and lo[1] - dj >= 0 + and hi[0] + di < n0 and hi[1] + dj < n1) + if bounded and length <= max_run: + out[i, j] = True + return out + + +@given( + n=st.integers(5, 21), + density=st.floats(0.0, 0.6), + seed=st.integers(0, 2**31 - 1), +) +@settings(max_examples=60, deadline=None) +def test_interpolable_defects_are_the_short_bounded_runs(n, density, seed): + """A defect pixel is interpolated exactly when its row or column run is + at most MAX_INTERPOLATED_RUN long and has clean pixels at both ends; the + rule commutes with quarter turns. + + Failure modes: a run touching the stamp border (edge band, corner) or a + wide hole is interpolated from one side; a 3-px bleed is left to noise; + a clean pixel is selected; one axis is ignored, so the rule has a + preferred direction. + """ + defect = np.random.RandomState(seed).uniform(size=(n, n)) < density + out = interpolable_defects(defect) + npt.assert_array_equal(out, _oracle(defect)) + assert not out[~defect].any() + for k in range(1, 4): + npt.assert_array_equal( + interpolable_defects(np.rot90(defect, k)), np.rot90(out, k) + ) + + +@pytest.mark.parametrize("kind,expected", [ + ("column", True), ("bleed3", True), ("finite_bleed3", True), + ("pixel", True), ("bleed4", False), ("blob5", False), + ("edge_band", False), ("corner", False), +]) +def test_calibrated_widths_are_interpolated_and_wider_holes_are_not( + kind, expected, +): + """Columns, 3-px bleeds and single pixels are interpolated; 4-px bleeds, + blobs, edge bands and corners are not.""" + n, c = 51, 25 + defect = np.zeros((n, n), dtype=bool) + region = { + "column": np.s_[:, c + 8], + "bleed3": np.s_[:, c + 8:c + 11], + "finite_bleed3": np.s_[c - 5:c + 6, c + 8:c + 11], + "pixel": np.s_[c, c + 8], + "bleed4": np.s_[:, c + 8:c + 12], + "blob5": np.s_[c - 2:c + 3, c + 8:c + 13], + "edge_band": np.s_[:, -3:], + "corner": np.s_[:2, :2], + }[kind] + defect[region] = True + out = interpolable_defects(defect) + assert out[defect].all() if expected else not out.any() + + +# --- fourfold --------------------------------------------------------------- + +@given(n=st.integers(2, 20), seed=st.integers(0, 2**31 - 1)) +@settings(max_examples=30, deadline=None) +def test_fourfold_is_the_quarter_turn_orbit(n, seed): + """The union contains the mask, is invariant under quarter turns, and is + the smallest such set (idempotent).""" + mask = np.random.RandomState(seed).uniform(size=(n, n)) < 0.1 + out = fourfold(mask) + assert out[mask].all() + for k in range(1, 4): + npt.assert_array_equal(np.rot90(out, k), out) + npt.assert_array_equal(fourfold(out), out) + npt.assert_array_equal( + out, mask | np.rot90(mask) | np.rot90(mask, 2) | np.rot90(mask, 3) + ) + + +def test_fourfold_rejects_rectangular_stamps(): + with pytest.raises(ValueError, match="square"): + fourfold(np.zeros((11, 12), dtype=bool)) + + +# --- interpolate_defects ---------------------------------------------------- + +def _mask(n=31): + """A column, a finite 3-px bleed and a single pixel, all bounded.""" + defect = np.zeros((n, n), dtype=bool) + defect[:, 19] = True + defect[4:12, 7:10] = True + defect[22, 11] = True + return defect + + +@given(seed=st.integers(0, 2**31 - 1)) +@settings(max_examples=20, deadline=None) +def test_interpolation_reproduces_planes_without_reading_defects(seed): + """Linear planes are reproduced at the interpolated pixels; clean pixels, + and defect pixels not selected, are returned untouched; the inputs are + not modified; and no defect value (NaN or a sentinel) is ever read. + + Failure modes: defect pixels enter the support; coordinates are + transposed or mis-rotated; the fill smooths clean pixels. + """ + n = 31 + rng = np.random.RandomState(seed) + rows, cols = np.indices((n, n)) + coeff = rng.uniform(-2, 2, (2, 3)) + planes = np.array([a + b * rows + c * cols for a, b, c in coeff]) + defect = _mask(n) + defect[:, -2:] = True # an edge band and a blob next to the column: + defect[13:18, 21:26] = True # defects that are not targets + target = interpolable_defects(defect) + assert target[15, 19] + assert not target[13:18, 21:26].any() and not target[:, -2:].any() + contaminated = planes.copy() + contaminated[:, defect] = np.nan + saved = contaminated.copy() + + out = interpolate_defects(contaminated, defect, target) + + npt.assert_allclose(out[:, target], planes[:, target], atol=1e-5) + npt.assert_array_equal(out[:, ~target], contaminated[:, ~target]) + npt.assert_array_equal(contaminated, saved) + contaminated[:, defect] = 1e30 + npt.assert_array_equal( + interpolate_defects(contaminated, defect, target)[:, target], + out[:, target], + ) + + +def test_interpolation_commutes_with_quarter_turns(): + """Rotating the stamp and its mask rotates the fill. A regular grid's + Delaunay triangulation has degenerate diagonals, so a single-orientation + interpolant does not commute; the four-orientation average does. + + Failure mode: the average over orientations is skipped, so the fill has a + preferred direction. + """ + n = 31 + rows, cols = np.indices((n, n)) + image = np.exp(-((rows - 15.3) ** 2 + (cols - 14.6) ** 2) / 10.0) + image += 0.05 * np.random.RandomState(3).normal(size=(n, n)) + planes = image[None] + defect = _mask(n) + target = interpolable_defects(defect) + out = interpolate_defects(planes, defect, target) + for k in range(1, 4): + rotated = interpolate_defects( + np.rot90(planes, k, axes=(1, 2)), np.rot90(defect, k), + np.rot90(target, k), + ) + npt.assert_allclose( + rotated, np.rot90(out, k, axes=(1, 2)), atol=1e-12, rtol=0 + ) + + +def test_every_plane_sees_the_same_operator(): + """The fill is one linear operator applied to every plane: filling + a * image + b * noise gives a * fill(image) + b * fill(noise), up to the + Clough-Tocher gradient solver's tolerance. + + Failure mode: the noise image is filled differently from the science + image (another support, triangulation or orientation set), so metacal's + fixnoise no longer mirrors the science image's correlated noise. + """ + n = 31 + rng = np.random.RandomState(11) + image, noise = rng.normal(size=(2, n, n)) + defect = _mask(n) + target = interpolable_defects(defect) + both = interpolate_defects(np.array([image, noise]), defect, target) + mixed = interpolate_defects( + np.array([2.0 * image - 3.0 * noise]), defect, target + ) + npt.assert_allclose(mixed[0], 2.0 * both[0] - 3.0 * both[1], atol=1e-5) + alone = interpolate_defects(noise[None], defect, target) + npt.assert_allclose(alone[0], both[1], atol=1e-5) + + +def test_no_target_is_a_no_op(): + planes = np.random.RandomState(5).normal(size=(2, 15, 15)) + defect = np.zeros((15, 15), dtype=bool) + defect[:, -3:] = True + out = interpolate_defects(planes, defect, np.zeros_like(defect)) + npt.assert_array_equal(out, planes) + + +def test_degenerate_support_gives_nan_not_an_error(): + """With fewer than three non-collinear clean pixels the target is NaN, + which the caller replaces by noise.""" + defect = np.ones((5, 5), dtype=bool) + defect[2, 1] = defect[2, 3] = False + target = np.zeros_like(defect) + target[2, 2] = True + out = interpolate_defects(np.ones((1, 5, 5)), defect, target) + assert np.isnan(out[0, 2, 2]) diff --git a/tests/module/test_ngmix_defect_fill.py b/tests/module/test_ngmix_defect_fill.py index 0480c7ef7..cbbbaa3b0 100644 --- a/tests/module/test_ngmix_defect_fill.py +++ b/tests/module/test_ngmix_defect_fill.py @@ -2,12 +2,14 @@ A defect is a stamp pixel with a nonzero flag, zero exposure weight or an invalid background RMS. Before metacal, :func:`prepare_ngmix_weights` gives -every defect weight 0 and fills it with noise at its background RMS, whatever -``BLEND_HANDLING`` is. Under uberseg, pixels on the neighbour side only lose -their weight, and their image values stay raw. The per-epoch cuts in -:func:`prepare_postage_stamps` act on the same defect set: the masked-fraction -cut counts it, and the central-defect veto drops an epoch with a defect near -the stamp centre. +every defect weight 0 and fills it, whatever ``BLEND_HANDLING`` is: with noise +at its background RMS (``DEFECT_FILL = noise``, the default) or, for short +defect runs, with an interpolant of the clean pixels around them +(``DEFECT_FILL = interpolate``). Under uberseg, pixels on the neighbour side +only lose their weight, and their image values stay raw. The per-epoch cuts +in :func:`prepare_postage_stamps` act on the same defect set: the +masked-fraction cut counts it, and the central-defect veto drops an epoch +with a defect near the stamp centre, at a radius set by the defect's fill. """ import re @@ -15,6 +17,7 @@ import numpy as np import numpy.testing as npt +import pytest from astropy.io import fits from astropy.wcs import WCS from hypothesis import given @@ -22,8 +25,14 @@ from sqlitedict import SqliteDict from shapepipe.modules.ngmix_package import ngmix as ngmix_module +from shapepipe.modules.ngmix_package.defect_interpolation import ( + fourfold, + interpolable_defects, + interpolate_defects, +) from shapepipe.modules.ngmix_package.ngmix import ( EPOCH_CENTRAL_DEFECT_RADIUS, + EPOCH_INTERPOLATED_DEFECT_RADIUS, Ngmix, make_ngmix_observation, prepare_ngmix_weights, @@ -461,8 +470,9 @@ def getboolean(self, _sec, key, fallback=False): def test_runner_threads_the_epoch_cut_options(tmp_path, monkeypatch): - """EPOCH_CENTRAL_DEFECT_RADIUS and EPOCH_MASKED_FRACTION_CUT reach Ngmix - as configured, and default to the module constants when absent. + """EPOCH_CENTRAL_DEFECT_RADIUS, EPOCH_MASKED_FRACTION_CUT, DEFECT_FILL and + EPOCH_INTERPOLATED_DEFECT_RADIUS reach Ngmix as configured, and default + to the module constants (and the noise fill) when absent. Failure mode: the runner drops or ignores an option, so a configured A/B arm silently runs the default cuts. @@ -483,11 +493,15 @@ def process(self): for path in inputs: SqliteDict(path).close() - for options, radius, fraction in ( + for options, radius, fraction, fill, interpolated_radius in ( ({"EPOCH_CENTRAL_DEFECT_RADIUS": "7.5", - "EPOCH_MASKED_FRACTION_CUT": "0.1"}, 7.5, 0.1), + "EPOCH_MASKED_FRACTION_CUT": "0.1", + "DEFECT_FILL": "interpolate", + "EPOCH_INTERPOLATED_DEFECT_RADIUS": "5.5"}, + 7.5, 0.1, "interpolate", 5.5), ({}, EPOCH_CENTRAL_DEFECT_RADIUS, - ngmix_module.EPOCH_MASKED_FRACTION_CUT), + ngmix_module.EPOCH_MASKED_FRACTION_CUT, "noise", + EPOCH_INTERPOLATED_DEFECT_RADIUS), ): runner_module.ngmix_runner( inputs, {"output": str(tmp_path)}, "-001-001", @@ -495,6 +509,9 @@ def process(self): ) assert captured[-1]["epoch_central_defect_radius"] == radius assert captured[-1]["epoch_masked_fraction_cut"] == fraction + assert captured[-1]["defect_fill"] == fill + assert (captured[-1]["epoch_interpolated_defect_radius"] + == interpolated_radius) # --- make_ngmix_observation: the HSM centroid reads the filled image ------- @@ -530,3 +547,187 @@ def test_hsm_centroid_ignores_raw_defect_values(): npt.assert_allclose( [row - centre, col - centre], [d_row, d_col], atol=0.05 ) + + +# --- DEFECT_FILL = interpolate ---------------------------------------------- + +def _hot_stamp(): + """A bright object with hot defects: a column and a finite 3-px bleed + (interpolated), an edge band and a 5x5 blob (noise-filled).""" + centre = N_STAMP // 2 + rows, cols = np.mgrid[:N_STAMP, :N_STAMP] + gal = 1e3 * np.exp( + -((rows - centre) ** 2 + (cols - centre) ** 2) / (2 * 3.0 ** 2) + ) + flag = np.zeros((N_STAMP, N_STAMP), dtype=np.int32) + flag[:, centre + 8] = 1 + flag[centre - 5:centre + 6, centre - 11:centre - 8] = 1 + flag[:, :4] = 1 + flag[centre + 9:centre + 14, centre + 12:centre + 17] = 1 + gal[flag != 0] = 5e4 + return gal, np.ones((N_STAMP, N_STAMP)), flag + + +@pytest.mark.parametrize("blend_handling", ["none", "uberseg"]) +def test_interpolated_fill_and_its_weights(blend_handling): + """Short defect runs take the interpolant of the clean image; other + defects take noise; the weight is zero on the defects and on the + quarter-turn orbit of the interpolated pixels, whose light stays; the + metacal noise image is interpolated with the same operator. + + Failure modes: the orbit is not zero-weighted (a one-sided hole in the + likelihood biases c), or its light is replaced; the fill mask is + symmetrized; wide defects or edge bands are extrapolated; raw defect + values leak; the noise image keeps independent noise where the science + image is smooth. + """ + gal, weight, flag = _hot_stamp() + defect = flag != 0 + target = interpolable_defects(defect) + assert target.any() and (defect & ~target).any() + kwargs = ( + dict(seg=_uberseg_seg(N_STAMP), object_number=1, dilate_neighbour=1) + if blend_handling == "uberseg" + else {} + ) + gal_out, w_out, noise_out = prepare_ngmix_weights( + gal, weight, flag, np.random.RandomState(4), + bkg_rms=np.ones((N_STAMP, N_STAMP)), blend_handling=blend_handling, + defect_fill="interpolate", **kwargs, + ) + + neighbour = ( + uberseg_weight(np.ones_like(gal), kwargs["seg"], 1, dilate_neighbour=1) + == 0.0 + if kwargs else np.zeros_like(defect) + ) + npt.assert_array_equal(w_out == 0.0, defect | fourfold(target) | neighbour) + npt.assert_array_equal(gal_out[~defect], gal[~defect]) + expected = interpolate_defects(gal[None], defect, target)[0] + npt.assert_allclose(gal_out[target], expected[target], rtol=1e-5) + assert np.all(np.abs(gal_out[defect & ~target]) < 10.0) + refilled = interpolate_defects(noise_out[None], defect, target)[0] + npt.assert_allclose(noise_out[target], refilled[target], atol=1e-5) + + +def test_noise_is_the_default_fill(): + """Leaving DEFECT_FILL unset is bit-identical to the noise fill.""" + gal, weight, flag = _hot_stamp() + rms = np.ones_like(gal) + default = prepare_ngmix_weights( + gal, weight, flag, np.random.RandomState(9), bkg_rms=rms + ) + noise = prepare_ngmix_weights( + gal, weight, flag, np.random.RandomState(9), bkg_rms=rms, + defect_fill="noise", + ) + for a, b in zip(default, noise): + npt.assert_array_equal(a, b) + + +def test_unknown_defect_fill_is_rejected(): + gal, weight, flag = _hot_stamp() + with pytest.raises(ValueError, match="DEFECT_FILL"): + prepare_ngmix_weights( + gal, weight, flag, np.random.RandomState(0), + defect_fill="interp", + ) + + +def _fill_veto_epochs(): + """A clean epoch; a column at the interpolated-defect radius; a column + one pixel inside it; a 5-px bleed (noise-filled) at the same radius.""" + centre = N_STAMP // 2 + radius = int(EPOCH_INTERPOLATED_DEFECT_RADIUS) + clean = np.zeros((N_STAMP, N_STAMP), dtype=np.int32) + ones = np.ones((N_STAMP, N_STAMP)) + at, inside, wide = clean.copy(), clean.copy(), clean.copy() + at[:, centre + radius] = 1 + inside[:, centre + radius - 1] = 1 + wide[:, centre + radius:centre + radius + 5] = 1 + return { + "2100001-10": (clean, ones), + "2100002-11": (at, ones), + "2100003-12": (inside, ones), + "2100004-13": (wide, ones), + } + + +def test_the_veto_radius_follows_the_fill(): + """Under interpolation, an interpolated defect is vetoed inside + EPOCH_INTERPOLATED_DEFECT_RADIUS and a noise-filled one inside + EPOCH_CENTRAL_DEFECT_RADIUS. Under the noise fill every defect keeps the + noise radius. The masked-fraction cut counts the raw defect set in + both modes. + + Failure modes: the interpolated radius is applied to noise-filled pixels + (a wide hole next to the object survives), or not applied at all; the + configured radius is ignored; the boundary is inclusive; the fraction + cut counts the zero-weight orbit. + """ + epochs = _fill_veto_epochs() + assert _surviving(epochs, defect_fill="interpolate") == [ + "2100001-10", "2100002-11" + ] + assert _surviving(epochs) == ["2100001-10"] + assert _surviving( + epochs, defect_fill="interpolate", + epoch_interpolated_defect_radius=EPOCH_INTERPOLATED_DEFECT_RADIUS + 1, + ) == ["2100001-10"] + assert _surviving(_epochs(), defect_fill="interpolate") == [ + "2100001-10", "2100002-11" + ] + + +def test_process_threads_the_defect_fill(tmp_path, monkeypatch): + """Ngmix hands DEFECT_FILL to both the epoch cuts and metacal: under + interpolation the column at the interpolated-defect radius survives and + metacal is asked to interpolate; under noise it is vetoed. + + Failure mode: the option is dropped on the way to either, so the tile + silently runs the noise fill or the noise cuts. + """ + epochs = {k: v for k, v in _fill_veto_epochs().items() + if k in ("2100001-10", "2100002-11")} + vignet, tile_cat, psf_obj, _ = _fake_inputs(epochs) + vignet.psf_vign_cat = {"1": psf_obj} + vignet.close = lambda: None + tile_cat.obj_id = np.array([1]) + tile_cat.flux = None + monkeypatch.setattr(ngmix_module, "Tile_cat", lambda *a, **k: tile_cat) + for method in ("compile_results", "save_results", "log_mean_ellipticity"): + monkeypatch.setattr(Ngmix, method, lambda *_a, **_k: None) + calls = [] + + def record(stamp, *_args, **kwargs): + calls.append((len(stamp.gals), kwargs.get("defect_fill"))) + raise RuntimeError("metacal is not under test") + + monkeypatch.setattr(ngmix_module, "do_ngmix_metacal", record) + paths = [tmp_path / f"{name}.sqlite" for name in + ("gal", "psf", "weight", "flag", "headers")] + for path in paths: + SqliteDict(str(path)).close() + for fill in ("interpolate", "noise"): + ngmix = Ngmix( + ["tile_cat.fits"] + [str(p) for p in paths[:4]], + str(tmp_path), "-001-001", 30.0, 0.186, str(paths[4]), + _RecordingLogger(), bkg_sub=False, defect_fill=fill, + ) + ngmix._vignet_cat.close() + ngmix._vignet_cat = vignet + ngmix.process() + assert calls == [(2, "interpolate"), (1, "noise")] + + +def test_ngmix_rejects_an_unknown_defect_fill(tmp_path): + paths = [tmp_path / f"{name}.sqlite" for name in + ("gal", "psf", "weight", "flag", "headers")] + for path in paths: + SqliteDict(str(path)).close() + with pytest.raises(ValueError, match="DEFECT_FILL"): + Ngmix( + ["tile_cat.fits"] + [str(p) for p in paths[:4]], + str(tmp_path), "-001-001", 30.0, 0.186, str(paths[4]), + _RecordingLogger(), bkg_sub=False, defect_fill="interp", + ) diff --git a/tests/science/test_defect_interpolation.py b/tests/science/test_defect_interpolation.py new file mode 100644 index 000000000..d36be39b0 --- /dev/null +++ b/tests/science/test_defect_interpolation.py @@ -0,0 +1,119 @@ +"""Shear recovery for the defects kept under ``DEFECT_FILL = interpolate``. + +Physics invariant: every defect the veto keeps leaves both additive terms +|c1|, |c2| < 5e-4 and both diagonal multiplicative terms |m11|, |m22| < 1%, +from the full 2x2 response matrix. Interpolated defects (columns, full and +finite 3-px bleeds, single pixels) are checked at +``EPOCH_INTERPOLATED_DEFECT_RADIUS`` and two pixels beyond it, on galaxies +with half-light radius 0.3" and 0.5" through a 0.7" PSF, round and with +ellipticity (0.05, 0.02), and on a 0.7" galaxy through a 0.9" PSF. Defects +too wide to interpolate are noise-filled, and are checked at +``EPOCH_CENTRAL_DEFECT_RADIUS``. The cases sit at the radii themselves, so +lowering either below its calibrated value turns this red. + +Positive control: a 3-px bleed three pixels inside the interpolated-defect +radius breaks the bound. +""" + +import json + +import numpy as np +import pytest + +from shapepipe.modules.ngmix_package.defect_interpolation import ( + interpolable_defects, +) +from shapepipe.modules.ngmix_package.ngmix import ( + EPOCH_CENTRAL_DEFECT_RADIUS, + EPOCH_INTERPOLATED_DEFECT_RADIUS, + EPOCH_MASKED_FRACTION_CUT, + central_defect_vetoes, + defect_mask, +) +from tests.helpers.defect_response import defect_response + +N = 51 +CENTRE = N // 2 +RI = int(np.ceil(EPOCH_INTERPOLATED_DEFECT_RADIUS)) +RN = int(np.ceil(EPOCH_CENTRAL_DEFECT_RADIUS)) +ROUND = (0.0, 0.0) +ELLIPTICAL = (0.05, 0.02) +SEEDS = range(6) +INTERPOLATE = {"defect_fill": "interpolate"} + + +def geometry(kind, distance): + """A detector defect whose nearest pixel is ``distance`` px from the + stamp centre.""" + bad = np.zeros((N, N), dtype=bool) + near = CENTRE + distance + if kind == "pixel": + bad[CENTRE, near] = True + elif kind == "column": + bad[:, near] = True + elif kind == "bleed": + bad[:, near:near + 3] = True + elif kind == "finite_bleed": + bad[CENTRE - 5:CENTRE + 6, near:near + 3] = True + elif kind == "wide_bleed": + bad[:, near:near + 5] = True + elif kind == "edge": + bad[:, near:] = True + return bad + + +INTERPOLATED = ("column", "bleed", "finite_bleed", "pixel") +CASES = ( + [(k, d, h, 0.7, ROUND) for k in INTERPOLATED + for d in (RI, RI + 2) for h in (0.3, 0.5)] + + [(k, RI, 0.5, 0.7, ELLIPTICAL) for k in INTERPOLATED] + + [(k, RI, 0.7, 0.9, psf) for k in ("bleed", "finite_bleed") + for psf in (ROUND, ELLIPTICAL)] + + [(k, RN, h, 0.7, ROUND) for k in ("wide_bleed", "edge") + for h in (0.3, 0.5)] +) + + +def case_id(case): + kind, distance, hlr, fwhm, psf_shear = case + psf = "elliptical" if any(psf_shear) else "round" + return f"{kind}-{distance}px-hlr{hlr}-psf{fwhm}-{psf}" + + +def recover(bad, hlr, fwhm, psf_shear, tmp_path): + result = defect_response(bad, hlr=hlr, psf=fwhm, seeds=SEEDS, + psf_shear=psf_shear, options=INTERPOLATE) + (tmp_path / "recovery.json").write_text(json.dumps(result, indent=2)) + return np.abs(result["m"]).max(), np.abs(result["c"]).max(), result + + +@pytest.mark.parametrize("kind,distance,hlr,fwhm,psf_shear", CASES, + ids=[case_id(c) for c in CASES]) +def test_kept_defects_recover_shear_on_both_axes(kind, distance, hlr, fwhm, + psf_shear, tmp_path): + """Failure modes: a veto radius is below its calibrated value; the + interpolated pixels' quarter-turn orbit keeps its weight (a one-sided + hole in the likelihood); the fill mask is symmetrized; a wide hole is + interpolated; raw defect values leak into metacal.""" + bad = geometry(kind, distance) + masked = defect_mask(np.ones((N, N)), bad.astype(np.int32)) + interpolated = interpolable_defects(masked) + assert interpolated.any() == (kind in INTERPOLATED) + assert masked.mean() <= EPOCH_MASKED_FRACTION_CUT + assert not central_defect_vetoes(masked, EPOCH_CENTRAL_DEFECT_RADIUS, + "interpolate") + m, c, result = recover(bad, hlr, fwhm, psf_shear, tmp_path) + assert m < 0.01, result + assert c < 5e-4, result + + +def test_vetoed_bleed_breaks_the_bound(tmp_path): + """Positive control: a 3-px bleed three pixels inside the + interpolated-defect radius, on the 0.3" galaxy, gives |c| > 1e-3 and + |m| > 1.5%. The veto drops it.""" + bad = geometry("bleed", RI - 3) + assert central_defect_vetoes(bad, EPOCH_CENTRAL_DEFECT_RADIUS, + "interpolate") + m, c, result = recover(bad, 0.3, 0.7, ROUND, tmp_path) + assert c > 1e-3, result + assert m > 0.015, result From 1bb6437afb0286ec6a084758e0549603f53b5b0a Mon Sep 17 00:00:00 2001 From: Cail Daley Date: Sat, 26 Sep 2026 07:22:49 +0200 Subject: [PATCH 06/30] feat(ngmix): DEFECT_FILL = interpolate, with a fill-dependent central veto DEFECT_FILL = interpolate fills short defect runs (at most 3 px along a row or column, with clean pixels at both ends: columns, 3-px bleeds, single pixels) with a Clough-Tocher interpolant of the clean pixels within 4 px. The interpolant is averaged over the four quarter turns of the stamp, and each turn's triangulation serves the science image and the metacal noise image alike. Wider holes and runs reaching the stamp border keep the noise fill. noise stays the default and is unchanged. Interpolation restores the object's light, so the remaining bias is the hole in the likelihood: a one-sided zero-weight column 8 px from a 0.5" galaxy still gives c1 = -1.3e-3. The weight is therefore also zero on the quarter-turn orbit of the interpolated pixels (their light stays), which brings that column to c1 = +2e-6. Symmetrizing the fill instead would trade true light for interpolated light over four times the area. Each defect pixel is vetoed at the radius of its own fill: EPOCH_INTERPOLATED_DEFECT_RADIUS (default 7 px) for interpolated pixels and EPOCH_CENTRAL_DEFECT_RADIUS (10 px) for noise-filled ones. 7 px is the smallest radius at which interpolated columns, 3-px bleeds and single pixels give |m11|, |m22| < 1% and |c1|, |c2| < 5e-4 on 0.3", 0.5" and 0.7" galaxies, round and elliptical PSFs; it does not scale with galaxy size. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01QAc1ywyd9Sbbra6po2iuaA --- .../ngmix_package/defect_interpolation.py | 168 ++++++++++++++ src/shapepipe/modules/ngmix_package/ngmix.py | 208 +++++++++++++++--- src/shapepipe/modules/ngmix_runner.py | 28 ++- 3 files changed, 375 insertions(+), 29 deletions(-) create mode 100644 src/shapepipe/modules/ngmix_package/defect_interpolation.py diff --git a/src/shapepipe/modules/ngmix_package/defect_interpolation.py b/src/shapepipe/modules/ngmix_package/defect_interpolation.py new file mode 100644 index 000000000..713e326f7 --- /dev/null +++ b/src/shapepipe/modules/ngmix_package/defect_interpolation.py @@ -0,0 +1,168 @@ +"""DEFECT INTERPOLATION. + +Clough-Tocher interpolation of short defect runs before metacal, selected +with ``DEFECT_FILL = interpolate`` (see +:func:`shapepipe.modules.ngmix_package.ngmix.prepare_ngmix_weights`). + +""" + +import numpy as np +from scipy.interpolate import CloughTocher2DInterpolator +from scipy.ndimage import binary_dilation, label +from scipy.spatial import QhullError + +# Longest row or column run of defect pixels that is interpolated. +MAX_INTERPOLATED_RUN = 3 + +# Clean pixels within this Chebyshev distance (pixels) of the interpolated +# pixels support the interpolant. +SUPPORT_RADIUS = 4 + +_ROW_RUNS = np.array([[0, 0, 0], [1, 1, 1], [0, 0, 0]]) + + +def _short_row_runs(defect, max_run): + """Pixels in row runs of at most ``max_run`` defects that stop short of + both stamp borders.""" + labels, n_runs = label(defect, structure=_ROW_RUNS) + if n_runs == 0: + return np.zeros_like(defect) + short = np.bincount(labels.ravel(), minlength=n_runs + 1) <= max_run + short[0] = False + short[labels[:, 0]] = False + short[labels[:, -1]] = False + return short[labels] + + +def interpolable_defects(defect, max_run=MAX_INTERPOLATED_RUN): + """Defect pixels that ``DEFECT_FILL = interpolate`` interpolates. + + @sc [decision:shape_measurement.defect_fill] interpolable-defects + A defect pixel is interpolated when its row or its column run of defect + pixels is at most ``max_run`` (3) long and has clean pixels at both + ends. That covers columns, 3-px bleeds and isolated pixels, the widths + whose shear recovery is calibrated. Wider holes, and runs that reach the + stamp border (edge bands, corners), have clean light on one side only; + they are noise-filled and vetoed at the noise-fill radius (see + :func:`~shapepipe.modules.ngmix_package.ngmix.central_defect_vetoes`). + The rule reads only the mask and commutes with quarter turns of the + stamp. + + Parameters + ---------- + defect : numpy.ndarray of bool + Defect mask of one epoch stamp. + max_run : int, optional + Longest interpolated run; the default is ``MAX_INTERPOLATED_RUN``. + + Returns + ------- + numpy.ndarray of bool + ``True`` on the defect pixels to interpolate. + """ + defect = np.asarray(defect, dtype=bool) + return ( + _short_row_runs(defect, max_run) + | _short_row_runs(defect.T, max_run).T + ) + + +def fourfold(mask): + """Union of a square stamp mask with its quarter turns. + + Parameters + ---------- + mask : numpy.ndarray of bool + Square mask. + + Returns + ------- + numpy.ndarray of bool + ``mask`` ORed with its rotations by 90, 180 and 270 degrees about + the stamp centre. + + Raises + ------ + ValueError + If ``mask`` is not square. + """ + mask = np.asarray(mask, dtype=bool) + if mask.ndim != 2 or mask.shape[0] != mask.shape[1]: + raise ValueError( + f"A quarter-turn orbit needs a square stamp, not {mask.shape}" + ) + return mask | np.rot90(mask) | np.rot90(mask, 2) | np.rot90(mask, 3) + + +def _interpolate_once(planes, defect, target): + """One Clough-Tocher interpolant of every plane at ``target``; NaN + elsewhere and where the support cannot reach.""" + out = np.full(planes.shape, np.nan) + support = binary_dilation( + target, structure=np.ones((3, 3), dtype=bool), + iterations=SUPPORT_RADIUS, + ) & ~defect + points = np.argwhere(support).astype(float) + if len(points) < 3: + return out + query = np.argwhere(target) + try: + interpolant = CloughTocher2DInterpolator( + points, planes[:, support].T, fill_value=np.nan, + ) + except QhullError: + return out + out[:, query[:, 0], query[:, 1]] = interpolant(query.astype(float)).T + return out + + +def interpolate_defects(planes, defect, target): + """Replace the ``target`` pixels of every plane by a Clough-Tocher + interpolant of the clean pixels around them. + + @sc [decision:shape_measurement.defect_fill] shared-rotation-averaged-interpolant + The support is the clean pixels within ``SUPPORT_RADIUS`` (4 px) of the + target; no defect pixel enters it, so defect values are never read. For + each quarter turn of the stamp, one Delaunay triangulation of the support + serves every plane, so the science image and the metacal noise image + see the same linear operator and fixnoise mirrors the science image's + interpolated noise. A regular grid's triangulation has degenerate + diagonals, so one orientation has a preferred direction; averaging the + four quarter-turned operators makes the fill commute with rotations of + the stamp. That removes up to 6e-5 of c2 for a column or 3-px bleed + 6 px from the object. + + Parameters + ---------- + planes : array_like + Stamp planes, shape ``(n, ny, nx)``. + defect : numpy.ndarray of bool + Every defect pixel, shape ``(ny, nx)``; none of them supports the + interpolant. + target : numpy.ndarray of bool + Defect pixels to interpolate (:func:`interpolable_defects`). + + Returns + ------- + numpy.ndarray + A copy of ``planes`` with ``target`` pixels interpolated; NaN at a + target pixel whose support is degenerate in some orientation. + """ + planes = np.asarray(planes, dtype=float) + defect = np.asarray(defect, dtype=bool) + target = np.asarray(target, dtype=bool) + out = planes.copy() + if not target.any(): + return out + turns = [ + np.rot90( + _interpolate_once( + np.rot90(planes, k, axes=(1, 2)), np.rot90(defect, k), + np.rot90(target, k), + ), + -k, axes=(1, 2), + )[:, target] + for k in range(4) + ] + out[:, target] = np.mean(turns, axis=0) + return out diff --git a/src/shapepipe/modules/ngmix_package/ngmix.py b/src/shapepipe/modules/ngmix_package/ngmix.py index a19d2a83e..acb425835 100644 --- a/src/shapepipe/modules/ngmix_package/ngmix.py +++ b/src/shapepipe/modules/ngmix_package/ngmix.py @@ -23,11 +23,20 @@ from scipy.spatial import cKDTree from sqlitedict import SqliteDict +from shapepipe.modules.ngmix_package.defect_interpolation import ( + fourfold, + interpolable_defects, + interpolate_defects, +) from shapepipe.pipeline import file_io # Neighbour treatments selectable with the BLEND_HANDLING option. BLEND_HANDLINGS = ("none", "uberseg") +# Defect fills selectable with the DEFECT_FILL option (see +# :func:`prepare_ngmix_weights`). +DEFECT_FILLS = ("noise", "interpolate") + # Default of the EPOCH_MASKED_FRACTION_CUT option: an epoch is dropped when # more than this fraction of its stamp is in :func:`defect_mask` (see # :func:`prepare_postage_stamps`). @@ -38,6 +47,11 @@ # stamp centre (see :func:`has_central_defect`). EPOCH_CENTRAL_DEFECT_RADIUS = 10 +# Default of the EPOCH_INTERPOLATED_DEFECT_RADIUS option (pixels): under +# DEFECT_FILL = interpolate, the veto radius for interpolated defect pixels +# (see :func:`central_defect_vetoes`). +EPOCH_INTERPOLATED_DEFECT_RADIUS = 7 + METACAL_TYPES = ('noshear', '1p', '1m', '2p', '2m') # Noise budget for the PSF observation's flat weight map (psf_wt = @@ -444,8 +458,7 @@ class Ngmix(object): Neighbour treatment. ``"none"`` (default) leaves neighbour pixels untouched; ``"uberseg"`` zeroes the weight of neighbour-side pixels from the coadd segmentation map and requires ``seg_cat_path``. Defect - pixels are noise-filled under both (see - :func:`prepare_ngmix_weights`). + pixels are filled under both (see :func:`prepare_ngmix_weights`). seg_cat_path : str, optional Path to the coadd-frame segmentation VIGNET catalogue (see :class:`Tile_cat`). Required when ``blend_handling="uberseg"``. @@ -460,6 +473,14 @@ class Ngmix(object): Drop an epoch when more than this fraction of its stamp is masked (see :func:`prepare_postage_stamps`); the default is ``EPOCH_MASKED_FRACTION_CUT``. + defect_fill : {"noise", "interpolate"}, optional + How defect pixels are filled before metacal (see + :func:`prepare_ngmix_weights`); the default is ``"noise"``. + epoch_interpolated_defect_radius : float, optional + Under ``defect_fill="interpolate"``, drop an epoch when an + interpolated defect pixel lies closer than this many pixels to the + stamp centre (see :func:`central_defect_vetoes`); the default is + ``EPOCH_INTERPOLATED_DEFECT_RADIUS``. Notes ----- @@ -471,8 +492,8 @@ class Ngmix(object): IndexError If the length of the input file list is incorrect ValueError - If ``blend_handling`` is unknown, or ``"uberseg"`` is selected without - ``seg_cat_path``. + If ``blend_handling`` or ``defect_fill`` is unknown, or ``"uberseg"`` + is selected without ``seg_cat_path``. """ @@ -496,6 +517,8 @@ def __init__( metacal_psf="fitgauss", epoch_central_defect_radius=EPOCH_CENTRAL_DEFECT_RADIUS, epoch_masked_fraction_cut=EPOCH_MASKED_FRACTION_CUT, + defect_fill="noise", + epoch_interpolated_defect_radius=EPOCH_INTERPOLATED_DEFECT_RADIUS, ): # Base count = catalogue + vignets, excluding the f_wcs headers (passed @@ -514,6 +537,11 @@ def __init__( f"Unknown BLEND_HANDLING '{blend_handling}'; expected one of" + f" {BLEND_HANDLINGS}" ) + if defect_fill not in DEFECT_FILLS: + raise ValueError( + f"Unknown DEFECT_FILL '{defect_fill}'; expected one of" + + f" {DEFECT_FILLS}" + ) # Fail fast at construction (not deep in the per-epoch loop) when # uberseg is requested without its segmentation input (shapepipe#776). @@ -567,6 +595,10 @@ def __init__( self._metacal_psf = metacal_psf self._epoch_central_defect_radius = epoch_central_defect_radius self._epoch_masked_fraction_cut = epoch_masked_fraction_cut + self._defect_fill = defect_fill + self._epoch_interpolated_defect_radius = ( + epoch_interpolated_defect_radius + ) self._w_log = w_log @@ -1046,6 +1078,10 @@ def process(self): gal_obj, epoch_central_defect_radius=self._epoch_central_defect_radius, epoch_masked_fraction_cut=self._epoch_masked_fraction_cut, + defect_fill=self._defect_fill, + epoch_interpolated_defect_radius=( + self._epoch_interpolated_defect_radius + ), ) epoch_cuts.update(stamp.epoch_cuts) @@ -1092,6 +1128,7 @@ def process(self): object_number=obj_id, dilate_neighbour=self._dilate_neighbour, metacal_psf=self._metacal_psf, + defect_fill=self._defect_fill, ) except Exception as ee: self._w_log.info( @@ -1193,6 +1230,8 @@ def prepare_postage_stamps( gal_obj=None, epoch_central_defect_radius=EPOCH_CENTRAL_DEFECT_RADIUS, epoch_masked_fraction_cut=EPOCH_MASKED_FRACTION_CUT, + defect_fill="noise", + epoch_interpolated_defect_radius=EPOCH_INTERPOLATED_DEFECT_RADIUS, ): """Gather one object's epoch stamps, dropping epochs its defects spoil. @@ -1200,8 +1239,9 @@ def prepare_postage_stamps( An epoch is dropped when more than ``epoch_masked_fraction_cut`` of its stamp lies in :func:`defect_mask`: flagged, zero-weight and invalid-RMS pixels, the set that :func:`prepare_ngmix_weights` zero-weights and - noise-fills. Counting flags alone would keep epochs whose filled area - exceeds the cut. The default is 1/3; 10%, the DES Y3 and Y6 value, is + fills. Counting flags alone would keep epochs whose filled area exceeds + the cut. The zero-weight orbit of interpolated pixels keeps its light and + is not counted. The default is 1/3; 10%, the DES Y3 and Y6 value, is the alternative to test. Parameters @@ -1225,6 +1265,14 @@ def prepare_postage_stamps( epoch_masked_fraction_cut : float, optional Drop an epoch with more than this fraction of its stamp in :func:`defect_mask`. The default is ``EPOCH_MASKED_FRACTION_CUT``. + defect_fill : {"noise", "interpolate"}, optional + The fill :func:`prepare_ngmix_weights` will apply; it sets the veto + radius of each defect pixel (:func:`central_defect_vetoes`). The + default is ``"noise"``. + epoch_interpolated_defect_radius : float, optional + Veto radius for interpolated defect pixels under + ``defect_fill="interpolate"``. The default is + ``EPOCH_INTERPOLATED_DEFECT_RADIUS``. Returns ------- @@ -1307,14 +1355,17 @@ def prepare_postage_stamps( else None ) # Drop the epoch when too much of it would be zero-weighted and - # noise-filled (epoch-cut-on-defect-mask), or when a filled pixel - # would sit near the object (epoch-central-defect-veto). + # filled (epoch-cut-on-defect-mask), or when a filled pixel would sit + # near the object (veto-radius-follows-the-fill). masked = defect_mask(weight_vign, flag_vign, bkg_rms_vign) stamp.epoch_cuts["considered"] += 1 if masked.mean() > epoch_masked_fraction_cut: stamp.epoch_cuts["masked_fraction"] += 1 continue - if has_central_defect(masked, epoch_central_defect_radius): + if central_defect_vetoes( + masked, epoch_central_defect_radius, defect_fill, + epoch_interpolated_defect_radius, + ): stamp.epoch_cuts["central_veto"] += 1 continue @@ -1734,6 +1785,59 @@ def has_central_defect(defect, radius): return bool(np.any(distance < radius)) +def central_defect_vetoes( + defect, radius, defect_fill="noise", + interpolated_radius=EPOCH_INTERPOLATED_DEFECT_RADIUS, +): + """Whether the central-defect veto drops an epoch under ``defect_fill``. + + @sc [decision:shape_measurement.central_defect_veto,decision:shape_measurement.defect_fill] veto-radius-follows-the-fill + Each defect pixel is vetoed (:func:`has_central_defect`) at the radius + calibrated for the operator that fills it. Under ``"noise"`` every + defect keeps ``radius`` (``EPOCH_CENTRAL_DEFECT_RADIUS``). Under + ``"interpolate"`` the pixels :func:`interpolable_defects` selects are + interpolated and vetoed at ``interpolated_radius`` + (``EPOCH_INTERPOLATED_DEFECT_RADIUS``); the others are noise-filled and + keep ``radius``. 7 px is the smallest interpolated radius at which + columns, full and finite 3-px bleeds and single pixels give + |m11|, |m22| < 1% and |c1|, |c2| < 5e-4 (full response matrix) on + galaxies with half-light radius 0.3" and 0.5" through a 0.7" PSF, round + or with ellipticity (0.05, 0.02), and on a 0.7" galaxy through a 0.9" + PSF. There the worst case is c1 = 3.6e-4 (3-px bleed, 0.7" galaxy, + elliptical PSF); on the 0.3" and 0.5" galaxies |c| <= 5e-5 and + |m| <= 0.21%. At 6 px those two still pass (|c| <= 3.1e-4), but a 3-px + bleed on the 0.7" galaxy gives c1 = 7.5e-4; at 5 px a 3-px bleed on the + 0.5" galaxy gives 9.5e-4. The radius does not scale with galaxy size: + interpolation needs one pixel more for the 0.7" galaxy (noise fill needs + four, 10 to 14 px), and a size-dependent veto would select on measured + size, which responds to shear. Guarded by + ``tests/science/test_defect_interpolation.py``. + + Parameters + ---------- + defect : numpy.ndarray of bool + Defect mask of one epoch stamp (:func:`defect_mask`). + radius : float + Veto radius in pixels for noise-filled defect pixels. + defect_fill : {"noise", "interpolate"}, optional + The fill the epoch will get; the default is ``"noise"``. + interpolated_radius : float, optional + Veto radius in pixels for interpolated defect pixels; the default is + ``EPOCH_INTERPOLATED_DEFECT_RADIUS``. + + Returns + ------- + bool + ``True`` if the epoch should be dropped. + """ + if defect_fill == "interpolate": + interpolated = interpolable_defects(defect) + return has_central_defect( + interpolated, interpolated_radius + ) or has_central_defect(defect & ~interpolated, radius) + return has_central_defect(defect, radius) + + def fill_defects(image, defect, noise): """Replace an image's defect pixels by a noise realisation. @@ -1766,24 +1870,42 @@ def fill_defects(image, defect, noise): def prepare_ngmix_weights( gal, weight, flag, rng, bkg_rms=None, blend_handling="none", seg=None, object_number=None, - dilate_neighbour=0, + dilate_neighbour=0, defect_fill="noise", ): """Build one epoch's image, weight map and noise image for ngmix. Defect pixels (:func:`defect_mask`: flagged, zero-weight or invalid-RMS - pixels) get weight 0 and are replaced by an independent noise - realisation at their background RMS (:func:`fill_defects`), under either - ``blend_handling``. ``blend_handling`` decides only the neighbour side. + pixels) get weight 0 and are filled under either ``blend_handling``: + by an independent noise realisation at their background RMS + (:func:`fill_defects`), or under ``defect_fill="interpolate"``, for + short defect runs, by an interpolant of the clean pixels around them. + ``blend_handling`` decides only the neighbour side. @sc [decision:shape_measurement.defect_fill,decision:shape_measurement.blend_handling] defects-filled-neighbours-raw - Every pixel of ``defect_mask`` is zero-weighted and noise-filled whatever - ``blend_handling`` is, and the filled set equals the zero-weight defect - set. ``blend_handling`` acts on the neighbour side only: uberseg zeroes + Every pixel of ``defect_mask`` is zero-weighted and filled whatever + ``blend_handling`` is, and the filled set equals the defect set. + ``blend_handling`` acts on the neighbour side only: uberseg zeroes the weight of pixels nearer a neighbour's footprint and leaves their image values raw, because that light is real sky that metacal shears along with the target; filling it with noise would cut the target along - an unsheared edge. Neighbour pixels are never noise-filled. The noise - image covers the whole stamp and is independent of both. + an unsheared edge. Neighbour pixels are never filled. The noise image + covers the whole stamp and is independent of both. + + @sc [decision:shape_measurement.defect_fill,label:physics] interpolated-defect-weight-orbit + Under ``defect_fill="interpolate"`` the short defect runs + (:func:`interpolable_defects`) take the interpolant of the clean pixels + around them (:func:`interpolate_defects`) in the image and in the noise + image alike, and the other defects are noise-filled. Interpolation + restores the object's light, which leaves the hole in the likelihood: + ngmix fits a Gaussian to a non-Gaussian profile, and a one-sided + zero-weight hole pulls that fit. Once interpolated, a column 8 px from a + galaxy with half-light radius 0.5" through a 0.7" PSF still gives + c1 = -1.3e-3. The weight is therefore also zero on the quarter-turn + orbit of the interpolated pixels, whose light stays, which cancels that + spin-2 term: c1 = +2e-6 for the same column. Only the weights are + symmetrized, not the fill: interpolating the orbit as well would trade + true light for interpolated light over four times the area, raising m11 + for a 3-px bleed 6 px from the same galaxy from +0.19% to +0.89%. Parameters ---------- @@ -1816,23 +1938,31 @@ def prepare_ngmix_weights( dilate_neighbour : int, optional Neighbour-mask dilation iterations, passed to :func:`uberseg_weight` under ``blend_handling="uberseg"``; ignored otherwise. + defect_fill : {"noise", "interpolate"}, optional + ``"noise"`` (default) noise-fills every defect; ``"interpolate"`` + interpolates the short defect runs and noise-fills the rest. Returns ------- numpy.ndarray - Galaxy image with defect pixels noise-filled. + Galaxy image with defect pixels filled. numpy.ndarray Inverse-variance weight map for ngmix. numpy.ndarray Noise image: an independent realisation over the whole stamp, for - metacal's ``fixnoise``. + metacal's ``fixnoise``, interpolated where the galaxy image is. Raises ------ ValueError - If ``blend_handling`` is unknown, or ``"uberseg"`` lacks ``seg`` or - ``object_number``. + If ``blend_handling`` or ``defect_fill`` is unknown, or + ``"uberseg"`` lacks ``seg`` or ``object_number``. """ + if defect_fill not in DEFECT_FILLS: + raise ValueError( + f"Unknown DEFECT_FILL '{defect_fill}'; expected one of" + + f" {DEFECT_FILLS}" + ) if blend_handling not in BLEND_HANDLINGS: raise ValueError( f"Unknown blend_handling '{blend_handling}'; expected one of" @@ -1848,19 +1978,29 @@ def prepare_ngmix_weights( defect = defect_mask(weight, flag, bkg_rms) clean = ~defect + interpolated = ( + interpolable_defects(defect) + if defect_fill == "interpolate" + else np.zeros_like(defect) + ) + # The quarter-turn orbit of the interpolated pixels carries no weight + # (interpolated-defect-weight-orbit). + weighted = ( + ~(defect | fourfold(interpolated)) if interpolated.any() else clean + ) if bkg_rms is None: sig_noise = sigma_mad(gal) # Guard the degenerate constant stamp (sigma_mad == 0): 0 * inf # would otherwise put NaN in a fully-masked weight map. weight_map = ( - clean.astype(float) / sig_noise ** 2 + weighted.astype(float) / sig_noise ** 2 if sig_noise > 0 else np.zeros_like(gal, dtype=float) ) else: weight_map = np.zeros_like(gal, dtype=float) - weight_map[clean] = 1.0 / bkg_rms[clean] ** 2 + weight_map[weighted] = 1.0 / bkg_rms[weighted] ** 2 # Per-pixel noise sigma for the realisations below: metacal's # fixnoise bookkeeping (1/w + 1/w_noise) assumes the noise image # is a faithful realisation of the per-pixel variance the weights @@ -1884,6 +2024,13 @@ def prepare_ngmix_weights( noise_img = rng.standard_normal(gal.shape) * sig_noise noise_img_gal = rng.standard_normal(gal.shape) * sig_noise gal_filled = fill_defects(gal, defect, noise_img_gal) + if interpolated.any(): + # One operator for the image and the noise image; a pixel whose + # support is degenerate keeps its noise fill. + filled = interpolate_defects([gal, noise_img], defect, interpolated) + done = interpolated & np.all(np.isfinite(filled), axis=0) + gal_filled[done] = filled[0][done] + noise_img = np.where(done, filled[1], noise_img) if blend_handling == "uberseg": weight_map = uberseg_weight( @@ -1897,7 +2044,7 @@ def make_ngmix_observation( gal, weight, flag, psf, wcs, rng, bkg_rms=None, centroid_source="wcs", offset=None, blend_handling="none", seg=None, object_number=None, - dilate_neighbour=0, + dilate_neighbour=0, defect_fill="noise", ): """Build an ngmix Observation for a single galaxy epoch. @@ -1951,6 +2098,9 @@ def make_ngmix_observation( dilate_neighbour : int, optional Neighbour-mask dilation iterations passed through to :func:`prepare_ngmix_weights` under ``blend_handling="uberseg"``. + defect_fill : {"noise", "interpolate"}, optional + Defect fill passed through to :func:`prepare_ngmix_weights`; the + default is ``"noise"``. Returns ------- @@ -1978,7 +2128,7 @@ def make_ngmix_observation( gal_masked, weight_map, noise_img = prepare_ngmix_weights( gal, weight, flag, rng, bkg_rms=bkg_rms, blend_handling=blend_handling, seg=seg, object_number=object_number, - dilate_neighbour=dilate_neighbour, + dilate_neighbour=dilate_neighbour, defect_fill=defect_fill, ) if centroid_source == "hsm": @@ -2204,7 +2354,7 @@ def make_runners(prior, flux_guess, rng): def do_ngmix_metacal( stamp, prior, flux_guess, rng, centroid_source="wcs", blend_handling="none", object_number=None, dilate_neighbour=0, - metacal_psf="fitgauss", + metacal_psf="fitgauss", defect_fill="noise", ): """Do Ngmix Metacal. @@ -2251,6 +2401,9 @@ def do_ngmix_metacal( round PSF that metacal reconvolves with after shearing, so it moves the metacal *response* (and therefore the recovered shear) but never reaches the deconvolution, which is by the PSF image. + defect_fill : {"noise", "interpolate"}, optional + Defect fill passed through to :func:`make_ngmix_observation`; the + default is ``"noise"``. Returns ------- @@ -2283,6 +2436,7 @@ def do_ngmix_metacal( seg=stamp.segs[n_e] if n_e < len(stamp.segs) else None, object_number=object_number, dilate_neighbour=dilate_neighbour, + defect_fill=defect_fill, ) gal_obs_list.append(gal_obs) diff --git a/src/shapepipe/modules/ngmix_runner.py b/src/shapepipe/modules/ngmix_runner.py index 605262395..158e14ef8 100644 --- a/src/shapepipe/modules/ngmix_runner.py +++ b/src/shapepipe/modules/ngmix_runner.py @@ -13,6 +13,7 @@ from shapepipe.modules.module_decorator import module_runner from shapepipe.modules.ngmix_package.ngmix import ( EPOCH_CENTRAL_DEFECT_RADIUS, + EPOCH_INTERPOLATED_DEFECT_RADIUS, EPOCH_MASKED_FRACTION_CUT, Ngmix, ) @@ -133,8 +134,8 @@ def ngmix_runner( # weighted and untouched; "uberseg" zeroes the weight of every pixel # closer to a neighbour than to the central object, from the segmentation # map, and leaves its image raw. Defect pixels (flagged, zero-weight or - # invalid-RMS) are zero-weighted and noise-filled under both; see - # prepare_ngmix_weights. + # invalid-RMS) are zero-weighted and filled under both (DEFECT_FILL + # below); see prepare_ngmix_weights. if config.has_option(module_config_sec, "BLEND_HANDLING"): blend_handling = config.get(module_config_sec, "BLEND_HANDLING") else: @@ -166,6 +167,27 @@ def ngmix_runner( else: epoch_masked_fraction_cut = EPOCH_MASKED_FRACTION_CUT + # DEFECT_FILL (optional): "noise" (default) noise-fills every defect; + # "interpolate" interpolates short defect runs (at most 3 px along a row + # or column) from the clean pixels around them and noise-fills the rest. + if config.has_option(module_config_sec, "DEFECT_FILL"): + defect_fill = config.get(module_config_sec, "DEFECT_FILL") + else: + defect_fill = "noise" + + # EPOCH_INTERPOLATED_DEFECT_RADIUS (optional, pixels): under + # DEFECT_FILL = interpolate, drop an epoch when an interpolated defect + # pixel lies closer than this to the stamp centre; noise-filled defect + # pixels keep EPOCH_CENTRAL_DEFECT_RADIUS. + if config.has_option( + module_config_sec, "EPOCH_INTERPOLATED_DEFECT_RADIUS" + ): + epoch_interpolated_defect_radius = config.getfloat( + module_config_sec, "EPOCH_INTERPOLATED_DEFECT_RADIUS" + ) + else: + epoch_interpolated_defect_radius = EPOCH_INTERPOLATED_DEFECT_RADIUS + # Check PSF vignets first: if all are empty dicts {}, the exposures for this # tile are absent from the PSF dictionary and no shape measurement is possible. # This check must come before reading image vignets to avoid a C-level malloc @@ -223,6 +245,8 @@ def ngmix_runner( metacal_psf=metacal_psf, epoch_central_defect_radius=epoch_central_defect_radius, epoch_masked_fraction_cut=epoch_masked_fraction_cut, + defect_fill=defect_fill, + epoch_interpolated_defect_radius=epoch_interpolated_defect_radius, ) # Process ngmix shape measurement and metacalibration From b9d92a9643538346289e16145834d0ef8d6d35f0 Mon Sep 17 00:00:00 2001 From: Cail Daley Date: Tue, 29 Sep 2026 16:14:13 +0200 Subject: [PATCH 07/30] ngmix: restore the BLEND_HANDLING name noisefill The neighbour option keeps develop's name, noisefill, in the code, the runner config key, tests, astra.yaml and the committed universe; the name none is dropped. Co-Authored-By: Claude Opus 5.5 --- astra.yaml | 8 +++--- src/shapepipe/modules/ngmix_package/ngmix.py | 30 ++++++++++---------- src/shapepipe/modules/ngmix_runner.py | 4 +-- tests/module/test_ngmix.py | 2 +- tests/module/test_ngmix_defect_fill.py | 4 +-- tests/module/test_ngmix_uberseg.py | 11 ++----- universes/committed.yaml | 2 +- 7 files changed, 27 insertions(+), 34 deletions(-) diff --git a/astra.yaml b/astra.yaml index 85ee5f48c..978fe94a7 100644 --- a/astra.yaml +++ b/astra.yaml @@ -1392,7 +1392,7 @@ analyses: rationale: >- Covers only pixels shared with a neighbour; defect_fill does not depend on it: defects are noise-filled under every BLEND_HANDLING. - none (the default; the committed config + noisefill (the default; the committed config sets no key) leaves neighbours fully weighted and untouched. uberseg zeroes the weight of pixels nearer a neighbour's coadd segmentation footprint than the target's (DILATE_NEIGHBOUR, default 1) @@ -1415,10 +1415,10 @@ analyses: instead cut the target's own light along an unsheared boundary, a sharp edge that rings in the FFTs. The recommended comparison arm is uberseg (weight-only), with defect_fill held equal across arms. - default: none + default: noisefill options: - none: - label: No neighbour treatment (BLEND_HANDLING = none) + noisefill: + label: No neighbour treatment (BLEND_HANDLING = noisefill) description: >- Neighbour pixels keep their full weight and image values, so the fit sees all neighbour light, which biases shapes toward diff --git a/src/shapepipe/modules/ngmix_package/ngmix.py b/src/shapepipe/modules/ngmix_package/ngmix.py index 9ebb018d3..5bd942913 100644 --- a/src/shapepipe/modules/ngmix_package/ngmix.py +++ b/src/shapepipe/modules/ngmix_package/ngmix.py @@ -31,7 +31,7 @@ from shapepipe.pipeline import file_io # Neighbour treatments selectable with the BLEND_HANDLING option. -BLEND_HANDLINGS = ("none", "uberseg") +BLEND_HANDLINGS = ("noisefill", "uberseg") # Defect fills selectable with the DEFECT_FILL option (see # :func:`prepare_ngmix_weights`). @@ -550,7 +550,7 @@ def get_data(self, cat_path): # Coadd-frame SExtractor segmentation stamp (integer labels), one per # object and row-aligned to the tile catalogue, overlaid unchanged on # every epoch for uberseg neighbour masking (shapepipe#776). None -> - # uberseg unavailable; ``"none"`` does not read it. + # uberseg unavailable; ``"noisefill"`` does not read it. self.seg = None if self.seg_cat_path: seg_cat = file_io.FITSCatalogue( @@ -605,7 +605,7 @@ def __init__( self.flags = [] self.bkg_rms = [] # Segmentation stamps, one per epoch, used only by the "uberseg" blend - # handling; empty under the default "none". All epochs carry the + # handling; empty under the default "noisefill". All epochs carry the # SAME coadd-frame seg stamp (shapepipe#776: one coadd seg per object, # no per-epoch reprojection), each MegaCam-flipped to match its galaxy # stamp so the overlay stays registered. @@ -753,8 +753,8 @@ class Ngmix(object): propagated on the vignette. ``"hsm"`` re-centers on the adaptive-moment centroid measured from the stamp pixels. See :func:`make_ngmix_observation`. - blend_handling : {"none", "uberseg"}, optional - Neighbour treatment. ``"none"`` (default) leaves neighbour pixels + blend_handling : {"noisefill", "uberseg"}, optional + Neighbour treatment. ``"noisefill"`` (default) leaves neighbour pixels untouched; ``"uberseg"`` zeroes the weight of neighbour-side pixels from the coadd segmentation map and requires ``seg_cat_path``. Defect pixels are filled under both (see :func:`prepare_ngmix_weights`). @@ -810,7 +810,7 @@ def __init__( id_obj_max=-1, bkg_sub=True, centroid_source="wcs", - blend_handling="none", + blend_handling="noisefill", seg_cat_path=None, dilate_neighbour=1, metacal_psf="fitgauss", @@ -2144,7 +2144,7 @@ def fill_defects(image, defect, noise): def prepare_ngmix_weights( gal, weight, flag, rng, bkg_rms=None, - blend_handling="none", seg=None, object_number=None, + blend_handling="noisefill", seg=None, object_number=None, dilate_neighbour=0, defect_fill="noise", ): """Build one epoch's image, weight map and noise image for ngmix. @@ -2198,8 +2198,8 @@ def prepare_ngmix_weights( ``1 / bkg_rms**2`` as the ngmix inverse variance, and non-finite or non-positive values mark defects. Otherwise every clean pixel gets ``1 / sigma_mad(gal)**2``. - blend_handling : {"none", "uberseg"}, optional - Neighbour treatment. ``"none"`` (default) leaves neighbour + blend_handling : {"noisefill", "uberseg"}, optional + Neighbour treatment. ``"noisefill"`` (default) leaves neighbour pixels weighted and untouched. ``"uberseg"`` zeroes the weight of every pixel closer to a neighbour's segmentation footprint than to the central object's and keeps its raw image value (see @@ -2319,7 +2319,7 @@ def prepare_ngmix_weights( def make_ngmix_observation( gal, weight, flag, psf, wcs, rng, bkg_rms=None, centroid_source="wcs", offset=None, - blend_handling="none", seg=None, object_number=None, + blend_handling="noisefill", seg=None, object_number=None, dilate_neighbour=0, defect_fill="noise", ): """Build an ngmix Observation for a single galaxy epoch. @@ -2362,9 +2362,9 @@ def make_ngmix_observation( Sub-pixel ``[row, col]`` coadd-centroid offset propagated from the stamp extractor. Required for ``centroid_source="wcs"`` (ignored for ``"hsm"``). - blend_handling : {"none", "uberseg"}, optional + blend_handling : {"noisefill", "uberseg"}, optional Neighbour treatment passed through to :func:`prepare_ngmix_weights`; - the default ``"none"`` leaves neighbour pixels untouched. + the default ``"noisefill"`` leaves neighbour pixels untouched. seg : numpy.ndarray, optional Segmentation map on the stamp grid. Required for ``blend_handling="uberseg"`` (ignored otherwise). @@ -2635,7 +2635,7 @@ def make_runners(prior, flux_guess, rng): def do_ngmix_metacal( stamp, prior, flux_guess, rng, centroid_source="wcs", - blend_handling="none", object_number=None, dilate_neighbour=0, + blend_handling="noisefill", object_number=None, dilate_neighbour=0, metacal_psf="fitgauss", defect_fill="noise", ): """Do Ngmix Metacal. @@ -2659,9 +2659,9 @@ def do_ngmix_metacal( coadd-centroid offset in ``stamp.offsets``, propagated from the stamp extractor); ``"hsm"`` uses the adaptive-moment centroid from the stamp pixels — see that function. - blend_handling : {"none", "uberseg"}, optional + blend_handling : {"noisefill", "uberseg"}, optional Neighbour treatment passed through to - :func:`make_ngmix_observation`; the default ``"none"`` leaves + :func:`make_ngmix_observation`; the default ``"noisefill"`` leaves neighbour pixels untouched. ``"uberseg"`` consumes ``stamp.segs`` and ``object_number``. object_number : int, optional diff --git a/src/shapepipe/modules/ngmix_runner.py b/src/shapepipe/modules/ngmix_runner.py index d350ba6fb..166e8cae5 100644 --- a/src/shapepipe/modules/ngmix_runner.py +++ b/src/shapepipe/modules/ngmix_runner.py @@ -144,7 +144,7 @@ def ngmix_runner( else: centroid_source = "wcs" - # Neighbour treatment: "none" (default) leaves neighbour pixels + # Neighbour treatment: "noisefill" (default) leaves neighbour pixels # weighted and untouched; "uberseg" zeroes the weight of every pixel # closer to a neighbour than to the central object, from the segmentation # map, and leaves its image raw. Defect pixels (flagged, zero-weight or @@ -153,7 +153,7 @@ def ngmix_runner( if config.has_option(module_config_sec, "BLEND_HANDLING"): blend_handling = config.get(module_config_sec, "BLEND_HANDLING") else: - blend_handling = "none" + blend_handling = "noisefill" # DILATE_NEIGHBOUR (optional): binary-dilation iterations enlarging the # uberseg neighbour mask, to absorb the few-pixel coadd-vs-epoch seg-overlay diff --git a/tests/module/test_ngmix.py b/tests/module/test_ngmix.py index edd54c118..a69aa0e69 100644 --- a/tests/module/test_ngmix.py +++ b/tests/module/test_ngmix.py @@ -660,7 +660,7 @@ def test_process_counts_flagged_fits_across_batches(tmp_path, monkeypatch, flags inst._id_obj_min = inst._id_obj_max = -1 inst._bkg_sub = True inst._pixel_scale = .186 - inst._blend_handling = "none" + inst._blend_handling = "noisefill" inst._dilate_neighbour = 1 inst._metacal_psf = "fitgauss" inst._epoch_central_defect_radius = module.EPOCH_CENTRAL_DEFECT_RADIUS diff --git a/tests/module/test_ngmix_defect_fill.py b/tests/module/test_ngmix_defect_fill.py index ac10057b7..74cb3045b 100644 --- a/tests/module/test_ngmix_defect_fill.py +++ b/tests/module/test_ngmix_defect_fill.py @@ -75,7 +75,7 @@ def _uberseg_seg(n): @given( stamp=defect_stamps(), - blend_handling=st.sampled_from(["none", "uberseg"]), + blend_handling=st.sampled_from(["noisefill", "uberseg"]), seed=st.integers(0, 2**31 - 1), ) def test_filled_set_is_the_defect_set(stamp, blend_handling, seed): @@ -572,7 +572,7 @@ def _hot_stamp(): return gal, np.ones((N_STAMP, N_STAMP)), flag -@pytest.mark.parametrize("blend_handling", ["none", "uberseg"]) +@pytest.mark.parametrize("blend_handling", ["noisefill", "uberseg"]) def test_interpolated_fill_and_its_weights(blend_handling): """Short defect runs take the interpolant of the clean image; other defects take noise; the weight is zero on the defects and on the diff --git a/tests/module/test_ngmix_uberseg.py b/tests/module/test_ngmix_uberseg.py index 4d6ebb32a..92616fde0 100644 --- a/tests/module/test_ngmix_uberseg.py +++ b/tests/module/test_ngmix_uberseg.py @@ -239,8 +239,8 @@ def _gal_flag_weight(npix=41, seed=7): return gal, flag, weight -def test_none_ignores_seg_and_dilate_kwargs(): - """Under BLEND_HANDLING = none, passing seg / dilate_neighbour changes +def test_noisefill_ignores_seg_and_dilate_kwargs(): + """Under BLEND_HANDLING = noisefill, passing seg / dilate_neighbour changes nothing: the result matches the plain default call on the same RNG stream.""" gal, flag, weight = _gal_flag_weight() seg, _, _ = two_object_seg(npix=gal.shape[0], sep=12) @@ -406,10 +406,3 @@ def test_runner_missing_seg_vignet_file_raises(tmp_path): _RecordingLogger(), ) - -def test_retired_noisefill_name_is_rejected(): - """Do not silently interpret the retired neighbour option.""" - with pytest.raises(ValueError, match="noisefill"): - prepare_ngmix_weights(np.ones((5, 5)), np.ones((5, 5)), - np.zeros((5, 5)), np.random.RandomState(3), - blend_handling="noisefill") diff --git a/universes/committed.yaml b/universes/committed.yaml index 4e3e02bec..6f2cf8da7 100644 --- a/universes/committed.yaml +++ b/universes/committed.yaml @@ -52,7 +52,7 @@ analyses: psf_likelihood_noise: psf_noise_1em5 megacam_ccd_flip: megapipe_flip defect_fill: noise - blend_handling: none + blend_handling: noisefill epoch_masked_fraction_cut: one_third central_defect_veto: fixed_radii catalogue_assembly: From 23cc5cb29e1035ac766d4cd1a047216a0548ad6c Mon Sep 17 00:00:00 2001 From: Cail Daley Date: Tue, 29 Sep 2026 16:16:34 +0200 Subject: [PATCH 08/30] test(ngmix): specify neighbour markers apart from defects SExtractor's -1e30 markers in the tile VIGNET form their own per-epoch mask (stamp.neighbours): a footprint 3 px from the centre, or covering 45% of the stamp, drops no epoch; noisefill zero-weights and noise-fills exactly the marked pixels, bit for bit as develop does; uberseg leaves them raw and weighted; a flagged column near the centre is still vetoed; and do_ngmix_metacal hands each epoch's mask to make_ngmix_observation. Co-Authored-By: Claude Opus 5.5 --- tests/module/test_ngmix_defect_fill.py | 267 +++++++++++++++++++++++++ 1 file changed, 267 insertions(+) diff --git a/tests/module/test_ngmix_defect_fill.py b/tests/module/test_ngmix_defect_fill.py index 74cb3045b..b9e3096c5 100644 --- a/tests/module/test_ngmix_defect_fill.py +++ b/tests/module/test_ngmix_defect_fill.py @@ -10,6 +10,9 @@ in :func:`prepare_postage_stamps` act on the same defect set: the masked-fraction cut counts it, and the central-defect veto drops an epoch with a defect near the stamp centre, at a radius set by the defect's fill. +SExtractor's -1e30 neighbour markers in the tile VIGNET are not defects: +noisefill zero-weights and noise-fills them, uberseg ignores them, and the +epoch cuts never count them. """ import re @@ -22,6 +25,7 @@ from astropy.wcs import WCS from hypothesis import given from hypothesis import strategies as st +from modopt.math.stats import sigma_mad from sqlitedict import SqliteDict from shapepipe.modules.ngmix_package import ngmix as ngmix_module @@ -735,3 +739,266 @@ def test_ngmix_rejects_an_unknown_defect_fill(tmp_path): str(tmp_path), "-001-001", 30.0, 0.186, str(paths[4]), _RecordingLogger(), bkg_sub=False, defect_fill="interp", ) + + +# --- SExtractor's -1e30 neighbour markers are not defects ------------------- +# +# The tile VIGNET carries -1e30 on the footprints of other detections. Every +# epoch shares that tile stamp, so a marker counted as a defect would drop +# every epoch of an object with a neighbour inside the veto radius. The +# markers are their own per-epoch mask (``stamp.neighbours``): noisefill +# zero-weights and noise-fills them, uberseg ignores them, and the epoch cuts +# never read them. + +_MARKER = -1.0e30 +_CENTRE = N_STAMP // 2 +# Flipped (ccd < 18) and unflipped (ccd >= 18) MegaCam CCDs. +_MARKER_EPOCH_NAMES = ["2100001-10", "2100002-20", "2100003-11"] + + +def _tile_with_neighbour(columns_from=_CENTRE + 3, rows=(_CENTRE - 2, + _CENTRE + 3)): + """Tile VIGNET with a -1e30 neighbour footprint whose nearest pixel is + 3 px from the stamp centre. The footprint is off-centre, so the MegaCam + flip moves it.""" + tile = np.random.default_rng(5).normal(0.0, 1.0, (N_STAMP, N_STAMP)) + tile[rows[0]:rows[1], columns_from:columns_from + 6] = _MARKER + return tile + + +def _marker_stamp(tile, epochs=None, **kwargs): + """Run prepare_postage_stamps on defect-free epochs under ``tile``.""" + clean = np.zeros((N_STAMP, N_STAMP), dtype=np.int32) + ones = np.ones((N_STAMP, N_STAMP)) + if epochs is None: + epochs = {name: (clean.copy(), ones) for name in _MARKER_EPOCH_NAMES} + vignet, tile_cat, psf_obj, gal_obj = _fake_inputs(epochs) + tile_cat.vign = tile[np.newaxis] + stamp = prepare_postage_stamps( + vignet, 1, 0, tile_cat, bkg_sub=False, + psf_obj=psf_obj, gal_obj=gal_obj, **kwargs, + ) + return stamp, epochs, gal_obj + + +def _expected_neighbours(tile, name): + return Ngmix.MegaCamFlip(tile, int(name.split("-")[1])) == _MARKER + + +def test_neighbour_markers_near_the_centre_keep_every_epoch(): + """A neighbour footprint 3 px from the centre, and one covering 45% of + the stamp, drop no epoch: the masked-fraction cut and the central veto + count no marker. + + Failure mode: the markers are written into the flag stamp and counted as + defects, so every epoch (they all share the tile VIGNET) is dropped by + the veto or the fraction cut and the object loses its shape + (neighbour-markers-are-not-defects). + """ + small = _tile_with_neighbour() + large = _tile_with_neighbour(rows=(0, N_STAMP)) + large[:, _CENTRE + 3:] = _MARKER + assert (large == _MARKER).mean() > 1 / 3 + for tile in (small, large): + stamp, _, _ = _marker_stamp(tile) + assert len(stamp.gals) == len(_MARKER_EPOCH_NAMES) + assert stamp.epoch_cuts["considered"] == len(_MARKER_EPOCH_NAMES) + assert stamp.epoch_cuts["masked_fraction"] == 0 + assert stamp.epoch_cuts["central_veto"] == 0 + + +def test_neighbour_markers_are_their_own_per_epoch_mask(): + """``stamp.neighbours`` holds the MegaCam-flipped marker mask of each + surviving epoch, and the flag stamps stay the exposure's own. + + Failure mode: the markers are merged into the flags, or the neighbour + mask is not flipped with its epoch and lands on the wrong pixels. + """ + tile = _tile_with_neighbour() + stamp, epochs, _ = _marker_stamp(tile) + names = {id(v[0]): name for name, v in epochs.items()} + assert len(stamp.neighbours) == len(stamp.flags) + for flag, neighbour in zip(stamp.flags, stamp.neighbours): + name = names[id(flag)] + npt.assert_array_equal(flag, 0) + npt.assert_array_equal(neighbour, _expected_neighbours(tile, name)) + assert not np.array_equal(stamp.neighbours[0], stamp.neighbours[1]) + + +def test_a_flagged_column_near_the_centre_is_still_vetoed(): + """With a neighbour footprint present, an epoch with a genuinely flagged + column 3 px from the centre is still dropped, and only that epoch. + + Failure mode: handling the markers apart also exempts real defects from + the central veto. + """ + clean = np.zeros((N_STAMP, N_STAMP), dtype=np.int32) + ones = np.ones((N_STAMP, N_STAMP)) + column = clean.copy() + column[:, _CENTRE - 3] = 1 + epochs = { + "2100001-10": (clean, ones), + "2100002-20": (column, ones), + "2100003-11": (clean.copy(), ones), + } + stamp, _, _ = _marker_stamp(_tile_with_neighbour(), epochs) + names = {id(v[0]): name for name, v in epochs.items()} + assert sorted(names[id(f)] for f in stamp.flags) == [ + "2100001-10", "2100003-11" + ] + assert stamp.epoch_cuts["central_veto"] == 1 + assert stamp.epoch_cuts["masked_fraction"] == 0 + + +def _stamp_epoch_weights(stamp, i, seed, **kwargs): + return prepare_ngmix_weights( + 1.0e3 + stamp.gals[i], stamp.weights[i], stamp.flags[i], + np.random.RandomState(seed), bkg_rms=stamp.bkg_rms[i], + neighbour=stamp.neighbours[i], **kwargs, + ) + + +def test_noisefill_fills_exactly_the_marked_pixels(): + """Under noisefill, a defect-free epoch has zero weight and noise + exactly on the marked neighbour pixels; every other pixel keeps its raw + value and its weight. + + Failure mode: the neighbour markers are dropped with the flags, so + noisefill no longer removes neighbour light (noisefill-fills-markers). + """ + tile = _tile_with_neighbour() + stamp, epochs, _ = _marker_stamp(tile) + assert len(stamp.gals) == len(_MARKER_EPOCH_NAMES) + for i in range(len(stamp.gals)): + gal = 1.0e3 + stamp.gals[i] + gal_out, w_out, _ = _stamp_epoch_weights( + stamp, i, seed=i, blend_handling="noisefill", + ) + neighbour = stamp.neighbours[i] + assert neighbour.any() + npt.assert_array_equal(gal_out != gal, neighbour) + npt.assert_array_equal(w_out == 0.0, neighbour) + assert np.all(np.abs(gal_out[neighbour]) < 10.0) + + +def test_uberseg_leaves_the_marked_pixels_raw(): + """Under uberseg, the markers mask nothing: with a seg map holding only + the central object, a defect-free epoch keeps every pixel raw and + weighted, marked or not. + + Failure mode: the markers reach the defect set or the fill, so uberseg + noise-fills the neighbour's light instead of leaving it to the seg-based + weight (uberseg-ignores-markers). + """ + tile = _tile_with_neighbour() + stamp, _, _ = _marker_stamp(tile) + seg = np.zeros((N_STAMP, N_STAMP), dtype=np.int32) + seg[_CENTRE - 1:_CENTRE + 2, _CENTRE - 1:_CENTRE + 2] = 1 + assert len(stamp.gals) == len(_MARKER_EPOCH_NAMES) + for i in range(len(stamp.gals)): + gal = 1.0e3 + stamp.gals[i] + gal_out, w_out, _ = _stamp_epoch_weights( + stamp, i, seed=i, blend_handling="uberseg", seg=seg, + object_number=1, + ) + npt.assert_array_equal(gal_out, gal) + assert np.all(w_out > 0.0) + + +def _develop_noisefill(gal, weight, flag, rng, bkg_rms=None): + """prepare_ngmix_weights under BLEND_HANDLING = noisefill on develop + (240b37e4), where the markers arrive as flag 2**10: the reference the + noisefill output is pinned to.""" + mask = np.copy(weight) != 0 + mask[flag != 0] = False + if bkg_rms is None: + sig_noise = sigma_mad(gal) + weight_map = mask.astype(float) / sig_noise ** 2 + else: + valid_rms = np.isfinite(bkg_rms) & (bkg_rms > 0) + mask &= valid_rms + weight_map = np.zeros_like(gal, dtype=float) + weight_map[mask] = 1.0 / bkg_rms[mask] ** 2 + sig_noise = np.where(valid_rms, bkg_rms, np.median(bkg_rms[mask])) + noise_img = rng.standard_normal(gal.shape) * sig_noise + noise_img_gal = rng.standard_normal(gal.shape) * sig_noise + gal_masked = np.copy(gal) + gal_masked[~mask] = noise_img_gal[~mask] + return gal_masked, weight_map, noise_img + + +@given( + seed=st.integers(0, 2**31 - 1), + with_rms=st.booleans(), + rms_scale=st.floats(0.5, 2.0), + with_defects=st.booleans(), +) +def test_noisefill_matches_develop_on_marked_neighbours( + seed, with_rms, rms_scale, with_defects, +): + """For an epoch with a marked neighbour, noisefill returns the image, + weight and noise image develop returned, bit for bit: with no defect, + and with a flagged column and a dead pixel under the default noise fill. + + Failure mode: carrying the markers apart from the flags changes what + noisefill does to neighbour pixels, their weights, the noise level or + the RNG stream (noisefill-matches-develop). + """ + rng = np.random.default_rng(seed) + n = 31 + gal = rng.normal(0.0, 1.0, (n, n)) + gal[n // 2 - 2:n // 2 + 3, n // 2 - 2:n // 2 + 3] += 50.0 + weight = np.ones((n, n)) + flag = np.zeros((n, n), dtype=np.int32) + neighbour = np.zeros((n, n), dtype=bool) + neighbour[n // 2 - 3:n // 2 + 4, n // 2 + 3:n // 2 + 9] = True + gal[neighbour] += 30.0 + if with_defects: + flag[:, 2] = 1 + weight[n - 3, n // 2] = 0.0 + bkg_rms = ( + rms_scale * (1.0 + 0.1 * rng.random((n, n))) if with_rms else None + ) + + new = prepare_ngmix_weights( + gal, weight, flag, np.random.RandomState(seed), bkg_rms=bkg_rms, + blend_handling="noisefill", neighbour=neighbour, + ) + old = _develop_noisefill( + gal, weight, np.where(neighbour, 2**10, flag), + np.random.RandomState(seed), bkg_rms=bkg_rms, + ) + for a, b in zip(new, old): + npt.assert_array_equal(a, b) + + +def test_do_ngmix_metacal_threads_each_epochs_neighbour_mask(monkeypatch): + """Each epoch's neighbour mask reaches make_ngmix_observation. + + Failure mode: the mask is built but never used, so noisefill silently + stops filling neighbours. + """ + tile = _tile_with_neighbour() + stamp, _, _ = _marker_stamp(tile) + assert len(stamp.gals) == len(_MARKER_EPOCH_NAMES) + seen = [] + + class _Stop(Exception): + pass + + def fake_observation(*args, **kwargs): + seen.append(kwargs["neighbour"]) + if len(seen) == len(stamp.gals): + raise _Stop + return None + + monkeypatch.setattr( + ngmix_module, "make_ngmix_observation", fake_observation, + ) + monkeypatch.setattr(ngmix_module, "ObsList", list) + with pytest.raises(_Stop): + ngmix_module.do_ngmix_metacal( + stamp, None, 1.0, np.random.RandomState(0), + ) + for got, want in zip(seen, stamp.neighbours): + assert got is want From 2517887cd49194969034df6948ba5bde44ba0a17 Mon Sep 17 00:00:00 2001 From: Cail Daley Date: Tue, 29 Sep 2026 16:22:58 +0200 Subject: [PATCH 09/30] fix(ngmix): keep SExtractor's neighbour markers out of the defect set prepare_postage_stamps wrote the tile VIGNET's -1e30 markers (other detections' footprints) into each epoch's flag stamp as 2**10, so defect_mask counted them: they fed the masked-fraction cut and the central veto. Every epoch shares the tile VIGNET, so a neighbour within EPOCH_CENTRAL_DEFECT_RADIUS dropped every epoch of the object. The markers are now their own per-epoch mask, stamp.neighbours (MegaCam-flipped with the epoch), threaded through do_ngmix_metacal and make_ngmix_observation to prepare_ngmix_weights. The epoch cuts count genuine defects only. Under noisefill the marked pixels get weight 0 and the defects' noise realisation, the output noisefill gave when they arrived as flags; under uberseg they are ignored and the seg-based weight handles neighbours. Defects are treated identically under both. Docstrings, @sc contracts, runner comments and the astra.yaml decisions say so. Co-Authored-By: Claude Opus 5.5 --- astra.yaml | 48 +++--- src/shapepipe/modules/ngmix_package/ngmix.py | 150 +++++++++++++------ src/shapepipe/modules/ngmix_runner.py | 15 +- tests/module/test_ngmix_uberseg.py | 3 +- 4 files changed, 143 insertions(+), 73 deletions(-) diff --git a/astra.yaml b/astra.yaml index 978fe94a7..493922a10 100644 --- a/astra.yaml +++ b/astra.yaml @@ -1329,10 +1329,11 @@ analyses: weighted pixels within about a PSF width. DES's ngmixer fills for that reason: "it may be important for codes that take moments or use FFTs". Under every BLEND_HANDLING and the default DEFECT_FILL = - noise, defect pixels are replaced with independent noise at the per-pixel background RMS when supplied, or - the stamp's robust noise scale otherwise; `BLEND_HANDLING = uberseg` - additionally zeroes the weight of neighbour-side pixels and leaves - their image values untouched. The committed fill + noise, defect pixels are replaced with independent noise at the + per-pixel background RMS when supplied, or the stamp's robust noise + scale otherwise. SExtractor's -1e30 neighbour markers in the tile + VIGNET are not defects; blend_handling decides their treatment. The + committed fill uses the unsymmetrized defect set: DES symmetrized its masks, but four-fold symmetrization quadruples m and still leaves an additive c1 (symmetrized_4fold_noise). The cost of not symmetrizing, a hole in @@ -1391,10 +1392,14 @@ analyses: label: Neighbour treatment before metacal rationale: >- Covers only pixels shared with a neighbour; defect_fill does not - depend on it: defects are noise-filled under every BLEND_HANDLING. - noisefill (the default; the committed config - sets no key) leaves neighbours fully weighted and untouched. - uberseg zeroes the weight of pixels nearer a neighbour's coadd + depend on it: defects are filled the same way under every + BLEND_HANDLING, and the epoch cuts never count neighbour pixels. + noisefill (the default; the committed config sets no key) gives + weight 0 to the pixels SExtractor marks -1e30 in the tile VIGNET + (other detections' footprints; about 93% of neighbour-footprint + pixels on a SExtractor-mode sims tile) and replaces them with noise; unmarked + neighbour pixels keep their weight and light. uberseg ignores the + markers and zeroes the weight of pixels nearer a neighbour's coadd segmentation footprint than the target's (DILATE_NEIGHBOUR, default 1) and leaves the image untouched, as official uberseg does: esheldon/meds get_uberseg returns a weight map (a @@ -1418,12 +1423,13 @@ analyses: default: noisefill options: noisefill: - label: No neighbour treatment (BLEND_HANDLING = noisefill) + label: Noise-fill SExtractor-marked neighbour pixels (BLEND_HANDLING = noisefill) description: >- - Neighbour pixels keep their full weight and image values, so the - fit sees all neighbour light, which biases shapes toward - neighbours (Jarvis et al. 2016); this masks less than even their - plain segmentation map. A candidate cause of the FLAGS=2 B-modes + Pixels marked -1e30 in the tile VIGNET get weight 0 and noise. + The fill stops at the marked footprint, so unmarked neighbour + pixels and the neighbour's wings keep their weight and light, + which biases shapes toward neighbours (Jarvis et al. 2016). A + candidate cause of the FLAGS=2 B-modes investigated in #814. insights: [mask_uberseg_neighbour_bias] uberseg: @@ -1456,9 +1462,10 @@ analyses: rationale: >- An epoch is dropped when a defect pixel lies strictly closer to the stamp centre than its fill's radius, beside the masked-fraction cut - in the epoch loop; EPOCH_CENTRAL_DEFECT_RADIUS = 0 disables it. The - veto reads only the - defect mask, so it selects on nothing shear-responsive; for the same + in the epoch loop; EPOCH_CENTRAL_DEFECT_RADIUS = 0 disables it. + SExtractor's -1e30 neighbour markers are not defects: all epochs + share the tile VIGNET, so a neighbour inside the radius would drop + every epoch. The veto reads only the defect mask, so it selects on nothing shear-responsive; for the same reason the radii are fixed rather than scaled by galaxy size: 10 px for noise-filled pixels (EPOCH_CENTRAL_DEFECT_RADIUS) and 7 px for interpolated ones (EPOCH_INTERPOLATED_DEFECT_RADIUS, @@ -1492,10 +1499,11 @@ analyses: rationale: >- An epoch whose stamp has more than EPOCH_MASKED_FRACTION_CUT (default 1/3) of its pixels in the raw, unsymmetrized defect set is - dropped from the multi-epoch fit: flagged pixels (any nonzero flag - bit, including the tile-coverage bit 2**10 set where the tile vignet - is off-image), zero-weight pixels and invalid-RMS pixels, the set - defect_fill noise-fills. An object with no surviving epoch has no + dropped from the multi-epoch fit: flagged pixels (any nonzero + exposure flag bit), zero-weight pixels and invalid-RMS pixels, the + set defect_fill fills. SExtractor's -1e30 neighbour markers in the + tile VIGNET are not counted: every epoch shares the tile VIGNET, so + a large neighbour would drop them all. An object with no surviving epoch has no shape. Before the cut, an epoch is dropped silently if its galaxy stamp is all zeros or its background-subtracted sigma_mad is not positive. DES was stricter: diff --git a/src/shapepipe/modules/ngmix_package/ngmix.py b/src/shapepipe/modules/ngmix_package/ngmix.py index 5bd942913..1e9d10797 100644 --- a/src/shapepipe/modules/ngmix_package/ngmix.py +++ b/src/shapepipe/modules/ngmix_package/ngmix.py @@ -603,6 +603,11 @@ def __init__( self.psfs = [] self.weights = [] self.flags = [] + # Neighbour masks, one per epoch: the pixels SExtractor marks -1e30 in + # the tile VIGNET (other detections' footprints), MegaCam-flipped to + # the epoch. noisefill zero-weights and noise-fills them; uberseg and + # the epoch cuts do not read them (see prepare_ngmix_weights). + self.neighbours = [] self.bkg_rms = [] # Segmentation stamps, one per epoch, used only by the "uberseg" blend # handling; empty under the default "noisefill". All epochs carry the @@ -754,10 +759,12 @@ class Ngmix(object): adaptive-moment centroid measured from the stamp pixels. See :func:`make_ngmix_observation`. blend_handling : {"noisefill", "uberseg"}, optional - Neighbour treatment. ``"noisefill"`` (default) leaves neighbour pixels - untouched; ``"uberseg"`` zeroes the weight of neighbour-side pixels - from the coadd segmentation map and requires ``seg_cat_path``. Defect - pixels are filled under both (see :func:`prepare_ngmix_weights`). + Neighbour treatment. ``"noisefill"`` (default) zero-weights and + noise-fills the pixels SExtractor marks -1e30 in the tile VIGNET + (other detections' footprints); ``"uberseg"`` ignores those markers, + zeroes the weight of neighbour-side pixels from the coadd + segmentation map and requires ``seg_cat_path``. Defect pixels are + filled under both (see :func:`prepare_ngmix_weights`). seg_cat_path : str, optional Path to the coadd-frame segmentation VIGNET catalogue (see :class:`Tile_cat`). Required when ``blend_handling="uberseg"``. @@ -765,11 +772,11 @@ class Ngmix(object): Neighbour-mask dilation iterations for ``"uberseg"`` (see :func:`uberseg_weight`); the default is ``1``. epoch_central_defect_radius : float, optional - Drop an epoch when a masked pixel lies closer than this many pixels + Drop an epoch when a defect pixel lies closer than this many pixels to the stamp centre (see :func:`has_central_defect`); the default is ``EPOCH_CENTRAL_DEFECT_RADIUS``. epoch_masked_fraction_cut : float, optional - Drop an epoch when more than this fraction of its stamp is masked + Drop an epoch when more than this fraction of its stamp is defects (see :func:`prepare_postage_stamps`); the default is ``EPOCH_MASKED_FRACTION_CUT``. defect_fill : {"noise", "interpolate"}, optional @@ -1511,10 +1518,19 @@ def prepare_postage_stamps( An epoch is dropped when more than ``epoch_masked_fraction_cut`` of its stamp lies in :func:`defect_mask`: flagged, zero-weight and invalid-RMS pixels, the set that :func:`prepare_ngmix_weights` zero-weights and - fills. Counting flags alone would keep epochs whose filled area exceeds - the cut. The zero-weight orbit of interpolated pixels keeps its light and - is not counted. The default is 1/3; 10%, the DES Y3 and Y6 value, is - the alternative to test. + fills under every ``blend_handling``. Counting flags alone would keep + epochs whose filled area exceeds the cut. The zero-weight orbit of + interpolated pixels keeps its light and is not counted. The default is + 1/3; 10%, the DES Y3 and Y6 value, is the alternative to test. + + @sc [decision:shape_measurement.central_defect_veto,decision:shape_measurement.epoch_masked_fraction_cut,decision:shape_measurement.blend_handling] neighbour-markers-are-not-defects + SExtractor writes -1e30 into the tile VIGNET on the footprints of other + detections (and beyond the tile's edge). These markers form the epoch's neighbour mask + (``stamp.neighbours``, MegaCam-flipped like the epoch), kept apart from + its flag stamp, so neither the masked-fraction cut nor the central + veto counts them. Every epoch shares the tile VIGNET: counting a + neighbour within the veto radius as a defect would drop every epoch of + the object. Parameters ---------- @@ -1618,8 +1634,12 @@ def prepare_postage_stamps( tile_seg = Ngmix.MegaCamFlip(tile_seg, int(ccd_n)) flag_vign = flag_obj[expccd_name]['VIGNET'] - if tile_vign is not None: - flag_vign[np.where(tile_vign == -1e30)] = 2**10 + # Neighbour markers (neighbour-markers-are-not-defects). + neighbour = ( + tile_vign == -1e30 + if tile_vign is not None + else np.zeros(np.shape(gal_vign), dtype=bool) + ) weight_vign = weight_obj[expccd_name]['VIGNET'] bkg_rms_vign = ( bkg_rms_obj[expccd_name]['VIGNET'] @@ -1673,6 +1693,7 @@ def prepare_postage_stamps( stamp.psfs.append(psf_obj[expccd_name]['VIGNET']) stamp.weights.append(weight_vign_scaled) stamp.flags.append(flag_vign) + stamp.neighbours.append(neighbour) stamp.bkg_rms.append(bkg_rms_vign_scaled) if tile_seg is not None: stamp.segs.append(tile_seg) @@ -1981,11 +2002,13 @@ def defect_mask(weight, flag, bkg_rms=None): """Defect pixels of one epoch stamp. @sc [decision:shape_measurement.defect_fill,label:physics] defect-set-unsymmetrized - A defect is a pixel with zero exposure weight, a nonzero flag, or (when a - background RMS map is given) a non-finite or non-positive RMS. This one - set is zero-weighted and noise-filled by :func:`prepare_ngmix_weights` - and counted by the epoch cuts in :func:`prepare_postage_stamps`. It is - not ORed with its rotations. For defects the central-defect veto keeps + A defect is a pixel with zero exposure weight, a nonzero exposure flag, + or (when a background RMS map is given) a non-finite or non-positive RMS. + This one set is zero-weighted and filled by :func:`prepare_ngmix_weights` + under every ``blend_handling`` and counted by the epoch cuts in + :func:`prepare_postage_stamps`. SExtractor's neighbour markers are not + in it (neighbour-markers-are-not-defects). It is not ORed with its + rotations. For defects the central-defect veto keeps (:func:`has_central_defect`), the unsymmetrized fill leaves |c| <= 3e-4 per affected epoch on galaxies with half-light radius 0.3" and 0.5" through a 0.7" PSF, round or elliptical. Symmetrizing would @@ -2001,7 +2024,7 @@ def defect_mask(weight, flag, bkg_rms=None): weight : numpy.ndarray Exposure weight stamp. flag : numpy.ndarray - Flag stamp. + Exposure flag stamp. bkg_rms : numpy.ndarray, optional Background RMS stamp. @@ -2130,7 +2153,8 @@ def fill_defects(image, defect, noise): image : numpy.ndarray Stamp image. defect : numpy.ndarray of bool - Pixels to replace (:func:`defect_mask`). + Pixels to replace: the defects (:func:`defect_mask`), and under + noisefill the marked neighbour pixels. noise : numpy.ndarray Noise realisation on the stamp grid. @@ -2145,7 +2169,7 @@ def fill_defects(image, defect, noise): def prepare_ngmix_weights( gal, weight, flag, rng, bkg_rms=None, blend_handling="noisefill", seg=None, object_number=None, - dilate_neighbour=0, defect_fill="noise", + dilate_neighbour=0, defect_fill="noise", neighbour=None, ): """Build one epoch's image, weight map and noise image for ngmix. @@ -2154,17 +2178,29 @@ def prepare_ngmix_weights( by an independent noise realisation at their background RMS (:func:`fill_defects`), or under ``defect_fill="interpolate"``, for short defect runs, by an interpolant of the clean pixels around them. - ``blend_handling`` decides only the neighbour side. + ``blend_handling`` decides only the neighbour pixels. - @sc [decision:shape_measurement.defect_fill,decision:shape_measurement.blend_handling] defects-filled-neighbours-raw + @sc [decision:shape_measurement.defect_fill,decision:shape_measurement.blend_handling] defects-filled-whatever-the-blend-handling Every pixel of ``defect_mask`` is zero-weighted and filled whatever - ``blend_handling`` is, and the filled set equals the defect set. - ``blend_handling`` acts on the neighbour side only: uberseg zeroes - the weight of pixels nearer a neighbour's footprint and leaves their - image values raw, because that light is real sky that metacal shears - along with the target; filling it with noise would cut the target along - an unsheared edge. Neighbour pixels are never filled. The noise image - covers the whole stamp and is independent of both. + ``blend_handling`` is, by the same operator. ``blend_handling`` acts on + the neighbour pixels only, and the noise image covers the whole stamp. + + @sc [decision:shape_measurement.blend_handling] noisefill-fills-markers + Under ``"noisefill"`` the pixels of ``neighbour``, SExtractor's -1e30 + neighbour markers, get weight 0 and are replaced by the same noise + realisation as the noise-filled defects, so no marked neighbour light + reaches metacal. Under the default noise fill, the image, weight map and + noise image are those the marked pixels would get as flagged defects; + only the epoch cuts treat them differently, by not counting them + (neighbour-markers-are-not-defects). Unmarked neighbour light stays + raw and weighted. + + @sc [decision:shape_measurement.blend_handling] uberseg-ignores-markers + Under ``"uberseg"`` the markers are ignored: :func:`uberseg_weight` + zeroes the weight of pixels nearer a neighbour's segmentation footprint + than the target's and leaves their image values raw, because that light + is real sky that metacal shears along with the target; filling it with + noise would cut the target along an unsheared edge. @sc [decision:shape_measurement.defect_fill,label:physics] interpolated-defect-weight-orbit Under ``defect_fill="interpolate"`` the short defect runs @@ -2189,7 +2225,7 @@ def prepare_ngmix_weights( weight : numpy.ndarray Exposure weight stamp; zero marks a defect. flag : numpy.ndarray - Flag stamp; nonzero marks a defect. + Exposure flag stamp; nonzero marks a defect. rng : numpy.random.RandomState Random state for the noise realisations (seeded per object; see :func:`position_seed`). @@ -2199,11 +2235,11 @@ def prepare_ngmix_weights( non-positive values mark defects. Otherwise every clean pixel gets ``1 / sigma_mad(gal)**2``. blend_handling : {"noisefill", "uberseg"}, optional - Neighbour treatment. ``"noisefill"`` (default) leaves neighbour - pixels weighted and untouched. ``"uberseg"`` zeroes the weight of - every pixel closer to a neighbour's segmentation footprint than to the - central object's and keeps its raw image value (see - :func:`uberseg_weight`). + Neighbour treatment. ``"noisefill"`` (default) zero-weights and + noise-fills the ``neighbour`` pixels. ``"uberseg"`` ignores + ``neighbour``, zeroes the weight of every pixel closer to a + neighbour's segmentation footprint than to the central object's and + keeps its raw image value (see :func:`uberseg_weight`). seg : numpy.ndarray, optional Segmentation map on the stamp grid (object NUMBERs). Required for ``blend_handling="uberseg"``; ignored otherwise. @@ -2216,11 +2252,15 @@ def prepare_ngmix_weights( defect_fill : {"noise", "interpolate"}, optional ``"noise"`` (default) noise-fills every defect; ``"interpolate"`` interpolates the short defect runs and noise-fills the rest. + neighbour : numpy.ndarray of bool, optional + The epoch's neighbour mask (``Postage_stamp.neighbours``); read only + under ``blend_handling="noisefill"``. ``None`` marks no pixel. Returns ------- numpy.ndarray - Galaxy image with defect pixels filled. + Galaxy image with defect pixels, and under noisefill marked + neighbour pixels, filled. numpy.ndarray Inverse-variance weight map for ngmix. numpy.ndarray @@ -2253,7 +2293,13 @@ def prepare_ngmix_weights( ) defect = defect_mask(weight, flag, bkg_rms) - clean = ~defect + # Marked neighbour pixels that noisefill removes (noisefill-fills-markers). + removed_neighbour = ( + np.asarray(neighbour, dtype=bool) + if blend_handling == "noisefill" and neighbour is not None + else np.zeros_like(defect) + ) + clean = ~(defect | removed_neighbour) interpolated = ( interpolable_defects(defect) if defect_fill == "interpolate" @@ -2262,7 +2308,7 @@ def prepare_ngmix_weights( # The quarter-turn orbit of the interpolated pixels carries no weight # (interpolated-defect-weight-orbit). weighted = ( - ~(defect | fourfold(interpolated)) if interpolated.any() else clean + clean & ~fourfold(interpolated) if interpolated.any() else clean ) if bkg_rms is None: @@ -2299,12 +2345,17 @@ def prepare_ngmix_weights( noise_img = rng.standard_normal(gal.shape) * sig_noise noise_img_gal = rng.standard_normal(gal.shape) * sig_noise - gal_filled = fill_defects(gal, defect, noise_img_gal) + gal_filled = fill_defects(gal, ~clean, noise_img_gal) if interpolated.any(): # One operator for the image and the noise image; a pixel whose - # support is degenerate keeps its noise fill. + # support is degenerate, or that noisefill removes as a neighbour, + # keeps its noise fill. filled = interpolate_defects([gal, noise_img], defect, interpolated) - done = interpolated & np.all(np.isfinite(filled), axis=0) + done = ( + interpolated + & ~removed_neighbour + & np.all(np.isfinite(filled), axis=0) + ) gal_filled[done] = filled[0][done] noise_img = np.where(done, filled[1], noise_img) @@ -2320,7 +2371,7 @@ def make_ngmix_observation( gal, weight, flag, psf, wcs, rng, bkg_rms=None, centroid_source="wcs", offset=None, blend_handling="noisefill", seg=None, object_number=None, - dilate_neighbour=0, defect_fill="noise", + dilate_neighbour=0, defect_fill="noise", neighbour=None, ): """Build an ngmix Observation for a single galaxy epoch. @@ -2364,7 +2415,8 @@ def make_ngmix_observation( ``"hsm"``). blend_handling : {"noisefill", "uberseg"}, optional Neighbour treatment passed through to :func:`prepare_ngmix_weights`; - the default ``"noisefill"`` leaves neighbour pixels untouched. + the default ``"noisefill"`` zero-weights and noise-fills the + ``neighbour`` pixels. seg : numpy.ndarray, optional Segmentation map on the stamp grid. Required for ``blend_handling="uberseg"`` (ignored otherwise). @@ -2377,6 +2429,8 @@ def make_ngmix_observation( defect_fill : {"noise", "interpolate"}, optional Defect fill passed through to :func:`prepare_ngmix_weights`; the default is ``"noise"``. + neighbour : numpy.ndarray of bool, optional + Neighbour mask passed through to :func:`prepare_ngmix_weights`. Returns ------- @@ -2406,6 +2460,7 @@ def make_ngmix_observation( gal, weight, flag, rng, bkg_rms=bkg_rms, blend_handling=blend_handling, seg=seg, object_number=object_number, dilate_neighbour=dilate_neighbour, defect_fill=defect_fill, + neighbour=neighbour, ) if centroid_source == "hsm": @@ -2661,9 +2716,9 @@ def do_ngmix_metacal( stamp pixels — see that function. blend_handling : {"noisefill", "uberseg"}, optional Neighbour treatment passed through to - :func:`make_ngmix_observation`; the default ``"noisefill"`` leaves - neighbour pixels untouched. ``"uberseg"`` consumes ``stamp.segs`` and - ``object_number``. + :func:`make_ngmix_observation`; the default ``"noisefill"`` + zero-weights and noise-fills the pixels of ``stamp.neighbours``. + ``"uberseg"`` consumes ``stamp.segs`` and ``object_number``. object_number : int, optional Central object's segmentation label — its SExtractor ``NUMBER`` (``obj_id``), authoritative because seg labels are the NUMBERs of the @@ -2720,6 +2775,9 @@ def do_ngmix_metacal( object_number=object_number, dilate_neighbour=dilate_neighbour, defect_fill=defect_fill, + neighbour=( + stamp.neighbours[n_e] if n_e < len(stamp.neighbours) else None + ), ) gal_obs_list.append(gal_obs) diff --git a/src/shapepipe/modules/ngmix_runner.py b/src/shapepipe/modules/ngmix_runner.py index 166e8cae5..dc27b8afc 100644 --- a/src/shapepipe/modules/ngmix_runner.py +++ b/src/shapepipe/modules/ngmix_runner.py @@ -144,10 +144,11 @@ def ngmix_runner( else: centroid_source = "wcs" - # Neighbour treatment: "noisefill" (default) leaves neighbour pixels - # weighted and untouched; "uberseg" zeroes the weight of every pixel - # closer to a neighbour than to the central object, from the segmentation - # map, and leaves its image raw. Defect pixels (flagged, zero-weight or + # Neighbour treatment: "noisefill" (default) zero-weights and noise-fills + # the pixels SExtractor marks -1e30 in the tile VIGNET (other detections' + # footprints); "uberseg" ignores those markers, zeroes the weight of every + # pixel closer to a neighbour than to the central object, from the + # segmentation map, and leaves its image raw. Defect pixels (flagged, zero-weight or # invalid-RMS) are zero-weighted and filled under both (DEFECT_FILL # below); see prepare_ngmix_weights. if config.has_option(module_config_sec, "BLEND_HANDLING"): @@ -164,7 +165,8 @@ def ngmix_runner( dilate_neighbour = 1 # EPOCH_CENTRAL_DEFECT_RADIUS (optional, pixels): drop an epoch when a - # masked pixel lies closer than this to the stamp centre; 0 disables. + # defect pixel (flagged, zero-weight or invalid-RMS; not a neighbour + # marker) lies closer than this to the stamp centre; 0 disables. if config.has_option(module_config_sec, "EPOCH_CENTRAL_DEFECT_RADIUS"): epoch_central_defect_radius = config.getfloat( module_config_sec, "EPOCH_CENTRAL_DEFECT_RADIUS" @@ -173,7 +175,8 @@ def ngmix_runner( epoch_central_defect_radius = EPOCH_CENTRAL_DEFECT_RADIUS # EPOCH_MASKED_FRACTION_CUT (optional): drop an epoch when more than this - # fraction of its stamp is masked (flagged, zero-weight or invalid-RMS). + # fraction of its stamp is defects (flagged, zero-weight or invalid-RMS; + # neighbour markers are not counted). if config.has_option(module_config_sec, "EPOCH_MASKED_FRACTION_CUT"): epoch_masked_fraction_cut = config.getfloat( module_config_sec, "EPOCH_MASKED_FRACTION_CUT" diff --git a/tests/module/test_ngmix_uberseg.py b/tests/module/test_ngmix_uberseg.py index 92616fde0..5c20789f6 100644 --- a/tests/module/test_ngmix_uberseg.py +++ b/tests/module/test_ngmix_uberseg.py @@ -263,7 +263,8 @@ def test_uberseg_fills_defects_and_leaves_neighbour_pixels_raw(): Failure modes: the defect fill is skipped under uberseg, so raw bad pixels reach metacal; or the neighbour side is noise-filled - (defects-filled-neighbours-raw). + (defects-filled-whatever-the-blend-handling, + uberseg-ignores-markers). """ npix = 41 gal, flag, weight = _gal_flag_weight(npix=npix) From 7faa9e78dcd7118a1308563a7c4c322741b08fc5 Mon Sep 17 00:00:00 2001 From: Cail Daley Date: Tue, 29 Sep 2026 16:45:43 +0200 Subject: [PATCH 10/30] fix(ngmix): off-tile pixels are defects, not neighbours SExtractor writes -1e30 into the tile VIGNET beyond the tile's edge as well as on other detections' footprints. Beyond the edge the epoch holds the object's own light, cut off, so treating those pixels as neighbours exempted edge objects from both epoch cuts and measured them with a noise-filled band near their centre. split_tile_markers classifies the markers geometrically on the flipped tile VIGNET: the union of stamp rows and columns that are entirely -1e30 (the off-image part of a rectangle clip) is off-tile, the rest is the neighbour mask. Off-tile pixels are flagged OFF_TILE_FLAG (2**10) in the epoch's flag stamp, so under both blend handlings they are zero-weighted, filled, and counted by the masked-fraction cut and the central veto. Docstrings, @sc contracts, runner comments and astra.yaml say so. Tests: an object 3 px from the tile edge loses every epoch; a band 12 px away passes the default cuts and is vetoed at radius 13; a corner's L-shaped off-tile region is flagged exactly and a border-touching neighbour stays a neighbour; off-tile pixels are filled under both blend handlings. Co-Authored-By: Claude Opus 5.5 --- astra.yaml | 33 +++-- src/shapepipe/modules/ngmix_package/ngmix.py | 77 +++++++++--- src/shapepipe/modules/ngmix_runner.py | 14 +-- tests/module/test_ngmix_defect_fill.py | 121 ++++++++++++++++++- 4 files changed, 206 insertions(+), 39 deletions(-) diff --git a/astra.yaml b/astra.yaml index 493922a10..fde5b8a70 100644 --- a/astra.yaml +++ b/astra.yaml @@ -1321,8 +1321,8 @@ analyses: defect_fill: label: Image content of defect pixels before metacal rationale: >- - A defect pixel (nonzero instrument flag, zero exposure weight or - invalid background RMS) gets weight 0 in prepare_ngmix_weights. Its + A defect pixel (nonzero instrument flag, zero exposure weight, + invalid background RMS, or off-tile) gets weight 0 in prepare_ngmix_weights. Its image value still matters: ngmix's metacal deconvolves, shears and reconvolves an InterpolatedImage of the whole image and copies the weights through, so a zero-weight pixel's content spreads into the @@ -1331,9 +1331,13 @@ analyses: FFTs". Under every BLEND_HANDLING and the default DEFECT_FILL = noise, defect pixels are replaced with independent noise at the per-pixel background RMS when supplied, or the stamp's robust noise - scale otherwise. SExtractor's -1e30 neighbour markers in the tile - VIGNET are not defects; blend_handling decides their treatment. The - committed fill + scale otherwise. SExtractor writes -1e30 into the tile VIGNET on + other detections' footprints and beyond the tile's edge. The stamp + rows and columns that are entirely -1e30 (the off-image part of a + rectangle clip) are off-tile: the epoch holds the object's own light + there, so they are defects, flagged 2**10. The other markers are + neighbour pixels, not defects; blend_handling decides their + treatment. The committed fill uses the unsymmetrized defect set: DES symmetrized its masks, but four-fold symmetrization quadruples m and still leaves an additive c1 (symmetrized_4fold_noise). The cost of not symmetrizing, a hole in @@ -1396,8 +1400,10 @@ analyses: BLEND_HANDLING, and the epoch cuts never count neighbour pixels. noisefill (the default; the committed config sets no key) gives weight 0 to the pixels SExtractor marks -1e30 in the tile VIGNET - (other detections' footprints; about 93% of neighbour-footprint - pixels on a SExtractor-mode sims tile) and replaces them with noise; unmarked + on other detections' footprints (about 93% of neighbour-footprint + pixels on a SExtractor-mode sims tile), excluding the off-tile + rows and columns, which are defects, and replaces them with noise; + unmarked neighbour pixels keep their weight and light. uberseg ignores the markers and zeroes the weight of pixels nearer a neighbour's coadd segmentation footprint than the target's (DILATE_NEIGHBOUR, default 1) @@ -1465,7 +1471,8 @@ analyses: in the epoch loop; EPOCH_CENTRAL_DEFECT_RADIUS = 0 disables it. SExtractor's -1e30 neighbour markers are not defects: all epochs share the tile VIGNET, so a neighbour inside the radius would drop - every epoch. The veto reads only the defect mask, so it selects on nothing shear-responsive; for the same + every epoch. Off-tile pixels are defects, so an object near the + tile edge is vetoed. The veto reads only the defect mask, so it selects on nothing shear-responsive; for the same reason the radii are fixed rather than scaled by galaxy size: 10 px for noise-filled pixels (EPOCH_CENTRAL_DEFECT_RADIUS) and 7 px for interpolated ones (EPOCH_INTERPOLATED_DEFECT_RADIUS, @@ -1500,10 +1507,12 @@ analyses: An epoch whose stamp has more than EPOCH_MASKED_FRACTION_CUT (default 1/3) of its pixels in the raw, unsymmetrized defect set is dropped from the multi-epoch fit: flagged pixels (any nonzero - exposure flag bit), zero-weight pixels and invalid-RMS pixels, the - set defect_fill fills. SExtractor's -1e30 neighbour markers in the - tile VIGNET are not counted: every epoch shares the tile VIGNET, so - a large neighbour would drop them all. An object with no surviving epoch has no + exposure flag bit), zero-weight pixels, invalid-RMS pixels and + off-tile pixels (whole -1e30 rows and columns of the tile VIGNET, + flag 2**10), the set defect_fill fills. On a 51-px stamp an object + within about 8.5 px of the tile edge fails the 1/3 cut. The other + -1e30 markers, neighbour footprints, are not counted: every epoch + shares the tile VIGNET, so a large neighbour would drop them all. An object with no surviving epoch has no shape. Before the cut, an epoch is dropped silently if its galaxy stamp is all zeros or its background-subtracted sigma_mad is not positive. DES was stricter: diff --git a/src/shapepipe/modules/ngmix_package/ngmix.py b/src/shapepipe/modules/ngmix_package/ngmix.py index 1e9d10797..9cd196743 100644 --- a/src/shapepipe/modules/ngmix_package/ngmix.py +++ b/src/shapepipe/modules/ngmix_package/ngmix.py @@ -30,6 +30,9 @@ ) from shapepipe.pipeline import file_io +# Flag bit set on an epoch's off-tile pixels (see :func:`split_tile_markers`). +OFF_TILE_FLAG = 2**10 + # Neighbour treatments selectable with the BLEND_HANDLING option. BLEND_HANDLINGS = ("noisefill", "uberseg") @@ -604,8 +607,9 @@ def __init__( self.weights = [] self.flags = [] # Neighbour masks, one per epoch: the pixels SExtractor marks -1e30 in - # the tile VIGNET (other detections' footprints), MegaCam-flipped to - # the epoch. noisefill zero-weights and noise-fills them; uberseg and + # the tile VIGNET on other detections' footprints (off-tile markers + # are flagged as defects instead; see split_tile_markers), + # MegaCam-flipped to the epoch. noisefill zero-weights and noise-fills them; uberseg and # the epoch cuts do not read them (see prepare_ngmix_weights). self.neighbours = [] self.bkg_rms = [] @@ -761,7 +765,7 @@ class Ngmix(object): blend_handling : {"noisefill", "uberseg"}, optional Neighbour treatment. ``"noisefill"`` (default) zero-weights and noise-fills the pixels SExtractor marks -1e30 in the tile VIGNET - (other detections' footprints); ``"uberseg"`` ignores those markers, + on other detections' footprints; ``"uberseg"`` ignores those markers, zeroes the weight of neighbour-side pixels from the coadd segmentation map and requires ``seg_cat_path``. Defect pixels are filled under both (see :func:`prepare_ngmix_weights`). @@ -1525,12 +1529,20 @@ def prepare_postage_stamps( @sc [decision:shape_measurement.central_defect_veto,decision:shape_measurement.epoch_masked_fraction_cut,decision:shape_measurement.blend_handling] neighbour-markers-are-not-defects SExtractor writes -1e30 into the tile VIGNET on the footprints of other - detections (and beyond the tile's edge). These markers form the epoch's neighbour mask - (``stamp.neighbours``, MegaCam-flipped like the epoch), kept apart from - its flag stamp, so neither the masked-fraction cut nor the central - veto counts them. Every epoch shares the tile VIGNET: counting a - neighbour within the veto radius as a defect would drop every epoch of - the object. + detections and beyond the tile's edge (:func:`split_tile_markers`). The + footprint markers form the epoch's neighbour mask (``stamp.neighbours``, + MegaCam-flipped like the epoch), kept apart from its flag stamp, so + neither the masked-fraction cut nor the central veto counts them. Every + epoch shares the tile VIGNET: counting a neighbour within the veto + radius as a defect would drop every epoch of the object. + + @sc [decision:shape_measurement.central_defect_veto,decision:shape_measurement.epoch_masked_fraction_cut,decision:shape_measurement.defect_fill] off-tile-pixels-are-defects + Beyond the tile's edge the epoch holds real data, the object's own light + cut off by the tile. Those pixels are flagged ``OFF_TILE_FLAG`` (2**10) + in the epoch's flag stamp and so join its defect set under every + ``blend_handling``: zero weight, the defect fill, and both epoch cuts. + On a 51-px stamp an object within about 8.5 px of the tile edge fails + the 1/3 cut. Parameters ---------- @@ -1634,12 +1646,10 @@ def prepare_postage_stamps( tile_seg = Ngmix.MegaCamFlip(tile_seg, int(ccd_n)) flag_vign = flag_obj[expccd_name]['VIGNET'] - # Neighbour markers (neighbour-markers-are-not-defects). - neighbour = ( - tile_vign == -1e30 - if tile_vign is not None - else np.zeros(np.shape(gal_vign), dtype=bool) - ) + # Off-tile pixels are defects (off-tile-pixels-are-defects); the + # other -1e30 markers are neighbours (neighbour-markers-are-not-defects). + neighbour, off_tile = split_tile_markers(tile_vign, np.shape(gal_vign)) + flag_vign[off_tile] = OFF_TILE_FLAG weight_vign = weight_obj[expccd_name]['VIGNET'] bkg_rms_vign = ( bkg_rms_obj[expccd_name]['VIGNET'] @@ -1714,6 +1724,38 @@ def prepare_postage_stamps( return stamp +def split_tile_markers(tile_vign, shape): + """Split the tile VIGNET's -1e30 markers into neighbour and off-tile. + + @sc [decision:shape_measurement.blend_handling,decision:shape_measurement.defect_fill] off-tile-is-whole-marked-rows-and-columns + SExtractor writes -1e30 on the footprints of other detections and on + stamp pixels beyond the tile's edge. A stamp clipped by the tile's + rectangle loses whole rows and whole columns, so the off-tile pixels are + the union of the stamp rows and columns that are entirely -1e30. The + remaining markers are neighbour pixels; a footprint touching the stamp + border stays a neighbour unless it fills a whole row or column. + + Parameters + ---------- + tile_vign : numpy.ndarray or None + Tile VIGNET stamp, oriented like the epoch; ``None`` marks nothing. + shape : tuple of int + Stamp shape, used when ``tile_vign`` is ``None``. + + Returns + ------- + numpy.ndarray of bool + Neighbour pixels. + numpy.ndarray of bool + Off-tile pixels. + """ + if tile_vign is None: + return np.zeros(shape, dtype=bool), np.zeros(shape, dtype=bool) + marker = tile_vign == -1e30 + off_tile = marker.all(axis=1)[:, None] | marker.all(axis=0)[None, :] + return marker & ~off_tile, off_tile + + def background_subtract(gal,bkg): """background subtraction. @@ -2007,7 +2049,8 @@ def defect_mask(weight, flag, bkg_rms=None): This one set is zero-weighted and filled by :func:`prepare_ngmix_weights` under every ``blend_handling`` and counted by the epoch cuts in :func:`prepare_postage_stamps`. SExtractor's neighbour markers are not - in it (neighbour-markers-are-not-defects). It is not ORed with its + in it (neighbour-markers-are-not-defects); off-tile pixels are, as flag + ``OFF_TILE_FLAG`` (off-tile-pixels-are-defects). It is not ORed with its rotations. For defects the central-defect veto keeps (:func:`has_central_defect`), the unsymmetrized fill leaves |c| <= 3e-4 per affected epoch on galaxies with half-light radius 0.3" @@ -2196,7 +2239,7 @@ def prepare_ngmix_weights( raw and weighted. @sc [decision:shape_measurement.blend_handling] uberseg-ignores-markers - Under ``"uberseg"`` the markers are ignored: :func:`uberseg_weight` + Under ``"uberseg"`` the neighbour markers are ignored: :func:`uberseg_weight` zeroes the weight of pixels nearer a neighbour's segmentation footprint than the target's and leaves their image values raw, because that light is real sky that metacal shears along with the target; filling it with diff --git a/src/shapepipe/modules/ngmix_runner.py b/src/shapepipe/modules/ngmix_runner.py index dc27b8afc..e217995c2 100644 --- a/src/shapepipe/modules/ngmix_runner.py +++ b/src/shapepipe/modules/ngmix_runner.py @@ -148,9 +148,9 @@ def ngmix_runner( # the pixels SExtractor marks -1e30 in the tile VIGNET (other detections' # footprints); "uberseg" ignores those markers, zeroes the weight of every # pixel closer to a neighbour than to the central object, from the - # segmentation map, and leaves its image raw. Defect pixels (flagged, zero-weight or - # invalid-RMS) are zero-weighted and filled under both (DEFECT_FILL - # below); see prepare_ngmix_weights. + # segmentation map, and leaves its image raw. Defect pixels (flagged, + # zero-weight, invalid-RMS or off-tile) are zero-weighted and filled + # under both (DEFECT_FILL below); see prepare_ngmix_weights. if config.has_option(module_config_sec, "BLEND_HANDLING"): blend_handling = config.get(module_config_sec, "BLEND_HANDLING") else: @@ -165,8 +165,8 @@ def ngmix_runner( dilate_neighbour = 1 # EPOCH_CENTRAL_DEFECT_RADIUS (optional, pixels): drop an epoch when a - # defect pixel (flagged, zero-weight or invalid-RMS; not a neighbour - # marker) lies closer than this to the stamp centre; 0 disables. + # defect pixel (flagged, zero-weight, invalid-RMS or off-tile; not a + # neighbour marker) lies closer than this to the stamp centre; 0 disables. if config.has_option(module_config_sec, "EPOCH_CENTRAL_DEFECT_RADIUS"): epoch_central_defect_radius = config.getfloat( module_config_sec, "EPOCH_CENTRAL_DEFECT_RADIUS" @@ -175,8 +175,8 @@ def ngmix_runner( epoch_central_defect_radius = EPOCH_CENTRAL_DEFECT_RADIUS # EPOCH_MASKED_FRACTION_CUT (optional): drop an epoch when more than this - # fraction of its stamp is defects (flagged, zero-weight or invalid-RMS; - # neighbour markers are not counted). + # fraction of its stamp is defects (flagged, zero-weight, invalid-RMS or + # off-tile; neighbour markers are not counted). if config.has_option(module_config_sec, "EPOCH_MASKED_FRACTION_CUT"): epoch_masked_fraction_cut = config.getfloat( module_config_sec, "EPOCH_MASKED_FRACTION_CUT" diff --git a/tests/module/test_ngmix_defect_fill.py b/tests/module/test_ngmix_defect_fill.py index b9e3096c5..9c15d1807 100644 --- a/tests/module/test_ngmix_defect_fill.py +++ b/tests/module/test_ngmix_defect_fill.py @@ -786,7 +786,7 @@ def _expected_neighbours(tile, name): def test_neighbour_markers_near_the_centre_keep_every_epoch(): - """A neighbour footprint 3 px from the centre, and one covering 45% of + """A neighbour footprint 3 px from the centre, and one covering 41% of the stamp, drop no epoch: the masked-fraction cut and the central veto count no marker. @@ -796,8 +796,10 @@ def test_neighbour_markers_near_the_centre_keep_every_epoch(): (neighbour-markers-are-not-defects). """ small = _tile_with_neighbour() - large = _tile_with_neighbour(rows=(0, N_STAMP)) - large[:, _CENTRE + 3:] = _MARKER + # Large, but short of the stamp border: no row or column is entirely + # marked, so it is a neighbour, not off-tile. + large = _tile_with_neighbour() + large[1:-1, _CENTRE + 3:-1] = _MARKER assert (large == _MARKER).mean() > 1 / 3 for tile in (small, large): stamp, _, _ = _marker_stamp(tile) @@ -939,6 +941,11 @@ def test_noisefill_matches_develop_on_marked_neighbours( """For an epoch with a marked neighbour, noisefill returns the image, weight and noise image develop returned, bit for bit: with no defect, and with a flagged column and a dead pixel under the default noise fill. + The stamps carry no off-tile pixels, and the equivalence is claimed for + such stamps only. Off-tile pixels reach this function as flag 2**10 + defects (off-tile-pixels-are-defects), as every marker did on develop; + which epochs survive the cuts differs from develop wherever there are + neighbour markers. Failure mode: carrying the markers apart from the flags changes what noisefill does to neighbour pixels, their weights, the noise level or @@ -1002,3 +1009,111 @@ def fake_observation(*args, **kwargs): ) for got, want in zip(seen, stamp.neighbours): assert got is want + + +# --- Off-tile pixels are defects ------------------------------------------- +# +# SExtractor also writes -1e30 beyond the tile's edge, where the epoch holds +# the object's own light, cut off. Those pixels are the stamp rows and +# columns that are entirely -1e30 (the off-image part of a rectangle clip); +# they join the epoch's defect set as flag 2**10. The other markers are the +# neighbour mask. + +_OFF_TILE = 2**10 + + +def _off_tile_expected(tile, name): + flipped = Ngmix.MegaCamFlip(tile, int(name.split("-")[1])) == _MARKER + return flipped.all(axis=1)[:, None] | flipped.all(axis=0)[None, :] + + +def test_object_three_px_from_the_tile_edge_is_dropped(): + """With the tile edge 3 px from the object, every epoch is dropped: the + off-tile band counts toward the epoch cuts. + + Failure mode: off-tile pixels are treated as neighbour markers, so an + edge object is measured with a noise-filled band through its own light + (off-tile-pixels-are-defects). + """ + tile = np.random.default_rng(5).normal(0.0, 1.0, (N_STAMP, N_STAMP)) + tile[:, :_CENTRE - 2] = _MARKER + stamp, _, _ = _marker_stamp(tile) + assert len(stamp.gals) == 0 + assert stamp.epoch_cuts["considered"] == len(_MARKER_EPOCH_NAMES) + assert ( + stamp.epoch_cuts["masked_fraction"] + stamp.epoch_cuts["central_veto"] + == len(_MARKER_EPOCH_NAMES) + ) + + +def test_the_central_veto_sees_off_tile_pixels(): + """An off-tile band 12 px from the object (27% of the stamp) passes the + default cuts and is vetoed at radius 13. + + Failure mode: the central veto does not read the off-tile set. + """ + tile = np.random.default_rng(5).normal(0.0, 1.0, (N_STAMP, N_STAMP)) + tile[:, :_CENTRE - 11] = _MARKER + kept, _, _ = _marker_stamp(tile) + assert len(kept.gals) == len(_MARKER_EPOCH_NAMES) + vetoed, _, _ = _marker_stamp(tile, epoch_central_defect_radius=13) + assert len(vetoed.gals) == 0 + assert vetoed.epoch_cuts["central_veto"] == len(_MARKER_EPOCH_NAMES) + + +def test_corner_off_tile_region_and_border_neighbour_are_classified(): + """At a tile corner, the L-shaped off-tile region is flagged 2**10 + exactly, and a neighbour footprint touching the stamp border without + filling a row or column stays in the neighbour mask. + + Failure modes: off-tile pixels are classified by something other than + whole marked rows and columns (the L is missed or a border-touching + neighbour is swallowed); the classification ignores the MegaCam flip. + """ + tile = np.random.default_rng(5).normal(0.0, 1.0, (N_STAMP, N_STAMP)) + tile[:5, :] = _MARKER + tile[:, -5:] = _MARKER + tile[40:, :6] = _MARKER # neighbour on the bottom-left border + stamp, epochs, _ = _marker_stamp(tile) + names = {id(v[0]): name for name, v in epochs.items()} + assert len(stamp.gals) == len(_MARKER_EPOCH_NAMES) + for flag, neighbour in zip(stamp.flags, stamp.neighbours): + name = names[id(flag)] + off_tile = _off_tile_expected(tile, name) + assert off_tile.sum() == 5 * N_STAMP * 2 - 25 + npt.assert_array_equal(flag == _OFF_TILE, off_tile) + npt.assert_array_equal( + neighbour, _expected_neighbours(tile, name) & ~off_tile + ) + assert neighbour.sum() == 11 * 6 + + +@pytest.mark.parametrize("blend_handling", ["noisefill", "uberseg"]) +def test_off_tile_pixels_are_zero_weighted_and_filled(blend_handling): + """Off-tile pixels are zero-weighted and noise-filled under either + BLEND_HANDLING; under uberseg the neighbour markers stay raw. + + Failure mode: under uberseg the off-tile band keeps its weight, or the + fill differs between blend handlings. + """ + tile = np.random.default_rng(5).normal(0.0, 1.0, (N_STAMP, N_STAMP)) + tile[:5, :] = _MARKER + tile[20:24, 35:40] = _MARKER + stamp, _, _ = _marker_stamp(tile) + seg = np.zeros((N_STAMP, N_STAMP), dtype=np.int32) + seg[_CENTRE - 1:_CENTRE + 2, _CENTRE - 1:_CENTRE + 2] = 1 + kwargs = ( + dict(seg=seg, object_number=1) if blend_handling == "uberseg" else {} + ) + for i in range(len(stamp.gals)): + gal = 1.0e3 + stamp.gals[i] + gal_out, w_out, _ = _stamp_epoch_weights( + stamp, i, seed=i, blend_handling=blend_handling, **kwargs, + ) + off_tile = stamp.flags[i] == _OFF_TILE + assert off_tile.sum() == 5 * N_STAMP + removed = off_tile | ( + stamp.neighbours[i] if blend_handling == "noisefill" else False + ) + npt.assert_array_equal(gal_out != gal, removed) + npt.assert_array_equal(w_out == 0.0, removed) From f0654053f5e300d9ea6293afb2b5022c0895b144 Mon Sep 17 00:00:00 2001 From: Cail Daley Date: Tue, 29 Sep 2026 16:47:26 +0200 Subject: [PATCH 11/30] fix(ngmix): interpolate defects from the pixels the image keeps Under DEFECT_FILL = interpolate and BLEND_HANDLING = noisefill, the Clough-Tocher support included marked neighbour pixels that noisefill replaces with noise, so a defect beside a bright neighbour was filled with light the image no longer contains. The support now excludes ~clean (the defects and the removed neighbour pixels); the NaN fallback to the noise fill covers degenerate supports. Under uberseg the neighbour light stays in the image and the support is unchanged. interpolate_defects names its second argument `excluded`. Test: a flagged pixel next to a 1e4 marked neighbour interpolates to the kept-pixel interpolant (1.4, against 3102 with the neighbour in the support) under noisefill, and from the raw neighbour light under uberseg. Co-Authored-By: Claude Opus 5.5 --- .../ngmix_package/defect_interpolation.py | 24 ++++----- src/shapepipe/modules/ngmix_package/ngmix.py | 18 +++++-- tests/module/test_ngmix_defect_fill.py | 52 +++++++++++++++++++ 3 files changed, 78 insertions(+), 16 deletions(-) diff --git a/src/shapepipe/modules/ngmix_package/defect_interpolation.py b/src/shapepipe/modules/ngmix_package/defect_interpolation.py index 713e326f7..ae0d53c41 100644 --- a/src/shapepipe/modules/ngmix_package/defect_interpolation.py +++ b/src/shapepipe/modules/ngmix_package/defect_interpolation.py @@ -94,14 +94,14 @@ def fourfold(mask): return mask | np.rot90(mask) | np.rot90(mask, 2) | np.rot90(mask, 3) -def _interpolate_once(planes, defect, target): +def _interpolate_once(planes, excluded, target): """One Clough-Tocher interpolant of every plane at ``target``; NaN elsewhere and where the support cannot reach.""" out = np.full(planes.shape, np.nan) support = binary_dilation( target, structure=np.ones((3, 3), dtype=bool), iterations=SUPPORT_RADIUS, - ) & ~defect + ) & ~excluded points = np.argwhere(support).astype(float) if len(points) < 3: return out @@ -116,14 +116,14 @@ def _interpolate_once(planes, defect, target): return out -def interpolate_defects(planes, defect, target): +def interpolate_defects(planes, excluded, target): """Replace the ``target`` pixels of every plane by a Clough-Tocher - interpolant of the clean pixels around them. + interpolant of the kept pixels around them. @sc [decision:shape_measurement.defect_fill] shared-rotation-averaged-interpolant - The support is the clean pixels within ``SUPPORT_RADIUS`` (4 px) of the - target; no defect pixel enters it, so defect values are never read. For - each quarter turn of the stamp, one Delaunay triangulation of the support + The support is the pixels within ``SUPPORT_RADIUS`` (4 px) of the target + outside ``excluded``; no excluded pixel enters it, so their values are + never read. For each quarter turn of the stamp, one Delaunay triangulation of the support serves every plane, so the science image and the metacal noise image see the same linear operator and fixnoise mirrors the science image's interpolated noise. A regular grid's triangulation has degenerate @@ -136,9 +136,9 @@ def interpolate_defects(planes, defect, target): ---------- planes : array_like Stamp planes, shape ``(n, ny, nx)``. - defect : numpy.ndarray of bool - Every defect pixel, shape ``(ny, nx)``; none of them supports the - interpolant. + excluded : numpy.ndarray of bool + Pixels that never support the interpolant, shape ``(ny, nx)``: every + defect, and any pixel whose light the image does not keep. target : numpy.ndarray of bool Defect pixels to interpolate (:func:`interpolable_defects`). @@ -149,7 +149,7 @@ def interpolate_defects(planes, defect, target): target pixel whose support is degenerate in some orientation. """ planes = np.asarray(planes, dtype=float) - defect = np.asarray(defect, dtype=bool) + excluded = np.asarray(excluded, dtype=bool) target = np.asarray(target, dtype=bool) out = planes.copy() if not target.any(): @@ -157,7 +157,7 @@ def interpolate_defects(planes, defect, target): turns = [ np.rot90( _interpolate_once( - np.rot90(planes, k, axes=(1, 2)), np.rot90(defect, k), + np.rot90(planes, k, axes=(1, 2)), np.rot90(excluded, k), np.rot90(target, k), ), -k, axes=(1, 2), diff --git a/src/shapepipe/modules/ngmix_package/ngmix.py b/src/shapepipe/modules/ngmix_package/ngmix.py index 9cd196743..bf9bf0b12 100644 --- a/src/shapepipe/modules/ngmix_package/ngmix.py +++ b/src/shapepipe/modules/ngmix_package/ngmix.py @@ -2228,6 +2228,15 @@ def prepare_ngmix_weights( ``blend_handling`` is, by the same operator. ``blend_handling`` acts on the neighbour pixels only, and the noise image covers the whole stamp. + @sc [decision:shape_measurement.defect_fill,decision:shape_measurement.blend_handling] interpolation-support-is-the-kept-image + Under ``defect_fill="interpolate"`` the interpolant is supported on the + pixels whose light the image keeps: never a defect, and under + ``"noisefill"`` never a marked neighbour pixel, whose light is replaced + by noise; a Clough-Tocher interpolant next to a bright neighbour would + otherwise build the defect's value from light the image no longer + contains. Under ``"uberseg"`` the neighbour light stays in the image and + supports the interpolant. + @sc [decision:shape_measurement.blend_handling] noisefill-fills-markers Under ``"noisefill"`` the pixels of ``neighbour``, SExtractor's -1e30 neighbour markers, get weight 0 and are replaced by the same noise @@ -2390,10 +2399,11 @@ def prepare_ngmix_weights( noise_img_gal = rng.standard_normal(gal.shape) * sig_noise gal_filled = fill_defects(gal, ~clean, noise_img_gal) if interpolated.any(): - # One operator for the image and the noise image; a pixel whose - # support is degenerate, or that noisefill removes as a neighbour, - # keeps its noise fill. - filled = interpolate_defects([gal, noise_img], defect, interpolated) + # One operator for the image and the noise image, supported on the + # pixels the image keeps (interpolation-support-is-the-kept-image); a + # pixel whose support is degenerate, or that noisefill removes as a + # neighbour, keeps its noise fill. + filled = interpolate_defects([gal, noise_img], ~clean, interpolated) done = ( interpolated & ~removed_neighbour diff --git a/tests/module/test_ngmix_defect_fill.py b/tests/module/test_ngmix_defect_fill.py index 9c15d1807..67aff0d20 100644 --- a/tests/module/test_ngmix_defect_fill.py +++ b/tests/module/test_ngmix_defect_fill.py @@ -1117,3 +1117,55 @@ def test_off_tile_pixels_are_zero_weighted_and_filled(blend_handling): ) npt.assert_array_equal(gal_out != gal, removed) npt.assert_array_equal(w_out == 0.0, removed) + + +# --- DEFECT_FILL = interpolate beside a removed neighbour ------------------- + +def _defect_beside_neighbour(): + """A single flagged pixel 6 px right of the centre, with a bright marked + neighbour footprint starting on the next column.""" + n = 31 + c = n // 2 + gal = np.random.default_rng(3).normal(0.0, 1.0, (n, n)) + flag = np.zeros((n, n), dtype=np.int32) + flag[c, c + 6] = 1 + neighbour = np.zeros((n, n), dtype=bool) + neighbour[c - 2:c + 3, c + 7:c + 10] = True + gal[neighbour] += 1.0e4 + seg = np.zeros((n, n), dtype=np.int32) + seg[c - 1:c + 2, c - 1:c + 2] = 1 + return gal, flag, neighbour, seg, (c, c + 6) + + +def test_noisefill_interpolation_does_not_read_removed_neighbour_light(): + """Under noisefill, the interpolant of a defect beside a marked neighbour + is built from the pixels the image keeps: the neighbour's light, which + noisefill removes, is not in its support. Under uberseg the neighbour + light is raw in the image and supports the interpolant. + + Failure mode: the support includes the removed neighbour pixels, so the + defect is filled with light the image no longer contains + (interpolation-support-is-the-kept-image). + """ + gal, flag, neighbour, seg, pix = _defect_beside_neighbour() + weight = np.ones_like(gal) + target = np.zeros_like(neighbour) + target[pix] = True + + out, _, _ = prepare_ngmix_weights( + gal, weight, flag, np.random.RandomState(0), + blend_handling="noisefill", neighbour=neighbour, + defect_fill="interpolate", + ) + expected = interpolate_defects(gal[None], (flag != 0) | neighbour, target) + assert out[pix] == pytest.approx(expected[0][pix]) + assert abs(out[pix]) < 100.0 + + out, _, _ = prepare_ngmix_weights( + gal, weight, flag, np.random.RandomState(0), + blend_handling="uberseg", seg=seg, object_number=1, + neighbour=neighbour, defect_fill="interpolate", + ) + expected = interpolate_defects(gal[None], flag != 0, target) + assert out[pix] == pytest.approx(expected[0][pix]) + assert out[pix] > 1000.0 From b27eca849426564bee89ce814bd788db728807e0 Mon Sep 17 00:00:00 2001 From: Cail Daley Date: Tue, 29 Sep 2026 16:48:52 +0200 Subject: [PATCH 12/30] fix(ngmix): the defect fill keeps the stamp's dtype fill_defects used np.where, which promotes a float32 stamp to float64; develop's noisefill kept the galaxy stamp's dtype. The filled image is cast back to the input dtype, so noisefill stays bit-identical to develop on float32 stamps. The develop-equivalence test gains a float32 case and checks dtypes. Co-Authored-By: Claude Opus 5.5 --- src/shapepipe/modules/ngmix_package/ngmix.py | 5 +++-- tests/module/test_ngmix_defect_fill.py | 9 +++++++-- 2 files changed, 10 insertions(+), 4 deletions(-) diff --git a/src/shapepipe/modules/ngmix_package/ngmix.py b/src/shapepipe/modules/ngmix_package/ngmix.py index bf9bf0b12..8f0ef66e1 100644 --- a/src/shapepipe/modules/ngmix_package/ngmix.py +++ b/src/shapepipe/modules/ngmix_package/ngmix.py @@ -2204,9 +2204,10 @@ def fill_defects(image, defect, noise): Returns ------- numpy.ndarray - ``image`` with ``defect`` pixels taken from ``noise``. + ``image`` with ``defect`` pixels taken from ``noise``, in the dtype + of ``image``. """ - return np.where(defect, noise, image) + return np.where(defect, noise, image).astype(image.dtype, copy=False) def prepare_ngmix_weights( diff --git a/tests/module/test_ngmix_defect_fill.py b/tests/module/test_ngmix_defect_fill.py index 67aff0d20..17f04a2a8 100644 --- a/tests/module/test_ngmix_defect_fill.py +++ b/tests/module/test_ngmix_defect_fill.py @@ -934,13 +934,16 @@ def _develop_noisefill(gal, weight, flag, rng, bkg_rms=None): with_rms=st.booleans(), rms_scale=st.floats(0.5, 2.0), with_defects=st.booleans(), + dtype=st.sampled_from([np.float64, np.float32]), ) def test_noisefill_matches_develop_on_marked_neighbours( - seed, with_rms, rms_scale, with_defects, + seed, with_rms, rms_scale, with_defects, dtype, ): """For an epoch with a marked neighbour, noisefill returns the image, weight and noise image develop returned, bit for bit: with no defect, - and with a flagged column and a dead pixel under the default noise fill. + and with a flagged column and a dead pixel under the default noise fill, + for float64 and float32 stamps (the filled image keeps the stamp's + dtype). The stamps carry no off-tile pixels, and the equivalence is claimed for such stamps only. Off-tile pixels reach this function as flag 2**10 defects (off-tile-pixels-are-defects), as every marker did on develop; @@ -963,6 +966,7 @@ def test_noisefill_matches_develop_on_marked_neighbours( if with_defects: flag[:, 2] = 1 weight[n - 3, n // 2] = 0.0 + gal = gal.astype(dtype) bkg_rms = ( rms_scale * (1.0 + 0.1 * rng.random((n, n))) if with_rms else None ) @@ -976,6 +980,7 @@ def test_noisefill_matches_develop_on_marked_neighbours( np.random.RandomState(seed), bkg_rms=bkg_rms, ) for a, b in zip(new, old): + assert a.dtype == b.dtype npt.assert_array_equal(a, b) From a5bfef10ca0fb905d1aa5658295208d57f573ff9 Mon Sep 17 00:00:00 2001 From: Cail Daley Date: Wed, 30 Sep 2026 02:01:47 +0200 Subject: [PATCH 13/30] fix(ngmix): off-tile pixels are the marked rows and columns at the stamp border Every entirely -1e30 stamp row or column counted as off-tile. In an edge stamp, a neighbour footprint that covers the on-image part of a row (a wide DR6 footprint running from the tile edge) completes that row, so the neighbour's pixels became OFF_TILE_FLAG defects: counted by the epoch cuts, interpolated under DEFECT_FILL = interpolate, filled under uberseg. Only the runs of marked rows and columns that start at a stamp border are off-tile now. Tests build tile VIGNETs with the catalogue-mode converter: on the real 202.301 DR6 patch every stamp splits exactly into off-image pixels and seg-map neighbour pixels, and the edge case above keeps its footprint in the neighbour mask through prepare_postage_stamps. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01BbT81hykcvtuefzB3FhYvY --- astra.yaml | 17 ++-- src/shapepipe/modules/ngmix_package/ngmix.py | 30 +++++-- tests/module/test_ngmix_defect_fill.py | 91 ++++++++++++++++++++ 3 files changed, 123 insertions(+), 15 deletions(-) diff --git a/astra.yaml b/astra.yaml index 8eea6bc4c..f83377c80 100644 --- a/astra.yaml +++ b/astra.yaml @@ -1386,13 +1386,16 @@ analyses: FFTs". Under every BLEND_HANDLING and the default DEFECT_FILL = noise, defect pixels are replaced with independent noise at the per-pixel background RMS when supplied, or the stamp's robust noise - scale otherwise. SExtractor writes -1e30 into the tile VIGNET on - other detections' footprints and beyond the tile's edge. The stamp - rows and columns that are entirely -1e30 (the off-image part of a - rectangle clip) are off-tile: the epoch holds the object's own light - there, so they are defects, flagged 2**10. The other markers are - neighbour pixels, not defects; blend_handling decides their - treatment. The committed fill + scale otherwise. The tile VIGNET holds -1e30 on other detections' + footprints and beyond the tile's edge (written by SExtractor, or by + the unions_catalogue converter from the catalogue's segmentation + map; catalogue_neighbour_marking). The runs of entirely -1e30 stamp + rows and columns that start at a stamp border (the off-image part of + a rectangle clip) are off-tile: the epoch holds the object's own + light there, so they are defects, flagged 2**10. The other markers, + including a neighbour footprint that completes an interior row + beside an off-tile band, are neighbour pixels, not defects; + blend_handling decides their treatment. The committed fill uses the unsymmetrized defect set: DES symmetrized its masks, but four-fold symmetrization quadruples m and still leaves an additive c1 (symmetrized_4fold_noise). The cost of not symmetrizing, a hole in diff --git a/src/shapepipe/modules/ngmix_package/ngmix.py b/src/shapepipe/modules/ngmix_package/ngmix.py index 2f172b4a9..4908fcd20 100644 --- a/src/shapepipe/modules/ngmix_package/ngmix.py +++ b/src/shapepipe/modules/ngmix_package/ngmix.py @@ -1754,13 +1754,17 @@ def prepare_postage_stamps( def split_tile_markers(tile_vign, shape): """Split the tile VIGNET's -1e30 markers into neighbour and off-tile. - @sc [decision:shape_measurement.blend_handling,decision:shape_measurement.defect_fill] off-tile-is-whole-marked-rows-and-columns - SExtractor writes -1e30 on the footprints of other detections and on - stamp pixels beyond the tile's edge. A stamp clipped by the tile's - rectangle loses whole rows and whole columns, so the off-tile pixels are - the union of the stamp rows and columns that are entirely -1e30. The - remaining markers are neighbour pixels; a footprint touching the stamp - border stays a neighbour unless it fills a whole row or column. + @sc [decision:shape_measurement.blend_handling,decision:shape_measurement.defect_fill] off-tile-is-marked-border-rows-and-columns + The tile VIGNET holds -1e30 on the footprints of other detections and on + stamp pixels beyond the tile's edge; SExtractor writes it, and in + catalogue mode the converter paints it from the catalogue's segmentation + map (``read_ext_sexcat._extract_vignets``). A stamp clipped by the tile's + rectangle loses whole rows and whole columns from its border, so the + off-tile pixels are the union of the runs of entirely -1e30 rows and + columns that start at a stamp border. The remaining markers are + neighbour pixels: a footprint touching the stamp border, and a footprint + that completes an interior row or column beside an off-tile band, stay + neighbours. Parameters ---------- @@ -1779,7 +1783,17 @@ def split_tile_markers(tile_vign, shape): if tile_vign is None: return np.zeros(shape, dtype=bool), np.zeros(shape, dtype=bool) marker = tile_vign == -1e30 - off_tile = marker.all(axis=1)[:, None] | marker.all(axis=0)[None, :] + + def border_runs(full): + # Lines in the unbroken run of marked lines from either border. + lead = np.logical_and.accumulate(full) + trail = np.logical_and.accumulate(full[::-1])[::-1] + return lead | trail + + off_tile = ( + border_runs(marker.all(axis=1))[:, None] + | border_runs(marker.all(axis=0))[None, :] + ) return marker & ~off_tile, off_tile diff --git a/tests/module/test_ngmix_defect_fill.py b/tests/module/test_ngmix_defect_fill.py index 17f04a2a8..8919453c2 100644 --- a/tests/module/test_ngmix_defect_fill.py +++ b/tests/module/test_ngmix_defect_fill.py @@ -16,6 +16,7 @@ """ import re +from pathlib import Path from types import SimpleNamespace import numpy as np @@ -41,8 +42,10 @@ make_ngmix_observation, prepare_ngmix_weights, prepare_postage_stamps, + split_tile_markers, uberseg_weight, ) +from shapepipe.modules.read_ext_sexcat_package import read_ext_sexcat # --- prepare_ngmix_weights: the filled set is the defect set --------------- @@ -1124,6 +1127,94 @@ def test_off_tile_pixels_are_zero_weighted_and_filled(blend_handling): npt.assert_array_equal(w_out == 0.0, removed) +# --- Catalogue mode: converter-built tile VIGNETs --------------------------- +# +# In catalogue mode the tile VIGNET comes from read_ext_sexcat, which paints +# -1e30 off the image and on other footprints of the DR6 segmentation map. +# The off-image pixels are the off-tile defects and the footprint pixels are +# the neighbour mask, exactly. + +DR6_PATCH = Path(__file__).parent / "data" / "dr6_202.301_seg_patch.fits" + + +def _converter_stamps(seg, number, x, y): + """Converter tile VIGNETs on a unit image, and each stamp's off-image + mask.""" + relabelled, _ = read_ext_sexcat.relabel_seg(seg, number, x, y) + vignets = read_ext_sexcat._extract_vignets( + np.ones(seg.shape, np.float32), x, y, N_STAMP, seg=relabelled, + number=number, + ) + half = N_STAMP // 2 + ny, nx = seg.shape + off_image = [] + for xi, yi in zip(x, y): + rows = int(np.rint(yi)) - 1 - half + np.arange(N_STAMP) + cols = int(np.rint(xi)) - 1 - half + np.arange(N_STAMP) + off_image.append( + ((rows < 0) | (rows >= ny))[:, None] + | ((cols < 0) | (cols >= nx))[None, :] + ) + return vignets, off_image + + +def test_dr6_converter_stamps_split_into_off_image_and_neighbours(): + """On the real 202.301 patch, every converter stamp splits into its + off-image pixels (off-tile) and its other -1e30 pixels (neighbours). + + Failure modes: the converter's float32 -1e30 is not recognised as a + marker; off-image pixels of an edge stamp land in the neighbour mask; + neighbour-footprint pixels become off-tile defects. + """ + with fits.open(DR6_PATCH) as hdul: + seg = hdul["SEG"].data + objects = hdul["OBJECTS"].data + number = np.array(objects["NUMBER"]) + x, y = np.array(objects["X_IMAGE"]), np.array(objects["Y_IMAGE"]) + vignets, off_image = _converter_stamps(seg, number, x, y) + assert vignets.dtype == np.float32 + n_edge = 0 + for vign, off in zip(vignets, off_image): + neighbour, off_tile = split_tile_markers(vign, vign.shape) + npt.assert_array_equal(off_tile, off) + npt.assert_array_equal(neighbour, (vign == _MARKER) & ~off) + n_edge += off.any() + assert n_edge >= 5 + assert sum( + split_tile_markers(v, v.shape)[0].sum() for v in vignets + ) > 1000 + + +def test_a_neighbour_completing_rows_beside_the_tile_edge_stays_a_neighbour(): + """An object 14 px from the tile's left edge, with a wide neighbour + footprint that runs from the tile edge across the stamp: in the rows of + that footprint every stamp pixel is -1e30, off the image or on the + neighbour. Only the off-image columns are off-tile; the footprint is the + neighbour mask, through prepare_postage_stamps. + + Failure mode: every entirely -1e30 row counts as off-tile, so the + neighbour's rows become defects that the epoch cuts count and the defect + fill interpolates (off-tile-is-marked-border-rows-and-columns). + """ + seg = np.zeros((80, 80), np.int32) + seg[20:24, 0:45] = 5 + seg[27:32, 13:18] = 1 + number, x, y = np.array([1, 5]), np.array([15.0, 30.0]), np.array([30.0, 22.0]) + vignets, off_image = _converter_stamps(seg, number, x, y) + tile, off = vignets[0], off_image[0] + footprint = (tile == _MARKER) & ~off + assert footprint.sum() == 4 * (N_STAMP - 11) + assert ((tile == _MARKER).all(axis=1) & ~off.all(axis=1)).sum() == 4 + + stamp, epochs, _ = _marker_stamp(tile) + names = {id(v[0]): name for name, v in epochs.items()} + assert len(stamp.flags) == len(_MARKER_EPOCH_NAMES) + for flag, neighbour in zip(stamp.flags, stamp.neighbours): + ccd = int(names[id(flag)].split("-")[1]) + npt.assert_array_equal(flag == _OFF_TILE, Ngmix.MegaCamFlip(off, ccd)) + npt.assert_array_equal(neighbour, Ngmix.MegaCamFlip(footprint, ccd)) + + # --- DEFECT_FILL = interpolate beside a removed neighbour ------------------- def _defect_beside_neighbour(): From 633425d94973521b1f1f7627cc69116b2e23fcc2 Mon Sep 17 00:00:00 2001 From: Cail Daley Date: Wed, 30 Sep 2026 02:02:35 +0200 Subject: [PATCH 14/30] docs: -1e30 tile-VIGNET markers come from SExtractor or the catalogue converter The neighbour-marking rationale and the converter's BIG comment describe the markers as ngmix neighbours under flag 2**10; ngmix splits them into off-tile defects and a neighbour mask. The masking text names the tile VIGNET, not SExtractor, as the markers' carrier, since catalogue mode paints them from the DR6 segmentation map. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01BbT81hykcvtuefzB3FhYvY --- astra.yaml | 22 ++++++++++--------- src/shapepipe/modules/ngmix_package/ngmix.py | 18 +++++++-------- src/shapepipe/modules/ngmix_runner.py | 2 +- .../read_ext_sexcat.py | 5 +++-- tests/module/test_ngmix_defect_fill.py | 11 +++++----- 5 files changed, 31 insertions(+), 27 deletions(-) diff --git a/astra.yaml b/astra.yaml index f83377c80..bf019aa51 100644 --- a/astra.yaml +++ b/astra.yaml @@ -637,9 +637,10 @@ analyses: catalogue_neighbour_marking: label: Neighbour pixels in the catalogue path's VIGNET rationale: >- - ngmix masks a neighbour only where the tile VIGNET is -1e30 (flag - 2**10: zero weight, noise-filled under noisefill), and SExtractor - writes -1e30 on neighbours' footprints and off the image. The + ngmix's noisefill masks a neighbour only where the tile VIGNET is + -1e30 (zero weight, noise-filled; the off-tile rows and columns are + defects instead), and SExtractor writes -1e30 on neighbours' + footprints and off the image. The unions_catalogue converter reproduces that from the catalogue's own r-band segmentation map (CFIS..r.seg.fits.fz, on the tile's pixel grid), so both tile_detection options mask the same way. @@ -1457,9 +1458,10 @@ analyses: depend on it: defects are filled the same way under every BLEND_HANDLING, and the epoch cuts never count neighbour pixels. noisefill (the default; the committed config sets no key) gives - weight 0 to the pixels SExtractor marks -1e30 in the tile VIGNET - on other detections' footprints (about 93% of neighbour-footprint - pixels on a SExtractor-mode sims tile), excluding the off-tile + weight 0 to the pixels marked -1e30 in the tile VIGNET on other + detections' footprints (about 93% of neighbour-footprint pixels on + a SExtractor-mode sims tile; every one in catalogue mode, from the + segmentation map), excluding the off-tile rows and columns, which are defects, and replaces them with noise; unmarked neighbour pixels keep their weight and light. uberseg ignores the @@ -1487,7 +1489,7 @@ analyses: default: noisefill options: noisefill: - label: Noise-fill SExtractor-marked neighbour pixels (BLEND_HANDLING = noisefill) + label: Noise-fill the marked neighbour pixels (BLEND_HANDLING = noisefill) description: >- Pixels marked -1e30 in the tile VIGNET get weight 0 and noise. The fill stops at the marked footprint, so unmarked neighbour @@ -1527,7 +1529,7 @@ analyses: An epoch is dropped when a defect pixel lies strictly closer to the stamp centre than its fill's radius, beside the masked-fraction cut in the epoch loop; EPOCH_CENTRAL_DEFECT_RADIUS = 0 disables it. - SExtractor's -1e30 neighbour markers are not defects: all epochs + The tile VIGNET's -1e30 neighbour markers are not defects: all epochs share the tile VIGNET, so a neighbour inside the radius would drop every epoch. Off-tile pixels are defects, so an object near the tile edge is vetoed. The veto reads only the defect mask, so it selects on nothing shear-responsive; for the same @@ -1566,8 +1568,8 @@ analyses: (default 1/3) of its pixels in the raw, unsymmetrized defect set is dropped from the multi-epoch fit: flagged pixels (any nonzero exposure flag bit), zero-weight pixels, invalid-RMS pixels and - off-tile pixels (whole -1e30 rows and columns of the tile VIGNET, - flag 2**10), the set defect_fill fills. On a 51-px stamp an object + off-tile pixels (whole -1e30 rows and columns at the tile VIGNET's + border, flag 2**10), the set defect_fill fills. On a 51-px stamp an object within about 8.5 px of the tile edge fails the 1/3 cut. The other -1e30 markers, neighbour footprints, are not counted: every epoch shares the tile VIGNET, so a large neighbour would drop them all. An object with no surviving epoch has no diff --git a/src/shapepipe/modules/ngmix_package/ngmix.py b/src/shapepipe/modules/ngmix_package/ngmix.py index 4908fcd20..471e62b2c 100644 --- a/src/shapepipe/modules/ngmix_package/ngmix.py +++ b/src/shapepipe/modules/ngmix_package/ngmix.py @@ -633,8 +633,8 @@ def __init__( self.psfs = [] self.weights = [] self.flags = [] - # Neighbour masks, one per epoch: the pixels SExtractor marks -1e30 in - # the tile VIGNET on other detections' footprints (off-tile markers + # Neighbour masks, one per epoch: the pixels marked -1e30 in the tile + # VIGNET on other detections' footprints (off-tile markers # are flagged as defects instead; see split_tile_markers), # MegaCam-flipped to the epoch. noisefill zero-weights and noise-fills them; uberseg and # the epoch cuts do not read them (see prepare_ngmix_weights). @@ -791,8 +791,8 @@ class Ngmix(object): :func:`make_ngmix_observation`. blend_handling : {"noisefill", "uberseg"}, optional Neighbour treatment. ``"noisefill"`` (default) zero-weights and - noise-fills the pixels SExtractor marks -1e30 in the tile VIGNET - on other detections' footprints; ``"uberseg"`` ignores those markers, + noise-fills the pixels marked -1e30 in the tile VIGNET on other + detections' footprints; ``"uberseg"`` ignores those markers, zeroes the weight of neighbour-side pixels from the coadd segmentation map and requires ``seg_cat_path``. Defect pixels are filled under both (see :func:`prepare_ngmix_weights`). @@ -1555,8 +1555,8 @@ def prepare_postage_stamps( 1/3; 10%, the DES Y3 and Y6 value, is the alternative to test. @sc [decision:shape_measurement.central_defect_veto,decision:shape_measurement.epoch_masked_fraction_cut,decision:shape_measurement.blend_handling] neighbour-markers-are-not-defects - SExtractor writes -1e30 into the tile VIGNET on the footprints of other - detections and beyond the tile's edge (:func:`split_tile_markers`). The + The tile VIGNET holds -1e30 on the footprints of other detections and + beyond the tile's edge (:func:`split_tile_markers`). The footprint markers form the epoch's neighbour mask (``stamp.neighbours``, MegaCam-flipped like the epoch), kept apart from its flag stamp, so neither the masked-fraction cut nor the central veto counts them. Every @@ -2089,7 +2089,7 @@ def defect_mask(weight, flag, bkg_rms=None): or (when a background RMS map is given) a non-finite or non-positive RMS. This one set is zero-weighted and filled by :func:`prepare_ngmix_weights` under every ``blend_handling`` and counted by the epoch cuts in - :func:`prepare_postage_stamps`. SExtractor's neighbour markers are not + :func:`prepare_postage_stamps`. The tile VIGNET's neighbour markers are not in it (neighbour-markers-are-not-defects); off-tile pixels are, as flag ``OFF_TILE_FLAG`` (off-tile-pixels-are-defects). It is not ORed with its rotations. For defects the central-defect veto keeps @@ -2280,8 +2280,8 @@ def prepare_ngmix_weights( supports the interpolant. @sc [decision:shape_measurement.blend_handling] noisefill-fills-markers - Under ``"noisefill"`` the pixels of ``neighbour``, SExtractor's -1e30 - neighbour markers, get weight 0 and are replaced by the same noise + Under ``"noisefill"`` the pixels of ``neighbour``, the tile VIGNET's + -1e30 neighbour markers, get weight 0 and are replaced by the same noise realisation as the noise-filled defects, so no marked neighbour light reaches metacal. Under the default noise fill, the image, weight map and noise image are those the marked pixels would get as flagged defects; diff --git a/src/shapepipe/modules/ngmix_runner.py b/src/shapepipe/modules/ngmix_runner.py index 205a38cab..08113aa84 100644 --- a/src/shapepipe/modules/ngmix_runner.py +++ b/src/shapepipe/modules/ngmix_runner.py @@ -145,7 +145,7 @@ def ngmix_runner( centroid_source = "wcs" # Neighbour treatment: "noisefill" (default) zero-weights and noise-fills - # the pixels SExtractor marks -1e30 in the tile VIGNET (other detections' + # the pixels marked -1e30 in the tile VIGNET (other detections' # footprints); "uberseg" ignores those markers, zeroes the weight of every # pixel closer to a neighbour than to the central object, from the # segmentation map, and leaves its image raw. Defect pixels (flagged, diff --git a/src/shapepipe/modules/read_ext_sexcat_package/read_ext_sexcat.py b/src/shapepipe/modules/read_ext_sexcat_package/read_ext_sexcat.py index ff4f58cc8..4b1d66012 100644 --- a/src/shapepipe/modules/read_ext_sexcat_package/read_ext_sexcat.py +++ b/src/shapepipe/modules/read_ext_sexcat_package/read_ext_sexcat.py @@ -48,8 +48,9 @@ def _build_ldac_imhead(img_header): # SExtractor's VIGNET value for pixels that are not the object's: off the -# image and on a neighbour's segmentation footprint. ngmix flags these pixels -# (tile VIGNET == -1e30) and noise-fills them at zero weight. +# image and on a neighbour's segmentation footprint. ngmix splits these pixels +# (tile VIGNET == -1e30, ``ngmix.split_tile_markers``): the off-image rows and +# columns are defects, the footprint pixels its neighbour mask. BIG = -1e30 # The label of a footprint no catalogue object claims in the relabelled diff --git a/tests/module/test_ngmix_defect_fill.py b/tests/module/test_ngmix_defect_fill.py index 8919453c2..f068c1290 100644 --- a/tests/module/test_ngmix_defect_fill.py +++ b/tests/module/test_ngmix_defect_fill.py @@ -10,7 +10,7 @@ in :func:`prepare_postage_stamps` act on the same defect set: the masked-fraction cut counts it, and the central-defect veto drops an epoch with a defect near the stamp centre, at a radius set by the defect's fill. -SExtractor's -1e30 neighbour markers in the tile VIGNET are not defects: +The tile VIGNET's -1e30 neighbour markers are not defects: noisefill zero-weights and noise-fills them, uberseg ignores them, and the epoch cuts never count them. """ @@ -744,7 +744,7 @@ def test_ngmix_rejects_an_unknown_defect_fill(tmp_path): ) -# --- SExtractor's -1e30 neighbour markers are not defects ------------------- +# --- The tile VIGNET's -1e30 neighbour markers are not defects ------------- # # The tile VIGNET carries -1e30 on the footprints of other detections. Every # epoch shares that tile stamp, so a marker counted as a defect would drop @@ -1021,9 +1021,10 @@ def fake_observation(*args, **kwargs): # --- Off-tile pixels are defects ------------------------------------------- # -# SExtractor also writes -1e30 beyond the tile's edge, where the epoch holds -# the object's own light, cut off. Those pixels are the stamp rows and -# columns that are entirely -1e30 (the off-image part of a rectangle clip); +# The tile VIGNET also holds -1e30 beyond the tile's edge, where the epoch +# holds the object's own light, cut off. Those pixels are the runs of +# entirely -1e30 stamp rows and columns that start at a stamp border (the +# off-image part of a rectangle clip); # they join the epoch's defect set as flag 2**10. The other markers are the # neighbour mask. From 74f5be39dd7fd60160a429afd9e280beb5e4368f Mon Sep 17 00:00:00 2001 From: Cail Daley Date: Fri, 2 Oct 2026 23:21:14 +0200 Subject: [PATCH 15/30] Masked-pixel figures for #922: pixel classes on a synthetic and a real DR6 blend, and a defect-type gallery Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01N4HLdzDyc3CiPLrgJ48GVo --- .../masked_pixels/defect_gallery.py | 125 ++++++++++++ scripts/validation/masked_pixels/dr6_blend.py | 149 ++++++++++++++ scripts/validation/masked_pixels/figstyle.py | 80 ++++++++ .../validation/masked_pixels/pixel_classes.py | 191 ++++++++++++++++++ 4 files changed, 545 insertions(+) create mode 100644 scripts/validation/masked_pixels/defect_gallery.py create mode 100644 scripts/validation/masked_pixels/dr6_blend.py create mode 100644 scripts/validation/masked_pixels/figstyle.py create mode 100644 scripts/validation/masked_pixels/pixel_classes.py diff --git a/scripts/validation/masked_pixels/defect_gallery.py b/scripts/validation/masked_pixels/defect_gallery.py new file mode 100644 index 000000000..f355f2426 --- /dev/null +++ b/scripts/validation/masked_pixels/defect_gallery.py @@ -0,0 +1,125 @@ +"""One defect per row: the raw epoch stamp, then the image and weights ngmix +receives under DEFECT_FILL = noise and DEFECT_FILL = interpolate. + +Run from the shapepipe checkout root inside the shapepipe container: + PYTHONPATH=src:scripts/validation/masked_pixels \ + python scripts/validation/masked_pixels/defect_gallery.py OUT.png +""" +import sys + +import matplotlib + +matplotlib.use("Agg") +import matplotlib.pyplot as plt +import numpy as np +from matplotlib.lines import Line2D + +from figstyle import (AQUA, INK, ORANGE, WEIGHT_LEGEND, clean, outline, + show_image, show_weight) +from shapepipe.modules.ngmix_package.defect_interpolation import ( + interpolable_defects, +) +from shapepipe.modules.ngmix_package.ngmix import ( + OFF_TILE_FLAG, + defect_mask, + prepare_ngmix_weights, +) + +N, C = 51, 25 +yy, xx = np.indices((N, N)) +RMS = 1.0 + + +def blob(r0, c0, sigma, flux, q=1.0, theta=0.0): + dy, dx = yy - r0, xx - c0 + ct, st = np.cos(theta), np.sin(theta) + u, v = ct * dx + st * dy, -st * dx + ct * dy + r = np.sqrt(u ** 2 + (v / q) ** 2) + return flux * np.exp(-r / sigma) / (2 * np.pi * sigma ** 2 * q) + + +def case(kind): + """Raw stamp, weight and flag for one defect type.""" + rng = np.random.RandomState(7) + gal = blob(C, C, 2.6, 4000, q=0.7, theta=0.5) + rng.normal(0, RMS, (N, N)) + weight = np.ones((N, N)) + flag = np.zeros((N, N), np.int32) + if kind == "column": + flag[:, C + 5] = 1 + gal[:, C + 5] = 60.0 + elif kind == "bleed": + flag[C - 2:, C - 7:C - 4] = 1 + gal[C - 2:, C - 7:C - 4] = 400.0 + elif kind == "cosmic": + track = [(C + 4 + i, C + 3 + i // 2) for i in range(9)] + for r, c in track: + flag[r, c] = 1 + gal[r, c] = 300.0 + flag[C - 9, C - 4] = 1 + gal[C - 9, C - 4] = 500.0 + elif kind == "edge": + flag[:6, :] = OFF_TILE_FLAG + elif kind == "hole": + weight[C - 3:C + 3, C + 4:C + 10] = 0.0 + gal[C - 3:C + 3, C + 4:C + 10] = 0.0 + return gal, weight, flag + + +ROWS = [ + ("column", "bad column"), + ("bleed", "3 px bleed"), + ("cosmic", "cosmic ray +\nsingle hot pixel"), + ("edge", "edge band\n(off-tile, 6 rows)"), + ("hole", "6×6 hole\n(too wide)"), +] +COLS = ["raw stamp", "image", "weight", "image", "weight"] + +plt.rcParams.update({"font.size": 10}) +fig, axs = plt.subplots(len(ROWS), 5, figsize=(10.2, 2.08 * len(ROWS)), + dpi=150, gridspec_kw=dict(hspace=0.06, wspace=0.05)) +for i, (kind, label) in enumerate(ROWS): + gal, weight, flag = case(kind) + defect = defect_mask(weight, flag) + interp = interpolable_defects(defect) + noisefilled = defect & ~interp + bkg_rms = np.full((N, N), RMS) + out = {} + for fill in ("noise", "interpolate"): + img, w, _ = prepare_ngmix_weights( + gal, weight, flag, np.random.RandomState(1), bkg_rms=bkg_rms, + defect_fill=fill, + ) + out[fill] = img, w + show_image(axs[i, 0], gal) + outline(axs[i, 0], defect, INK, lw=0.9) + show_image(axs[i, 1], out["noise"][0]) + show_weight(axs[i, 2], out["noise"][1]) + show_image(axs[i, 3], out["interpolate"][0]) + outline(axs[i, 3], interp, AQUA, lw=0.9) + outline(axs[i, 3], noisefilled, ORANGE, lw=0.9) + show_weight(axs[i, 4], out["interpolate"][1], interp) + axs[i, 0].set_ylabel(label, color=INK, fontsize=10) +for j, t in enumerate(COLS): + axs[0, j].set_title(t, color=INK, fontsize=10) +for ax in axs.flat: + clean(ax) + +top = axs[0, 1].get_position().y1 + 0.035 +for (a, b), t in (((1, 2), "DEFECT_FILL = noise"), + ((3, 4), "DEFECT_FILL = interpolate")): + x0 = axs[0, a].get_position().x0 + x1 = axs[0, b].get_position().x1 + fig.text((x0 + x1) / 2, top, t, ha="center", va="bottom", color=INK, + fontsize=11, family="monospace") + fig.add_artist(Line2D([x0 + 0.005, x1 - 0.005], [top - 0.004] * 2, + color=INK, lw=0.8)) + +handles = [ + Line2D([], [], color=INK, lw=1.6, label="defect (raw stamp)"), + Line2D([], [], color=ORANGE, lw=1.6, label="defect, noise-filled"), + Line2D([], [], color=AQUA, lw=1.6, label="defect, interpolated"), +] + WEIGHT_LEGEND +fig.legend(handles=handles, loc="upper center", ncol=3, frameon=False, + fontsize=9, handlelength=1.4, + bbox_to_anchor=(0.5, axs[-1, 0].get_position().y0 - 0.005)) +fig.savefig(sys.argv[1], bbox_inches="tight") diff --git a/scripts/validation/masked_pixels/dr6_blend.py b/scripts/validation/masked_pixels/dr6_blend.py new file mode 100644 index 000000000..e7d63052f --- /dev/null +++ b/scripts/validation/masked_pixels/dr6_blend.py @@ -0,0 +1,149 @@ +"""Cut one real DR6 epoch of a blended object into the arrays that +prepare_ngmix_weights takes. + +Inputs: + * one MegaCam exposure with its weight and flag maps (vos:cfis/pitcairn, + vos:cfis/weights, vos:cfis/flags), e.g. 2086786p.fits.fz; + * the DR6 tile catalogue with VIGNET (-1e30 neighbour and off-tile + markers) and SEG_VIGNET columns, as read_ext_sexcat writes it from + CFIS..r.cat and CFIS..r.seg.fits.fz. + +The epoch stamps are cut as vignetmaker does: 51x51 around the rounded +pixel of the object's (XWIN_WORLD, YWIN_WORLD) in the CCD's own WCS. The +tile VIGNET and seg stamp are MegaCam-flipped onto the epoch and split into +neighbour and off-tile pixels (split_tile_markers). The background is a +sigma-clipped median over the clean pixels of a 257x257 box around the +object, and the background RMS is constant over the stamp; the pipeline +takes both from SExtractor's maps instead. Only the survey flag map marks +defects here: the pipeline's mask step adds star halos and borders. + +Run from the shapepipe checkout root inside the shapepipe container: + PYTHONPATH=src python scripts/validation/masked_pixels/dr6_blend.py \ + scan EXP WEIGHT FLAG SEXCAT # rank candidate epochs + PYTHONPATH=src python scripts/validation/masked_pixels/dr6_blend.py \ + cut EXP WEIGHT FLAG SEXCAT NUMBER OUT.npz # write one epoch +""" +import sys + +import fitsio +import numpy as np +from astropy.io import fits +from astropy.stats import sigma_clipped_stats +from astropy.wcs import WCS + +from shapepipe.modules.ngmix_package.ngmix import ( + OFF_TILE_FLAG, + Ngmix, + defect_mask, + split_tile_markers, +) +from shapepipe.modules.vignetmaker_package.vignetmaker import get_stamps + +RAD = 25 +N = 2 * RAD + 1 +_rr = np.hypot(*(np.indices((N, N)) - RAD)) +BOX = 128 + + +def read_tile(sexcat): + with fitsio.FITS(sexcat) as f: + hdu = [h for h in f if h.get_extname() == "LDAC_OBJECTS"][0] + return hdu.read(columns=["NUMBER", "XWIN_WORLD", "YWIN_WORLD", + "MAG_AUTO", "VIGNET", "SEG_VIGNET"]) + + +def blend_rows(tile): + """Rows of objects with one sizeable neighbour footprint in their stamp, + no off-tile pixels and a target footprint of at least 40 pixels.""" + rows = [] + for i, (num, vign, seg) in enumerate( + zip(tile["NUMBER"], tile["VIGNET"], tile["SEG_VIGNET"])): + nb, off = split_tile_markers(vign, (N, N)) + if off.any() or (seg == num).sum() < 40: + continue + labels = [lab for lab in set(np.unique(seg[nb])) - {0, num} + if (seg == lab).sum() >= 40] + if len(labels) == 1: + rows.append(i) + return np.array(rows) + + +def ccd_hits(hdus, ra, dec, _wcs={}): + """(extension, [row, col]) of every CCD of the exposure holding (ra, dec) + at least RAD pixels from its border.""" + hits = [] + for ext in range(1, len(hdus)): + h = hdus[ext].header + if (id(hdus), ext) not in _wcs: + _wcs[id(hdus), ext] = WCS(h) + x, y = _wcs[id(hdus), ext].all_world2pix(ra, dec, 1) + if RAD <= x - 1 < h["NAXIS1"] - RAD and RAD <= y - 1 < h["NAXIS2"] - RAD: + hits.append((ext, np.array([y - 1, x - 1]))) + return hits + + +def cut(img_hdus, wgt_hdus, flg_hdus, ext, pos, tile, i): + """One epoch of tile row ``i`` from CCD extension ``ext``.""" + img, wgt, flg = (h[ext].data for h in (img_hdus, wgt_hdus, flg_hdus)) + ccd = ext - 1 # split_exp names CCD files by HDU index - 1 + stamps = {} + for key, a, dtype in (("gal", img, float), ("weight", wgt, float), + ("flag", flg, np.int32)): + s, int_pos, offset = get_stamps(a, pos[None, :], RAD) + stamps[key] = s[0].astype(dtype) + r0, c0 = int_pos[0] + box = (slice(max(r0 - BOX, 0), r0 + BOX + 1), + slice(max(c0 - BOX, 0), c0 + BOX + 1)) + good = (wgt[box] > 0) & (flg[box] == 0) + _, bkg, rms = sigma_clipped_stats(img[box][good], sigma=3.0) + stamps["gal"] = stamps["gal"] - bkg + tile_vign = Ngmix.MegaCamFlip(np.copy(tile["VIGNET"][i]), ccd) + seg = Ngmix.MegaCamFlip(np.copy(tile["SEG_VIGNET"][i]), ccd) + neighbour, off_tile = split_tile_markers(tile_vign, (N, N)) + stamps["flag"][off_tile] = OFF_TILE_FLAG + stamps.update( + bkg_rms=np.full((N, N), rms), seg=seg.astype(np.int32), + neighbour=neighbour, object_number=int(tile["NUMBER"][i]), + ccd=ccd, offset=offset[0], int_pos=int_pos[0], + ) + return stamps + + +def describe(ep): + defect = defect_mask(ep["weight"], ep["flag"], ep["bkg_rms"]) + d_def = _rr[defect].min() if defect.any() else np.inf + d_nb = _rr[ep["neighbour"]].min() if ep["neighbour"].any() else np.inf + return defect.sum(), d_def, d_nb + + +def main(): + mode, exp, wgt, flg, sexcat = sys.argv[1:6] + tile = read_tile(sexcat) + img_hdus, wgt_hdus, flg_hdus = (fits.open(p) for p in (exp, wgt, flg)) + if mode == "scan": + for i in blend_rows(tile): + for ext, pos in ccd_hits(img_hdus, tile["XWIN_WORLD"][i], + tile["YWIN_WORLD"][i]): + ep = cut(img_hdus, wgt_hdus, flg_hdus, ext, pos, tile, i) + n_def, d_def, d_nb = describe(ep) + print(f"NUMBER {tile['NUMBER'][i]} mag " + f"{tile['MAG_AUTO'][i]:.2f} ccd {ep['ccd']} " + f"defects {n_def} at >= {d_def:.1f} px, " + f"neighbour at >= {d_nb:.1f} px", flush=True) + return + number, out = int(sys.argv[6]), sys.argv[7] + i = int(np.flatnonzero(tile["NUMBER"] == number)[0]) + ext, pos = ccd_hits(img_hdus, tile["XWIN_WORLD"][i], + tile["YWIN_WORLD"][i])[0] + ep = cut(img_hdus, wgt_hdus, flg_hdus, ext, pos, tile, i) + expname = exp.split("/")[-1].split(".")[0] + ep["label"] = f"tile 202.301, object {number}" + ep["source"] = f"exposure {expname}, CCD {ep['ccd']}" + ep.update(ra=tile["XWIN_WORLD"][i], dec=tile["YWIN_WORLD"][i], + mag=tile["MAG_AUTO"][i]) + np.savez(out, **ep) + print(ep["label"], describe(ep)) + + +if __name__ == "__main__": + main() diff --git a/scripts/validation/masked_pixels/figstyle.py b/scripts/validation/masked_pixels/figstyle.py new file mode 100644 index 000000000..ecbe0fcd5 --- /dev/null +++ b/scripts/validation/masked_pixels/figstyle.py @@ -0,0 +1,80 @@ +"""Shared look of the masked-pixel figures (image stretch, weight classes).""" +import numpy as np +from matplotlib.colors import ListedColormap +from matplotlib.patches import Patch + +from shapepipe.modules.ngmix_package.defect_interpolation import fourfold + +INK, INK2 = "#0b0b0b", "#52514e" +PAPER, DARK = "#f4f3ef", "#3a3a38" +BLUE, ORANGE, AQUA, VIOLET = "#2a78d6", "#eb6834", "#1baf7a", "#8e5bd0" +TARGET = "#c3c2b7" + + +def stretch(a, soft=3.0): + return np.arcsinh(np.asarray(a, dtype=float) / soft) + + +def show_image(ax, img, vmax=120.0, soft=3.0): + ax.imshow(stretch(img, soft), origin="lower", cmap="gray_r", + vmin=stretch(-3 * soft / 3.0, soft), vmax=stretch(vmax, soft), + interpolation="nearest") + + +# Weight panel: 0 weighted, 1 zero weight, 2 zero weight on a rotated copy +# of an interpolated pixel (its light is kept). +WEIGHT_CMAP = ListedColormap([PAPER, DARK, VIOLET]) + + +def weight_classes(w, interpolated): + """0 = weighted, 1 = zero weight, 2 = zero weight only as a quarter-turn + copy of an interpolated pixel.""" + cls = np.where(w > 0, 0, 1) + copies = fourfold(interpolated) & ~interpolated + cls[(w == 0) & copies] = 2 + return cls + + +def show_weight(ax, w, interpolated=None): + if interpolated is None: + interpolated = np.zeros(w.shape, bool) + ax.imshow(weight_classes(w, interpolated), origin="lower", + cmap=WEIGHT_CMAP, vmin=-0.5, vmax=2.5, interpolation="nearest") + + +WEIGHT_LEGEND = [ + Patch(color=PAPER, ec=INK2, lw=0.5, label="weighted"), + Patch(color=DARK, label="zero weight"), + Patch(color=VIOLET, label="zero weight, light kept: rotated copy of an " + "interpolated pixel"), +] + + +def outline(ax, mask, color, lw=1.0): + """Draw the pixel-edge outline of a boolean mask.""" + mask = np.asarray(mask, bool) + ny, nx = mask.shape + pad = np.pad(mask, 1) + segs = [] + for r in range(ny): + for c in range(nx): + if not mask[r, c]: + continue + if not pad[r, c + 1]: + segs.append(((c - .5, c + .5), (r - .5, r - .5))) + if not pad[r + 2, c + 1]: + segs.append(((c - .5, c + .5), (r + .5, r + .5))) + if not pad[r + 1, c]: + segs.append(((c - .5, c - .5), (r - .5, r + .5))) + if not pad[r + 1, c + 2]: + segs.append(((c + .5, c + .5), (r - .5, r + .5))) + for xs, ys in segs: + ax.plot(xs, ys, color=color, lw=lw, solid_capstyle="butt") + + +def clean(ax): + ax.set_xticks([]) + ax.set_yticks([]) + for s in ax.spines.values(): + s.set_color(INK2) + s.set_linewidth(0.6) diff --git a/scripts/validation/masked_pixels/pixel_classes.py b/scripts/validation/masked_pixels/pixel_classes.py new file mode 100644 index 000000000..2f42b70c8 --- /dev/null +++ b/scripts/validation/masked_pixels/pixel_classes.py @@ -0,0 +1,191 @@ +"""Pixel classes of one epoch stamp and what ngmix receives: a synthetic +blend under noisefill + noise and uberseg + interpolate, then a real DR6 +blend under uberseg + interpolate. + +Run from the shapepipe checkout root inside the shapepipe container: + PYTHONPATH=src:scripts/validation/masked_pixels \ + python scripts/validation/masked_pixels/pixel_classes.py REAL.npz OUT.png + +REAL.npz is one real epoch written by dr6_blend.py. +""" +import sys + +import matplotlib + +matplotlib.use("Agg") +import matplotlib.pyplot as plt +import numpy as np +from matplotlib.colors import ListedColormap +from matplotlib.lines import Line2D +from matplotlib.patches import Circle, Patch + +from figstyle import (AQUA, BLUE, INK, INK2, ORANGE, PAPER, TARGET, + WEIGHT_LEGEND, clean, show_image, show_weight) +from shapepipe.modules.ngmix_package.defect_interpolation import ( + interpolable_defects, +) +from shapepipe.modules.ngmix_package.ngmix import ( + EPOCH_CENTRAL_DEFECT_RADIUS, + EPOCH_INTERPOLATED_DEFECT_RADIUS, + OFF_TILE_FLAG, + defect_mask, + prepare_ngmix_weights, +) + +SETTINGS = { + "noisefill_noise": dict(blend_handling="noisefill", defect_fill="noise"), + "uberseg_interpolate": dict(blend_handling="uberseg", + defect_fill="interpolate"), +} + + +def synthetic(): + """The synthetic epoch: a target, a neighbour 15 px away, a bad column, + a hot pixel and an off-tile band.""" + N, C = 51, 25 + rng = np.random.RandomState(4) + yy, xx = np.indices((N, N)) + + def blob(r0, c0, sigma, flux, q=1.0, theta=0.0): + dy, dx = yy - r0, xx - c0 + ct, st = np.cos(theta), np.sin(theta) + u, v = ct * dx + st * dy, -st * dx + ct * dy + r = np.sqrt(u ** 2 + (v / q) ** 2) + return flux * np.exp(-r / sigma) / (2 * np.pi * sigma ** 2 * q) + + rms = 1.0 + target = blob(C, C, 2.2, 3000, q=0.7, theta=0.5) + nbr = blob(C - 10, C - 11, 2.0, 4000, q=0.8, theta=-0.3) + gal = target + nbr + rng.normal(0, rms, (N, N)) + # Segmentation: each pixel above 2 sigma joins the brighter profile. + above = (target + nbr) > 2 * rms + seg = np.where(above, np.where(target >= nbr, 1, 2), 0).astype(np.int32) + flag = np.zeros((N, N), np.int32) + flag[:, C + 12] = 1 # bad column, 12 px right of centre + flag[C + 9, C - 9] = 1 # hot pixel, 12.7 px from centre + flag[N - 5:, :] = OFF_TILE_FLAG # off-tile band at the stamp's edge + gal[:, C + 12] = 400.0 + gal[C + 9, C - 9] = 400.0 + return dict(gal=gal, weight=np.ones((N, N)), flag=flag, + bkg_rms=np.full((N, N), rms), seg=seg, object_number=1, + neighbour=seg == 2) + + +def run(ep, setting): + kw = dict(SETTINGS[setting]) + if kw["blend_handling"] == "uberseg": + kw.update(seg=ep["seg"], object_number=int(ep["object_number"])) + img, w, _ = prepare_ngmix_weights( + ep["gal"], ep["weight"], ep["flag"], np.random.RandomState(1), + bkg_rms=ep["bkg_rms"], neighbour=ep["neighbour"], **kw, + ) + defect = defect_mask(ep["weight"], ep["flag"], ep["bkg_rms"]) + interp = (interpolable_defects(defect) if kw["defect_fill"] == + "interpolate" else np.zeros_like(defect)) + return img, w, interp + + +CLASS_CMAP = ListedColormap([PAPER, BLUE, ORANGE, AQUA, TARGET]) + + +def classes(ep): + cls = np.zeros(ep["gal"].shape, int) + target = ep["seg"] == int(ep["object_number"]) + cls[target] = 4 + cls[ep["neighbour"]] = 1 + defect = defect_mask(ep["weight"], ep["flag"], ep["bkg_rms"]) + cls[defect] = 2 + cls[ep["flag"] == OFF_TILE_FLAG] = 3 + return cls + + +def class_panel(ax, ep, radii=True): + ax.imshow(classes(ep), origin="lower", cmap=CLASS_CMAP, vmin=-0.5, + vmax=4.5, interpolation="nearest") + if radii: + n = ep["gal"].shape[0] + c = (n - 1) / 2 + for r, ls in ((EPOCH_CENTRAL_DEFECT_RADIUS, "--"), + (EPOCH_INTERPOLATED_DEFECT_RADIUS, ":")): + ax.add_patch(Circle((c, c), r, fill=False, ec=INK, ls=ls, + lw=1.0)) + + +def main(real_path, out_path): + syn = synthetic() + real = dict(np.load(real_path, allow_pickle=True)) + # The synthetic stamp has unit noise; scale the real one's stretch by + # its background RMS so both read alike. + rms = float(np.median(real["bkg_rms"])) + vmax_real, soft_real = 120.0 * rms, 3.0 * rms + + plt.rcParams.update({"font.size": 10}) + fig, axs = plt.subplots( + 2, 5, figsize=(13.6, 6.1), dpi=150, + gridspec_kw=dict(hspace=0.08, wspace=0.05, + width_ratios=[1, 1, 1, 1, 1]), + ) + # Open a gap between the synthetic and the real columns. + for ax in axs[:, 3:].flat: + p = ax.get_position() + ax.set_position([p.x0 + 0.025, p.y0, p.width, p.height]) + + # (a) synthetic epoch and its pixel classes + show_image(axs[0, 0], syn["gal"]) + class_panel(axs[1, 0], syn) + axs[0, 0].set_title("(a) one epoch's stamp", color=INK, fontsize=10) + axs[0, 0].set_ylabel("image ngmix sees", color=INK, fontsize=11) + axs[1, 0].set_ylabel("pixel class / weight", color=INK, fontsize=11) + c = 25 + axs[1, 0].text(c, c + 10.8, "10 px", ha="center", va="bottom", + fontsize=8, color=INK) + axs[1, 0].text(c, c - 6.3, "7 px", ha="center", va="bottom", + fontsize=8, color=INK) + + for j, (setting, title) in enumerate( + (("noisefill_noise", "(b) noisefill + noise\n(the defaults)"), + ("uberseg_interpolate", "(c) uberseg + interpolate")), start=1): + img, w, interp = run(syn, setting) + show_image(axs[0, j], img) + show_weight(axs[1, j], w, interp) + axs[0, j].set_title(title, color=INK, fontsize=10) + + # (d) the real DR6 epoch: its classes, then uberseg + interpolate + show_image(axs[0, 3], real["gal"], vmax=vmax_real, soft=soft_real) + class_panel(axs[1, 3], real) + axs[0, 3].set_title(f"(d) {real['label']}\n{real['source']}", + color=INK, fontsize=10) + img, w, interp = run(real, "uberseg_interpolate") + show_image(axs[0, 4], img, vmax=vmax_real, soft=soft_real) + show_weight(axs[1, 4], w, interp) + axs[0, 4].set_title("uberseg + interpolate", color=INK, fontsize=10) + for ax in axs.flat: + clean(ax) + + y = axs[0, 0].get_position().y1 + 0.075 + for (a, b), t in (((0, 2), "synthetic blend"), + ((3, 4), "real DR6 blend")): + x0 = axs[0, a].get_position().x0 + x1 = axs[0, b].get_position().x1 + fig.text((x0 + x1) / 2, y, t, ha="center", va="bottom", color=INK, + fontsize=11) + fig.add_artist(Line2D([x0 + 0.004, x1 - 0.004], [y - 0.006] * 2, + color=INK2, lw=0.8)) + + handles = [ + Patch(color=TARGET, label="target footprint"), + Patch(color=BLUE, label="neighbour (−1e30 in tile)"), + Patch(color=ORANGE, label="defect (flag / weight / RMS)"), + Patch(color=AQUA, label="off-tile (a defect)"), + Line2D([], [], color=INK, ls="--", lw=1, label="10 px veto"), + Line2D([], [], color=INK, ls=":", lw=1, + label="7 px veto (interpolated)"), + ] + WEIGHT_LEGEND + fig.legend(handles=handles, loc="upper center", ncol=5, frameon=False, + fontsize=9, handlelength=1.4, + bbox_to_anchor=(0.5, axs[1, 0].get_position().y0 - 0.01)) + fig.savefig(out_path, bbox_inches="tight") + + +if __name__ == "__main__": + main(sys.argv[1], sys.argv[2]) From b482d750c0b5f3e10ad7f047abdb0d79b3cd17fd Mon Sep 17 00:00:00 2001 From: Cail Daley Date: Fri, 2 Oct 2026 23:40:50 +0200 Subject: [PATCH 16/30] Defect gallery from real DR6 epochs; dr6_epochs.py scans an exposure for every defect type Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01N4HLdzDyc3CiPLrgJ48GVo --- .../masked_pixels/defect_gallery.py | 164 +++++++--------- scripts/validation/masked_pixels/dr6_blend.py | 149 -------------- .../validation/masked_pixels/dr6_epochs.py | 184 ++++++++++++++++++ .../validation/masked_pixels/pixel_classes.py | 2 +- 4 files changed, 251 insertions(+), 248 deletions(-) delete mode 100644 scripts/validation/masked_pixels/dr6_blend.py create mode 100644 scripts/validation/masked_pixels/dr6_epochs.py diff --git a/scripts/validation/masked_pixels/defect_gallery.py b/scripts/validation/masked_pixels/defect_gallery.py index f355f2426..1aed130f4 100644 --- a/scripts/validation/masked_pixels/defect_gallery.py +++ b/scripts/validation/masked_pixels/defect_gallery.py @@ -1,9 +1,12 @@ -"""One defect per row: the raw epoch stamp, then the image and weights ngmix -receives under DEFECT_FILL = noise and DEFECT_FILL = interpolate. +"""One real DR6 epoch per defect type: the raw epoch stamp, then the image +and weights ngmix receives under DEFECT_FILL = noise and +DEFECT_FILL = interpolate (BLEND_HANDLING = noisefill throughout). -Run from the shapepipe checkout root inside the shapepipe container: +Each epoch is an npz written by ``dr6_epochs.py cut``. Run from the +shapepipe checkout root inside the shapepipe container: PYTHONPATH=src:scripts/validation/masked_pixels \ - python scripts/validation/masked_pixels/defect_gallery.py OUT.png + python scripts/validation/masked_pixels/defect_gallery.py OUT.png \ + "bad column=ep_4202.npz" "3-column cluster=ep_28647.npz" ... """ import sys @@ -14,112 +17,77 @@ import numpy as np from matplotlib.lines import Line2D -from figstyle import (AQUA, INK, ORANGE, WEIGHT_LEGEND, clean, outline, +from figstyle import (AQUA, BLUE, INK, ORANGE, WEIGHT_LEGEND, clean, outline, show_image, show_weight) from shapepipe.modules.ngmix_package.defect_interpolation import ( interpolable_defects, ) from shapepipe.modules.ngmix_package.ngmix import ( - OFF_TILE_FLAG, defect_mask, prepare_ngmix_weights, ) -N, C = 51, 25 -yy, xx = np.indices((N, N)) -RMS = 1.0 - - -def blob(r0, c0, sigma, flux, q=1.0, theta=0.0): - dy, dx = yy - r0, xx - c0 - ct, st = np.cos(theta), np.sin(theta) - u, v = ct * dx + st * dy, -st * dx + ct * dy - r = np.sqrt(u ** 2 + (v / q) ** 2) - return flux * np.exp(-r / sigma) / (2 * np.pi * sigma ** 2 * q) - +COLS = ["raw stamp", "image", "weight", "image", "weight"] -def case(kind): - """Raw stamp, weight and flag for one defect type.""" - rng = np.random.RandomState(7) - gal = blob(C, C, 2.6, 4000, q=0.7, theta=0.5) + rng.normal(0, RMS, (N, N)) - weight = np.ones((N, N)) - flag = np.zeros((N, N), np.int32) - if kind == "column": - flag[:, C + 5] = 1 - gal[:, C + 5] = 60.0 - elif kind == "bleed": - flag[C - 2:, C - 7:C - 4] = 1 - gal[C - 2:, C - 7:C - 4] = 400.0 - elif kind == "cosmic": - track = [(C + 4 + i, C + 3 + i // 2) for i in range(9)] - for r, c in track: - flag[r, c] = 1 - gal[r, c] = 300.0 - flag[C - 9, C - 4] = 1 - gal[C - 9, C - 4] = 500.0 - elif kind == "edge": - flag[:6, :] = OFF_TILE_FLAG - elif kind == "hole": - weight[C - 3:C + 3, C + 4:C + 10] = 0.0 - gal[C - 3:C + 3, C + 4:C + 10] = 0.0 - return gal, weight, flag +def main(out_path, rows): + plt.rcParams.update({"font.size": 10}) + fig, axs = plt.subplots(len(rows), 5, figsize=(11.2, 2.2 * len(rows)), + dpi=150, + gridspec_kw=dict(hspace=0.06, wspace=0.05)) + for i, (kind, path) in enumerate(rows): + ep = dict(np.load(path, allow_pickle=True)) + rms = float(np.median(ep["bkg_rms"])) + stretch = dict(vmax=120.0 * rms, soft=3.0 * rms) + defect = defect_mask(ep["weight"], ep["flag"], ep["bkg_rms"]) + interp = interpolable_defects(defect) + out = {} + for fill in ("noise", "interpolate"): + out[fill] = prepare_ngmix_weights( + ep["gal"], ep["weight"], ep["flag"], + np.random.RandomState(1), bkg_rms=ep["bkg_rms"], + neighbour=ep["neighbour"], defect_fill=fill, + )[:2] + show_image(axs[i, 0], ep["gal"], **stretch) + outline(axs[i, 0], ep["neighbour"], BLUE, lw=0.9) + outline(axs[i, 0], defect, INK, lw=0.9) + show_image(axs[i, 1], out["noise"][0], **stretch) + show_weight(axs[i, 2], out["noise"][1]) + show_image(axs[i, 3], out["interpolate"][0], **stretch) + outline(axs[i, 3], interp, AQUA, lw=0.9) + outline(axs[i, 3], defect & ~interp, ORANGE, lw=0.9) + show_weight(axs[i, 4], out["interpolate"][1], interp) + axs[i, 0].set_ylabel( + f"{kind}\n{ep['label']}\n{ep['source']}", color=INK, fontsize=8, + linespacing=1.15, + ) + for j, t in enumerate(COLS): + axs[0, j].set_title(t, color=INK, fontsize=10) + for ax in axs.flat: + clean(ax) -ROWS = [ - ("column", "bad column"), - ("bleed", "3 px bleed"), - ("cosmic", "cosmic ray +\nsingle hot pixel"), - ("edge", "edge band\n(off-tile, 6 rows)"), - ("hole", "6×6 hole\n(too wide)"), -] -COLS = ["raw stamp", "image", "weight", "image", "weight"] + top = axs[0, 1].get_position().y1 + 0.03 + for (a, b), t in (((1, 2), "DEFECT_FILL = noise"), + ((3, 4), "DEFECT_FILL = interpolate")): + x0 = axs[0, a].get_position().x0 + x1 = axs[0, b].get_position().x1 + fig.text((x0 + x1) / 2, top, t, ha="center", va="bottom", color=INK, + fontsize=11, family="monospace") + fig.add_artist(Line2D([x0 + 0.005, x1 - 0.005], [top - 0.004] * 2, + color=INK, lw=0.8)) -plt.rcParams.update({"font.size": 10}) -fig, axs = plt.subplots(len(ROWS), 5, figsize=(10.2, 2.08 * len(ROWS)), - dpi=150, gridspec_kw=dict(hspace=0.06, wspace=0.05)) -for i, (kind, label) in enumerate(ROWS): - gal, weight, flag = case(kind) - defect = defect_mask(weight, flag) - interp = interpolable_defects(defect) - noisefilled = defect & ~interp - bkg_rms = np.full((N, N), RMS) - out = {} - for fill in ("noise", "interpolate"): - img, w, _ = prepare_ngmix_weights( - gal, weight, flag, np.random.RandomState(1), bkg_rms=bkg_rms, - defect_fill=fill, - ) - out[fill] = img, w - show_image(axs[i, 0], gal) - outline(axs[i, 0], defect, INK, lw=0.9) - show_image(axs[i, 1], out["noise"][0]) - show_weight(axs[i, 2], out["noise"][1]) - show_image(axs[i, 3], out["interpolate"][0]) - outline(axs[i, 3], interp, AQUA, lw=0.9) - outline(axs[i, 3], noisefilled, ORANGE, lw=0.9) - show_weight(axs[i, 4], out["interpolate"][1], interp) - axs[i, 0].set_ylabel(label, color=INK, fontsize=10) -for j, t in enumerate(COLS): - axs[0, j].set_title(t, color=INK, fontsize=10) -for ax in axs.flat: - clean(ax) + handles = [ + Line2D([], [], color=INK, lw=1.6, label="defect (raw stamp)"), + Line2D([], [], color=BLUE, lw=1.6, + label="neighbour (raw stamp; noise-filled, zero weight)"), + Line2D([], [], color=ORANGE, lw=1.6, label="defect, noise-filled"), + Line2D([], [], color=AQUA, lw=1.6, label="defect, interpolated"), + ] + WEIGHT_LEGEND + fig.legend(handles=handles, loc="upper center", ncol=3, frameon=False, + fontsize=9, handlelength=1.4, + bbox_to_anchor=(0.5, axs[-1, 0].get_position().y0 - 0.005)) + fig.savefig(out_path, bbox_inches="tight") -top = axs[0, 1].get_position().y1 + 0.035 -for (a, b), t in (((1, 2), "DEFECT_FILL = noise"), - ((3, 4), "DEFECT_FILL = interpolate")): - x0 = axs[0, a].get_position().x0 - x1 = axs[0, b].get_position().x1 - fig.text((x0 + x1) / 2, top, t, ha="center", va="bottom", color=INK, - fontsize=11, family="monospace") - fig.add_artist(Line2D([x0 + 0.005, x1 - 0.005], [top - 0.004] * 2, - color=INK, lw=0.8)) -handles = [ - Line2D([], [], color=INK, lw=1.6, label="defect (raw stamp)"), - Line2D([], [], color=ORANGE, lw=1.6, label="defect, noise-filled"), - Line2D([], [], color=AQUA, lw=1.6, label="defect, interpolated"), -] + WEIGHT_LEGEND -fig.legend(handles=handles, loc="upper center", ncol=3, frameon=False, - fontsize=9, handlelength=1.4, - bbox_to_anchor=(0.5, axs[-1, 0].get_position().y0 - 0.005)) -fig.savefig(sys.argv[1], bbox_inches="tight") +if __name__ == "__main__": + main(sys.argv[1], [a.split("=", 1) for a in sys.argv[2:]]) diff --git a/scripts/validation/masked_pixels/dr6_blend.py b/scripts/validation/masked_pixels/dr6_blend.py deleted file mode 100644 index e7d63052f..000000000 --- a/scripts/validation/masked_pixels/dr6_blend.py +++ /dev/null @@ -1,149 +0,0 @@ -"""Cut one real DR6 epoch of a blended object into the arrays that -prepare_ngmix_weights takes. - -Inputs: - * one MegaCam exposure with its weight and flag maps (vos:cfis/pitcairn, - vos:cfis/weights, vos:cfis/flags), e.g. 2086786p.fits.fz; - * the DR6 tile catalogue with VIGNET (-1e30 neighbour and off-tile - markers) and SEG_VIGNET columns, as read_ext_sexcat writes it from - CFIS..r.cat and CFIS..r.seg.fits.fz. - -The epoch stamps are cut as vignetmaker does: 51x51 around the rounded -pixel of the object's (XWIN_WORLD, YWIN_WORLD) in the CCD's own WCS. The -tile VIGNET and seg stamp are MegaCam-flipped onto the epoch and split into -neighbour and off-tile pixels (split_tile_markers). The background is a -sigma-clipped median over the clean pixels of a 257x257 box around the -object, and the background RMS is constant over the stamp; the pipeline -takes both from SExtractor's maps instead. Only the survey flag map marks -defects here: the pipeline's mask step adds star halos and borders. - -Run from the shapepipe checkout root inside the shapepipe container: - PYTHONPATH=src python scripts/validation/masked_pixels/dr6_blend.py \ - scan EXP WEIGHT FLAG SEXCAT # rank candidate epochs - PYTHONPATH=src python scripts/validation/masked_pixels/dr6_blend.py \ - cut EXP WEIGHT FLAG SEXCAT NUMBER OUT.npz # write one epoch -""" -import sys - -import fitsio -import numpy as np -from astropy.io import fits -from astropy.stats import sigma_clipped_stats -from astropy.wcs import WCS - -from shapepipe.modules.ngmix_package.ngmix import ( - OFF_TILE_FLAG, - Ngmix, - defect_mask, - split_tile_markers, -) -from shapepipe.modules.vignetmaker_package.vignetmaker import get_stamps - -RAD = 25 -N = 2 * RAD + 1 -_rr = np.hypot(*(np.indices((N, N)) - RAD)) -BOX = 128 - - -def read_tile(sexcat): - with fitsio.FITS(sexcat) as f: - hdu = [h for h in f if h.get_extname() == "LDAC_OBJECTS"][0] - return hdu.read(columns=["NUMBER", "XWIN_WORLD", "YWIN_WORLD", - "MAG_AUTO", "VIGNET", "SEG_VIGNET"]) - - -def blend_rows(tile): - """Rows of objects with one sizeable neighbour footprint in their stamp, - no off-tile pixels and a target footprint of at least 40 pixels.""" - rows = [] - for i, (num, vign, seg) in enumerate( - zip(tile["NUMBER"], tile["VIGNET"], tile["SEG_VIGNET"])): - nb, off = split_tile_markers(vign, (N, N)) - if off.any() or (seg == num).sum() < 40: - continue - labels = [lab for lab in set(np.unique(seg[nb])) - {0, num} - if (seg == lab).sum() >= 40] - if len(labels) == 1: - rows.append(i) - return np.array(rows) - - -def ccd_hits(hdus, ra, dec, _wcs={}): - """(extension, [row, col]) of every CCD of the exposure holding (ra, dec) - at least RAD pixels from its border.""" - hits = [] - for ext in range(1, len(hdus)): - h = hdus[ext].header - if (id(hdus), ext) not in _wcs: - _wcs[id(hdus), ext] = WCS(h) - x, y = _wcs[id(hdus), ext].all_world2pix(ra, dec, 1) - if RAD <= x - 1 < h["NAXIS1"] - RAD and RAD <= y - 1 < h["NAXIS2"] - RAD: - hits.append((ext, np.array([y - 1, x - 1]))) - return hits - - -def cut(img_hdus, wgt_hdus, flg_hdus, ext, pos, tile, i): - """One epoch of tile row ``i`` from CCD extension ``ext``.""" - img, wgt, flg = (h[ext].data for h in (img_hdus, wgt_hdus, flg_hdus)) - ccd = ext - 1 # split_exp names CCD files by HDU index - 1 - stamps = {} - for key, a, dtype in (("gal", img, float), ("weight", wgt, float), - ("flag", flg, np.int32)): - s, int_pos, offset = get_stamps(a, pos[None, :], RAD) - stamps[key] = s[0].astype(dtype) - r0, c0 = int_pos[0] - box = (slice(max(r0 - BOX, 0), r0 + BOX + 1), - slice(max(c0 - BOX, 0), c0 + BOX + 1)) - good = (wgt[box] > 0) & (flg[box] == 0) - _, bkg, rms = sigma_clipped_stats(img[box][good], sigma=3.0) - stamps["gal"] = stamps["gal"] - bkg - tile_vign = Ngmix.MegaCamFlip(np.copy(tile["VIGNET"][i]), ccd) - seg = Ngmix.MegaCamFlip(np.copy(tile["SEG_VIGNET"][i]), ccd) - neighbour, off_tile = split_tile_markers(tile_vign, (N, N)) - stamps["flag"][off_tile] = OFF_TILE_FLAG - stamps.update( - bkg_rms=np.full((N, N), rms), seg=seg.astype(np.int32), - neighbour=neighbour, object_number=int(tile["NUMBER"][i]), - ccd=ccd, offset=offset[0], int_pos=int_pos[0], - ) - return stamps - - -def describe(ep): - defect = defect_mask(ep["weight"], ep["flag"], ep["bkg_rms"]) - d_def = _rr[defect].min() if defect.any() else np.inf - d_nb = _rr[ep["neighbour"]].min() if ep["neighbour"].any() else np.inf - return defect.sum(), d_def, d_nb - - -def main(): - mode, exp, wgt, flg, sexcat = sys.argv[1:6] - tile = read_tile(sexcat) - img_hdus, wgt_hdus, flg_hdus = (fits.open(p) for p in (exp, wgt, flg)) - if mode == "scan": - for i in blend_rows(tile): - for ext, pos in ccd_hits(img_hdus, tile["XWIN_WORLD"][i], - tile["YWIN_WORLD"][i]): - ep = cut(img_hdus, wgt_hdus, flg_hdus, ext, pos, tile, i) - n_def, d_def, d_nb = describe(ep) - print(f"NUMBER {tile['NUMBER'][i]} mag " - f"{tile['MAG_AUTO'][i]:.2f} ccd {ep['ccd']} " - f"defects {n_def} at >= {d_def:.1f} px, " - f"neighbour at >= {d_nb:.1f} px", flush=True) - return - number, out = int(sys.argv[6]), sys.argv[7] - i = int(np.flatnonzero(tile["NUMBER"] == number)[0]) - ext, pos = ccd_hits(img_hdus, tile["XWIN_WORLD"][i], - tile["YWIN_WORLD"][i])[0] - ep = cut(img_hdus, wgt_hdus, flg_hdus, ext, pos, tile, i) - expname = exp.split("/")[-1].split(".")[0] - ep["label"] = f"tile 202.301, object {number}" - ep["source"] = f"exposure {expname}, CCD {ep['ccd']}" - ep.update(ra=tile["XWIN_WORLD"][i], dec=tile["YWIN_WORLD"][i], - mag=tile["MAG_AUTO"][i]) - np.savez(out, **ep) - print(ep["label"], describe(ep)) - - -if __name__ == "__main__": - main() diff --git a/scripts/validation/masked_pixels/dr6_epochs.py b/scripts/validation/masked_pixels/dr6_epochs.py new file mode 100644 index 000000000..6faf1dec4 --- /dev/null +++ b/scripts/validation/masked_pixels/dr6_epochs.py @@ -0,0 +1,184 @@ +"""Cut real DR6 epoch stamps into the arrays prepare_ngmix_weights takes, +and find epochs with each kind of defect. + +Inputs: + * one MegaCam exposure with its weight and flag maps (vos:cfis/pitcairn, + vos:cfis/weights, vos:cfis/flags), e.g. 2086786p.fits.fz; + * the DR6 tile catalogue with VIGNET (-1e30 neighbour and off-tile + markers) and SEG_VIGNET columns, as read_ext_sexcat writes it from + CFIS..r.cat and CFIS..r.seg.fits.fz + (sexcat--.fits). + +The epoch stamps are cut as vignetmaker does: 51x51 around the rounded +pixel of the object's (XWIN_WORLD, YWIN_WORLD) in the CCD's own WCS, CCDs +numbered as split_exp names them (HDU index - 1). The tile VIGNET and seg +stamp are MegaCam-flipped onto the epoch and split into neighbour and +off-tile pixels (split_tile_markers). The background is a sigma-clipped +median over the clean pixels of a 257x257 box around the object, and the +background RMS is constant over the stamp; the pipeline takes both from +SExtractor's maps instead. Only the survey flag and weight maps mark +defects here: the pipeline's mask step adds star halos and borders. + +Run from the shapepipe checkout root inside the shapepipe container: + PYTHONPATH=src python scripts/validation/masked_pixels/dr6_epochs.py \ + scan EXP WEIGHT FLAG SEXCAT > scan.txt # one line per epoch + PYTHONPATH=src python scripts/validation/masked_pixels/dr6_epochs.py \ + cut EXP WEIGHT FLAG SEXCAT NUMBER OUT.npz +""" +import re +import sys + +import fitsio +import numpy as np +from astropy.io import fits +from astropy.stats import sigma_clipped_stats +from astropy.wcs import WCS +from scipy.ndimage import find_objects, label + +from shapepipe.modules.ngmix_package.ngmix import ( + OFF_TILE_FLAG, + Ngmix, + defect_mask, + split_tile_markers, +) +from shapepipe.modules.vignetmaker_package.vignetmaker import get_stamps + +RAD = 25 +N = 2 * RAD + 1 +_rr = np.hypot(*(np.indices((N, N)) - RAD)) +BOX = 128 + + +def read_tile(sexcat): + with fitsio.FITS(sexcat) as f: + hdu = [h for h in f if h.get_extname() == "LDAC_OBJECTS"][0] + return hdu.read(columns=["NUMBER", "XWIN_WORLD", "YWIN_WORLD", + "MAG_AUTO", "FLUX_RADIUS", "VIGNET", + "SEG_VIGNET"]) + + +def tile_name(sexcat): + return ".".join(re.search(r"(\d{3})-(\d{3})", sexcat).groups()) + + +def ccd_positions(hdus, ra, dec): + """For each CCD extension, the tile rows whose stamps fit on it and their + 0-indexed [row, col] positions.""" + out = [] + for ext in range(1, len(hdus)): + h = hdus[ext].header + x, y = WCS(h).all_world2pix(ra, dec, 1) + inside = ((x - 1 >= RAD) & (x - 1 < h["NAXIS1"] - RAD) + & (y - 1 >= RAD) & (y - 1 < h["NAXIS2"] - RAD)) + rows = np.flatnonzero(inside) + out.append((ext, rows, np.column_stack([y[rows] - 1, x[rows] - 1]))) + return out + + +def tile_overlay(tile, i, ccd): + tile_vign = Ngmix.MegaCamFlip(np.copy(tile["VIGNET"][i]), ccd) + seg = Ngmix.MegaCamFlip(np.copy(tile["SEG_VIGNET"][i]), ccd) + neighbour, off_tile = split_tile_markers(tile_vign, (N, N)) + return seg.astype(np.int32), neighbour, off_tile + + +def components(defect, flag, off_tile): + """Defect components outside the off-tile band, nearest first: (kind, + distance, height, width, size, flag values).""" + labels, _ = label(defect & ~off_tile, structure=np.ones((3, 3))) + found = [] + for k, sl in enumerate(find_objects(labels), start=1): + part = labels == k + h, w = sl[0].stop - sl[0].start, sl[1].stop - sl[1].start + size = int(part.sum()) + if h == N: + kind = {1: "column"}.get(w, f"columns{w}" if w <= 4 else "wide") + elif size <= 6: + kind = "pixel" + elif w <= 4 and h >= 8: + kind = "trail" + else: + kind = "blob" + values = sorted(set(np.unique(flag[part]).tolist()) - {0}) + found.append((kind, float(_rr[part].min()), h, w, size, values)) + return sorted(found, key=lambda c: c[1]) + + +def cut(hdus, ext, pos, tile, i): + """One epoch of tile row ``i`` from CCD extension ``ext``.""" + img, wgt, flg = (h[ext].data for h in hdus) + ccd = ext - 1 + stamps = {} + for key, a, dtype in (("gal", img, float), ("weight", wgt, float), + ("flag", flg, np.int32)): + s, int_pos, offset = get_stamps(a, pos[None, :], RAD) + stamps[key] = s[0].astype(dtype) + r0, c0 = int_pos[0] + box = (slice(max(r0 - BOX, 0), r0 + BOX + 1), + slice(max(c0 - BOX, 0), c0 + BOX + 1)) + good = (wgt[box] > 0) & (flg[box] == 0) + _, bkg, rms = sigma_clipped_stats(img[box][good], sigma=3.0) + stamps["gal"] = stamps["gal"] - bkg + seg, neighbour, off_tile = tile_overlay(tile, i, ccd) + stamps["flag"][off_tile] = OFF_TILE_FLAG + stamps.update( + bkg_rms=np.full((N, N), rms), seg=seg, neighbour=neighbour, + object_number=int(tile["NUMBER"][i]), ccd=ccd, + offset=offset[0], int_pos=int_pos[0], + ) + return stamps + + +def scan(hdus, tile): + """Print one line per epoch with any defect pixel.""" + for ext, rows, pos in ccd_positions(hdus[0], tile["XWIN_WORLD"], + tile["YWIN_WORLD"]): + if not len(rows): + continue + wgt = get_stamps(hdus[1][ext].data, pos, RAD)[0] + flg = get_stamps(hdus[2][ext].data, pos, RAD)[0].astype(np.int32) + for i, w, f in zip(rows, wgt, flg): + seg, neighbour, off_tile = tile_overlay(tile, i, ext - 1) + f[off_tile] = OFF_TILE_FLAG + defect = defect_mask(w, f) + if not defect.any(): + continue + d_nb = _rr[neighbour].min() if neighbour.any() else np.inf + parts = ";".join( + f"{k}@{d:.1f}:{h}x{wd}:{n}:{'/'.join(map(str, v))}" + for k, d, h, wd, n, v in components(defect, f, off_tile) + ) + print(f"{tile['NUMBER'][i]} {tile['MAG_AUTO'][i]:.2f} " + f"{tile['FLUX_RADIUS'][i]:.2f} {ext - 1} " + f"{int(defect.sum())} {int(off_tile.sum())} {d_nb:.1f} " + f"{parts or '-'}", flush=True) + + +def main(): + mode, exp, wgt, flg, sexcat = sys.argv[1:6] + tile = read_tile(sexcat) + hdus = [fits.open(p) for p in (exp, wgt, flg)] + if mode == "scan": + print("NUMBER MAG_AUTO FLUX_RADIUS CCD N_DEFECT N_OFF_TILE " + "D_NEIGHBOUR COMPONENTS(kind@dist:hxw:size:flags)") + scan(hdus, tile) + return + number, out = int(sys.argv[6]), sys.argv[7] + i = int(np.flatnonzero(tile["NUMBER"] == number)[0]) + ext, rows, pos = next( + (e, r, p) for e, r, p in ccd_positions( + hdus[0], tile["XWIN_WORLD"][[i]], tile["YWIN_WORLD"][[i]]) + if len(r) + ) + ep = cut(hdus, ext, pos[0], tile, i) + expname = exp.split("/")[-1].split(".")[0] + ep["label"] = f"tile {tile_name(sexcat)}, object {number}" + ep["source"] = f"exposure {expname}, CCD {ep['ccd']}" + ep.update(ra=tile["XWIN_WORLD"][i], dec=tile["YWIN_WORLD"][i], + mag=tile["MAG_AUTO"][i]) + np.savez(out, **ep) + print(ep["label"], ep["source"]) + + +if __name__ == "__main__": + main() diff --git a/scripts/validation/masked_pixels/pixel_classes.py b/scripts/validation/masked_pixels/pixel_classes.py index 2f42b70c8..b12c72084 100644 --- a/scripts/validation/masked_pixels/pixel_classes.py +++ b/scripts/validation/masked_pixels/pixel_classes.py @@ -6,7 +6,7 @@ PYTHONPATH=src:scripts/validation/masked_pixels \ python scripts/validation/masked_pixels/pixel_classes.py REAL.npz OUT.png -REAL.npz is one real epoch written by dr6_blend.py. +REAL.npz is one real epoch written by dr6_epochs.py. """ import sys From 575ed162d697effc5241bdbfdef16e555ad024b9 Mon Sep 17 00:00:00 2001 From: Cail Daley Date: Sat, 3 Oct 2026 00:05:03 +0200 Subject: [PATCH 17/30] ngmix: one weight-symmetrization step, SYMMETRIZE_WEIGHTS names the classes it covers prepare_ngmix_weights sorts each stamp pixel into clean, light-replaced (noise-filled defects, noisefill neighbours) and light-kept-at-zero-weight (interpolated defects, the uberseg neighbour side). One step zeroes the weight on the quarter-turn copies, about the stamp centre, of the light-kept holes that SYMMETRIZE_WEIGHTS names: - interpolated (default): the interpolated defects, as before; - interpolated_and_neighbours: also the uberseg neighbour side (#815); - none. Light-replaced pixels are never symmetrized. Under noisefill + noise every value is a no-op, so the defaults are unchanged. uberseg_weight becomes uberseg_mask, returning the neighbour-side pixels, so the weight map is built once from the pixel classes. Measured with defect_response (6 seeds, 0.5" galaxy, 0.7" PSF, neighbour at 10 px): a lightless neighbour footprint gives c1 = -7.8e-3 under uberseg and 2e-6 symmetrized, but with neighbour light (0.1x flux) c1 goes from +1.1e-3 to +1.2e-2, because the one-sided hole partly offsets the neighbour light in the target's cell. Symmetrizing also keeps 117 of 1580 weighted pixels (97% of S/N) at 10 px and 25 (71%) at 7 px. Hence the neighbour side stays opt-in. tests/science/test_weight_symmetrization.py guards both results; defect_response gains a neighbour and the fit S/N. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01N4HLdzDyc3CiPLrgJ48GVo --- astra.yaml | 54 +++- src/shapepipe/modules/ngmix_package/ngmix.py | 268 ++++++++++++------- src/shapepipe/modules/ngmix_runner.py | 15 ++ tests/helpers/defect_response.py | 59 +++- tests/module/test_ngmix.py | 2 + tests/module/test_ngmix_defect_fill.py | 111 +++++++- tests/module/test_ngmix_uberseg.py | 96 +++---- tests/science/test_weight_symmetrization.py | 54 ++++ universes/committed.yaml | 1 + 9 files changed, 488 insertions(+), 172 deletions(-) create mode 100644 tests/science/test_weight_symmetrization.py diff --git a/astra.yaml b/astra.yaml index a56a46748..588c5cf54 100644 --- a/astra.yaml +++ b/astra.yaml @@ -1223,7 +1223,8 @@ analyses: metacal_scheme, centroid_source, epoch_flux_rescaling, psf_epoch_averaging, galaxy_pixel_weights, psf_likelihood_noise, megacam_ccd_flip, defect_fill, blend_handling, - epoch_masked_fraction_cut, central_defect_veto] + weight_symmetrization, epoch_masked_fraction_cut, + central_defect_veto] decisions: ngmix_seed_mode: label: Per-object RNG seeded from sky position @@ -1564,6 +1565,57 @@ analyses: the ~2% uberseg-only bias in DES Y1 simulations, although Sheldon et al. 2020 found similar blend biases with and without MOF. insights: [mask_des_y1_uberseg_only, mask_blend_bias_detection] + weight_symmetrization: + label: Four-fold symmetric weight holes (SYMMETRIZE_WEIGHTS) + rationale: >- + ngmix fits a Gaussian to a profile that is not one, so a zero-weight + hole on one side of the object pulls the fit: a spin-2 term. One + step in prepare_ngmix_weights cancels it: the weight is also zeroed + on the three quarter-turn copies, about the stamp centre, of the + zero-weight pixels whose light stays, of the classes + SYMMETRIZE_WEIGHTS names. Pixels whose light is replaced + (noise-filled defects, noisefill neighbours) are never symmetrized: + the hole is then in the image metacal shears, and a weight orbit + leaves m unchanged (-0.53% against -0.54% for a noise-filled column + 10 px from a 0.5 arcsec galaxy) while moving c by about 1e-4 either + way. So under noisefill with the noise fill the option changes + nothing. The rotation is about the stamp centre, where the + extractor places the object to within half a pixel per axis; the + measurements draw the object uniformly within that half pixel. + Measured with tests/helpers/defect_response.py (6 seeds, full + response matrix, galaxy half-light radius 0.5 arcsec, 0.7 arcsec + PSF). + default: interpolated + options: + interpolated: + label: Symmetrize the interpolated defects + description: >- + SYMMETRIZE_WEIGHTS = interpolated. An interpolated column 8 px + from the galaxy gives c1 = -1.3e-3 unsymmetrized and +2e-6 + symmetrized (PR #916). The uberseg neighbour side stays + one-sided. + insights: [mask_bad_column_symmetrize] + interpolated_and_neighbours: + label: Also symmetrize the uberseg neighbour side (#815) + description: >- + SYMMETRIZE_WEIGHTS = interpolated_and_neighbours. Alone, the + neighbour-side hole is a strong pull: a neighbour footprint with + no light 10 px away gives c1 = -7.8e-3 unsymmetrized and 2e-6 + symmetrized. A real neighbour leaves light in the target's + uberseg cell, which pulls the other way, and the one-sided hole + partly offsets it: for a neighbour of a tenth of the target's + flux at 10 px, c1 = +1.1e-3 unsymmetrized and +1.2e-2 + symmetrized; at equal flux, +0.13 and +0.21. The symmetrized + hole keeps 117 of 1580 weighted pixels and 97% of the target's + S/N at 10 px (99% unsymmetrized), 25 pixels and 71% at 7 px, + none at 5 px. For a neighbour-subtracted image, or as an A/B + arm beside uberseg. + insights: [mask_bad_column_symmetrize, mask_uberseg_neighbour_bias] + none: + label: No symmetrization (SYMMETRIZE_WEIGHTS = none) + description: >- + Every zero-weight hole stays one-sided; under DEFECT_FILL = + interpolate the column above gives c1 = -1.3e-3. central_defect_veto: label: Per-epoch veto on a defect near the stamp centre rationale: >- diff --git a/src/shapepipe/modules/ngmix_package/ngmix.py b/src/shapepipe/modules/ngmix_package/ngmix.py index e897fce26..7733c0689 100644 --- a/src/shapepipe/modules/ngmix_package/ngmix.py +++ b/src/shapepipe/modules/ngmix_package/ngmix.py @@ -39,6 +39,11 @@ # :func:`prepare_ngmix_weights`). DEFECT_FILLS = ("noise", "interpolate") +# Zero-weight pixel classes whose quarter-turn copies also lose their weight, +# selectable with the SYMMETRIZE_WEIGHTS option (see +# :func:`prepare_ngmix_weights`). +WEIGHT_SYMMETRIZATIONS = ("interpolated", "interpolated_and_neighbours", "none") + # Default of the EPOCH_MASKED_FRACTION_CUT option: an epoch is dropped when # more than this fraction of its stamp is in :func:`defect_mask` (see # :func:`prepare_postage_stamps`). @@ -758,7 +763,7 @@ class Ngmix(object): :class:`Tile_cat`). Required when ``blend_handling="uberseg"``. dilate_neighbour : int, optional Neighbour-mask dilation iterations for ``"uberseg"`` (see - :func:`uberseg_weight`); the default is ``1``. + :func:`uberseg_mask`); the default is ``1``. epoch_central_defect_radius : float, optional Drop an epoch when a defect pixel lies closer than this many pixels to the stamp centre (see :func:`has_central_defect`); the default is @@ -775,6 +780,10 @@ class Ngmix(object): interpolated defect pixel lies closer than this many pixels to the stamp centre (see :func:`central_defect_vetoes`); the default is ``EPOCH_INTERPOLATED_DEFECT_RADIUS``. + symmetrize_weights : {"interpolated", "interpolated_and_neighbours", "none"}, optional + Which zero-weight pixels also zero the weight of their quarter-turn + copies (see :func:`prepare_ngmix_weights`); the default is + ``"interpolated"``. Notes ----- @@ -786,8 +795,8 @@ class Ngmix(object): IndexError If the length of the input file list is incorrect ValueError - If ``blend_handling`` or ``defect_fill`` is unknown, or ``"uberseg"`` - is selected without ``seg_cat_path``. + If ``blend_handling``, ``defect_fill`` or ``symmetrize_weights`` is + unknown, or ``"uberseg"`` is selected without ``seg_cat_path``. """ @@ -812,6 +821,7 @@ def __init__( epoch_masked_fraction_cut=EPOCH_MASKED_FRACTION_CUT, defect_fill="noise", epoch_interpolated_defect_radius=EPOCH_INTERPOLATED_DEFECT_RADIUS, + symmetrize_weights="interpolated", ): # Base count = catalogue + vignets, excluding the f_wcs headers (passed @@ -835,6 +845,11 @@ def __init__( f"Unknown DEFECT_FILL '{defect_fill}'; expected one of" + f" {DEFECT_FILLS}" ) + if symmetrize_weights not in WEIGHT_SYMMETRIZATIONS: + raise ValueError( + f"Unknown SYMMETRIZE_WEIGHTS '{symmetrize_weights}'; expected" + + f" one of {WEIGHT_SYMMETRIZATIONS}" + ) # Fail fast at construction (not deep in the per-epoch loop) when # uberseg is requested without its segmentation input (shapepipe#776). @@ -892,6 +907,7 @@ def __init__( self._epoch_interpolated_defect_radius = ( epoch_interpolated_defect_radius ) + self._symmetrize_weights = symmetrize_weights self._w_log = w_log @@ -1380,6 +1396,7 @@ def process(self): dilate_neighbour=self._dilate_neighbour, metacal_psf=self._metacal_psf, defect_fill=self._defect_fill, + symmetrize_weights=self._symmetrize_weights, ) except Exception as ee: self._w_log.info( @@ -1488,8 +1505,9 @@ def prepare_postage_stamps( stamp lies in :func:`defect_mask`: flagged, zero-weight and invalid-RMS pixels, the set that :func:`prepare_ngmix_weights` zero-weights and fills under every ``blend_handling``. Counting flags alone would keep - epochs whose filled area exceeds the cut. The zero-weight orbit of - interpolated pixels keeps its light and is not counted. The default is + epochs whose filled area exceeds the cut. The quarter-turn copies that + weight symmetrization zero-weights keep their light and are not counted + (symmetrized-weight-holes). The default is 1/3; 10%, the DES Y3 and Y6 value, is the alternative to test. @sc [decision:shape_measurement.central_defect_veto,decision:shape_measurement.epoch_masked_fraction_cut,decision:shape_measurement.blend_handling] neighbour-markers-are-not-defects @@ -1949,16 +1967,16 @@ def seg_has_neighbour(seg, object_number): return bool(labels.size and np.any(labels != object_number)) -def uberseg_weight(weight, seg, object_number, dilate_neighbour=0): - """Zero a stamp's weight on neighbour-side pixels — the UberSeg treatment. +def uberseg_mask(seg, object_number, dilate_neighbour=0): + """Neighbour-side pixels of a stamp: the UberSeg mask. UberSeg is Erin Sheldon's MEDS/ngmix neighbour mask (``esheldon/meds``, https://github.com/esheldon/meds — ``MEDS.get_uberseg`` / ``meds._uberseg.uberseg_tree``, in MEDS itself "adapted from Niall MacCrann and Joe Zuntz", and used in the DES shear pipeline). Each stamp pixel is assigned to the object whose segmentation footprint it lies nearest to — a - nearest-segment Voronoi partition — and only pixels assigned to the central - object keep their weight. + nearest-segment Voronoi partition — and the pixels assigned to a neighbour + are masked. This function **reimplements** the partition rather than depending on ``meds``: it is a five-line ``scipy.spatial.cKDTree`` nearest-neighbour @@ -1976,69 +1994,61 @@ def uberseg_weight(weight, seg, object_number, dilate_neighbour=0): Because the partition is by distance to the nearest footprint, the pixels surviving around a compact central object form a single connected, roughly circular core; the "circularisation" is emergent geometry, not a - separate aperture. Only the weight changes: neighbour-side pixels are - handed to ngmix as a hard mask (weight = 0) and keep their image values, - so metacal shears the neighbour's light along with the target's. + separate aperture. :func:`prepare_ngmix_weights` zeroes the weight of the + masked pixels and keeps their image values, so metacal shears the + neighbour's light along with the target's. Parameters ---------- - weight : numpy.ndarray - Per-pixel weight (inverse variance) map for the stamp. seg : numpy.ndarray - Segmentation map on the same grid as ``weight``: 0 for sky, the - SExtractor object number for each detected object's footprint. + Segmentation stamp: 0 for sky, the SExtractor object number for each + detected object's footprint. object_number : int Segmentation label of the central object — its SExtractor ``NUMBER`` (``obj_id``), authoritative because seg labels are the NUMBERs of the same SE run. Not the centre-pixel label, which a few-pixel coadd-vs- epoch offset can steal for a neighbour and so invert this mask. dilate_neighbour : int, optional - Enlarge the neighbour mask by this many binary-dilation iterations - (4-connected, ~one pixel per iteration) on top of the base Voronoi - partition, to absorb the few-pixel coadd-vs-epoch registration offset - the coadd-seg overlay accepts (shapepipe#776, decision on seg source). - ``0`` (the default) recovers the pure Sheldon UberSeg mask, byte-for- - byte. The dilation is additive: it only ever zeros more pixels, never - restores a base-masked one, so over-masking a boundary pixel costs a - little central-object S/N but never leaks neighbour flux into the fit. + Enlarge the mask by this many binary-dilation iterations of the + neighbour footprints (4-connected, ~one pixel per iteration) on top of + the Voronoi partition, to absorb the few-pixel coadd-vs-epoch + registration offset the coadd-seg overlay accepts (shapepipe#776, + decision on seg source). ``0`` (the default) is the pure Sheldon + UberSeg mask. The dilation only ever adds pixels, so over-masking a + boundary pixel costs a little central-object S/N but never leaks + neighbour flux into the fit. Returns ------- - numpy.ndarray - Copy of ``weight`` with neighbour-side pixels zeroed. + numpy.ndarray of bool + ``True`` on pixels nearer a neighbour's footprint than the central + object's; all ``False`` when the stamp holds no neighbour. @sc [decision:shape_measurement.blend_handling] """ - weight = np.copy(weight) + seg = np.asarray(seg) + masked = np.zeros(seg.shape, dtype=bool) obj_pix = np.argwhere(seg != 0) - # No detected footprint, or only sky plus the central object: there is - # no neighbour to mask, so the weight is unchanged (cf. MEDS' early - # ``len(np.unique(seg)) == 2`` return). - if obj_pix.shape[0] == 0: - return weight labels = seg[seg != 0] + # No neighbour footprint on the stamp: nothing to mask (cf. MEDS' early + # ``len(np.unique(seg)) == 2`` return). if np.all(labels == object_number): - return weight + return masked - # Nearest segmentation pixel for every stamp pixel; zero the weight - # wherever that nearest footprint belongs to a neighbour rather than to - # the central object. + # Nearest segmentation pixel for every stamp pixel; mask wherever that + # nearest footprint belongs to a neighbour rather than to the central + # object. grid = np.indices(seg.shape).reshape(2, -1).T _, nearest = cKDTree(obj_pix).query(grid) - nearest_label = labels[nearest].reshape(seg.shape) - weight[nearest_label != object_number] = 0.0 + masked |= labels[nearest].reshape(seg.shape) != object_number - # Enlarge the neighbour mask to absorb the coadd-vs-epoch offset: zero any - # pixel within ``dilate_neighbour`` of a neighbour footprint. Purely - # additive over the base Voronoi result above. if dilate_neighbour > 0: - neighbour_mask = binary_dilation( + masked |= binary_dilation( (seg != 0) & (seg != object_number), iterations=dilate_neighbour, ) - weight[neighbour_mask] = 0.0 - return weight + return masked def defect_mask(weight, flag, bkg_rms=None): @@ -2215,15 +2225,17 @@ def prepare_ngmix_weights( gal, weight, flag, rng, bkg_rms=None, blend_handling="noisefill", seg=None, object_number=None, dilate_neighbour=0, defect_fill="noise", neighbour=None, + symmetrize_weights="interpolated", ): """Build one epoch's image, weight map and noise image for ngmix. - Defect pixels (:func:`defect_mask`: flagged, zero-weight or invalid-RMS - pixels) get weight 0 and are filled under either ``blend_handling``: - by an independent noise realisation at their background RMS - (:func:`fill_defects`), or under ``defect_fill="interpolate"``, for - short defect runs, by an interpolant of the clean pixels around them. - ``blend_handling`` decides only the neighbour pixels. + Every stamp pixel falls in one of three classes. A clean pixel keeps its + light and its weight. A pixel whose light is replaced keeps neither: a + noise-filled defect, or under ``"noisefill"`` a marked neighbour pixel. + A pixel whose light stays but whose weight is zero is a hole in the + likelihood only: an interpolated defect, or under ``"uberseg"`` a + neighbour-side pixel. ``symmetrize_weights`` names which holes of this + third class are made four-fold symmetric. @sc [decision:shape_measurement.defect_fill,decision:shape_measurement.blend_handling] defects-filled-whatever-the-blend-handling Every pixel of ``defect_mask`` is zero-weighted and filled whatever @@ -2250,27 +2262,58 @@ def prepare_ngmix_weights( raw and weighted. @sc [decision:shape_measurement.blend_handling] uberseg-ignores-markers - Under ``"uberseg"`` the neighbour markers are ignored: :func:`uberseg_weight` - zeroes the weight of pixels nearer a neighbour's segmentation footprint - than the target's and leaves their image values raw, because that light - is real sky that metacal shears along with the target; filling it with - noise would cut the target along an unsheared edge. + Under ``"uberseg"`` the neighbour markers are ignored: the pixels of + :func:`uberseg_mask`, nearer a neighbour's segmentation footprint than + the target's, get weight 0 and keep their raw image values, because that + light is real sky that metacal shears along with the target; filling it + with noise would cut the target along an unsheared edge. - @sc [decision:shape_measurement.defect_fill,label:physics] interpolated-defect-weight-orbit + @sc [decision:shape_measurement.defect_fill,label:physics] interpolated-defects-keep-light Under ``defect_fill="interpolate"`` the short defect runs (:func:`interpolable_defects`) take the interpolant of the clean pixels around them (:func:`interpolate_defects`) in the image and in the noise image alike, and the other defects are noise-filled. Interpolation - restores the object's light, which leaves the hole in the likelihood: - ngmix fits a Gaussian to a non-Gaussian profile, and a one-sided - zero-weight hole pulls that fit. Once interpolated, a column 8 px from a - galaxy with half-light radius 0.5" through a 0.7" PSF still gives - c1 = -1.3e-3. The weight is therefore also zero on the quarter-turn - orbit of the interpolated pixels, whose light stays, which cancels that - spin-2 term: c1 = +2e-6 for the same column. Only the weights are - symmetrized, not the fill: interpolating the orbit as well would trade - true light for interpolated light over four times the area, raising m11 - for a 3-px bleed 6 px from the same galaxy from +0.19% to +0.89%. + restores the object's light and keeps the pixel's zero weight. Only the + weights are symmetrized (symmetrized-weight-holes), not the fill: + interpolating the orbit as well would trade true light for interpolated + light over four times the area, raising m11 for a 3-px bleed 6 px from a + galaxy with half-light radius 0.5" through a 0.7" PSF from +0.19% to + +0.89%. + + @sc [decision:shape_measurement.weight_symmetrization,label:physics] symmetrized-weight-holes + ngmix fits a Gaussian to a profile that is not one, so a zero-weight hole + on one side of the object pulls the fit along that side: a spin-2 term. + One step cancels it: the weight is also zeroed on the three quarter-turn + copies, about the stamp centre, of the holes ``symmetrize_weights`` + names, and their light stays, so the likelihood has no preferred axis. + Under ``"interpolated"`` (the default) those are the interpolated + defects: an interpolated column 8 px from a galaxy with half-light + radius 0.5" through a 0.7" PSF gives c1 = -1.3e-3 unsymmetrized and + +2e-6 symmetrized. ``"interpolated_and_neighbours"`` adds the uberseg + neighbour side. On its own that hole is a strong pull: a neighbour + footprint with no light 10 px from the same galaxy gives c1 = -7.8e-3, + and 2e-6 symmetrized. But a real neighbour leaves light in the target's + cell, which pulls the other way, and the one-sided hole partly offsets + it. With a neighbour of a tenth of the target's flux at 10 px, c1 goes + from +1.1e-3 unsymmetrized to +1.2e-2 symmetrized; at equal flux, from + +0.13 to +0.21. The symmetrized hole also keeps few pixels: at 10 px, + 117 weighted pixels of 1580 and 97% of the target's S/N (99% without); + at 7 px, 25 pixels and 71%; at 5 px, none. So the neighbour side stays + unsymmetrized by default; the option is for a neighbour-subtracted + image or an A/B arm. Pixels whose light is replaced (noise-filled + defects, noisefill neighbours) are never symmetrized: their hole is in + the image metacal shears, and a weight orbit leaves it there. For a + noise-filled column 10 px from the 0.5" galaxy, m11 stays at -0.53% + with the orbit (-0.54% without) while c1 moves from -1.1e-4 to + +0.3e-4; for a 3-px bleed, c1 moves from +1.4e-4 to +3.2e-4. Under + ``"noisefill"`` with the noise fill no pixel is in a symmetrized class, + so ``symmetrize_weights`` changes nothing there. The rotations are + about the stamp centre, where the extractor puts the object to within + half a pixel per axis (the coadd-centroid offset); the measurements + above draw the object uniformly within that half pixel, so the + cancellation they show holds at the offsets the pipeline produces. + Measured with ``tests/helpers/defect_response.py``, 6 seeds, full + response matrix; guarded by ``tests/science/test_weight_symmetrization.py``. Parameters ---------- @@ -2291,9 +2334,8 @@ def prepare_ngmix_weights( blend_handling : {"noisefill", "uberseg"}, optional Neighbour treatment. ``"noisefill"`` (default) zero-weights and noise-fills the ``neighbour`` pixels. ``"uberseg"`` ignores - ``neighbour``, zeroes the weight of every pixel closer to a - neighbour's segmentation footprint than to the central object's and - keeps its raw image value (see :func:`uberseg_weight`). + ``neighbour``, zeroes the weight of the pixels of + :func:`uberseg_mask` and keeps their raw image values. seg : numpy.ndarray, optional Segmentation map on the stamp grid (object NUMBERs). Required for ``blend_handling="uberseg"``; ignored otherwise. @@ -2301,7 +2343,7 @@ def prepare_ngmix_weights( Central object's segmentation label. Required for ``blend_handling="uberseg"``; ignored otherwise. dilate_neighbour : int, optional - Neighbour-mask dilation iterations, passed to :func:`uberseg_weight` + Neighbour-mask dilation iterations, passed to :func:`uberseg_mask` under ``blend_handling="uberseg"``; ignored otherwise. defect_fill : {"noise", "interpolate"}, optional ``"noise"`` (default) noise-fills every defect; ``"interpolate"`` @@ -2309,6 +2351,11 @@ def prepare_ngmix_weights( neighbour : numpy.ndarray of bool, optional The epoch's neighbour mask (``Postage_stamp.neighbours``); read only under ``blend_handling="noisefill"``. ``None`` marks no pixel. + symmetrize_weights : {"interpolated", "interpolated_and_neighbours", "none"}, optional + Which zero-weight pixels whose light stays also zero the weight of + their quarter-turn copies (symmetrized-weight-holes): the + interpolated defects (the default), those and the uberseg neighbour + side, or none. Symmetrizing needs a square stamp. Returns ------- @@ -2324,9 +2371,9 @@ def prepare_ngmix_weights( Raises ------ ValueError - If ``blend_handling`` or ``defect_fill`` is unknown, or - ``"uberseg"`` lacks ``seg`` or ``object_number``. - @sc [decision:masking.pixel_mask_source,decision:shape_measurement.blend_handling,decision:shape_measurement.defect_fill,decision:shape_measurement.galaxy_pixel_weights] + If ``blend_handling``, ``defect_fill`` or ``symmetrize_weights`` is + unknown, or ``"uberseg"`` lacks ``seg`` or ``object_number``. + @sc [decision:masking.pixel_mask_source,decision:shape_measurement.blend_handling,decision:shape_measurement.defect_fill,decision:shape_measurement.galaxy_pixel_weights,decision:shape_measurement.weight_symmetrization] """ if defect_fill not in DEFECT_FILLS: raise ValueError( @@ -2338,6 +2385,11 @@ def prepare_ngmix_weights( f"Unknown blend_handling '{blend_handling}'; expected one of" + f" {BLEND_HANDLINGS}" ) + if symmetrize_weights not in WEIGHT_SYMMETRIZATIONS: + raise ValueError( + f"Unknown SYMMETRIZE_WEIGHTS '{symmetrize_weights}'; expected one" + + f" of {WEIGHT_SYMMETRIZATIONS}" + ) if blend_handling == "uberseg" and (seg is None or object_number is None): raise ValueError( "blend_handling='uberseg' requires a segmentation map and the" @@ -2347,23 +2399,17 @@ def prepare_ngmix_weights( ) defect = defect_mask(weight, flag, bkg_rms) - # Marked neighbour pixels that noisefill removes (noisefill-fills-markers). - removed_neighbour = ( - np.asarray(neighbour, dtype=bool) - if blend_handling == "noisefill" and neighbour is not None - else np.zeros_like(defect) - ) + no_pixel = np.zeros_like(defect) + # Neighbour pixels whose light noisefill replaces (noisefill-fills-markers) + # or whose weight uberseg zeroes (uberseg-ignores-markers). + removed_neighbour, neighbour_side = no_pixel, no_pixel + if blend_handling == "noisefill" and neighbour is not None: + removed_neighbour = np.asarray(neighbour, dtype=bool) + elif blend_handling == "uberseg": + neighbour_side = uberseg_mask(seg, object_number, dilate_neighbour) + # Pixels whose light the image keeps raw. clean = ~(defect | removed_neighbour) - interpolated = ( - interpolable_defects(defect) - if defect_fill == "interpolate" - else np.zeros_like(defect) - ) - # The quarter-turn orbit of the interpolated pixels carries no weight - # (interpolated-defect-weight-orbit). - weighted = ( - clean & ~fourfold(interpolated) if interpolated.any() else clean - ) + weighted = clean & ~neighbour_side if bkg_rms is None: sig_noise = sigma_mad(gal) @@ -2400,24 +2446,29 @@ def prepare_ngmix_weights( noise_img = rng.standard_normal(gal.shape) * sig_noise noise_img_gal = rng.standard_normal(gal.shape) * sig_noise gal_filled = fill_defects(gal, ~clean, noise_img_gal) - if interpolated.any(): + interpolated = no_pixel + if defect_fill == "interpolate" and defect.any(): # One operator for the image and the noise image, supported on the # pixels the image keeps (interpolation-support-is-the-kept-image); a # pixel whose support is degenerate, or that noisefill removes as a # neighbour, keeps its noise fill. - filled = interpolate_defects([gal, noise_img], ~clean, interpolated) - done = ( - interpolated - & ~removed_neighbour - & np.all(np.isfinite(filled), axis=0) - ) - gal_filled[done] = filled[0][done] - noise_img = np.where(done, filled[1], noise_img) - - if blend_handling == "uberseg": - weight_map = uberseg_weight( - weight_map, seg, object_number, dilate_neighbour=dilate_neighbour + target = interpolable_defects(defect) + filled = interpolate_defects([gal, noise_img], ~clean, target) + interpolated = ( + target & ~removed_neighbour & np.all(np.isfinite(filled), axis=0) ) + gal_filled[interpolated] = filled[0][interpolated] + noise_img = np.where(interpolated, filled[1], noise_img) + + # The zero-weight pixels whose light stays, of the classes + # symmetrize_weights names (symmetrized-weight-holes). + holes = { + "interpolated": interpolated, + "interpolated_and_neighbours": interpolated | neighbour_side, + "none": no_pixel, + }[symmetrize_weights] + if holes.any(): + weight_map[fourfold(holes)] = 0.0 return gal_filled, weight_map, noise_img @@ -2427,6 +2478,7 @@ def make_ngmix_observation( bkg_rms=None, centroid_source="wcs", offset=None, blend_handling="noisefill", seg=None, object_number=None, dilate_neighbour=0, defect_fill="noise", neighbour=None, + symmetrize_weights="interpolated", ): """Build an ngmix Observation for a single galaxy epoch. @@ -2486,6 +2538,9 @@ def make_ngmix_observation( default is ``"noise"``. neighbour : numpy.ndarray of bool, optional Neighbour mask passed through to :func:`prepare_ngmix_weights`. + symmetrize_weights : {"interpolated", "interpolated_and_neighbours", "none"}, optional + Weight symmetrization passed through to + :func:`prepare_ngmix_weights`; the default is ``"interpolated"``. Returns ------- @@ -2515,7 +2570,7 @@ def make_ngmix_observation( gal, weight, flag, rng, bkg_rms=bkg_rms, blend_handling=blend_handling, seg=seg, object_number=object_number, dilate_neighbour=dilate_neighbour, defect_fill=defect_fill, - neighbour=neighbour, + neighbour=neighbour, symmetrize_weights=symmetrize_weights, ) if centroid_source == "hsm": @@ -2747,6 +2802,7 @@ def do_ngmix_metacal( stamp, prior, flux_guess, rng, centroid_source="wcs", blend_handling="noisefill", object_number=None, dilate_neighbour=0, metacal_psf="fitgauss", defect_fill="noise", + symmetrize_weights="interpolated", ): """Do Ngmix Metacal. @@ -2796,6 +2852,9 @@ def do_ngmix_metacal( defect_fill : {"noise", "interpolate"}, optional Defect fill passed through to :func:`make_ngmix_observation`; the default is ``"noise"``. + symmetrize_weights : {"interpolated", "interpolated_and_neighbours", "none"}, optional + Weight symmetrization passed through to + :func:`make_ngmix_observation`; the default is ``"interpolated"``. Returns ------- @@ -2806,7 +2865,7 @@ def do_ngmix_metacal( dict (:func:`average_original_psf`). The two PSF dicts share keys but describe different PSFs; the named fields guard against transposing them. Unpacks positionally as ``resdict, psf_res, psf_orig_res``. - @sc [decision:shape_measurement.defect_fill,decision:shape_measurement.metacal_scheme] + @sc [decision:shape_measurement.defect_fill,decision:shape_measurement.metacal_scheme,decision:shape_measurement.weight_symmetrization] """ n_epoch = len(stamp.gals) if n_epoch == 0: @@ -2833,6 +2892,7 @@ def do_ngmix_metacal( neighbour=( stamp.neighbours[n_e] if n_e < len(stamp.neighbours) else None ), + symmetrize_weights=symmetrize_weights, ) gal_obs_list.append(gal_obs) diff --git a/src/shapepipe/modules/ngmix_runner.py b/src/shapepipe/modules/ngmix_runner.py index 0b971492d..227097a73 100644 --- a/src/shapepipe/modules/ngmix_runner.py +++ b/src/shapepipe/modules/ngmix_runner.py @@ -197,6 +197,20 @@ def ngmix_runner( else: epoch_interpolated_defect_radius = EPOCH_INTERPOLATED_DEFECT_RADIUS + # SYMMETRIZE_WEIGHTS (optional): which zero-weight pixels whose light + # stays also zero the weight of their quarter-turn copies about the stamp + # centre, so no one-sided hole in the likelihood pulls the fit. + # "interpolated" (default): the interpolated defects; + # "interpolated_and_neighbours": those and the uberseg neighbour side; + # "none". Nothing changes under BLEND_HANDLING = noisefill with + # DEFECT_FILL = noise. See prepare_ngmix_weights. + if config.has_option(module_config_sec, "SYMMETRIZE_WEIGHTS"): + symmetrize_weights = config.get( + module_config_sec, "SYMMETRIZE_WEIGHTS" + ) + else: + symmetrize_weights = "interpolated" + # Check PSF vignets first: if all are empty dicts {}, the exposures for this # tile are absent from the PSF dictionary and no shape measurement is possible. # This check must come before reading image vignets to avoid a C-level malloc @@ -261,6 +275,7 @@ def ngmix_runner( epoch_masked_fraction_cut=epoch_masked_fraction_cut, defect_fill=defect_fill, epoch_interpolated_defect_radius=epoch_interpolated_defect_radius, + symmetrize_weights=symmetrize_weights, ) # Process ngmix shape measurement and metacalibration diff --git a/tests/helpers/defect_response.py b/tests/helpers/defect_response.py index c3e3ac157..3d10d1dbf 100644 --- a/tests/helpers/defect_response.py +++ b/tests/helpers/defect_response.py @@ -1,5 +1,6 @@ -"""Full-matrix metacal recovery for fixed detector defects.""" +"""Full-matrix metacal recovery for fixed detector defects and neighbours.""" +import galsim import numpy as np from shapepipe.modules.ngmix_package import ngmix as ngm @@ -7,9 +8,41 @@ from tests.helpers.metacal_sim import build_stamp +N = 51 +PIXEL_SCALE = 0.1857 +FLUX = 1000.0 + + +def neighbour_scene(offset, shear, psf, psf_shear, hlr=0.5, flux_ratio=1.0, + seg_radius=3): + """A neighbour's image and the stamp's segmentation map. + + The neighbour is an exponential galaxy whose centre lies ``offset`` + (row, col) pixels from the stamp centre, sheared with the scene (its + profile and its position) and convolved with the detector-frame PSF. + The segmentation map labels a disk of ``seg_radius`` px around the + stamp centre 1 (the target) and one around the unsheared neighbour + position 2. + """ + row, col = offset + matrix = galsim.Shear(g1=shear[0], g2=shear[1]).getMatrix() + dx, dy = matrix @ np.array([col, row]) * PIXEL_SCALE + image = galsim.Convolve( + galsim.Moffat(beta=2.5, fwhm=psf).shear( + g1=psf_shear[0], g2=psf_shear[1]), + galsim.Exponential(half_light_radius=hlr, flux=FLUX * flux_ratio) + .shear(g1=shear[0], g2=shear[1]), + ).shift(dx, dy).drawImage(nx=N, ny=N, scale=PIXEL_SCALE).array + rows, cols = np.indices((N, N)) - N // 2 + seg = np.zeros((N, N), dtype=np.int32) + seg[rows ** 2 + cols ** 2 <= seg_radius ** 2] = 1 + seg[(rows - row) ** 2 + (cols - col) ** 2 <= seg_radius ** 2] = 2 + return image, seg + + def defect_response(bad, hlr=0.5, psf=0.7, seeds=range(4), psf_shear=(0.0, 0.0), options=None, known_rms=True, - defect_value=1e3): + defect_value=1e3, neighbour=None): """Recover c and M = inverse(mean R) A - I with paired seed errors. Null pairs rotate the pixels by 90 degrees while pre-rotating the PSF @@ -20,10 +53,20 @@ def defect_response(bad, hlr=0.5, psf=0.7, seeds=range(4), The flagged pixels ``bad`` hold ``defect_value`` (far above the galaxy's peak) rather than sky, as a hot column or bleed would, so any defect value that reaches metacal shows up as a bias. + + ``neighbour``, if given, holds the keyword arguments of + :func:`neighbour_scene` but ``shear``, ``psf`` and ``psf_shear``: a + neighbour fixed in detector coordinates like the defect. Its + segmentation footprint is the stamp's neighbour mask under + ``blend_handling="noisefill"``, and the segmentation map rides the stamp + for ``"uberseg"`` (pass ``object_number=1`` in ``options``). + + The result also holds ``s2n``, the mean over arms of the no-shear fit's + S/N, the price of the pixels a treatment drops. """ gamma = 0.02 options = {} if options is None else options - samples = [] + samples, s2n = [], [] for seed in seeds: def arm(axis, sign, rotation=0): shear = [0.0, 0.0] @@ -41,10 +84,18 @@ def arm(axis, sign, rotation=0): data[0] = [np.rot90(a).copy() for a in data[0]] data[1] = [np.rot90(a).copy() for a in data[1]] offset = np.array([-offset[1], offset[0]]) + seg = None + if neighbour is not None: + light, seg = neighbour_scene( + shear=shear, psf=psf, psf_shear=psf_shear, **neighbour) + data[0] = [a + light for a in data[0]] data[0] = [np.where(bad, defect_value, a) for a in data[0]] data[4] = [bad.astype(np.int32)] stamp = build_stamp(data) stamp.offsets = [offset] + if seg is not None: + stamp.segs = [seg] + stamp.neighbours = [seg == 2] if known_rms: stamp.bkg_rms = [np.full((51, 51), 1e-4)] rng = np.random.RandomState(seed) @@ -53,6 +104,7 @@ def arm(axis, sign, rotation=0): centroid_source="wcs", **options, ) assert all(result[t]["flags"] == 0 for t in ngm.METACAL_TYPES) + s2n.append(result["noshear"]["s2n"]) e = np.asarray(result["noshear"]["g"]) response = np.column_stack([ (np.asarray(result[p]["g"]) - result[m]["g"]) / 0.02 @@ -84,6 +136,7 @@ def arm(axis, sign, rotation=0): c_err=cb.std(axis=0).tolist(), m_err=np.diag(mb.std(axis=0)).tolist(), response=r.mean(axis=0).tolist(), + s2n=float(np.mean(s2n)), seed_groups=[dict(e=s[0].tolist(), R=s[1].tolist(), A=s[2].tolist(), Rm=s[3].tolist()) for s in samples]) diff --git a/tests/module/test_ngmix.py b/tests/module/test_ngmix.py index 2e9fa9a52..fadc4b991 100644 --- a/tests/module/test_ngmix.py +++ b/tests/module/test_ngmix.py @@ -669,6 +669,7 @@ def test_process_counts_flagged_fits_across_batches(tmp_path, monkeypatch, flags inst._epoch_interpolated_defect_radius = ( module.EPOCH_INTERPOLATED_DEFECT_RADIUS ) + inst._symmetrize_weights = "interpolated" inst._save_batch = 1 inst._zero_point = 30. inst._output_dir = str(tmp_path) @@ -772,6 +773,7 @@ def capture(stamp, prior, flux_guess, rng, **kwargs): inst._epoch_interpolated_defect_radius = ( module.EPOCH_INTERPOLATED_DEFECT_RADIUS ) + inst._symmetrize_weights = "interpolated" inst._save_batch = -1 inst._w_log = _RecordingLogger() diff --git a/tests/module/test_ngmix_defect_fill.py b/tests/module/test_ngmix_defect_fill.py index 3f242d3a8..ed59ef2f6 100644 --- a/tests/module/test_ngmix_defect_fill.py +++ b/tests/module/test_ngmix_defect_fill.py @@ -43,7 +43,7 @@ prepare_ngmix_weights, prepare_postage_stamps, split_tile_markers, - uberseg_weight, + uberseg_mask, ) @@ -87,7 +87,8 @@ def _uberseg_seg(n): def test_filled_set_is_the_defect_set(stamp, blend_handling, seed): """Filled pixels are exactly the defects (flag, zero weight, bad RMS). They carry zero weight and look like noise. Every other pixel keeps its - raw value, under either BLEND_HANDLING. + raw value, under either BLEND_HANDLING. Under uberseg the weight is also + zero on the neighbour side, whose light stays. Failure modes: * a defect source (flag, zero weight, bad RMS) is left out of the fill; @@ -118,8 +119,7 @@ def test_filled_set_is_the_defect_set(stamp, blend_handling, seed): npt.assert_array_equal(filled, defect, "filled set is not the defect set") assert np.all(np.abs(gal_out[filled]) < 10.0), "fill is not unit noise" neighbour = ( - uberseg_weight(np.ones((n, n)), kwargs["seg"], 1, dilate_neighbour=1) - == 0.0 + uberseg_mask(kwargs["seg"], 1, dilate_neighbour=1) if blend_handling == "uberseg" else np.zeros((n, n), dtype=bool) ) @@ -607,8 +607,7 @@ def test_interpolated_fill_and_its_weights(blend_handling): ) neighbour = ( - uberseg_weight(np.ones_like(gal), kwargs["seg"], 1, dilate_neighbour=1) - == 0.0 + uberseg_mask(kwargs["seg"], 1, dilate_neighbour=1) if kwargs else np.zeros_like(defect) ) npt.assert_array_equal(w_out == 0.0, defect | fourfold(target) | neighbour) @@ -620,6 +619,106 @@ def test_interpolated_fill_and_its_weights(blend_handling): npt.assert_allclose(noise_out[target], refilled[target], atol=1e-5) +# --- SYMMETRIZE_WEIGHTS ---------------------------------------------------- + +def _symmetrization_case(blend_handling, defect_fill): + """The hot stamp with a neighbour in one corner, and its pixel classes: + those whose light is replaced, the interpolated defects and the uberseg + neighbour side (the last two keep their light at zero weight).""" + gal, weight, flag = _hot_stamp() + defect = flag != 0 + seg = np.zeros_like(flag) + seg[N_STAMP // 2, N_STAMP // 2] = 1 + seg[-12:, -12:] = 2 + marked = seg == 2 + kwargs = dict( + blend_handling=blend_handling, defect_fill=defect_fill, + seg=seg, object_number=1, dilate_neighbour=1, neighbour=marked, + ) + interpolated = ( + interpolable_defects(defect) & ~marked + if defect_fill == "interpolate" else np.zeros_like(defect) + ) + side = ( + uberseg_mask(seg, 1, dilate_neighbour=1) + if blend_handling == "uberseg" else np.zeros_like(defect) + ) + replaced = defect & ~interpolated + if blend_handling == "noisefill": + replaced |= marked + return gal, weight, flag, kwargs, replaced, interpolated, side + + +@pytest.mark.parametrize("defect_fill", ["noise", "interpolate"]) +@pytest.mark.parametrize("blend_handling", ["noisefill", "uberseg"]) +@pytest.mark.parametrize( + "symmetrize", ["interpolated", "interpolated_and_neighbours", "none"] +) +def test_symmetrized_weight_holes(blend_handling, defect_fill, symmetrize): + """The weight is zero on the pixels whose light is replaced, on the + interpolated defects and on the uberseg neighbour side, and on the + quarter turns of the classes SYMMETRIZE_WEIGHTS names. The switch moves + no image value and no noise draw. + + Failure modes: a noise-filled defect or noisefill neighbour is + symmetrized; the default symmetrizes the uberseg neighbour side (its + neighbour light then dominates the bias, see symmetrized-weight-holes), + or the opt-in value does not; the switch moves an image value or a + noise draw. + """ + gal, weight, flag, kwargs, replaced, interpolated, side = ( + _symmetrization_case(blend_handling, defect_fill) + ) + rms = np.ones_like(gal) + out = prepare_ngmix_weights( + gal, weight, flag, np.random.RandomState(3), bkg_rms=rms, + symmetrize_weights=symmetrize, **kwargs, + ) + unsymmetrized = prepare_ngmix_weights( + gal, weight, flag, np.random.RandomState(3), bkg_rms=rms, + symmetrize_weights="none", **kwargs, + ) + holes = { + "interpolated": interpolated, + "interpolated_and_neighbours": interpolated | side, + "none": np.zeros_like(side), + }[symmetrize] + npt.assert_array_equal( + out[1] == 0.0, replaced | interpolated | side | fourfold(holes) + ) + npt.assert_array_equal(out[0], unsymmetrized[0]) + npt.assert_array_equal(out[2], unsymmetrized[2]) + + +def test_symmetrization_changes_nothing_under_the_defaults(): + """Under noisefill with the noise fill every zero-weight pixel has its + light replaced, so every SYMMETRIZE_WEIGHTS value gives the same + image, weight map and noise image.""" + gal, weight, flag, kwargs, _, _, _ = _symmetrization_case( + "noisefill", "noise" + ) + rms = np.ones_like(gal) + outputs = [ + prepare_ngmix_weights( + gal, weight, flag, np.random.RandomState(8), bkg_rms=rms, + symmetrize_weights=value, **kwargs, + ) + for value in ("interpolated", "interpolated_and_neighbours", "none") + ] + for other in outputs[1:]: + for a, b in zip(outputs[0], other): + npt.assert_array_equal(a, b) + + +def test_unknown_symmetrize_weights_is_rejected(): + gal, weight, flag = _hot_stamp() + with pytest.raises(ValueError, match="SYMMETRIZE_WEIGHTS"): + prepare_ngmix_weights( + gal, weight, flag, np.random.RandomState(0), + symmetrize_weights=True, + ) + + def test_noise_is_the_default_fill(): """Leaving DEFECT_FILL unset is bit-identical to the noise fill.""" gal, weight, flag = _hot_stamp() diff --git a/tests/module/test_ngmix_uberseg.py b/tests/module/test_ngmix_uberseg.py index 2c8f54a77..a9d95486a 100644 --- a/tests/module/test_ngmix_uberseg.py +++ b/tests/module/test_ngmix_uberseg.py @@ -2,10 +2,10 @@ Covers the ``BLEND_HANDLING = uberseg`` option added to the ngmix module: -* :func:`uberseg_weight` — the Sheldon/MEDS nearest-segment Voronoi mask. +* :func:`uberseg_mask` — the Sheldon/MEDS nearest-segment Voronoi mask. Geometry assertions on a synthetic two-object stamp: neighbour-side pixels - are zeroed, the surviving central core is a *single connected* region (the - emergent "circularisation"), and the neighbour footprint is fully removed. + are masked, the surviving central core is a *single connected* region (the + emergent "circularisation"), and the neighbour footprint is fully masked. * :func:`prepare_ngmix_weights` under ``uberseg`` — neighbour-side pixels lose their weight and keep their raw image values, while defect pixels are noise-filled as under any blend handling (the defect fill itself is covered @@ -25,7 +25,7 @@ central_seg_label, prepare_ngmix_weights, seg_has_neighbour, - uberseg_weight, + uberseg_mask, ) @@ -53,33 +53,24 @@ def two_object_seg(npix=41, sep=12, r_central=3, r_neighbour=3): return seg, centre, neigh -def test_uberseg_zeros_neighbour_side_keeps_centre(): - """Neighbour-side pixels lose their weight; the central pixel keeps it.""" +def test_uberseg_masks_neighbour_side_keeps_centre(): + """Neighbour-side pixels are masked; the central footprint is not.""" seg, centre, neigh = two_object_seg() - weight = np.ones_like(seg, dtype=float) - out = uberseg_weight(weight, seg, object_number=1) + masked = uberseg_mask(seg, object_number=1) - # Central object pixel kept; deep-neighbour pixel zeroed. - assert out[centre] == 1.0 - assert out[neigh] == 0.0 - # Every neighbour-footprint pixel is removed from the fit. - assert np.all(out[seg == 2] == 0.0) - # Every central-footprint pixel survives. - assert np.all(out[seg == 1] == 1.0) - # The input weight is not mutated in place. - assert np.all(weight == 1.0) + assert not masked[centre] + assert masked[neigh] + assert np.all(masked[seg == 2]) + assert not np.any(masked[seg == 1]) def test_uberseg_core_is_single_connected_region(): """The surviving (kept-weight) region is one connected component — the emergent circular core of the nearest-segment Voronoi partition.""" seg, _, _ = two_object_seg() - weight = np.ones_like(seg, dtype=float) - out = uberseg_weight(weight, seg, object_number=1) - - kept = out > 0 + kept = ~uberseg_mask(seg, object_number=1) _, n_components = ndimage.label(kept) assert n_components == 1 # The partition splits the stamp: some pixels survive, some are masked. @@ -91,28 +82,26 @@ def test_uberseg_partition_is_the_perpendicular_bisector(): half-plane beyond the footprint bisector: the left edge survives, the column past the neighbour is gone.""" seg, centre, neigh = two_object_seg(npix=41, sep=12) - weight = np.ones_like(seg, dtype=float) - out = uberseg_weight(weight, seg, object_number=1) + masked = uberseg_mask(seg, object_number=1) c_row, c_col = centre - assert out[c_row, 0] == 1.0 # far side from the neighbour: kept - assert out[c_row, -1] == 0.0 # neighbour side edge: masked + assert not masked[c_row, 0] # far side from the neighbour: kept + assert masked[c_row, -1] # neighbour side edge: masked def test_uberseg_no_neighbour_is_passthrough(): - """A stamp with only the central object (or empty seg) is unchanged.""" + """A stamp with only the central object (or empty seg) masks nothing.""" npix = 21 - weight = np.random.default_rng(0).random((npix, npix)) + 0.1 # Only the central object present. seg = np.zeros((npix, npix), dtype=np.int32) seg[8:13, 8:13] = 1 - npt.assert_array_equal(uberseg_weight(weight, seg, object_number=1), weight) + assert not uberseg_mask(seg, object_number=1, dilate_neighbour=2).any() # Wholly empty seg (no detections). empty = np.zeros((npix, npix), dtype=np.int32) - npt.assert_array_equal(uberseg_weight(weight, empty, object_number=1), weight) + assert not uberseg_mask(empty, object_number=1).any() # --- central_seg_label: centre-pixel identification (#776 decision 3) ------- @@ -162,7 +151,7 @@ def test_seg_has_neighbour(): # --- uberseg dilation: additive neighbour-mask enlargement (#776 dec. 2) ---- def test_uberseg_dilation_grows_neighbour_mask(): - """dilate_neighbour>0 zeros a strict superset of the base (dilate=0) mask, + """dilate_neighbour>0 masks a strict superset of the base (dilate=0) mask, and every base-masked pixel stays masked (additive-only). Geometric subtlety: for well-separated objects the base Voronoi cut (the @@ -171,59 +160,50 @@ def test_uberseg_dilation_grows_neighbour_mask(): crosses the bisector into the central Voronoi cell. A few iterations guarantee that crossing here (sep=8, r=3 -> ~2px footprint gap).""" seg, _, _ = two_object_seg(npix=41, sep=8) - weight = np.ones_like(seg, dtype=float) - out0 = uberseg_weight(weight, seg, object_number=1, dilate_neighbour=0) - out3 = uberseg_weight(weight, seg, object_number=1, dilate_neighbour=3) + out0 = uberseg_mask(seg, object_number=1, dilate_neighbour=0) + out3 = uberseg_mask(seg, object_number=1, dilate_neighbour=3) - # dilate=0 reproduces the validated no-dilation result byte-for-byte. - npt.assert_array_equal( - out0, uberseg_weight(weight, seg, object_number=1) - ) + # dilate=0 reproduces the validated no-dilation result. + npt.assert_array_equal(out0, uberseg_mask(seg, object_number=1)) # Every pixel masked at dilate=0 is still masked at dilate=3 (additive). - assert np.all(out3[out0 == 0.0] == 0.0) + assert np.all(out3[out0]) # And strictly more pixels are masked once the dilation crosses the # bisector into the central cell. - assert (out3 == 0.0).sum() > (out0 == 0.0).sum() + assert out3.sum() > out0.sum() def test_uberseg_dilation_zero_is_pure_sheldon(): """dilate_neighbour=0 is bit-identical to the default (no-kwarg) call and to the O(N^2) brute-force nearest-segment rule.""" seg, _, _ = two_object_seg(npix=25, sep=8) - weight = np.ones_like(seg, dtype=float) - out = uberseg_weight(weight, seg, object_number=1, dilate_neighbour=0) - npt.assert_array_equal(out, uberseg_weight(weight, seg, object_number=1)) - - obj = np.argwhere(seg != 0) - labels = seg[seg != 0] - brute = np.ones_like(weight) - for i in range(seg.shape[0]): - for j in range(seg.shape[1]): - d2 = (i - obj[:, 0]) ** 2 + (j - obj[:, 1]) ** 2 - if labels[np.argmin(d2)] != 1: - brute[i, j] = 0.0 - npt.assert_array_equal(out, brute) + out = uberseg_mask(seg, object_number=1, dilate_neighbour=0) + npt.assert_array_equal(out, uberseg_mask(seg, object_number=1)) + npt.assert_array_equal(out, _brute_force_uberseg(seg)) def test_uberseg_matches_bruteforce_nearest_segment(): """The cKDTree result equals the O(N^2) brute-force nearest-segment rule (Sheldon's non-C fallback) it stands in for.""" seg, _, _ = two_object_seg(npix=25, sep=8) - weight = np.ones_like(seg, dtype=float) - out = uberseg_weight(weight, seg, object_number=1) + npt.assert_array_equal( + uberseg_mask(seg, object_number=1), _brute_force_uberseg(seg) + ) + +def _brute_force_uberseg(seg): + """O(N^2) nearest-segment rule: True where the nearest footprint pixel + is not the central object's (label 1).""" obj = np.argwhere(seg != 0) labels = seg[seg != 0] - brute = np.ones_like(weight) + brute = np.zeros(seg.shape, dtype=bool) for i in range(seg.shape[0]): for j in range(seg.shape[1]): d2 = (i - obj[:, 0]) ** 2 + (j - obj[:, 1]) ** 2 - if labels[np.argmin(d2)] != 1: - brute[i, j] = 0.0 - npt.assert_array_equal(out, brute) + brute[i, j] = labels[np.argmin(d2)] != 1 + return brute # --- prepare_ngmix_weights: uberseg zeroes neighbour weights only --------- diff --git a/tests/science/test_weight_symmetrization.py b/tests/science/test_weight_symmetrization.py new file mode 100644 index 000000000..53e8982ab --- /dev/null +++ b/tests/science/test_weight_symmetrization.py @@ -0,0 +1,54 @@ +"""Weight symmetrization on the uberseg neighbour side. + +Physics invariant: a one-sided zero-weight hole pulls the Gaussian fit, and +zeroing the weight on its three quarter-turn copies cancels the pull. A +neighbour footprint with no light 10 px from a galaxy with half-light radius +0.5" through a 0.7" PSF gives c1 = -7.8e-3 under uberseg, and 2e-6 with +``symmetrize_weights="interpolated_and_neighbours"``. + +Why the default leaves the neighbour side unsymmetrized: a real neighbour +leaves light in the target's cell, which pulls the fit the other way, and +the one-sided hole partly offsets it. With a neighbour of a tenth of the +target's flux at 10 px, c1 = +1.1e-3 unsymmetrized and +1.2e-2 symmetrized. +If this second test turns red, the default deserves another look. +""" + +import numpy as np + +from tests.helpers.defect_response import defect_response + +N = 51 +SEEDS = range(6) +NO_DEFECT = np.zeros((N, N), dtype=bool) + + +def uberseg_response(flux_ratio, symmetrize): + return defect_response( + NO_DEFECT, hlr=0.5, psf=0.7, seeds=SEEDS, + options=dict(blend_handling="uberseg", object_number=1, + dilate_neighbour=1, symmetrize_weights=symmetrize), + neighbour=dict(offset=(0, 10), flux_ratio=flux_ratio, hlr=0.5, + seg_radius=4), + ) + + +def test_symmetrization_cancels_the_neighbour_side_hole(): + """Failure modes: the neighbour side is not symmetrized under + "interpolated_and_neighbours", or the rotation is not about the stamp + centre (the hole's pull survives). Positive control: the unsymmetrized + hole gives |c1| > 5e-3.""" + unsymmetrized = uberseg_response(0.0, "interpolated") + symmetrized = uberseg_response(0.0, "interpolated_and_neighbours") + assert abs(unsymmetrized["c"][0]) > 5e-3, unsymmetrized + assert np.abs(symmetrized["c"]).max() < 5e-5, symmetrized + assert np.abs(symmetrized["m"]).max() < 0.01, symmetrized + + +def test_symmetrizing_the_neighbour_side_exposes_neighbour_light(): + """The default ("interpolated") leaves the uberseg neighbour side + unsymmetrized because, with neighbour light in the image, symmetrizing + it raises c1 several-fold.""" + unsymmetrized = uberseg_response(0.1, "interpolated") + symmetrized = uberseg_response(0.1, "interpolated_and_neighbours") + assert abs(unsymmetrized["c"][0]) < 3e-3, unsymmetrized + assert symmetrized["c"][0] > 5e-3, symmetrized diff --git a/universes/committed.yaml b/universes/committed.yaml index b6ca2a830..74e873f0a 100644 --- a/universes/committed.yaml +++ b/universes/committed.yaml @@ -54,6 +54,7 @@ analyses: megacam_ccd_flip: megapipe_flip defect_fill: noise blend_handling: noisefill + weight_symmetrization: interpolated epoch_masked_fraction_cut: one_third central_defect_veto: fixed_radii catalogue_assembly: From da73f3a5c9249044dc0b247c3af5b5242298bba1 Mon Sep 17 00:00:00 2001 From: Cail Daley Date: Sat, 3 Oct 2026 01:29:40 +0200 Subject: [PATCH 18/30] ngmix: interpolation is the only defect fill, and the veto reads the fill's interpolated set Short defect runs are always interpolated and the rest noise-filled; DEFECT_FILL, SYMMETRIZE_WEIGHTS and the three epoch-cut keys are gone, and the radii and the masked fraction are module constants. central_defect_vetoes takes the interpolated set prepare_ngmix_weights uses, removed noisefill neighbours included, so a run that ends on a neighbour is vetoed at the noise-fill radius it gets. A non-finite interpolant raises instead of falling back to noise. Off-tile pixels get zero weight instead of a flag sentinel. Only interpolated holes are weight-symmetrized; the #815 neighbour-side option is dropped. The defect science tests merge into tests/science/test_defect_recovery.py, and the module tests lose their option plumbing. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01N4HLdzDyc3CiPLrgJ48GVo --- astra.yaml | 272 ++++---- .../ngmix_package/defect_interpolation.py | 55 +- src/shapepipe/modules/ngmix_package/ngmix.py | 521 ++++----------- src/shapepipe/modules/ngmix_runner.py | 68 +- tests/module/test_defect_interpolation.py | 75 ++- tests/module/test_ngmix.py | 20 +- tests/module/test_ngmix_defect_fill.py | 615 ++++-------------- tests/module/test_ngmix_uberseg.py | 8 +- ...terpolation.py => test_defect_recovery.py} | 42 +- tests/science/test_defect_veto.py | 104 --- tests/science/test_weight_symmetrization.py | 54 -- universes/committed.yaml | 2 +- 12 files changed, 540 insertions(+), 1296 deletions(-) rename tests/science/{test_defect_interpolation.py => test_defect_recovery.py} (68%) delete mode 100644 tests/science/test_defect_veto.py delete mode 100644 tests/science/test_weight_symmetrization.py diff --git a/astra.yaml b/astra.yaml index 588c5cf54..f67c84804 100644 --- a/astra.yaml +++ b/astra.yaml @@ -1423,54 +1423,67 @@ analyses: label: Image content of defect pixels before metacal rationale: >- A defect pixel (nonzero instrument flag, zero exposure weight, - invalid background RMS, or off-tile) gets weight 0 in prepare_ngmix_weights. Its - image value still matters: ngmix's metacal deconvolves, shears and - reconvolves an InterpolatedImage of the whole image and copies the - weights through, so a zero-weight pixel's content spreads into the - weighted pixels within about a PSF width. DES's ngmixer fills for - that reason: "it may be important for codes that take moments or use - FFTs". Under every BLEND_HANDLING and the default DEFECT_FILL = - noise, defect pixels are replaced with independent noise at the - per-pixel background RMS when supplied, or the stamp's robust noise - scale otherwise. The tile VIGNET holds -1e30 on other detections' - footprints and beyond the tile's edge (written by SExtractor). The runs of entirely -1e30 stamp - rows and columns that start at a stamp border (the off-image part of - a rectangle clip) are off-tile: the epoch holds the object's own - light there, so they are defects, flagged 2**10. The other markers, - including a neighbour footprint that completes an interior row - beside an off-tile band, are neighbour pixels, not defects; - blend_handling decides their treatment. The committed fill - uses the unsymmetrized defect set: DES symmetrized its masks, but - four-fold symmetrization quadruples m and still leaves an additive c1 + invalid background RMS, or off-tile) gets weight 0 in + prepare_ngmix_weights. Its image value still matters: ngmix's + metacal deconvolves, shears and reconvolves an InterpolatedImage of + the whole image and copies the weights through, so a zero-weight + pixel's content spreads into the weighted pixels within about a PSF + width. DES's ngmixer fills for that reason: "it may be important for + codes that take moments or use FFTs". [HARDCODED] Under every + BLEND_HANDLING, short defect runs are interpolated and the rest are + noise-filled (interpolate). The tile VIGNET holds -1e30 on other + detections' footprints and beyond the tile's edge (written by + SExtractor). The runs of entirely -1e30 stamp rows and columns that + start at a stamp border (the off-image part of a rectangle clip) are + off-tile: the epoch holds the object's own light there, so they are + defects, at zero exposure weight. The other markers, including a + neighbour footprint that completes an interior row beside an + off-tile band, are neighbour pixels, not defects; blend_handling + decides their treatment. The fill uses the unsymmetrized defect + set: DES symmetrized its masks, but four-fold symmetrization + quadruples m and still leaves an additive c1 (symmetrized_4fold_noise). The cost of not symmetrizing, a hole in - the galaxy light, is bounded by central_defect_veto. Noise stays the - default until a survey A/B against interpolation. + the galaxy light, is bounded by central_defect_veto. Interpolation + is the only fill because it lets interpolated defects come to 7 px + of the object, against a size-blind 10 px for noise fill, which a + 0.7 arcsec galaxy through a 0.9 arcsec PSF needs widened to 14 px. Values: VIGNETMAKER_RUNNER_RUN_1.MASKING = False; VIGNETMAKER_RUNNER_RUN_2.MASKING = False. - default: noise + default: interpolate options: - noise: - label: Independent noise on the unsymmetrized defect set - description: >- - Masked pixels get independent noise using the per-pixel - background RMS when supplied, or the stamp's robust noise scale - otherwise; their inverse-variance weights stay zero. Metacal's - fixnoise noise image covers every pixel. - insights: [mask_metacal_acts_on_whole_stamp, mask_bad_column_symmetrize] interpolate: label: Interpolate short bounded runs; noise-fill the rest description: >- - DEFECT_FILL = interpolate: row or column runs of at most - MAX_INTERPOLATED_RUN = 3 defect pixels that stop short of the - stamp border are Clough-Tocher interpolated from the clean pixels - within SUPPORT_RADIUS = 4 px; other defects are noise-filled. - PR #916 measures c1 = -1.3e-3 for a column - 8 px from a 0.5 arcsec galaxy and +2e-6 when the weight is also - zeroed on its quarter-turn orbit. For a 3-px bleed 6 px from that - galaxy, PR #916 measures m11 = +0.89% when the fill is - symmetrized, against +0.19% otherwise. + Row or column runs of at most MAX_INTERPOLATED_RUN = 3 defect + pixels whose neighbours beyond both ends are kept pixels (on + the stamp, not defects, and not noisefill neighbours) are + Clough-Tocher interpolated from the kept pixels within + SUPPORT_RADIUS = 4 px, averaged over the stamp's quarter turns, + in the image and the metacal noise image alike. Other defects + (wider holes, edge bands, runs ending on a noisefill neighbour) + get independent noise at the per-pixel background RMS when + supplied, or the stamp's robust noise scale otherwise. The + interpolated pixels keep weight 0 (weight_symmetrization). For a + 3-px bleed 6 px from a 0.5 arcsec galaxy, PR #916 measures + m11 = +0.89% when the fill is symmetrized, against +0.19% + otherwise. insights: [mask_interpolate_with_noise, mask_sharp_edges_ring] + noise: + label: Independent noise on every defect + description: >- + Every defect pixel gets independent noise; their + inverse-variance weights stay zero. + excluded: true + excluded_reason: >- + A noise-filled defect leaves a hole in the object's light that + metacal shears and the sky does not. Columns, 3-px bleeds and + single pixels then need the 10 px veto on 0.3 and 0.5 arcsec + galaxies through a 0.7 arcsec PSF, and 14 px on a 0.7 arcsec + galaxy through a 0.9 arcsec PSF (a 3-px bleed at 10 px gives + m11 = -6.8%). Interpolated, the same defects pass from 7 px on + all three (PR #922 bias scan). + insights: [mask_metacal_acts_on_whole_stamp, mask_bad_column_symmetrize] symmetrized_4fold_noise: label: Four-fold-symmetrized defect set (M | rot90 | rot180 | rot270), then noise fill description: Not implemented. @@ -1527,7 +1540,7 @@ analyses: mask_blend_bias_detection). Noise-filling the neighbour side would instead cut the target's own light along an unsheared boundary, a sharp edge that rings in the FFTs. The recommended comparison arm is - uberseg (weight-only), with defect_fill held equal across arms. + uberseg (weight-only); defect_fill is the same in both arms. default: noisefill options: noisefill: @@ -1566,88 +1579,93 @@ analyses: et al. 2020 found similar blend biases with and without MOF. insights: [mask_des_y1_uberseg_only, mask_blend_bias_detection] weight_symmetrization: - label: Four-fold symmetric weight holes (SYMMETRIZE_WEIGHTS) + label: Four-fold symmetric weight holes of interpolated defects rationale: >- ngmix fits a Gaussian to a profile that is not one, so a zero-weight - hole on one side of the object pulls the fit: a spin-2 term. One - step in prepare_ngmix_weights cancels it: the weight is also zeroed - on the three quarter-turn copies, about the stamp centre, of the - zero-weight pixels whose light stays, of the classes - SYMMETRIZE_WEIGHTS names. Pixels whose light is replaced - (noise-filled defects, noisefill neighbours) are never symmetrized: - the hole is then in the image metacal shears, and a weight orbit - leaves m unchanged (-0.53% against -0.54% for a noise-filled column - 10 px from a 0.5 arcsec galaxy) while moving c by about 1e-4 either - way. So under noisefill with the noise fill the option changes - nothing. The rotation is about the stamp centre, where the - extractor places the object to within half a pixel per axis; the - measurements draw the object uniformly within that half pixel. - Measured with tests/helpers/defect_response.py (6 seeds, full - response matrix, galaxy half-light radius 0.5 arcsec, 0.7 arcsec - PSF). + hole on one side of the object pulls the fit: a spin-2 term. + [HARDCODED] prepare_ngmix_weights cancels it for the interpolated + defects, whose light stays: the weight is also zeroed on their three + quarter-turn copies about the stamp centre, which keep their light. + Pixels whose light is replaced (noise-filled defects, noisefill + neighbours) are not symmetrized: the hole is then in the image + metacal shears, and a weight orbit leaves m unchanged (-0.53% + against -0.54% for a noise-filled column 10 px from a 0.5 arcsec + galaxy) while moving c by about 1e-4 either way. The uberseg + neighbour side is not symmetrized either + (interpolated_and_neighbours). The rotation is about the stamp + centre, where the extractor places the object to within half a + pixel per axis; the measurements draw the object uniformly within + that half pixel. Measured with tests/helpers/defect_response.py (6 + seeds, full response matrix, galaxy half-light radius 0.5 arcsec, + 0.7 arcsec PSF). default: interpolated options: interpolated: label: Symmetrize the interpolated defects description: >- - SYMMETRIZE_WEIGHTS = interpolated. An interpolated column 8 px - from the galaxy gives c1 = -1.3e-3 unsymmetrized and +2e-6 - symmetrized (PR #916). The uberseg neighbour side stays - one-sided. + An interpolated column 8 px from the galaxy gives c1 = -1.3e-3 + unsymmetrized and +2e-6 symmetrized (PR #916). insights: [mask_bad_column_symmetrize] interpolated_and_neighbours: label: Also symmetrize the uberseg neighbour side (#815) - description: >- - SYMMETRIZE_WEIGHTS = interpolated_and_neighbours. Alone, the - neighbour-side hole is a strong pull: a neighbour footprint with - no light 10 px away gives c1 = -7.8e-3 unsymmetrized and 2e-6 - symmetrized. A real neighbour leaves light in the target's - uberseg cell, which pulls the other way, and the one-sided hole - partly offsets it: for a neighbour of a tenth of the target's - flux at 10 px, c1 = +1.1e-3 unsymmetrized and +1.2e-2 - symmetrized; at equal flux, +0.13 and +0.21. The symmetrized - hole keeps 117 of 1580 weighted pixels and 97% of the target's - S/N at 10 px (99% unsymmetrized), 25 pixels and 71% at 7 px, - none at 5 px. For a neighbour-subtracted image, or as an A/B - arm beside uberseg. + excluded: true + excluded_reason: >- + Alone, the neighbour-side hole is a strong pull: a neighbour + footprint with no light 10 px away gives c1 = -7.8e-3 + unsymmetrized and 2e-6 symmetrized. But a real neighbour leaves + light in the target's uberseg cell, which pulls the other way, + and the one-sided hole partly offsets it: for a neighbour of a + tenth of the target's flux at 10 px, c1 = +1.1e-3 unsymmetrized + and +1.2e-2 symmetrized; at equal flux, +0.13 and +0.21. The + symmetrized hole also keeps 117 of 1580 weighted pixels and 97% + of the target's S/N at 10 px (99% unsymmetrized), 25 pixels and + 71% at 7 px, none at 5 px. Worth revisiting only on a + neighbour-subtracted image (blend_handling.mof_subtract). insights: [mask_bad_column_symmetrize, mask_uberseg_neighbour_bias] none: - label: No symmetrization (SYMMETRIZE_WEIGHTS = none) - description: >- - Every zero-weight hole stays one-sided; under DEFECT_FILL = - interpolate the column above gives c1 = -1.3e-3. + label: No symmetrization + excluded: true + excluded_reason: >- + Every zero-weight hole stays one-sided: the interpolated column + above gives c1 = -1.3e-3. central_defect_veto: label: Per-epoch veto on a defect near the stamp centre rationale: >- - An epoch is dropped when a defect pixel lies strictly closer to the - stamp centre than its fill's radius, beside the masked-fraction cut - in the epoch loop; EPOCH_CENTRAL_DEFECT_RADIUS = 0 disables it. - The tile VIGNET's -1e30 neighbour markers are not defects: all epochs - share the tile VIGNET, so a neighbour inside the radius would drop - every epoch. Off-tile pixels are defects, so an object near the - tile edge is vetoed. The veto reads only the defect mask, so it selects on nothing shear-responsive; for the same - reason the radii are fixed rather than scaled by galaxy size: 10 px - for noise-filled pixels (EPOCH_CENTRAL_DEFECT_RADIUS) and 7 px for - interpolated ones (EPOCH_INTERPOLATED_DEFECT_RADIUS, - feat/defect-interpolation). Calibrated on 51-px stamps at known RMS - and high S/N, through a round and a (0.05, 0.02) elliptical 0.7 - arcsec PSF, on galaxies of half-light radius 0.3 and 0.5 arcsec, and - for interpolation also 0.7 and 0.9 arcsec. Known limits: at 10 px, - wide defects through the elliptical PSF sit at the 1% bound (m11 = - -0.98%; -0.24% at 11 px), and noise fill needs 14 px for 0.7 and 0.9 - arcsec galaxies. Measured on feat/defect-fill-veto and - feat/defect-interpolation. The 7 px radius applies only under - DEFECT_FILL = interpolate; with the committed noise fill every defect - takes the 10 px radius. + [HARDCODED] An epoch is dropped when a defect pixel lies strictly + closer to the stamp centre than the radius of its fill: 10 px for + noise-filled pixels (EPOCH_CENTRAL_DEFECT_RADIUS) and 7 px for + interpolated ones (EPOCH_INTERPOLATED_DEFECT_RADIUS). The veto + reads the same interpolated set as the fill, including the runs + that end on a noisefill neighbour and so are noise-filled. The tile + VIGNET's -1e30 neighbour markers are not defects: all epochs share + the tile VIGNET, so a neighbour inside the radius would drop every + epoch. Off-tile pixels are defects, so an object near the tile edge + is vetoed. The veto reads only masks, so it selects on nothing + shear-responsive; for the same reason the radii are fixed rather + than scaled by galaxy size. Each radius is the smallest at which + the defects it keeps give |m11|, |m22| < 1% and |c1|, |c2| < 5e-4, + on 51-px stamps at known RMS and high S/N, through a round and a + (0.05, 0.02) elliptical 0.7 arcsec PSF, on galaxies of half-light + radius 0.3 and 0.5 arcsec, and for interpolated defects also 0.7 + arcsec through a 0.9 arcsec PSF. Known limit: at 10 px the widest + edge band the veto keeps sits at the 1% bound through the + elliptical PSF (m11 = -0.94% +/- 0.06%). Guarded by + tests/science/test_defect_recovery.py. Values: EPOCH_CENTRAL_DEFECT_RADIUS = 10; EPOCH_INTERPOLATED_DEFECT_RADIUS = 7. default: fixed_radii options: - disabled: - label: No central veto (EPOCH_CENTRAL_DEFECT_RADIUS = 0) fixed_radii: label: Fixed radii, 10 px for noise-filled and 7 px for interpolated defects + disabled: + label: No central veto + excluded: true + excluded_reason: >- + A column two pixels inside the 10 px radius, noise-filled, gives + |m| > 2% and |c| > 1e-3 on a 0.5 arcsec galaxy; a 3-px bleed + three pixels inside the 7 px radius, interpolated, gives + |c| > 1e-3 on a 0.3 arcsec galaxy. size_scaled_radius: label: Veto radius scaled by galaxy size excluded: true @@ -1657,25 +1675,31 @@ analyses: epoch_masked_fraction_cut: label: Per-epoch masked-fraction cut rationale: >- - An epoch whose stamp has more than EPOCH_MASKED_FRACTION_CUT - (default 1/3) of its pixels in the raw, unsymmetrized defect set is - dropped from the multi-epoch fit: flagged pixels (any nonzero - exposure flag bit), zero-weight pixels, invalid-RMS pixels and - off-tile pixels (whole -1e30 rows and columns at the tile VIGNET's - border, flag 2**10), the set defect_fill fills. On a 51-px stamp an object - within about 8.5 px of the tile edge fails the 1/3 cut. The other - -1e30 markers, neighbour footprints, are not counted: every epoch - shares the tile VIGNET, so a large neighbour would drop them all. An object with no surviving epoch has no - shape. Before the cut, an epoch is - dropped silently if its galaxy stamp is all zeros or its - background-subtracted sigma_mad is not positive. DES was stricter: - Y1 rejected any epoch with a masked or zero-weight pixel, or with - its central 4-pixel region masked; Y3 cut at 10% raw zero-weight - pixels; Y6 drops images more than 10% missing and keeps objects at - mfrac < 0.1 (mask_multi_epoch_drop). UNIONS has fewer epochs than - DES, so the cost in effective number density has to be measured, - not assumed. A defect near the centre is handled separately - (central_defect_veto). + [HARDCODED] An epoch whose stamp has more than + EPOCH_MASKED_FRACTION_CUT = 1/3 of its pixels in the raw, + unsymmetrized defect set is dropped from the multi-epoch fit: + flagged pixels (any nonzero exposure flag bit), zero-weight pixels + (off-tile pixels included) and invalid-RMS pixels, the set + defect_fill fills. On a 51-px stamp an object within about 8.5 px + of the tile edge fails the 1/3 cut. Neighbour footprints are not + counted: every epoch shares the tile VIGNET, so a large neighbour + would drop them all. An object with no surviving epoch has no + shape. Before the cut, an epoch is dropped silently if its galaxy + stamp is all zeros or its background-subtracted sigma_mad is not + positive. The cut guards no shear bias that was measured: defects + the central veto keeps, covering up to 58% of the stamp (two-sided + bands, corner Ls and frames 12 px or more from the centre), give + |m11|, |m22| <= 0.05% and |c| < 7e-5 on 0.3 and 0.5 arcsec galaxies + through a round and an elliptical 0.7 arcsec PSF (PR #922). The one + biased far-field case, a one-sided edge band reaching 10 px (31%), + is the veto radius's own bound (m11 = -0.83%). So 1/3 is a + robustness margin, and what it costs is the number of epochs the + real-tile tally shows it dropping. DES was stricter: Y1 rejected any + epoch with a masked or zero-weight pixel, or with its central + 4-pixel region masked; Y3 cut at 10% raw zero-weight pixels; Y6 + drops images more than 10% missing and keeps objects at + mfrac < 0.1 (mask_multi_epoch_drop). A defect near the centre is + handled separately (central_defect_veto). default: one_third options: one_third: @@ -1685,10 +1709,12 @@ analyses: defects. ten_percent: label: 10% (DES Y3 / Y6) - description: >- - Not a default; EPOCH_MASKED_FRACTION_CUT = 0.1. Matches DES Y3 - max_zero_weight_frac and - Y6 max_masked_fraction. + excluded: true + excluded_reason: >- + Matches DES Y3 max_zero_weight_frac and Y6 max_masked_fraction, + but far-field defects bias nothing measured up to 58% of the + stamp, so 10% would only discard epochs, among them every edge + band more than 5 px wide. insights: [mask_multi_epoch_drop] any_masked: label: Any masked pixel (DES Y1) diff --git a/src/shapepipe/modules/ngmix_package/defect_interpolation.py b/src/shapepipe/modules/ngmix_package/defect_interpolation.py index ae0d53c41..07bea48db 100644 --- a/src/shapepipe/modules/ngmix_package/defect_interpolation.py +++ b/src/shapepipe/modules/ngmix_package/defect_interpolation.py @@ -1,7 +1,6 @@ """DEFECT INTERPOLATION. -Clough-Tocher interpolation of short defect runs before metacal, selected -with ``DEFECT_FILL = interpolate`` (see +Clough-Tocher interpolation of short defect runs before metacal (see :func:`shapepipe.modules.ngmix_package.ngmix.prepare_ngmix_weights`). """ @@ -21,37 +20,46 @@ _ROW_RUNS = np.array([[0, 0, 0], [1, 1, 1], [0, 0, 0]]) -def _short_row_runs(defect, max_run): - """Pixels in row runs of at most ``max_run`` defects that stop short of - both stamp borders.""" - labels, n_runs = label(defect, structure=_ROW_RUNS) +def _short_row_runs(run, blocked, max_run): + """Pixels of ``run`` in row runs of at most ``max_run`` pixels whose + neighbours beyond both ends lie on the stamp and outside ``blocked``.""" + labels, n_runs = label(run, structure=_ROW_RUNS) if n_runs == 0: - return np.zeros_like(defect) + return np.zeros_like(run) short = np.bincount(labels.ravel(), minlength=n_runs + 1) <= max_run short[0] = False - short[labels[:, 0]] = False - short[labels[:, -1]] = False + # A run pixel's left (right) neighbour must be in the run or open. + in_run = np.pad(run, ((0, 0), (1, 1))) + is_open = np.pad(~blocked, ((0, 0), (1, 1))) + closed_end = run & ~( + (in_run[:, :-2] | is_open[:, :-2]) & (in_run[:, 2:] | is_open[:, 2:]) + ) + short[labels[closed_end]] = False return short[labels] -def interpolable_defects(defect, max_run=MAX_INTERPOLATED_RUN): - """Defect pixels that ``DEFECT_FILL = interpolate`` interpolates. +def interpolable_defects(defect, removed=None, max_run=MAX_INTERPOLATED_RUN): + """Defect pixels that :func:`interpolate_defects` fills. @sc [decision:shape_measurement.defect_fill] interpolable-defects A defect pixel is interpolated when its row or its column run of defect - pixels is at most ``max_run`` (3) long and has clean pixels at both - ends. That covers columns, 3-px bleeds and isolated pixels, the widths - whose shear recovery is calibrated. Wider holes, and runs that reach the - stamp border (edge bands, corners), have clean light on one side only; - they are noise-filled and vetoed at the noise-fill radius (see - :func:`~shapepipe.modules.ngmix_package.ngmix.central_defect_vetoes`). - The rule reads only the mask and commutes with quarter turns of the - stamp. + pixels is at most ``max_run`` (3) long and the pixels beyond both ends + support the interpolant: on the stamp, not a defect, and not + ``removed``. That covers columns, 3-px bleeds and isolated pixels. + Wider holes, edge bands and runs that end on a removed pixel are + noise-filled and vetoed at the noise-fill radius + (:func:`~shapepipe.modules.ngmix_package.ngmix.central_defect_vetoes`). + Each interpolated pixel lies between two support pixels, so the + interpolant reaches it. The rule reads only the masks and commutes with + quarter turns of the stamp. Parameters ---------- defect : numpy.ndarray of bool Defect mask of one epoch stamp. + removed : numpy.ndarray of bool, optional + Pixels whose light the image replaces (noisefill neighbours). They + neither support the interpolant nor are interpolated. max_run : int, optional Longest interpolated run; the default is ``MAX_INTERPOLATED_RUN``. @@ -61,9 +69,14 @@ def interpolable_defects(defect, max_run=MAX_INTERPOLATED_RUN): ``True`` on the defect pixels to interpolate. """ defect = np.asarray(defect, dtype=bool) + removed = ( + np.zeros_like(defect) if removed is None + else np.asarray(removed, dtype=bool) + ) + run, blocked = defect & ~removed, defect | removed return ( - _short_row_runs(defect, max_run) - | _short_row_runs(defect.T, max_run).T + _short_row_runs(run, blocked, max_run) + | _short_row_runs(run.T, blocked.T, max_run).T ) diff --git a/src/shapepipe/modules/ngmix_package/ngmix.py b/src/shapepipe/modules/ngmix_package/ngmix.py index 7733c0689..a0530dd27 100644 --- a/src/shapepipe/modules/ngmix_package/ngmix.py +++ b/src/shapepipe/modules/ngmix_package/ngmix.py @@ -29,35 +29,19 @@ ) from shapepipe.pipeline import file_io -# Flag bit set on an epoch's off-tile pixels (see :func:`split_tile_markers`). -OFF_TILE_FLAG = 2**10 - # Neighbour treatments selectable with the BLEND_HANDLING option. BLEND_HANDLINGS = ("noisefill", "uberseg") -# Defect fills selectable with the DEFECT_FILL option (see -# :func:`prepare_ngmix_weights`). -DEFECT_FILLS = ("noise", "interpolate") - -# Zero-weight pixel classes whose quarter-turn copies also lose their weight, -# selectable with the SYMMETRIZE_WEIGHTS option (see -# :func:`prepare_ngmix_weights`). -WEIGHT_SYMMETRIZATIONS = ("interpolated", "interpolated_and_neighbours", "none") - -# Default of the EPOCH_MASKED_FRACTION_CUT option: an epoch is dropped when -# more than this fraction of its stamp is in :func:`defect_mask` (see -# :func:`prepare_postage_stamps`). +# The epoch cuts (see :func:`prepare_postage_stamps`). Calibration outputs, +# not options: an epoch is dropped when more than EPOCH_MASKED_FRACTION_CUT +# of its stamp is in :func:`defect_mask`, or when a noise-filled +# (interpolated) defect pixel lies closer than EPOCH_CENTRAL_DEFECT_RADIUS +# (EPOCH_INTERPOLATED_DEFECT_RADIUS) pixels to the stamp centre (see +# :func:`central_defect_vetoes`). +# @sc [decision:shape_measurement.epoch_masked_fraction_cut] EPOCH_MASKED_FRACTION_CUT = 1 / 3 - -# Default of the EPOCH_CENTRAL_DEFECT_RADIUS option (pixels): an epoch is -# dropped when a pixel of :func:`defect_mask` lies closer than this to the -# stamp centre (see :func:`has_central_defect`). # @sc [decision:shape_measurement.central_defect_veto] EPOCH_CENTRAL_DEFECT_RADIUS = 10 - -# Default of the EPOCH_INTERPOLATED_DEFECT_RADIUS option (pixels): under -# DEFECT_FILL = interpolate, the veto radius for interpolated defect pixels -# (see :func:`central_defect_vetoes`). # @sc [decision:shape_measurement.central_defect_veto] EPOCH_INTERPOLATED_DEFECT_RADIUS = 7 @@ -638,10 +622,10 @@ def __init__( self.weights = [] self.flags = [] # Neighbour masks, one per epoch: the pixels marked -1e30 in the tile - # VIGNET on other detections' footprints (off-tile markers - # are flagged as defects instead; see split_tile_markers), - # MegaCam-flipped to the epoch. noisefill zero-weights and noise-fills them; uberseg and - # the epoch cuts do not read them (see prepare_ngmix_weights). + # VIGNET on other detections' footprints (off-tile markers get zero + # weight instead; see split_tile_markers), MegaCam-flipped to the + # epoch. noisefill zero-weights and noise-fills them, and the epoch + # cuts do not count them (see prepare_ngmix_weights). self.neighbours = [] self.bkg_rms = [] # Segmentation stamps, one per epoch, used only by the "uberseg" blend @@ -764,26 +748,6 @@ class Ngmix(object): dilate_neighbour : int, optional Neighbour-mask dilation iterations for ``"uberseg"`` (see :func:`uberseg_mask`); the default is ``1``. - epoch_central_defect_radius : float, optional - Drop an epoch when a defect pixel lies closer than this many pixels - to the stamp centre (see :func:`has_central_defect`); the default is - ``EPOCH_CENTRAL_DEFECT_RADIUS``. - epoch_masked_fraction_cut : float, optional - Drop an epoch when more than this fraction of its stamp is defects - (see :func:`prepare_postage_stamps`); the default is - ``EPOCH_MASKED_FRACTION_CUT``. - defect_fill : {"noise", "interpolate"}, optional - How defect pixels are filled before metacal (see - :func:`prepare_ngmix_weights`); the default is ``"noise"``. - epoch_interpolated_defect_radius : float, optional - Under ``defect_fill="interpolate"``, drop an epoch when an - interpolated defect pixel lies closer than this many pixels to the - stamp centre (see :func:`central_defect_vetoes`); the default is - ``EPOCH_INTERPOLATED_DEFECT_RADIUS``. - symmetrize_weights : {"interpolated", "interpolated_and_neighbours", "none"}, optional - Which zero-weight pixels also zero the weight of their quarter-turn - copies (see :func:`prepare_ngmix_weights`); the default is - ``"interpolated"``. Notes ----- @@ -795,8 +759,8 @@ class Ngmix(object): IndexError If the length of the input file list is incorrect ValueError - If ``blend_handling``, ``defect_fill`` or ``symmetrize_weights`` is - unknown, or ``"uberseg"`` is selected without ``seg_cat_path``. + If ``blend_handling`` is unknown, or ``"uberseg"`` is selected + without ``seg_cat_path``. """ @@ -817,11 +781,6 @@ def __init__( seg_cat_path=None, dilate_neighbour=1, metacal_psf="fitgauss", - epoch_central_defect_radius=EPOCH_CENTRAL_DEFECT_RADIUS, - epoch_masked_fraction_cut=EPOCH_MASKED_FRACTION_CUT, - defect_fill="noise", - epoch_interpolated_defect_radius=EPOCH_INTERPOLATED_DEFECT_RADIUS, - symmetrize_weights="interpolated", ): # Base count = catalogue + vignets, excluding the f_wcs headers (passed @@ -840,16 +799,6 @@ def __init__( f"Unknown BLEND_HANDLING '{blend_handling}'; expected one of" + f" {BLEND_HANDLINGS}" ) - if defect_fill not in DEFECT_FILLS: - raise ValueError( - f"Unknown DEFECT_FILL '{defect_fill}'; expected one of" - + f" {DEFECT_FILLS}" - ) - if symmetrize_weights not in WEIGHT_SYMMETRIZATIONS: - raise ValueError( - f"Unknown SYMMETRIZE_WEIGHTS '{symmetrize_weights}'; expected" - + f" one of {WEIGHT_SYMMETRIZATIONS}" - ) # Fail fast at construction (not deep in the per-epoch loop) when # uberseg is requested without its segmentation input (shapepipe#776). @@ -901,13 +850,6 @@ def __init__( self._seg_cat_path = seg_cat_path self._dilate_neighbour = dilate_neighbour self._metacal_psf = metacal_psf - self._epoch_central_defect_radius = epoch_central_defect_radius - self._epoch_masked_fraction_cut = epoch_masked_fraction_cut - self._defect_fill = defect_fill - self._epoch_interpolated_defect_radius = ( - epoch_interpolated_defect_radius - ) - self._symmetrize_weights = symmetrize_weights self._w_log = w_log @@ -1342,12 +1284,7 @@ def process(self): self._bkg_sub, psf_obj, gal_obj, - epoch_central_defect_radius=self._epoch_central_defect_radius, - epoch_masked_fraction_cut=self._epoch_masked_fraction_cut, - defect_fill=self._defect_fill, - epoch_interpolated_defect_radius=( - self._epoch_interpolated_defect_radius - ), + blend_handling=self._blend_handling, ) epoch_cuts.update(stamp.epoch_cuts) @@ -1395,8 +1332,6 @@ def process(self): object_number=obj_id, dilate_neighbour=self._dilate_neighbour, metacal_psf=self._metacal_psf, - defect_fill=self._defect_fill, - symmetrize_weights=self._symmetrize_weights, ) except Exception as ee: self._w_log.info( @@ -1493,39 +1428,28 @@ def prepare_postage_stamps( bkg_sub=True, psf_obj=None, gal_obj=None, - epoch_central_defect_radius=EPOCH_CENTRAL_DEFECT_RADIUS, - epoch_masked_fraction_cut=EPOCH_MASKED_FRACTION_CUT, - defect_fill="noise", - epoch_interpolated_defect_radius=EPOCH_INTERPOLATED_DEFECT_RADIUS, + blend_handling="noisefill", ): """Gather one object's epoch stamps, dropping epochs its defects spoil. - @sc [decision:shape_measurement.central_defect_veto,decision:shape_measurement.epoch_masked_fraction_cut,decision:shape_measurement.defect_fill] epoch-cut-on-defect-mask - An epoch is dropped when more than ``epoch_masked_fraction_cut`` of its - stamp lies in :func:`defect_mask`: flagged, zero-weight and invalid-RMS - pixels, the set that :func:`prepare_ngmix_weights` zero-weights and - fills under every ``blend_handling``. Counting flags alone would keep - epochs whose filled area exceeds the cut. The quarter-turn copies that - weight symmetrization zero-weights keep their light and are not counted - (symmetrized-weight-holes). The default is - 1/3; 10%, the DES Y3 and Y6 value, is the alternative to test. + @sc [decision:shape_measurement.central_defect_veto,decision:shape_measurement.epoch_masked_fraction_cut] epoch-cut-on-defect-mask + An epoch is dropped when more than ``EPOCH_MASKED_FRACTION_CUT`` of its + stamp lies in :func:`defect_mask`, the set :func:`prepare_ngmix_weights` + zero-weights and fills, or when :func:`central_defect_vetoes` finds a + defect near the object. The quarter-turn weight copies of interpolated + pixels keep their light and are not counted. @sc [decision:shape_measurement.central_defect_veto,decision:shape_measurement.epoch_masked_fraction_cut,decision:shape_measurement.blend_handling] neighbour-markers-are-not-defects - The tile VIGNET holds -1e30 on the footprints of other detections and - beyond the tile's edge (:func:`split_tile_markers`). The - footprint markers form the epoch's neighbour mask (``stamp.neighbours``, - MegaCam-flipped like the epoch), kept apart from its flag stamp, so - neither the masked-fraction cut nor the central veto counts them. Every - epoch shares the tile VIGNET: counting a neighbour within the veto - radius as a defect would drop every epoch of the object. + The tile VIGNET's -1e30 markers on other detections' footprints + (:func:`split_tile_markers`) are the epoch's neighbour mask + (``stamp.neighbours``), not defects: every epoch shares the tile VIGNET, + so counting them would drop every epoch of a blended object. @sc [decision:shape_measurement.central_defect_veto,decision:shape_measurement.epoch_masked_fraction_cut,decision:shape_measurement.defect_fill] off-tile-pixels-are-defects - Beyond the tile's edge the epoch holds real data, the object's own light - cut off by the tile. Those pixels are flagged ``OFF_TILE_FLAG`` (2**10) - in the epoch's flag stamp and so join its defect set under every - ``blend_handling``: zero weight, the defect fill, and both epoch cuts. - On a 51-px stamp an object within about 8.5 px of the tile edge fails - the 1/3 cut. + Beyond the tile's edge the epoch holds the object's own light, cut off by + the tile, so those pixels get zero exposure weight and join the defect + set under every ``blend_handling``. On a 51-px stamp an object within + about 8.5 px of the tile edge fails the masked-fraction cut. Parameters ---------- @@ -1541,21 +1465,10 @@ def prepare_postage_stamps( Subtract the background vignet; the default is ``True``. psf_obj, gal_obj : dict, optional The object's PSF and galaxy vignet dicts, if already read. - epoch_central_defect_radius : float, optional - Drop an epoch with a defect pixel closer than this many pixels to the - stamp centre (:func:`has_central_defect`); 0 disables the veto. The - default is ``EPOCH_CENTRAL_DEFECT_RADIUS``. - epoch_masked_fraction_cut : float, optional - Drop an epoch with more than this fraction of its stamp in - :func:`defect_mask`. The default is ``EPOCH_MASKED_FRACTION_CUT``. - defect_fill : {"noise", "interpolate"}, optional - The fill :func:`prepare_ngmix_weights` will apply; it sets the veto - radius of each defect pixel (:func:`central_defect_vetoes`). The - default is ``"noise"``. - epoch_interpolated_defect_radius : float, optional - Veto radius for interpolated defect pixels under - ``defect_fill="interpolate"``. The default is - ``EPOCH_INTERPOLATED_DEFECT_RADIUS``. + blend_handling : {"noisefill", "uberseg"}, optional + The neighbour treatment :func:`prepare_ngmix_weights` will apply, + which decides which defects it can interpolate and so their veto + radius; the default is ``"noisefill"``. Returns ------- @@ -1632,24 +1545,20 @@ def prepare_postage_stamps( # Off-tile pixels are defects (off-tile-pixels-are-defects); the # other -1e30 markers are neighbours (neighbour-markers-are-not-defects). neighbour, off_tile = split_tile_markers(tile_vign, np.shape(gal_vign)) - flag_vign[off_tile] = OFF_TILE_FLAG - weight_vign = weight_obj[expccd_name]['VIGNET'] + weight_vign = np.where(off_tile, 0, weight_obj[expccd_name]['VIGNET']) bkg_rms_vign = ( bkg_rms_obj[expccd_name]['VIGNET'] if bkg_rms_obj is not None else None ) - # Drop the epoch when too much of it would be zero-weighted and - # filled (epoch-cut-on-defect-mask), or when a filled pixel would sit - # near the object (veto-radius-follows-the-fill). - masked = defect_mask(weight_vign, flag_vign, bkg_rms_vign) + defect = defect_mask(weight_vign, flag_vign, bkg_rms_vign) stamp.epoch_cuts["considered"] += 1 - if masked.mean() > epoch_masked_fraction_cut: + if defect.mean() > EPOCH_MASKED_FRACTION_CUT: stamp.epoch_cuts["masked_fraction"] += 1 continue + removed = neighbour if blend_handling == "noisefill" else None if central_defect_vetoes( - masked, epoch_central_defect_radius, defect_fill, - epoch_interpolated_defect_radius, + defect, interpolable_defects(defect, removed) ): stamp.epoch_cuts["central_veto"] += 1 continue @@ -2055,23 +1964,13 @@ def defect_mask(weight, flag, bkg_rms=None): """Defect pixels of one epoch stamp. @sc [decision:shape_measurement.defect_fill,label:physics] defect-set-unsymmetrized - A defect is a pixel with zero exposure weight, a nonzero exposure flag, - or (when a background RMS map is given) a non-finite or non-positive RMS. - This one set is zero-weighted and filled by :func:`prepare_ngmix_weights` - under every ``blend_handling`` and counted by the epoch cuts in - :func:`prepare_postage_stamps`. The tile VIGNET's neighbour markers are not - in it (neighbour-markers-are-not-defects); off-tile pixels are, as flag - ``OFF_TILE_FLAG`` (off-tile-pixels-are-defects). It is not ORed with its - rotations. For defects the central-defect veto keeps - (:func:`has_central_defect`), the unsymmetrized fill leaves - |c| <= 3e-4 per affected epoch on galaxies with half-light radius 0.3" - and 0.5" through a 0.7" PSF, round or elliptical. Symmetrizing would - quadruple the filled area near the object, and with it m: a 3-px bleed at - 10 px on the 0.5" galaxy gives m11 = -2.7% four-fold against -0.64% - unsymmetrized. Through a PSF with ellipticity (0.05, 0.02) it would not - cancel c either: c1 = 7.3e-4 four-fold against 0.7e-4. It would also - turn a 5-px edge band (9.8% of the stamp), which biases nothing, into a - 35.4% frame that fails the masked-fraction cut. + A defect is a pixel with zero exposure weight (off-tile pixels + included), a nonzero exposure flag, or, when a background RMS map is + given, a non-finite or non-positive RMS. This one set is zero-weighted + and filled by :func:`prepare_ngmix_weights` and counted by the epoch + cuts. It is not ORed with its rotations: a four-fold fill quadruples the + filled area near the object and with it m (astra option + ``defect_fill.symmetrized_4fold_noise``). Parameters ---------- @@ -2093,227 +1992,87 @@ def defect_mask(weight, flag, bkg_rms=None): return defect -def has_central_defect(defect, radius): - """Whether a defect pixel lies closer than ``radius`` to the stamp centre. - - @sc [decision:shape_measurement.central_defect_veto,decision:shape_measurement.defect_fill] epoch-central-defect-veto - An epoch is dropped when any pixel of :func:`defect_mask` lies closer - than ``radius`` pixels to the stamp centre. A noise-filled hole in the - object's light is sheared by metacal but not by the sky, so the metacal - response is wrong for that epoch, and a one-sided hole adds an additive - term. The default radius (``EPOCH_CENTRAL_DEFECT_RADIUS``, 10 px or - 1.9") is the smallest at which columns, 3-px bleeds, single pixels and - edge bands give |m11|, |m22| < 1% and |c1|, |c2| < 5e-4 (full response - matrix) on galaxies with half-light radius 0.3" and 0.5" through a 0.7" - PSF. The bias is anisotropic: a column at 10 px on the 0.5" galaxy gives - m11 = -0.60%, m22 = +0.01% and c1 = -1.4e-4; at 9 px, m11 = -1.75% and - c1 = -1.1e-3. Through a PSF with ellipticity (0.05, 0.02), wide defects - at 10 px on the 0.5" galaxy sit at the bound: a 3-px bleed gives - m11 = -0.98% +/- 0.04% and the widest edge band the veto keeps (16 px) - -0.94% +/- 0.06%; at 11 px the bleed gives -0.24%. Larger galaxies need - more: at hlr 0.7" through a 0.9" PSF a 3-px bleed at 10 px gives - m11 = -6.8% and c1 = -4.9e-3, and passes from 14 px. The distance is - measured from the stamp centre, where the extractor places the object to - within half a pixel. The veto reads only the defect mask, never the - object's pixels, so it selects on nothing that responds to shear. - Radius 0 disables it. Guarded by ``tests/science/test_defect_veto.py``. - - Parameters - ---------- - defect : numpy.ndarray of bool - Defect mask of one epoch stamp (:func:`defect_mask`). - radius : float - Veto radius in pixels. - - Returns - ------- - bool - ``True`` if the epoch should be dropped. - """ - rows, cols = np.nonzero(defect) - centre_row = (defect.shape[0] - 1) / 2 - centre_col = (defect.shape[1] - 1) / 2 - distance = np.hypot(rows - centre_row, cols - centre_col) - return bool(np.any(distance < radius)) - - -def central_defect_vetoes( - defect, radius, defect_fill="noise", - interpolated_radius=EPOCH_INTERPOLATED_DEFECT_RADIUS, -): - """Whether the central-defect veto drops an epoch under ``defect_fill``. +def central_defect_vetoes(defect, interpolated): + """Whether a defect near the stamp centre drops the epoch. @sc [decision:shape_measurement.central_defect_veto,decision:shape_measurement.defect_fill] veto-radius-follows-the-fill - Each defect pixel is vetoed (:func:`has_central_defect`) at the radius - calibrated for the operator that fills it. Under ``"noise"`` every - defect keeps ``radius`` (``EPOCH_CENTRAL_DEFECT_RADIUS``). Under - ``"interpolate"`` the pixels :func:`interpolable_defects` selects are - interpolated and vetoed at ``interpolated_radius`` - (``EPOCH_INTERPOLATED_DEFECT_RADIUS``); the others are noise-filled and - keep ``radius``. 7 px is the smallest interpolated radius at which - columns, full and finite 3-px bleeds and single pixels give - |m11|, |m22| < 1% and |c1|, |c2| < 5e-4 (full response matrix) on - galaxies with half-light radius 0.3" and 0.5" through a 0.7" PSF, round - or with ellipticity (0.05, 0.02), and on a 0.7" galaxy through a 0.9" - PSF. There the worst case is c1 = 3.6e-4 (3-px bleed, 0.7" galaxy, - elliptical PSF); on the 0.3" and 0.5" galaxies |c| <= 5e-5 and - |m| <= 0.21%. At 6 px those two still pass (|c| <= 3.1e-4), but a 3-px - bleed on the 0.7" galaxy gives c1 = 7.5e-4; at 5 px a 3-px bleed on the - 0.5" galaxy gives 9.5e-4. The radius does not scale with galaxy size: - interpolation needs one pixel more for the 0.7" galaxy (noise fill needs - four, 10 to 14 px), and a size-dependent veto would select on measured - size, which responds to shear. Guarded by - ``tests/science/test_defect_interpolation.py``. + The epoch is dropped when an interpolated defect pixel lies closer than + ``EPOCH_INTERPOLATED_DEFECT_RADIUS`` to the stamp centre, or a + noise-filled one closer than ``EPOCH_CENTRAL_DEFECT_RADIUS``: the + smallest radii at which the defects kept recover shear within + |m| < 1% and |c| < 5e-4. ``interpolated`` is the set + :func:`prepare_ngmix_weights` interpolates, so each pixel is vetoed at + the radius of the fill it gets. The veto reads only masks, so it selects + on nothing that responds to shear. Calibration: astra decision + ``shape_measurement.central_defect_veto``; guarded by + ``tests/science/test_defect_recovery.py``. Parameters ---------- defect : numpy.ndarray of bool Defect mask of one epoch stamp (:func:`defect_mask`). - radius : float - Veto radius in pixels for noise-filled defect pixels. - defect_fill : {"noise", "interpolate"}, optional - The fill the epoch will get; the default is ``"noise"``. - interpolated_radius : float, optional - Veto radius in pixels for interpolated defect pixels; the default is - ``EPOCH_INTERPOLATED_DEFECT_RADIUS``. + interpolated : numpy.ndarray of bool + The defect pixels that are interpolated (:func:`interpolable_defects` + with the epoch's removed neighbour pixels); the rest are noise-filled. Returns ------- bool ``True`` if the epoch should be dropped. """ - if defect_fill == "interpolate": - interpolated = interpolable_defects(defect) - return has_central_defect( - interpolated, interpolated_radius - ) or has_central_defect(defect & ~interpolated, radius) - return has_central_defect(defect, radius) - - -def fill_defects(image, defect, noise): - """Replace an image's defect pixels by a noise realisation. - - @sc [decision:shape_measurement.defect_fill,label:physics] defect-noise-fill - Defect pixels are replaced by ``noise``, an independent realisation at - the per-pixel background RMS. Metacal deconvolves, shears and reconvolves - the whole image without reading the weights, so a raw defect value (bad - column, bleed, cosmic ray) would leak into the fit. The noise fill leaves - a hole in the object's light that metacal shears and the sky does not, - which biases m for an epoch with a defect near the object; the - central-defect veto drops those epochs (epoch-central-defect-veto). - - Parameters - ---------- - image : numpy.ndarray - Stamp image. - defect : numpy.ndarray of bool - Pixels to replace: the defects (:func:`defect_mask`), and under - noisefill the marked neighbour pixels. - noise : numpy.ndarray - Noise realisation on the stamp grid. - - Returns - ------- - numpy.ndarray - ``image`` with ``defect`` pixels taken from ``noise``, in the dtype - of ``image``. - """ - return np.where(defect, noise, image).astype(image.dtype, copy=False) + rows, cols = np.indices(defect.shape) + distance = np.hypot( + rows - (defect.shape[0] - 1) / 2, cols - (defect.shape[1] - 1) / 2 + ) + radius = np.where( + interpolated, + EPOCH_INTERPOLATED_DEFECT_RADIUS, + EPOCH_CENTRAL_DEFECT_RADIUS, + ) + return bool(np.any(defect & (distance < radius))) def prepare_ngmix_weights( gal, weight, flag, rng, bkg_rms=None, blend_handling="noisefill", seg=None, object_number=None, - dilate_neighbour=0, defect_fill="noise", neighbour=None, - symmetrize_weights="interpolated", + dilate_neighbour=0, neighbour=None, ): """Build one epoch's image, weight map and noise image for ngmix. Every stamp pixel falls in one of three classes. A clean pixel keeps its - light and its weight. A pixel whose light is replaced keeps neither: a - noise-filled defect, or under ``"noisefill"`` a marked neighbour pixel. - A pixel whose light stays but whose weight is zero is a hole in the - likelihood only: an interpolated defect, or under ``"uberseg"`` a - neighbour-side pixel. ``symmetrize_weights`` names which holes of this - third class are made four-fold symmetric. + light and its weight. A pixel whose light is replaced by noise keeps + neither: a defect that is not interpolated, or under ``"noisefill"`` a + marked neighbour pixel. A pixel whose light stays at zero weight is a + hole in the likelihood only: an interpolated defect and its quarter-turn + copies, or under ``"uberseg"`` a neighbour-side pixel. @sc [decision:shape_measurement.defect_fill,decision:shape_measurement.blend_handling] defects-filled-whatever-the-blend-handling - Every pixel of ``defect_mask`` is zero-weighted and filled whatever - ``blend_handling`` is, by the same operator. ``blend_handling`` acts on - the neighbour pixels only, and the noise image covers the whole stamp. - - @sc [decision:shape_measurement.defect_fill,decision:shape_measurement.blend_handling] interpolation-support-is-the-kept-image - Under ``defect_fill="interpolate"`` the interpolant is supported on the - pixels whose light the image keeps: never a defect, and under - ``"noisefill"`` never a marked neighbour pixel, whose light is replaced - by noise; a Clough-Tocher interpolant next to a bright neighbour would - otherwise build the defect's value from light the image no longer - contains. Under ``"uberseg"`` the neighbour light stays in the image and - supports the interpolant. + Every pixel of :func:`defect_mask` gets weight 0 and is filled the same + way under every ``blend_handling``. The short runs of + :func:`interpolable_defects` take a Clough-Tocher interpolant of the + pixels whose light the image keeps (:func:`interpolate_defects`), in the + image and the noise image alike; the other defects take noise at the + background RMS. Metacal shears the whole image whatever the weights, so + no raw defect value may reach it. @sc [decision:shape_measurement.blend_handling] noisefill-fills-markers - Under ``"noisefill"`` the pixels of ``neighbour``, the tile VIGNET's - -1e30 neighbour markers, get weight 0 and are replaced by the same noise - realisation as the noise-filled defects, so no marked neighbour light - reaches metacal. Under the default noise fill, the image, weight map and - noise image are those the marked pixels would get as flagged defects; - only the epoch cuts treat them differently, by not counting them - (neighbour-markers-are-not-defects). Unmarked neighbour light stays - raw and weighted. + Under ``"noisefill"`` the ``neighbour`` pixels (the tile VIGNET's -1e30 + neighbour markers) get weight 0 and noise. Their light is gone, so they + neither support the interpolant nor receive it. @sc [decision:shape_measurement.blend_handling] uberseg-ignores-markers - Under ``"uberseg"`` the neighbour markers are ignored: the pixels of - :func:`uberseg_mask`, nearer a neighbour's segmentation footprint than - the target's, get weight 0 and keep their raw image values, because that - light is real sky that metacal shears along with the target; filling it - with noise would cut the target along an unsheared edge. - - @sc [decision:shape_measurement.defect_fill,label:physics] interpolated-defects-keep-light - Under ``defect_fill="interpolate"`` the short defect runs - (:func:`interpolable_defects`) take the interpolant of the clean pixels - around them (:func:`interpolate_defects`) in the image and in the noise - image alike, and the other defects are noise-filled. Interpolation - restores the object's light and keeps the pixel's zero weight. Only the - weights are symmetrized (symmetrized-weight-holes), not the fill: - interpolating the orbit as well would trade true light for interpolated - light over four times the area, raising m11 for a 3-px bleed 6 px from a - galaxy with half-light radius 0.5" through a 0.7" PSF from +0.19% to - +0.89%. + Under ``"uberseg"`` the markers are ignored: the pixels of + :func:`uberseg_mask` get weight 0 and keep their light, which is real + sky that metacal shears with the target and which supports the + interpolant. @sc [decision:shape_measurement.weight_symmetrization,label:physics] symmetrized-weight-holes - ngmix fits a Gaussian to a profile that is not one, so a zero-weight hole - on one side of the object pulls the fit along that side: a spin-2 term. - One step cancels it: the weight is also zeroed on the three quarter-turn - copies, about the stamp centre, of the holes ``symmetrize_weights`` - names, and their light stays, so the likelihood has no preferred axis. - Under ``"interpolated"`` (the default) those are the interpolated - defects: an interpolated column 8 px from a galaxy with half-light - radius 0.5" through a 0.7" PSF gives c1 = -1.3e-3 unsymmetrized and - +2e-6 symmetrized. ``"interpolated_and_neighbours"`` adds the uberseg - neighbour side. On its own that hole is a strong pull: a neighbour - footprint with no light 10 px from the same galaxy gives c1 = -7.8e-3, - and 2e-6 symmetrized. But a real neighbour leaves light in the target's - cell, which pulls the other way, and the one-sided hole partly offsets - it. With a neighbour of a tenth of the target's flux at 10 px, c1 goes - from +1.1e-3 unsymmetrized to +1.2e-2 symmetrized; at equal flux, from - +0.13 to +0.21. The symmetrized hole also keeps few pixels: at 10 px, - 117 weighted pixels of 1580 and 97% of the target's S/N (99% without); - at 7 px, 25 pixels and 71%; at 5 px, none. So the neighbour side stays - unsymmetrized by default; the option is for a neighbour-subtracted - image or an A/B arm. Pixels whose light is replaced (noise-filled - defects, noisefill neighbours) are never symmetrized: their hole is in - the image metacal shears, and a weight orbit leaves it there. For a - noise-filled column 10 px from the 0.5" galaxy, m11 stays at -0.53% - with the orbit (-0.54% without) while c1 moves from -1.1e-4 to - +0.3e-4; for a 3-px bleed, c1 moves from +1.4e-4 to +3.2e-4. Under - ``"noisefill"`` with the noise fill no pixel is in a symmetrized class, - so ``symmetrize_weights`` changes nothing there. The rotations are - about the stamp centre, where the extractor puts the object to within - half a pixel per axis (the coadd-centroid offset); the measurements - above draw the object uniformly within that half pixel, so the - cancellation they show holds at the offsets the pipeline produces. - Measured with ``tests/helpers/defect_response.py``, 6 seeds, full - response matrix; guarded by ``tests/science/test_weight_symmetrization.py``. + A one-sided zero-weight hole pulls the Gaussian fit toward its side, so + the weight is also zeroed on the three quarter-turn copies, about the + stamp centre, of every interpolated pixel, and their light stays. + Noise-filled pixels and the uberseg neighbour side are not symmetrized. + Measurements: astra decision ``shape_measurement.weight_symmetrization``; + guarded by ``tests/science/test_defect_recovery.py``. Parameters ---------- @@ -2345,17 +2104,9 @@ def prepare_ngmix_weights( dilate_neighbour : int, optional Neighbour-mask dilation iterations, passed to :func:`uberseg_mask` under ``blend_handling="uberseg"``; ignored otherwise. - defect_fill : {"noise", "interpolate"}, optional - ``"noise"`` (default) noise-fills every defect; ``"interpolate"`` - interpolates the short defect runs and noise-fills the rest. neighbour : numpy.ndarray of bool, optional The epoch's neighbour mask (``Postage_stamp.neighbours``); read only under ``blend_handling="noisefill"``. ``None`` marks no pixel. - symmetrize_weights : {"interpolated", "interpolated_and_neighbours", "none"}, optional - Which zero-weight pixels whose light stays also zero the weight of - their quarter-turn copies (symmetrized-weight-holes): the - interpolated defects (the default), those and the uberseg neighbour - side, or none. Symmetrizing needs a square stamp. Returns ------- @@ -2371,25 +2122,18 @@ def prepare_ngmix_weights( Raises ------ ValueError - If ``blend_handling``, ``defect_fill`` or ``symmetrize_weights`` is - unknown, or ``"uberseg"`` lacks ``seg`` or ``object_number``. + If ``blend_handling`` is unknown, or ``"uberseg"`` lacks ``seg`` or + ``object_number``. + RuntimeError + If the interpolant does not reach a pixel :func:`interpolable_defects` + selected (degenerate support, or non-finite image values in it). @sc [decision:masking.pixel_mask_source,decision:shape_measurement.blend_handling,decision:shape_measurement.defect_fill,decision:shape_measurement.galaxy_pixel_weights,decision:shape_measurement.weight_symmetrization] """ - if defect_fill not in DEFECT_FILLS: - raise ValueError( - f"Unknown DEFECT_FILL '{defect_fill}'; expected one of" - + f" {DEFECT_FILLS}" - ) if blend_handling not in BLEND_HANDLINGS: raise ValueError( f"Unknown blend_handling '{blend_handling}'; expected one of" + f" {BLEND_HANDLINGS}" ) - if symmetrize_weights not in WEIGHT_SYMMETRIZATIONS: - raise ValueError( - f"Unknown SYMMETRIZE_WEIGHTS '{symmetrize_weights}'; expected one" - + f" of {WEIGHT_SYMMETRIZATIONS}" - ) if blend_handling == "uberseg" and (seg is None or object_number is None): raise ValueError( "blend_handling='uberseg' requires a segmentation map and the" @@ -2445,30 +2189,22 @@ def prepare_ngmix_weights( noise_img = rng.standard_normal(gal.shape) * sig_noise noise_img_gal = rng.standard_normal(gal.shape) * sig_noise - gal_filled = fill_defects(gal, ~clean, noise_img_gal) - interpolated = no_pixel - if defect_fill == "interpolate" and defect.any(): - # One operator for the image and the noise image, supported on the - # pixels the image keeps (interpolation-support-is-the-kept-image); a - # pixel whose support is degenerate, or that noisefill removes as a - # neighbour, keeps its noise fill. - target = interpolable_defects(defect) - filled = interpolate_defects([gal, noise_img], ~clean, target) - interpolated = ( - target & ~removed_neighbour & np.all(np.isfinite(filled), axis=0) - ) - gal_filled[interpolated] = filled[0][interpolated] - noise_img = np.where(interpolated, filled[1], noise_img) - - # The zero-weight pixels whose light stays, of the classes - # symmetrize_weights names (symmetrized-weight-holes). - holes = { - "interpolated": interpolated, - "interpolated_and_neighbours": interpolated | neighbour_side, - "none": no_pixel, - }[symmetrize_weights] - if holes.any(): - weight_map[fourfold(holes)] = 0.0 + gal_filled = np.where(clean, gal, noise_img_gal).astype(gal.dtype) + interpolated = interpolable_defects(defect, removed_neighbour) + if interpolated.any(): + # One operator for the image and the noise image, so fixnoise + # mirrors the science image's interpolated noise. + filled = interpolate_defects( + [gal, noise_img], ~clean, interpolated + )[:, interpolated] + if not np.all(np.isfinite(filled)): + raise RuntimeError( + "The defect interpolant is not finite on pixels" + + " interpolable_defects selected." + ) + gal_filled[interpolated] = filled[0] + noise_img[interpolated] = filled[1] + weight_map[fourfold(interpolated)] = 0.0 return gal_filled, weight_map, noise_img @@ -2477,8 +2213,7 @@ def make_ngmix_observation( gal, weight, flag, psf, wcs, rng, bkg_rms=None, centroid_source="wcs", offset=None, blend_handling="noisefill", seg=None, object_number=None, - dilate_neighbour=0, defect_fill="noise", neighbour=None, - symmetrize_weights="interpolated", + dilate_neighbour=0, neighbour=None, ): """Build an ngmix Observation for a single galaxy epoch. @@ -2533,14 +2268,8 @@ def make_ngmix_observation( dilate_neighbour : int, optional Neighbour-mask dilation iterations passed through to :func:`prepare_ngmix_weights` under ``blend_handling="uberseg"``. - defect_fill : {"noise", "interpolate"}, optional - Defect fill passed through to :func:`prepare_ngmix_weights`; the - default is ``"noise"``. neighbour : numpy.ndarray of bool, optional Neighbour mask passed through to :func:`prepare_ngmix_weights`. - symmetrize_weights : {"interpolated", "interpolated_and_neighbours", "none"}, optional - Weight symmetrization passed through to - :func:`prepare_ngmix_weights`; the default is ``"interpolated"``. Returns ------- @@ -2569,8 +2298,7 @@ def make_ngmix_observation( gal_masked, weight_map, noise_img = prepare_ngmix_weights( gal, weight, flag, rng, bkg_rms=bkg_rms, blend_handling=blend_handling, seg=seg, object_number=object_number, - dilate_neighbour=dilate_neighbour, defect_fill=defect_fill, - neighbour=neighbour, symmetrize_weights=symmetrize_weights, + dilate_neighbour=dilate_neighbour, neighbour=neighbour, ) if centroid_source == "hsm": @@ -2801,8 +2529,7 @@ def make_runners(prior, flux_guess, rng): def do_ngmix_metacal( stamp, prior, flux_guess, rng, centroid_source="wcs", blend_handling="noisefill", object_number=None, dilate_neighbour=0, - metacal_psf="fitgauss", defect_fill="noise", - symmetrize_weights="interpolated", + metacal_psf="fitgauss", ): """Do Ngmix Metacal. @@ -2849,12 +2576,6 @@ def do_ngmix_metacal( round PSF that metacal reconvolves with after shearing, so it moves the metacal *response* (and therefore the recovered shear) but never reaches the deconvolution, which is by the PSF image. - defect_fill : {"noise", "interpolate"}, optional - Defect fill passed through to :func:`make_ngmix_observation`; the - default is ``"noise"``. - symmetrize_weights : {"interpolated", "interpolated_and_neighbours", "none"}, optional - Weight symmetrization passed through to - :func:`make_ngmix_observation`; the default is ``"interpolated"``. Returns ------- @@ -2888,11 +2609,9 @@ def do_ngmix_metacal( seg=stamp.segs[n_e] if n_e < len(stamp.segs) else None, object_number=object_number, dilate_neighbour=dilate_neighbour, - defect_fill=defect_fill, neighbour=( stamp.neighbours[n_e] if n_e < len(stamp.neighbours) else None ), - symmetrize_weights=symmetrize_weights, ) gal_obs_list.append(gal_obs) diff --git a/src/shapepipe/modules/ngmix_runner.py b/src/shapepipe/modules/ngmix_runner.py index 227097a73..ebe4f670c 100644 --- a/src/shapepipe/modules/ngmix_runner.py +++ b/src/shapepipe/modules/ngmix_runner.py @@ -12,9 +12,6 @@ from shapepipe.modules.module_decorator import module_runner from shapepipe.modules.ngmix_package.ngmix import ( - EPOCH_CENTRAL_DEFECT_RADIUS, - EPOCH_INTERPOLATED_DEFECT_RADIUS, - EPOCH_MASKED_FRACTION_CUT, Ngmix, write_empty_tile_output, ) @@ -141,8 +138,9 @@ def ngmix_runner( # footprints); "uberseg" ignores those markers, zeroes the weight of every # pixel closer to a neighbour than to the central object, from the # segmentation map, and leaves its image raw. Defect pixels (flagged, - # zero-weight, invalid-RMS or off-tile) are zero-weighted and filled - # under both (DEFECT_FILL below); see prepare_ngmix_weights. + # zero-weight, invalid-RMS or off-tile) are zero-weighted and filled the + # same way under both, and the epoch cuts are fixed; see + # prepare_postage_stamps and prepare_ngmix_weights. if config.has_option(module_config_sec, "BLEND_HANDLING"): blend_handling = config.get(module_config_sec, "BLEND_HANDLING") else: @@ -156,61 +154,6 @@ def ngmix_runner( else: dilate_neighbour = 1 - # EPOCH_CENTRAL_DEFECT_RADIUS (optional, pixels): drop an epoch when a - # defect pixel (flagged, zero-weight, invalid-RMS or off-tile; not a - # neighbour marker) lies closer than this to the stamp centre; 0 disables. - if config.has_option(module_config_sec, "EPOCH_CENTRAL_DEFECT_RADIUS"): - epoch_central_defect_radius = config.getfloat( - module_config_sec, "EPOCH_CENTRAL_DEFECT_RADIUS" - ) - else: - epoch_central_defect_radius = EPOCH_CENTRAL_DEFECT_RADIUS - - # EPOCH_MASKED_FRACTION_CUT (optional): drop an epoch when more than this - # fraction of its stamp is defects (flagged, zero-weight, invalid-RMS or - # off-tile; neighbour markers are not counted). - if config.has_option(module_config_sec, "EPOCH_MASKED_FRACTION_CUT"): - epoch_masked_fraction_cut = config.getfloat( - module_config_sec, "EPOCH_MASKED_FRACTION_CUT" - ) - else: - epoch_masked_fraction_cut = EPOCH_MASKED_FRACTION_CUT - - # DEFECT_FILL (optional): "noise" (default) noise-fills every defect; - # "interpolate" interpolates short defect runs (at most 3 px along a row - # or column) from the clean pixels around them and noise-fills the rest. - if config.has_option(module_config_sec, "DEFECT_FILL"): - defect_fill = config.get(module_config_sec, "DEFECT_FILL") - else: - defect_fill = "noise" - - # EPOCH_INTERPOLATED_DEFECT_RADIUS (optional, pixels): under - # DEFECT_FILL = interpolate, drop an epoch when an interpolated defect - # pixel lies closer than this to the stamp centre; noise-filled defect - # pixels keep EPOCH_CENTRAL_DEFECT_RADIUS. - if config.has_option( - module_config_sec, "EPOCH_INTERPOLATED_DEFECT_RADIUS" - ): - epoch_interpolated_defect_radius = config.getfloat( - module_config_sec, "EPOCH_INTERPOLATED_DEFECT_RADIUS" - ) - else: - epoch_interpolated_defect_radius = EPOCH_INTERPOLATED_DEFECT_RADIUS - - # SYMMETRIZE_WEIGHTS (optional): which zero-weight pixels whose light - # stays also zero the weight of their quarter-turn copies about the stamp - # centre, so no one-sided hole in the likelihood pulls the fit. - # "interpolated" (default): the interpolated defects; - # "interpolated_and_neighbours": those and the uberseg neighbour side; - # "none". Nothing changes under BLEND_HANDLING = noisefill with - # DEFECT_FILL = noise. See prepare_ngmix_weights. - if config.has_option(module_config_sec, "SYMMETRIZE_WEIGHTS"): - symmetrize_weights = config.get( - module_config_sec, "SYMMETRIZE_WEIGHTS" - ) - else: - symmetrize_weights = "interpolated" - # Check PSF vignets first: if all are empty dicts {}, the exposures for this # tile are absent from the PSF dictionary and no shape measurement is possible. # This check must come before reading image vignets to avoid a C-level malloc @@ -271,11 +214,6 @@ def ngmix_runner( seg_cat_path=seg_vignet_path, dilate_neighbour=dilate_neighbour, metacal_psf=metacal_psf, - epoch_central_defect_radius=epoch_central_defect_radius, - epoch_masked_fraction_cut=epoch_masked_fraction_cut, - defect_fill=defect_fill, - epoch_interpolated_defect_radius=epoch_interpolated_defect_radius, - symmetrize_weights=symmetrize_weights, ) # Process ngmix shape measurement and metacalibration diff --git a/tests/module/test_defect_interpolation.py b/tests/module/test_defect_interpolation.py index 1368e78cc..b550b7f57 100644 --- a/tests/module/test_defect_interpolation.py +++ b/tests/module/test_defect_interpolation.py @@ -1,8 +1,8 @@ -"""Defect interpolation (``DEFECT_FILL = interpolate``): which pixels are -interpolated, and the properties of the interpolant. +"""Defect interpolation: which pixels are interpolated, and the properties +of the interpolant. :func:`interpolable_defects` picks the defect pixels that lie in a short row -or column run with clean pixels at both ends; :func:`interpolate_defects` +or column run with support pixels at both ends; :func:`interpolate_defects` fills them from the nearby clean pixels with a Clough-Tocher interpolant, averaged over the four quarter turns of the stamp and shared by every plane (science image and metacal noise image). @@ -24,9 +24,11 @@ # --- interpolable_defects --------------------------------------------------- -def _oracle(defect, max_run=MAX_INTERPOLATED_RUN): - """Brute force: walk each defect pixel's row and column run.""" +def _oracle(defect, removed, max_run=MAX_INTERPOLATED_RUN): + """Brute force: walk each kept defect pixel's row and column run.""" n0, n1 = defect.shape + blocked = defect | removed + defect = defect & ~removed out = np.zeros_like(defect) for i, j in zip(*np.nonzero(defect)): for di, dj in ((0, 1), (1, 0)): @@ -39,7 +41,9 @@ def _oracle(defect, max_run=MAX_INTERPOLATED_RUN): hi = (hi[0] + di, hi[1] + dj) length = hi[0] - lo[0] + hi[1] - lo[1] + 1 bounded = (lo[0] - di >= 0 and lo[1] - dj >= 0 - and hi[0] + di < n0 and hi[1] + dj < n1) + and hi[0] + di < n0 and hi[1] + dj < n1 + and not blocked[lo[0] - di, lo[1] - dj] + and not blocked[hi[0] + di, hi[1] + dj]) if bounded and length <= max_run: out[i, j] = True return out @@ -48,29 +52,62 @@ def _oracle(defect, max_run=MAX_INTERPOLATED_RUN): @given( n=st.integers(5, 21), density=st.floats(0.0, 0.6), + removed_density=st.sampled_from([0.0, 0.1, 0.4]), seed=st.integers(0, 2**31 - 1), ) @settings(max_examples=60, deadline=None) -def test_interpolable_defects_are_the_short_bounded_runs(n, density, seed): +def test_interpolable_defects_are_the_short_bounded_runs( + n, density, removed_density, seed, +): """A defect pixel is interpolated exactly when its row or column run is - at most MAX_INTERPOLATED_RUN long and has clean pixels at both ends; the - rule commutes with quarter turns. - - Failure modes: a run touching the stamp border (edge band, corner) or a - wide hole is interpolated from one side; a 3-px bleed is left to noise; - a clean pixel is selected; one axis is ignored, so the rule has a - preferred direction. + at most MAX_INTERPOLATED_RUN long and the pixels beyond both ends are on + the stamp and neither defects nor removed; the rule commutes with + quarter turns. + + Failure modes: a run touching the stamp border (edge band, corner), a + wide hole, or a run ending on a removed neighbour pixel is interpolated + from one side; a 3-px bleed is left to noise; a clean or removed pixel + is selected; one axis is ignored, so the rule has a preferred direction. """ - defect = np.random.RandomState(seed).uniform(size=(n, n)) < density - out = interpolable_defects(defect) - npt.assert_array_equal(out, _oracle(defect)) - assert not out[~defect].any() + rng = np.random.RandomState(seed) + defect = rng.uniform(size=(n, n)) < density + removed = rng.uniform(size=(n, n)) < removed_density + out = interpolable_defects(defect, removed) + npt.assert_array_equal(out, _oracle(defect, removed)) + assert not out[~defect | removed].any() for k in range(1, 4): npt.assert_array_equal( - interpolable_defects(np.rot90(defect, k)), np.rot90(out, k) + interpolable_defects(np.rot90(defect, k), np.rot90(removed, k)), + np.rot90(out, k), ) +@given( + n=st.integers(5, 31), + density=st.floats(0.0, 0.6), + removed_density=st.sampled_from([0.0, 0.1, 0.4]), + seed=st.integers(0, 2**31 - 1), +) +@settings(max_examples=60, deadline=None) +def test_the_interpolant_reaches_every_interpolable_pixel( + n, density, removed_density, seed, +): + """Each selected pixel lies between two support pixels, so the + interpolant is finite there; prepare_ngmix_weights relies on this to + veto and fill the same pixels. + + Failure mode: a selected pixel falls outside the support's hull and gets + NaN, so the fill and the central-defect veto disagree on it. + """ + rng = np.random.RandomState(seed) + defect = rng.uniform(size=(n, n)) < density + removed = rng.uniform(size=(n, n)) < removed_density + target = interpolable_defects(defect, removed) + planes = rng.normal(size=(2, n, n)) + out = interpolate_defects(planes, defect | removed, target) + assert np.all(np.isfinite(out[:, target])) + + @pytest.mark.parametrize("kind,expected", [ ("column", True), ("bleed3", True), ("finite_bleed3", True), ("pixel", True), ("bleed4", False), ("blob5", False), diff --git a/tests/module/test_ngmix.py b/tests/module/test_ngmix.py index fadc4b991..89acb58ea 100644 --- a/tests/module/test_ngmix.py +++ b/tests/module/test_ngmix.py @@ -663,13 +663,6 @@ def test_process_counts_flagged_fits_across_batches(tmp_path, monkeypatch, flags inst._blend_handling = "noisefill" inst._dilate_neighbour = 1 inst._metacal_psf = "fitgauss" - inst._epoch_central_defect_radius = module.EPOCH_CENTRAL_DEFECT_RADIUS - inst._epoch_masked_fraction_cut = module.EPOCH_MASKED_FRACTION_CUT - inst._defect_fill = "noise" - inst._epoch_interpolated_defect_radius = ( - module.EPOCH_INTERPOLATED_DEFECT_RADIUS - ) - inst._symmetrize_weights = "interpolated" inst._save_batch = 1 inst._zero_point = 30. inst._output_dir = str(tmp_path) @@ -767,13 +760,6 @@ def capture(stamp, prior, flux_guess, rng, **kwargs): inst._blend_handling = "noisefill" inst._dilate_neighbour = 1 inst._metacal_psf = "fitgauss" - inst._epoch_central_defect_radius = module.EPOCH_CENTRAL_DEFECT_RADIUS - inst._epoch_masked_fraction_cut = module.EPOCH_MASKED_FRACTION_CUT - inst._defect_fill = "noise" - inst._epoch_interpolated_defect_radius = ( - module.EPOCH_INTERPOLATED_DEFECT_RADIUS - ) - inst._symmetrize_weights = "interpolated" inst._save_batch = -1 inst._w_log = _RecordingLogger() @@ -1132,10 +1118,12 @@ def test_background_rms_builds_per_pixel_inverse_variance(): gal, weight, flag, np.random.RandomState(0), bkg_rms=bkg_rms ) + # The bad-RMS pixel (1, 2) is interpolated from (0, 2) and (2, 2), so + # its quarter turns (0, 1) and (1, 0) lose their weight too. expected = np.array( [ - [1.0, 0.25, 0.0625], - [4.0, 0.0, 0.0], + [1.0, 0.0, 0.0625], + [0.0, 0.0, 0.0], [0.0, 0.0, 1.0], ] ) diff --git a/tests/module/test_ngmix_defect_fill.py b/tests/module/test_ngmix_defect_fill.py index ed59ef2f6..29d534e46 100644 --- a/tests/module/test_ngmix_defect_fill.py +++ b/tests/module/test_ngmix_defect_fill.py @@ -1,18 +1,18 @@ """Defect fill and the epoch cuts (ngmix module). -A defect is a stamp pixel with a nonzero flag, zero exposure weight or an -invalid background RMS. Before metacal, :func:`prepare_ngmix_weights` gives -every defect weight 0 and fills it, whatever ``BLEND_HANDLING`` is: with noise -at its background RMS (``DEFECT_FILL = noise``, the default) or, for short -defect runs, with an interpolant of the clean pixels around them -(``DEFECT_FILL = interpolate``). Under uberseg, pixels on the neighbour side -only lose their weight, and their image values stay raw. The per-epoch cuts -in :func:`prepare_postage_stamps` act on the same defect set: the +A defect is a stamp pixel with a nonzero flag, zero exposure weight +(off-tile pixels included) or an invalid background RMS. Before metacal, +:func:`prepare_ngmix_weights` gives every defect weight 0 and fills it the +same way whatever ``BLEND_HANDLING`` is: short defect runs take an +interpolant of the kept pixels around them, and the rest take noise at the +background RMS. Under uberseg, pixels on the neighbour side only lose their +weight, and their image values stay raw. The per-epoch cuts in +:func:`prepare_postage_stamps` act on the same defect set: the masked-fraction cut counts it, and the central-defect veto drops an epoch -with a defect near the stamp centre, at a radius set by the defect's fill. -The tile VIGNET's -1e30 neighbour markers are not defects: -noisefill zero-weights and noise-fills them, uberseg ignores them, and the -epoch cuts never count them. +with a defect near the stamp centre, at the radius of the fill that defect +gets. The tile VIGNET's -1e30 neighbour markers are not defects: noisefill +zero-weights and noise-fills them, uberseg ignores them, and the epoch cuts +never count them. """ import re @@ -24,9 +24,8 @@ import pytest from astropy.io import fits from astropy.wcs import WCS -from hypothesis import given +from hypothesis import given, settings from hypothesis import strategies as st -from modopt.math.stats import sigma_mad from sqlitedict import SqliteDict from shapepipe.modules.ngmix_package import ngmix as ngmix_module @@ -65,9 +64,13 @@ def defect_stamps(draw): flag[i, j] = draw(st.sampled_from([1, 2, 2**10])) for i, j in draw(pixels): bkg_rms[i, j] = draw(st.sampled_from([0.0, -1.0, np.nan, np.inf])) - # Raw values far outside the unit-RMS noise, so a filled pixel is - # unambiguous. - gal = 1.0e3 + np.arange(n * n, dtype=float).reshape(n, n) + # Unit-noise sky with raw defect values far above it, so a filled pixel + # is unambiguous and a leaked raw value is obvious. + seed = draw(st.integers(0, 2**31 - 1)) + gal = np.random.RandomState(seed).normal(size=(n, n)) + valid_rms = np.isfinite(bkg_rms) & (bkg_rms > 0) + defect = (weight == 0) | (flag != 0) | ~valid_rms + gal[defect] = 1.0e6 return gal, weight, flag, bkg_rms @@ -84,16 +87,19 @@ def _uberseg_seg(n): blend_handling=st.sampled_from(["noisefill", "uberseg"]), seed=st.integers(0, 2**31 - 1), ) +@settings(deadline=None) def test_filled_set_is_the_defect_set(stamp, blend_handling, seed): - """Filled pixels are exactly the defects (flag, zero weight, bad RMS). - They carry zero weight and look like noise. Every other pixel keeps its - raw value, under either BLEND_HANDLING. Under uberseg the weight is also - zero on the neighbour side, whose light stays. + """Filled pixels are exactly the defects (flag, zero weight, bad RMS), + and no raw defect value survives. Every other pixel keeps its raw value, + under either BLEND_HANDLING. The weight is zero on the defects, on the + quarter-turn copies of the interpolated ones, and under uberseg on the + neighbour side; those copies and the neighbour side keep their light. Failure modes: * a defect source (flag, zero weight, bad RMS) is left out of the fill; * the filled set grows beyond the defects (for example symmetrized); - * the filled set and the zero-weight defect set differ; + * the zero-weight set is not the defects plus the interpolated pixels' + quarter turns (and the uberseg neighbour side); * the fill is skipped under uberseg; * neighbour-side pixels are filled. """ @@ -117,16 +123,17 @@ def test_filled_set_is_the_defect_set(stamp, blend_handling, seed): return # fully masked: no clean pixel left to set the noise level filled = gal_out != gal npt.assert_array_equal(filled, defect, "filled set is not the defect set") - assert np.all(np.abs(gal_out[filled]) < 10.0), "fill is not unit noise" + assert np.all(np.abs(gal_out[filled]) < 100.0), "a raw defect value leaks" neighbour = ( uberseg_mask(kwargs["seg"], 1, dilate_neighbour=1) if blend_handling == "uberseg" else np.zeros((n, n), dtype=bool) ) - # Zero weight on defects and neighbour side; neighbour pixels that are - # not defects keep their raw values (filled-set equality above). - npt.assert_array_equal(w_out == 0.0, defect | neighbour) - npt.assert_array_equal(w_out[~(defect | neighbour)], 1.0) + # Neighbour pixels that are not defects keep their raw values + # (filled-set equality above). + zero = defect | neighbour | fourfold(interpolable_defects(defect)) + npt.assert_array_equal(w_out == 0.0, zero) + npt.assert_array_equal(w_out[~zero], 1.0) def test_uberseg_defect_in_neighbour_region_is_filled(): @@ -140,6 +147,7 @@ def test_uberseg_defect_in_neighbour_region_is_filled(): weight = np.ones((n, n)) flag = np.zeros((n, n), dtype=np.int32) flag[1, 1] = 1 # inside the neighbour footprint + gal[1, 1] = 1.0e6 gal_out, w_out, _ = prepare_ngmix_weights( gal, weight, flag, np.random.RandomState(0), bkg_rms=np.ones((n, n)), blend_handling="uberseg", seg=_uberseg_seg(n), object_number=1, @@ -261,62 +269,51 @@ def test_epoch_cut_counts_the_defect_set(): assert _surviving(_epochs()) == ["2100001-10", "2100002-11"] -def test_epoch_cut_threshold_is_the_configured_fraction(): - """At a 10% cut (the DES Y3 and Y6 value), the 19.6% band is dropped as - well, and only the clean epoch survives. - - Failure mode: the configured threshold is ignored. - """ - assert _surviving(_epochs(), epoch_masked_fraction_cut=0.1) == [ - "2100001-10" - ] - - # --- prepare_postage_stamps: the central-defect veto ----------------------- -def _veto_epochs(radius): - """A clean epoch, one with a single flagged pixel just inside - ``radius`` of the stamp centre, and one with a flagged column exactly - ``radius`` away. Both masked epochs are far below the fraction cut. - """ +def _veto_epochs(): + """A clean epoch and five with one defect each, all far below the + fraction cut: a column at EPOCH_INTERPOLATED_DEFECT_RADIUS, a column one + pixel inside it, a single pixel at 9 px (all interpolated), and a 5-px + bleed (noise-filled) at the interpolated radius and at + EPOCH_CENTRAL_DEFECT_RADIUS.""" centre = N_STAMP // 2 + ri = int(EPOCH_INTERPOLATED_DEFECT_RADIUS) + rn = int(EPOCH_CENTRAL_DEFECT_RADIUS) clean = np.zeros((N_STAMP, N_STAMP), dtype=np.int32) ones = np.ones((N_STAMP, N_STAMP)) - near, far = clean.copy(), clean.copy() - near[centre, centre + radius - 1] = 1 - far[:, centre + radius] = 1 + column_at, column_inside, pixel, wide_at, wide_far = ( + clean.copy() for _ in range(5) + ) + column_at[:, centre + ri] = 1 + column_inside[:, centre + ri - 1] = 1 + pixel[centre, centre + 9] = 1 + wide_at[:, centre + ri:centre + ri + 5] = 1 + wide_far[:, centre + rn:centre + rn + 5] = 1 return { "2100001-10": (clean, ones), - "2100002-11": (near, ones), - "2100003-12": (far, ones), + "2100002-11": (column_at, ones), + "2100003-12": (column_inside, ones), + "2100004-13": (pixel, ones), + "2100005-14": (wide_at, ones), + "2100006-15": (wide_far, ones), } -def test_central_defect_vetoes_the_epoch(): - """At the default radius, a single defect pixel inside it drops the - epoch, and a column at the radius does not. - - Failure mode: the veto is skipped, so an epoch whose filled hole overlaps - the object's light enters the fit; or the boundary is inclusive, dropping - the column at the radius (epoch-central-defect-veto). - """ - epochs = _veto_epochs(EPOCH_CENTRAL_DEFECT_RADIUS) - assert _surviving(epochs) == ["2100001-10", "2100003-12"] - - -def test_central_defect_radius_is_the_configured_value(): - """Radius 0 disables the veto; a radius one pixel larger than the far - column's distance drops that epoch too. - - Failure mode: the configured radius is ignored. +def test_the_veto_radius_follows_the_fill(): + """An interpolated defect drops the epoch inside + EPOCH_INTERPOLATED_DEFECT_RADIUS, a noise-filled one inside + EPOCH_CENTRAL_DEFECT_RADIUS; a defect at its radius is kept. + + Failure modes: the veto is skipped, so a filled hole in the object's + light enters the fit; the interpolated radius is applied to noise-filled + pixels (a wide hole next to the object survives), or not applied at all + (a 9-px pixel is dropped); the boundary is inclusive + (veto-radius-follows-the-fill). """ - epochs = _veto_epochs(EPOCH_CENTRAL_DEFECT_RADIUS) - assert _surviving(epochs, epoch_central_defect_radius=0) == [ - "2100001-10", "2100002-11", "2100003-12" + assert _surviving(_veto_epochs()) == [ + "2100001-10", "2100002-11", "2100004-13", "2100006-15" ] - assert _surviving( - epochs, epoch_central_defect_radius=EPOCH_CENTRAL_DEFECT_RADIUS + 1 - ) == ["2100001-10"] # --- prepare_postage_stamps: per-epoch OFFSET ------------------------------ @@ -360,7 +357,7 @@ def info(self, msg, *_args, **_kwargs): def test_process_logs_the_epoch_cut_tally(tmp_path, monkeypatch): """One tile, four objects; the end-of-tile line counts each cut's drops. - * object 1: clean, 18-column edge band, defect inside the radius -> one + * object 1: clean, 18-column edge band, defect 3 px from the centre -> one epoch each for considered, masked_fraction, central_veto; survives. * object 2: edge band and central defect -> both epochs dropped; emptied. * object 3: one all-zero stamp, skipped before the cuts -> not considered, @@ -372,13 +369,12 @@ def test_process_logs_the_epoch_cut_tally(tmp_path, monkeypatch): object with no epoch at all is reported as emptied by the cuts; counts from one object overwrite another's. """ - radius = EPOCH_CENTRAL_DEFECT_RADIUS centre = N_STAMP // 2 clean = np.zeros((N_STAMP, N_STAMP), dtype=np.int32) ones = np.ones((N_STAMP, N_STAMP)) wide18, near = clean.copy(), clean.copy() wide18[:, -18:] = 1 - near[centre, centre + radius - 1] = 1 + near[centre, centre + 3] = 1 objects = { 1: {"2100001-10": (clean, ones), "2100002-11": (wide18, ones), "2100003-12": (near, ones)}, @@ -450,80 +446,6 @@ def no_fit(*_args, **_kwargs): ), lines[0] -# --- ngmix_runner: the epoch-cut options reach Ngmix ------------------------ - -class _OptionConfig: - """Config stub answering from a dict; absent options take the caller's - fallback.""" - - def __init__(self, options): - self._options = {"MAG_ZP": "30.0", "ID_OBJ_MIN": "-1", - "ID_OBJ_MAX": "-1", **options} - - def has_option(self, _sec, key): - return key in self._options - - def get(self, _sec, key): - return self._options[key] - - def getexpanded(self, _sec, key): - return self._options[key] - - def getfloat(self, _sec, key): - return float(self._options[key]) - - def getint(self, _sec, key): - return int(self._options[key]) - - def getboolean(self, _sec, key, fallback=False): - return fallback - - -def test_runner_threads_the_epoch_cut_options(tmp_path, monkeypatch): - """EPOCH_CENTRAL_DEFECT_RADIUS, EPOCH_MASKED_FRACTION_CUT, DEFECT_FILL and - EPOCH_INTERPOLATED_DEFECT_RADIUS reach Ngmix as configured, and default - to the module constants (and the noise fill) when absent. - - Failure mode: the runner drops or ignores an option, so a configured A/B - arm silently runs the default cuts. - """ - from shapepipe.modules import ngmix_runner as runner_module - - captured = [] - - class _Capture: - def __init__(self, *_args, **kwargs): - captured.append(kwargs) - - def process(self): - pass - - monkeypatch.setattr(runner_module, "Ngmix", _Capture) - inputs = [str(tmp_path / f"in{i}.sqlite") for i in range(7)] - for path in inputs: - SqliteDict(path).close() - - for options, radius, fraction, fill, interpolated_radius in ( - ({"EPOCH_CENTRAL_DEFECT_RADIUS": "7.5", - "EPOCH_MASKED_FRACTION_CUT": "0.1", - "DEFECT_FILL": "interpolate", - "EPOCH_INTERPOLATED_DEFECT_RADIUS": "5.5"}, - 7.5, 0.1, "interpolate", 5.5), - ({}, EPOCH_CENTRAL_DEFECT_RADIUS, - ngmix_module.EPOCH_MASKED_FRACTION_CUT, "noise", - EPOCH_INTERPOLATED_DEFECT_RADIUS), - ): - runner_module.ngmix_runner( - inputs, {"output": str(tmp_path)}, "-001-001", - _OptionConfig(options), "NGMIX_RUNNER", _RecordingLogger(), - ) - assert captured[-1]["epoch_central_defect_radius"] == radius - assert captured[-1]["epoch_masked_fraction_cut"] == fraction - assert captured[-1]["defect_fill"] == fill - assert (captured[-1]["epoch_interpolated_defect_radius"] - == interpolated_radius) - - # --- make_ngmix_observation: the HSM centroid reads the filled image ------- def test_hsm_centroid_ignores_raw_defect_values(): @@ -559,7 +481,7 @@ def test_hsm_centroid_ignores_raw_defect_values(): ) -# --- DEFECT_FILL = interpolate ---------------------------------------------- +# --- Interpolated defects --------------------------------------------------- def _hot_stamp(): """A bright object with hot defects: a column and a finite 3-px bleed @@ -603,7 +525,7 @@ def test_interpolated_fill_and_its_weights(blend_handling): gal_out, w_out, noise_out = prepare_ngmix_weights( gal, weight, flag, np.random.RandomState(4), bkg_rms=np.ones((N_STAMP, N_STAMP)), blend_handling=blend_handling, - defect_fill="interpolate", **kwargs, + **kwargs, ) neighbour = ( @@ -619,229 +541,6 @@ def test_interpolated_fill_and_its_weights(blend_handling): npt.assert_allclose(noise_out[target], refilled[target], atol=1e-5) -# --- SYMMETRIZE_WEIGHTS ---------------------------------------------------- - -def _symmetrization_case(blend_handling, defect_fill): - """The hot stamp with a neighbour in one corner, and its pixel classes: - those whose light is replaced, the interpolated defects and the uberseg - neighbour side (the last two keep their light at zero weight).""" - gal, weight, flag = _hot_stamp() - defect = flag != 0 - seg = np.zeros_like(flag) - seg[N_STAMP // 2, N_STAMP // 2] = 1 - seg[-12:, -12:] = 2 - marked = seg == 2 - kwargs = dict( - blend_handling=blend_handling, defect_fill=defect_fill, - seg=seg, object_number=1, dilate_neighbour=1, neighbour=marked, - ) - interpolated = ( - interpolable_defects(defect) & ~marked - if defect_fill == "interpolate" else np.zeros_like(defect) - ) - side = ( - uberseg_mask(seg, 1, dilate_neighbour=1) - if blend_handling == "uberseg" else np.zeros_like(defect) - ) - replaced = defect & ~interpolated - if blend_handling == "noisefill": - replaced |= marked - return gal, weight, flag, kwargs, replaced, interpolated, side - - -@pytest.mark.parametrize("defect_fill", ["noise", "interpolate"]) -@pytest.mark.parametrize("blend_handling", ["noisefill", "uberseg"]) -@pytest.mark.parametrize( - "symmetrize", ["interpolated", "interpolated_and_neighbours", "none"] -) -def test_symmetrized_weight_holes(blend_handling, defect_fill, symmetrize): - """The weight is zero on the pixels whose light is replaced, on the - interpolated defects and on the uberseg neighbour side, and on the - quarter turns of the classes SYMMETRIZE_WEIGHTS names. The switch moves - no image value and no noise draw. - - Failure modes: a noise-filled defect or noisefill neighbour is - symmetrized; the default symmetrizes the uberseg neighbour side (its - neighbour light then dominates the bias, see symmetrized-weight-holes), - or the opt-in value does not; the switch moves an image value or a - noise draw. - """ - gal, weight, flag, kwargs, replaced, interpolated, side = ( - _symmetrization_case(blend_handling, defect_fill) - ) - rms = np.ones_like(gal) - out = prepare_ngmix_weights( - gal, weight, flag, np.random.RandomState(3), bkg_rms=rms, - symmetrize_weights=symmetrize, **kwargs, - ) - unsymmetrized = prepare_ngmix_weights( - gal, weight, flag, np.random.RandomState(3), bkg_rms=rms, - symmetrize_weights="none", **kwargs, - ) - holes = { - "interpolated": interpolated, - "interpolated_and_neighbours": interpolated | side, - "none": np.zeros_like(side), - }[symmetrize] - npt.assert_array_equal( - out[1] == 0.0, replaced | interpolated | side | fourfold(holes) - ) - npt.assert_array_equal(out[0], unsymmetrized[0]) - npt.assert_array_equal(out[2], unsymmetrized[2]) - - -def test_symmetrization_changes_nothing_under_the_defaults(): - """Under noisefill with the noise fill every zero-weight pixel has its - light replaced, so every SYMMETRIZE_WEIGHTS value gives the same - image, weight map and noise image.""" - gal, weight, flag, kwargs, _, _, _ = _symmetrization_case( - "noisefill", "noise" - ) - rms = np.ones_like(gal) - outputs = [ - prepare_ngmix_weights( - gal, weight, flag, np.random.RandomState(8), bkg_rms=rms, - symmetrize_weights=value, **kwargs, - ) - for value in ("interpolated", "interpolated_and_neighbours", "none") - ] - for other in outputs[1:]: - for a, b in zip(outputs[0], other): - npt.assert_array_equal(a, b) - - -def test_unknown_symmetrize_weights_is_rejected(): - gal, weight, flag = _hot_stamp() - with pytest.raises(ValueError, match="SYMMETRIZE_WEIGHTS"): - prepare_ngmix_weights( - gal, weight, flag, np.random.RandomState(0), - symmetrize_weights=True, - ) - - -def test_noise_is_the_default_fill(): - """Leaving DEFECT_FILL unset is bit-identical to the noise fill.""" - gal, weight, flag = _hot_stamp() - rms = np.ones_like(gal) - default = prepare_ngmix_weights( - gal, weight, flag, np.random.RandomState(9), bkg_rms=rms - ) - noise = prepare_ngmix_weights( - gal, weight, flag, np.random.RandomState(9), bkg_rms=rms, - defect_fill="noise", - ) - for a, b in zip(default, noise): - npt.assert_array_equal(a, b) - - -def test_unknown_defect_fill_is_rejected(): - gal, weight, flag = _hot_stamp() - with pytest.raises(ValueError, match="DEFECT_FILL"): - prepare_ngmix_weights( - gal, weight, flag, np.random.RandomState(0), - defect_fill="interp", - ) - - -def _fill_veto_epochs(): - """A clean epoch; a column at the interpolated-defect radius; a column - one pixel inside it; a 5-px bleed (noise-filled) at the same radius.""" - centre = N_STAMP // 2 - radius = int(EPOCH_INTERPOLATED_DEFECT_RADIUS) - clean = np.zeros((N_STAMP, N_STAMP), dtype=np.int32) - ones = np.ones((N_STAMP, N_STAMP)) - at, inside, wide = clean.copy(), clean.copy(), clean.copy() - at[:, centre + radius] = 1 - inside[:, centre + radius - 1] = 1 - wide[:, centre + radius:centre + radius + 5] = 1 - return { - "2100001-10": (clean, ones), - "2100002-11": (at, ones), - "2100003-12": (inside, ones), - "2100004-13": (wide, ones), - } - - -def test_the_veto_radius_follows_the_fill(): - """Under interpolation, an interpolated defect is vetoed inside - EPOCH_INTERPOLATED_DEFECT_RADIUS and a noise-filled one inside - EPOCH_CENTRAL_DEFECT_RADIUS. Under the noise fill every defect keeps the - noise radius. The masked-fraction cut counts the raw defect set in - both modes. - - Failure modes: the interpolated radius is applied to noise-filled pixels - (a wide hole next to the object survives), or not applied at all; the - configured radius is ignored; the boundary is inclusive; the fraction - cut counts the zero-weight orbit. - """ - epochs = _fill_veto_epochs() - assert _surviving(epochs, defect_fill="interpolate") == [ - "2100001-10", "2100002-11" - ] - assert _surviving(epochs) == ["2100001-10"] - assert _surviving( - epochs, defect_fill="interpolate", - epoch_interpolated_defect_radius=EPOCH_INTERPOLATED_DEFECT_RADIUS + 1, - ) == ["2100001-10"] - assert _surviving(_epochs(), defect_fill="interpolate") == [ - "2100001-10", "2100002-11" - ] - - -def test_process_threads_the_defect_fill(tmp_path, monkeypatch): - """Ngmix hands DEFECT_FILL to both the epoch cuts and metacal: under - interpolation the column at the interpolated-defect radius survives and - metacal is asked to interpolate; under noise it is vetoed. - - Failure mode: the option is dropped on the way to either, so the tile - silently runs the noise fill or the noise cuts. - """ - epochs = {k: v for k, v in _fill_veto_epochs().items() - if k in ("2100001-10", "2100002-11")} - vignet, tile_cat, psf_obj, _ = _fake_inputs(epochs) - vignet.psf_vign_cat = {"1": psf_obj} - vignet.close = lambda: None - tile_cat.obj_id = np.array([1]) - tile_cat.flux = None - monkeypatch.setattr(ngmix_module, "Tile_cat", lambda *a, **k: tile_cat) - for method in ("compile_results", "save_results", "log_mean_ellipticity"): - monkeypatch.setattr(Ngmix, method, lambda *_a, **_k: None) - calls = [] - - def record(stamp, *_args, **kwargs): - calls.append((len(stamp.gals), kwargs.get("defect_fill"))) - raise RuntimeError("metacal is not under test") - - monkeypatch.setattr(ngmix_module, "do_ngmix_metacal", record) - paths = [tmp_path / f"{name}.sqlite" for name in - ("gal", "psf", "weight", "flag", "headers")] - for path in paths: - SqliteDict(str(path)).close() - for fill in ("interpolate", "noise"): - ngmix = Ngmix( - ["tile_cat.fits"] + [str(p) for p in paths[:4]], - str(tmp_path), "-001-001", 30.0, str(paths[4]), - _RecordingLogger(), bkg_sub=False, defect_fill=fill, - ) - ngmix._vignet_cat.close() - ngmix._vignet_cat = vignet - ngmix.process() - assert calls == [(2, "interpolate"), (1, "noise")] - - -def test_ngmix_rejects_an_unknown_defect_fill(tmp_path): - paths = [tmp_path / f"{name}.sqlite" for name in - ("gal", "psf", "weight", "flag", "headers")] - for path in paths: - SqliteDict(str(path)).close() - with pytest.raises(ValueError, match="DEFECT_FILL"): - Ngmix( - ["tile_cat.fits"] + [str(p) for p in paths[:4]], - str(tmp_path), "-001-001", 30.0, str(paths[4]), - _RecordingLogger(), bkg_sub=False, defect_fill="interp", - ) - - # --- The tile VIGNET's -1e30 neighbour markers are not defects ------------- # # The tile VIGNET carries -1e30 on the footprints of other detections. Every @@ -1008,83 +707,6 @@ def test_uberseg_leaves_the_marked_pixels_raw(): assert np.all(w_out > 0.0) -def _develop_noisefill(gal, weight, flag, rng, bkg_rms=None): - """prepare_ngmix_weights under BLEND_HANDLING = noisefill on develop - (240b37e4), where the markers arrive as flag 2**10: the reference the - noisefill output is pinned to.""" - mask = np.copy(weight) != 0 - mask[flag != 0] = False - if bkg_rms is None: - sig_noise = sigma_mad(gal) - weight_map = mask.astype(float) / sig_noise ** 2 - else: - valid_rms = np.isfinite(bkg_rms) & (bkg_rms > 0) - mask &= valid_rms - weight_map = np.zeros_like(gal, dtype=float) - weight_map[mask] = 1.0 / bkg_rms[mask] ** 2 - sig_noise = np.where(valid_rms, bkg_rms, np.median(bkg_rms[mask])) - noise_img = rng.standard_normal(gal.shape) * sig_noise - noise_img_gal = rng.standard_normal(gal.shape) * sig_noise - gal_masked = np.copy(gal) - gal_masked[~mask] = noise_img_gal[~mask] - return gal_masked, weight_map, noise_img - - -@given( - seed=st.integers(0, 2**31 - 1), - with_rms=st.booleans(), - rms_scale=st.floats(0.5, 2.0), - with_defects=st.booleans(), - dtype=st.sampled_from([np.float64, np.float32]), -) -def test_noisefill_matches_develop_on_marked_neighbours( - seed, with_rms, rms_scale, with_defects, dtype, -): - """For an epoch with a marked neighbour, noisefill returns the image, - weight and noise image develop returned, bit for bit: with no defect, - and with a flagged column and a dead pixel under the default noise fill, - for float64 and float32 stamps (the filled image keeps the stamp's - dtype). - The stamps carry no off-tile pixels, and the equivalence is claimed for - such stamps only. Off-tile pixels reach this function as flag 2**10 - defects (off-tile-pixels-are-defects), as every marker did on develop; - which epochs survive the cuts differs from develop wherever there are - neighbour markers. - - Failure mode: carrying the markers apart from the flags changes what - noisefill does to neighbour pixels, their weights, the noise level or - the RNG stream (noisefill-matches-develop). - """ - rng = np.random.default_rng(seed) - n = 31 - gal = rng.normal(0.0, 1.0, (n, n)) - gal[n // 2 - 2:n // 2 + 3, n // 2 - 2:n // 2 + 3] += 50.0 - weight = np.ones((n, n)) - flag = np.zeros((n, n), dtype=np.int32) - neighbour = np.zeros((n, n), dtype=bool) - neighbour[n // 2 - 3:n // 2 + 4, n // 2 + 3:n // 2 + 9] = True - gal[neighbour] += 30.0 - if with_defects: - flag[:, 2] = 1 - weight[n - 3, n // 2] = 0.0 - gal = gal.astype(dtype) - bkg_rms = ( - rms_scale * (1.0 + 0.1 * rng.random((n, n))) if with_rms else None - ) - - new = prepare_ngmix_weights( - gal, weight, flag, np.random.RandomState(seed), bkg_rms=bkg_rms, - blend_handling="noisefill", neighbour=neighbour, - ) - old = _develop_noisefill( - gal, weight, np.where(neighbour, 2**10, flag), - np.random.RandomState(seed), bkg_rms=bkg_rms, - ) - for a, b in zip(new, old): - assert a.dtype == b.dtype - npt.assert_array_equal(a, b) - - def test_do_ngmix_metacal_threads_each_epochs_neighbour_mask(monkeypatch): """Each epoch's neighbour mask reaches make_ngmix_observation. @@ -1122,11 +744,8 @@ def fake_observation(*args, **kwargs): # The tile VIGNET also holds -1e30 beyond the tile's edge, where the epoch # holds the object's own light, cut off. Those pixels are the runs of # entirely -1e30 stamp rows and columns that start at a stamp border (the -# off-image part of a rectangle clip); -# they join the epoch's defect set as flag 2**10. The other markers are the -# neighbour mask. - -_OFF_TILE = 2**10 +# off-image part of a rectangle clip); they join the epoch's defect set with +# zero exposure weight. The other markers are the neighbour mask. def _off_tile_expected(tile, name): @@ -1154,23 +773,27 @@ def test_object_three_px_from_the_tile_edge_is_dropped(): def test_the_central_veto_sees_off_tile_pixels(): - """An off-tile band 12 px from the object (27% of the stamp) passes the - default cuts and is vetoed at radius 13. + """An off-tile band 12 px from the object passes both cuts; one 9 px + away (17 columns, exactly 1/3 of the stamp, which the fraction cut + keeps) is noise-filled inside the 10-px radius and vetoed. Failure mode: the central veto does not read the off-tile set. """ - tile = np.random.default_rng(5).normal(0.0, 1.0, (N_STAMP, N_STAMP)) - tile[:, :_CENTRE - 11] = _MARKER - kept, _, _ = _marker_stamp(tile) + sky = np.random.default_rng(5).normal(0.0, 1.0, (N_STAMP, N_STAMP)) + far, near = sky.copy(), sky.copy() + far[:, :_CENTRE - 11] = _MARKER + near[:, :_CENTRE - 8] = _MARKER + assert (near == _MARKER).mean() == 1 / 3 + kept, _, _ = _marker_stamp(far) assert len(kept.gals) == len(_MARKER_EPOCH_NAMES) - vetoed, _, _ = _marker_stamp(tile, epoch_central_defect_radius=13) + vetoed, _, _ = _marker_stamp(near) assert len(vetoed.gals) == 0 assert vetoed.epoch_cuts["central_veto"] == len(_MARKER_EPOCH_NAMES) def test_corner_off_tile_region_and_border_neighbour_are_classified(): - """At a tile corner, the L-shaped off-tile region is flagged 2**10 - exactly, and a neighbour footprint touching the stamp border without + """At a tile corner, exactly the L-shaped off-tile region gets zero + weight, and a neighbour footprint touching the stamp border without filling a row or column stays in the neighbour mask. Failure modes: off-tile pixels are classified by something other than @@ -1184,11 +807,14 @@ def test_corner_off_tile_region_and_border_neighbour_are_classified(): stamp, epochs, _ = _marker_stamp(tile) names = {id(v[0]): name for name, v in epochs.items()} assert len(stamp.gals) == len(_MARKER_EPOCH_NAMES) - for flag, neighbour in zip(stamp.flags, stamp.neighbours): + for flag, weight, neighbour in zip( + stamp.flags, stamp.weights, stamp.neighbours + ): name = names[id(flag)] off_tile = _off_tile_expected(tile, name) assert off_tile.sum() == 5 * N_STAMP * 2 - 25 - npt.assert_array_equal(flag == _OFF_TILE, off_tile) + npt.assert_array_equal(weight == 0, off_tile) + npt.assert_array_equal(flag, 0) npt.assert_array_equal( neighbour, _expected_neighbours(tile, name) & ~off_tile ) @@ -1217,7 +843,7 @@ def test_off_tile_pixels_are_zero_weighted_and_filled(blend_handling): gal_out, w_out, _ = _stamp_epoch_weights( stamp, i, seed=i, blend_handling=blend_handling, **kwargs, ) - off_tile = stamp.flags[i] == _OFF_TILE + off_tile = stamp.weights[i] == 0 assert off_tile.sum() == 5 * N_STAMP removed = off_tile | ( stamp.neighbours[i] if blend_handling == "noisefill" else False @@ -1311,13 +937,15 @@ def test_a_neighbour_completing_rows_beside_the_tile_edge_stays_a_neighbour(): stamp, epochs, _ = _marker_stamp(tile) names = {id(v[0]): name for name, v in epochs.items()} assert len(stamp.flags) == len(_MARKER_EPOCH_NAMES) - for flag, neighbour in zip(stamp.flags, stamp.neighbours): + for flag, weight, neighbour in zip( + stamp.flags, stamp.weights, stamp.neighbours + ): ccd = int(names[id(flag)].split("-")[1]) - npt.assert_array_equal(flag == _OFF_TILE, Ngmix.MegaCamFlip(off, ccd)) + npt.assert_array_equal(weight == 0, Ngmix.MegaCamFlip(off, ccd)) npt.assert_array_equal(neighbour, Ngmix.MegaCamFlip(footprint, ccd)) -# --- DEFECT_FILL = interpolate beside a removed neighbour ------------------- +# --- Interpolation beside a removed neighbour ------------------------------- def _defect_beside_neighbour(): """A single flagged pixel 6 px right of the centre, with a bright marked @@ -1353,7 +981,6 @@ def test_noisefill_interpolation_does_not_read_removed_neighbour_light(): out, _, _ = prepare_ngmix_weights( gal, weight, flag, np.random.RandomState(0), blend_handling="noisefill", neighbour=neighbour, - defect_fill="interpolate", ) expected = interpolate_defects(gal[None], (flag != 0) | neighbour, target) assert out[pix] == pytest.approx(expected[0][pix]) @@ -1362,8 +989,54 @@ def test_noisefill_interpolation_does_not_read_removed_neighbour_light(): out, _, _ = prepare_ngmix_weights( gal, weight, flag, np.random.RandomState(0), blend_handling="uberseg", seg=seg, object_number=1, - neighbour=neighbour, defect_fill="interpolate", + neighbour=neighbour, ) expected = interpolate_defects(gal[None], flag != 0, target) assert out[pix] == pytest.approx(expected[0][pix]) assert out[pix] > 1000.0 + + +def test_a_column_beside_a_noisefill_neighbour_is_vetoed_as_noise_filled(): + """A column 8 px from the centre is interpolated, and kept, unless a + marked neighbour lies against it. Under noisefill the neighbour's light + is removed, so the column's rows that end on the neighbour cannot be + interpolated; the fill noise-fills them, and the veto drops the epoch at + the noise-fill radius. Under uberseg the neighbour's light stays and + supports the interpolant, so the epoch is kept. + + Failure mode: the veto decides which pixels are interpolated from the + defect mask alone, so it keeps the epoch at the 7-px radius while the + fill noise-fills pixels 8 px from the object, inside the 10 px that + noise fill needs. + """ + column = np.zeros((N_STAMP, N_STAMP), dtype=np.int32) + column[:, _CENTRE + 8] = 1 + # CCD 20 is not flipped, so tile and epoch share their orientation. + epochs = {"2100001-20": (column, np.ones((N_STAMP, N_STAMP)))} + sky = np.random.default_rng(5).normal(0.0, 1.0, (N_STAMP, N_STAMP)) + tile = sky.copy() + tile[_CENTRE - 3:_CENTRE + 4, _CENTRE + 9:_CENTRE + 14] = _MARKER + for tile_vign, blend_handling, kept in ( + (sky, "noisefill", 1), + (tile, "noisefill", 0), + (tile, "uberseg", 1), + ): + stamp, _, _ = _marker_stamp( + tile_vign, epochs, blend_handling=blend_handling, + ) + assert len(stamp.gals) == kept + assert stamp.epoch_cuts["central_veto"] == 1 - kept + + # The fill agrees: under noisefill only the rows clear of the neighbour + # are interpolated (their quarter turns lose weight too); the rows + # against it are noise-filled. + neighbour = tile == _MARKER + defect = column != 0 + interpolated = defect & ~(neighbour[:, _CENTRE + 9][:, None]) + _, weights, _ = prepare_ngmix_weights( + sky, np.ones_like(sky), column, np.random.RandomState(0), + bkg_rms=np.ones_like(sky), neighbour=neighbour, + ) + npt.assert_array_equal( + weights == 0.0, defect | neighbour | fourfold(interpolated) + ) diff --git a/tests/module/test_ngmix_uberseg.py b/tests/module/test_ngmix_uberseg.py index a9d95486a..183fc046b 100644 --- a/tests/module/test_ngmix_uberseg.py +++ b/tests/module/test_ngmix_uberseg.py @@ -8,8 +8,8 @@ emergent "circularisation"), and the neighbour footprint is fully masked. * :func:`prepare_ngmix_weights` under ``uberseg`` — neighbour-side pixels lose their weight and keep their raw image values, while defect pixels are - noise-filled as under any blend handling (the defect fill itself is covered - in ``test_ngmix_defect_fill.py``). + filled as under any blend handling (the defect fill itself is covered in + ``test_ngmix_defect_fill.py``). * The error contract when ``uberseg`` is selected without a segmentation map (the seg-map source is plumbing-gated upstream). """ @@ -237,7 +237,7 @@ def test_noisefill_ignores_seg_and_dilate_kwargs(): def test_uberseg_fills_defects_and_leaves_neighbour_pixels_raw(): - """uberseg: flagged pixels are noise-filled at weight 0; neighbour-side + """uberseg: flagged pixels are filled at weight 0; neighbour-side pixels get weight 0 and keep their raw image values; the central core keeps weight and image. @@ -255,7 +255,7 @@ def test_uberseg_fills_defects_and_leaves_neighbour_pixels_raw(): blend_handling="uberseg", seg=seg, object_number=1, ) - # Flagged pixels: zero weight, image replaced by noise. + # Flagged pixels: zero weight, raw value replaced. for pix in [(5, 5), (30, 12)]: assert w_out[pix] == 0.0 assert gal_out[pix] != gal[pix] diff --git a/tests/science/test_defect_interpolation.py b/tests/science/test_defect_recovery.py similarity index 68% rename from tests/science/test_defect_interpolation.py rename to tests/science/test_defect_recovery.py index d36be39b0..539a6834d 100644 --- a/tests/science/test_defect_interpolation.py +++ b/tests/science/test_defect_recovery.py @@ -1,15 +1,19 @@ -"""Shear recovery for the defects kept under ``DEFECT_FILL = interpolate``. +"""Shear recovery for the defects the epoch cuts keep. -Physics invariant: every defect the veto keeps leaves both additive terms +Physics invariant: every defect the cuts keep leaves both additive terms |c1|, |c2| < 5e-4 and both diagonal multiplicative terms |m11|, |m22| < 1%, -from the full 2x2 response matrix. Interpolated defects (columns, full and -finite 3-px bleeds, single pixels) are checked at -``EPOCH_INTERPOLATED_DEFECT_RADIUS`` and two pixels beyond it, on galaxies -with half-light radius 0.3" and 0.5" through a 0.7" PSF, round and with -ellipticity (0.05, 0.02), and on a 0.7" galaxy through a 0.9" PSF. Defects -too wide to interpolate are noise-filled, and are checked at -``EPOCH_CENTRAL_DEFECT_RADIUS``. The cases sit at the radii themselves, so -lowering either below its calibrated value turns this red. +from the full 2x2 response matrix. A filled defect biases m anisotropically +(m11 and m22 can differ tenfold), so a scalar m would hide it. +Interpolated defects (columns, full and finite 3-px bleeds, single pixels) +are checked at ``EPOCH_INTERPOLATED_DEFECT_RADIUS`` and two pixels beyond +it, on galaxies with half-light radius 0.3" and 0.5" through a 0.7" PSF, +round and with ellipticity (0.05, 0.02), and on a 0.7" galaxy through a +0.9" PSF. These pass only because the interpolated pixels' quarter turns +also lose their weight: an unsymmetrized column at 8 px gives +c1 = -1.3e-3. Defects too wide to interpolate (a 5-px bleed, edge bands) +are noise-filled, and are checked at ``EPOCH_CENTRAL_DEFECT_RADIUS``. The +cases sit at the radii themselves, so lowering either below its calibrated +value turns this red. Positive control: a 3-px bleed three pixels inside the interpolated-defect radius breaks the bound. @@ -39,7 +43,6 @@ ROUND = (0.0, 0.0) ELLIPTICAL = (0.05, 0.02) SEEDS = range(6) -INTERPOLATE = {"defect_fill": "interpolate"} def geometry(kind, distance): @@ -71,6 +74,11 @@ def geometry(kind, distance): for psf in (ROUND, ELLIPTICAL)] + [(k, RN, h, 0.7, ROUND) for k in ("wide_bleed", "edge") for h in (0.3, 0.5)] + # The widest edge band the veto keeps, on the 0.5" galaxy through the + # elliptical PSF, sits at the bound at the radius itself + # (m11 = -0.94% +/- 0.06%), so it is checked one pixel out. + + [("edge", RN + 1, 0.5, 0.7, ELLIPTICAL)] + + [("edge", N - CENTRE - 5, 0.5, 0.7, psf) for psf in (ROUND, ELLIPTICAL)] ) @@ -82,7 +90,7 @@ def case_id(case): def recover(bad, hlr, fwhm, psf_shear, tmp_path): result = defect_response(bad, hlr=hlr, psf=fwhm, seeds=SEEDS, - psf_shear=psf_shear, options=INTERPOLATE) + psf_shear=psf_shear) (tmp_path / "recovery.json").write_text(json.dumps(result, indent=2)) return np.abs(result["m"]).max(), np.abs(result["c"]).max(), result @@ -94,14 +102,15 @@ def test_kept_defects_recover_shear_on_both_axes(kind, distance, hlr, fwhm, """Failure modes: a veto radius is below its calibrated value; the interpolated pixels' quarter-turn orbit keeps its weight (a one-sided hole in the likelihood); the fill mask is symmetrized; a wide hole is - interpolated; raw defect values leak into metacal.""" + interpolated; raw defect values leak into metacal; the check reads m11 + alone and misses m22, or c1 alone.""" bad = geometry(kind, distance) masked = defect_mask(np.ones((N, N)), bad.astype(np.int32)) + np.testing.assert_array_equal(masked, bad) interpolated = interpolable_defects(masked) assert interpolated.any() == (kind in INTERPOLATED) assert masked.mean() <= EPOCH_MASKED_FRACTION_CUT - assert not central_defect_vetoes(masked, EPOCH_CENTRAL_DEFECT_RADIUS, - "interpolate") + assert not central_defect_vetoes(masked, interpolated) m, c, result = recover(bad, hlr, fwhm, psf_shear, tmp_path) assert m < 0.01, result assert c < 5e-4, result @@ -112,8 +121,7 @@ def test_vetoed_bleed_breaks_the_bound(tmp_path): interpolated-defect radius, on the 0.3" galaxy, gives |c| > 1e-3 and |m| > 1.5%. The veto drops it.""" bad = geometry("bleed", RI - 3) - assert central_defect_vetoes(bad, EPOCH_CENTRAL_DEFECT_RADIUS, - "interpolate") + assert central_defect_vetoes(bad, interpolable_defects(bad)) m, c, result = recover(bad, 0.3, 0.7, ROUND, tmp_path) assert c > 1e-3, result assert m > 0.015, result diff --git a/tests/science/test_defect_veto.py b/tests/science/test_defect_veto.py deleted file mode 100644 index 3384263e4..000000000 --- a/tests/science/test_defect_veto.py +++ /dev/null @@ -1,104 +0,0 @@ -"""Shear recovery for the defects the central-defect veto keeps (noise fill). - -Physics invariant: a defect the veto keeps -- a column, a 3-px bleed or a -single pixel at the veto radius or beyond, or an edge band -- leaves both -additive terms |c1|, |c2| < 5e-4 and both diagonal multiplicative terms -|m11|, |m22| < 1%, from the full 2x2 response matrix. The grid spans galaxies -with half-light radius 0.3" and 0.5" through a 0.7" PSF, round and with -ellipticity (0.05, 0.02). A noise-filled defect biases m anisotropically -(m11 and m22 differ by up to a factor of ten), so a scalar m would hide it. -The cases sit at ``EPOCH_CENTRAL_DEFECT_RADIUS`` itself, so lowering the -radius below the calibrated value turns this red. - -Positive control: the same column two pixels inside the radius breaks the -bound, so the recovery check can see the bias the veto removes. -""" - -import json - -import numpy as np -import pytest - -from shapepipe.modules.ngmix_package.ngmix import ( - EPOCH_CENTRAL_DEFECT_RADIUS, - EPOCH_MASKED_FRACTION_CUT, - defect_mask, - has_central_defect, -) -from tests.helpers.defect_response import defect_response - -N = 51 -CENTRE = N // 2 -R = int(np.ceil(EPOCH_CENTRAL_DEFECT_RADIUS)) -ELLIPTICAL = (0.05, 0.02) -SEEDS = range(6) - - -def geometry(kind, distance): - """A detector defect whose nearest pixel is ``distance`` px from the - stamp centre; an edge band reaches in to that distance.""" - bad = np.zeros((N, N), dtype=bool) - if kind == "pixel": - bad[CENTRE, CENTRE + distance] = True - elif kind == "column": - bad[:, CENTRE + distance] = True - elif kind == "bleed": - bad[:, CENTRE + distance:CENTRE + distance + 3] = True - elif kind == "edge": - bad[:, CENTRE + distance:] = True - return bad - - -# Wide defects (a 3-px bleed, the widest edge band the veto keeps) at the -# radius itself on the 0.5" galaxy through the elliptical PSF sit at the -# bound (m11 = -0.98% +/- 0.04% and -0.94% +/- 0.06%; see -# :func:`has_central_defect`), so those two are checked one pixel out. -CASES = ( - [(k, d, h, (0.0, 0.0)) for k in ("column", "bleed", "pixel") - for d in (R, R + 2) for h in (0.3, 0.5)] - + [(k, R, 0.5, ELLIPTICAL) for k in ("column", "pixel")] - + [(k, R + 1, 0.5, ELLIPTICAL) for k in ("bleed", "edge")] - + [("edge", R, 0.5, (0.0, 0.0))] - + [("edge", N - CENTRE - 5, 0.5, psf) for psf in ((0.0, 0.0), ELLIPTICAL)] -) - - -def recover(bad, hlr, psf_shear, tmp_path): - result = defect_response(bad, hlr=hlr, psf=0.7, seeds=SEEDS, - psf_shear=psf_shear) - (tmp_path / "recovery.json").write_text(json.dumps(result, indent=2)) - return np.abs(result["m"]).max(), np.abs(result["c"]).max(), result - - -def case_id(case): - kind, distance, hlr, psf_shear = case - psf = "elliptical" if any(psf_shear) else "round" - return f"{kind}-{distance}px-hlr{hlr}-{psf}" - - -@pytest.mark.parametrize("kind,distance,hlr,psf_shear", CASES, - ids=[case_id(c) for c in CASES]) -def test_kept_defects_recover_shear_on_both_axes(kind, distance, hlr, - psf_shear, tmp_path): - """Failure modes: the veto radius is below the calibrated value; the - fill leaks raw defect values or leaves more of the object's light - missing; the check reads m11 alone and misses m22, or c1 alone. - """ - bad = geometry(kind, distance) - masked = defect_mask(np.ones((N, N)), bad.astype(np.int32)) - np.testing.assert_array_equal(masked, bad) - assert masked.mean() <= EPOCH_MASKED_FRACTION_CUT - assert not has_central_defect(masked, EPOCH_CENTRAL_DEFECT_RADIUS) - m, c, result = recover(bad, hlr, psf_shear, tmp_path) - assert m < 0.01, result - assert c < 5e-4, result - - -def test_vetoed_column_breaks_the_bound(tmp_path): - """Positive control: a column two pixels inside the radius, on the - 0.5" galaxy, gives |m| > 2% and |c| > 1e-3. The veto drops it.""" - bad = geometry("column", R - 2) - assert has_central_defect(bad, EPOCH_CENTRAL_DEFECT_RADIUS) - m, c, result = recover(bad, 0.5, (0.0, 0.0), tmp_path) - assert m > 0.02, result - assert c > 1e-3, result diff --git a/tests/science/test_weight_symmetrization.py b/tests/science/test_weight_symmetrization.py deleted file mode 100644 index 53e8982ab..000000000 --- a/tests/science/test_weight_symmetrization.py +++ /dev/null @@ -1,54 +0,0 @@ -"""Weight symmetrization on the uberseg neighbour side. - -Physics invariant: a one-sided zero-weight hole pulls the Gaussian fit, and -zeroing the weight on its three quarter-turn copies cancels the pull. A -neighbour footprint with no light 10 px from a galaxy with half-light radius -0.5" through a 0.7" PSF gives c1 = -7.8e-3 under uberseg, and 2e-6 with -``symmetrize_weights="interpolated_and_neighbours"``. - -Why the default leaves the neighbour side unsymmetrized: a real neighbour -leaves light in the target's cell, which pulls the fit the other way, and -the one-sided hole partly offsets it. With a neighbour of a tenth of the -target's flux at 10 px, c1 = +1.1e-3 unsymmetrized and +1.2e-2 symmetrized. -If this second test turns red, the default deserves another look. -""" - -import numpy as np - -from tests.helpers.defect_response import defect_response - -N = 51 -SEEDS = range(6) -NO_DEFECT = np.zeros((N, N), dtype=bool) - - -def uberseg_response(flux_ratio, symmetrize): - return defect_response( - NO_DEFECT, hlr=0.5, psf=0.7, seeds=SEEDS, - options=dict(blend_handling="uberseg", object_number=1, - dilate_neighbour=1, symmetrize_weights=symmetrize), - neighbour=dict(offset=(0, 10), flux_ratio=flux_ratio, hlr=0.5, - seg_radius=4), - ) - - -def test_symmetrization_cancels_the_neighbour_side_hole(): - """Failure modes: the neighbour side is not symmetrized under - "interpolated_and_neighbours", or the rotation is not about the stamp - centre (the hole's pull survives). Positive control: the unsymmetrized - hole gives |c1| > 5e-3.""" - unsymmetrized = uberseg_response(0.0, "interpolated") - symmetrized = uberseg_response(0.0, "interpolated_and_neighbours") - assert abs(unsymmetrized["c"][0]) > 5e-3, unsymmetrized - assert np.abs(symmetrized["c"]).max() < 5e-5, symmetrized - assert np.abs(symmetrized["m"]).max() < 0.01, symmetrized - - -def test_symmetrizing_the_neighbour_side_exposes_neighbour_light(): - """The default ("interpolated") leaves the uberseg neighbour side - unsymmetrized because, with neighbour light in the image, symmetrizing - it raises c1 several-fold.""" - unsymmetrized = uberseg_response(0.1, "interpolated") - symmetrized = uberseg_response(0.1, "interpolated_and_neighbours") - assert abs(unsymmetrized["c"][0]) < 3e-3, unsymmetrized - assert symmetrized["c"][0] > 5e-3, symmetrized diff --git a/universes/committed.yaml b/universes/committed.yaml index 74e873f0a..9a49ba6ec 100644 --- a/universes/committed.yaml +++ b/universes/committed.yaml @@ -52,7 +52,7 @@ analyses: galaxy_pixel_weights: rms_vignet_weights psf_likelihood_noise: psf_noise_1em5 megacam_ccd_flip: megapipe_flip - defect_fill: noise + defect_fill: interpolate blend_handling: noisefill weight_symmetrization: interpolated epoch_masked_fraction_cut: one_third From 0226edf662bba5c2435c855789f8f888dfaec2e9 Mon Sep 17 00:00:00 2001 From: Cail Daley Date: Sat, 3 Oct 2026 01:29:40 +0200 Subject: [PATCH 19/30] CI: run tests/science once, in its own step Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01N4HLdzDyc3CiPLrgJ48GVo --- .github/workflows/deploy-image.yml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/.github/workflows/deploy-image.yml b/.github/workflows/deploy-image.yml index 6e8eb1206..9e420fd2b 100644 --- a/.github/workflows/deploy-image.yml +++ b/.github/workflows/deploy-image.yml @@ -120,9 +120,10 @@ jobs: docker run --rm -e HYPOTHESIS_PROFILE=ci "$IMAGE" \ pytest -rX -m "not slow" --no-cov tests/science - # The actual test suite, run inside the shipped image — replacing the + # The rest of the test suite, run inside the shipped image — replacing the # retired conda-based suite. pytest exercises the same wheels, binaries, # and Python (3.12) that production runs on, not a parallel environment. + # tests/science ran in the step above, so it is left out here. # pyproject's addopts add `--cov=shapepipe`; COVERAGE_FILE is set to /tmp # in the image so it works on read-only filesystems too. The Hypothesis # profile is explicit here so CI always uses the deterministic, capped @@ -130,7 +131,7 @@ jobs: - name: Test — pytest suite run: | IMAGE=$(echo "${{ steps.meta.outputs.tags }}" | head -n1) - docker run --rm -e HYPOTHESIS_PROFILE=ci -e SHAPEPIPE_ON_CANDIDE=0 "$IMAGE" pytest -rX + docker run --rm -e HYPOTHESIS_PROFILE=ci -e SHAPEPIPE_ON_CANDIDE=0 "$IMAGE" pytest -rX --ignore=tests/science # tests/workflow drives the Snakefile through snakemake's API. Snakemake # is a host tool and stays out of the image (it wraps each job in the From 6c75fd48c994e80fd8455a665cc5d5f6477e84d1 Mon Sep 17 00:00:00 2001 From: Cail Daley Date: Sat, 3 Oct 2026 01:56:41 +0200 Subject: [PATCH 20/30] ngmix: one-orientation defect interpolant; interpolated_defects names the set interpolate_defects fits one Clough-Tocher interpolant instead of averaging four quarter-turned ones. For the defects the veto keeps the average moves c by at most 5e-5 and m by at most 4e-4 (test_defect_recovery cases), and the quarter-turn weight copies already make the likelihood symmetric. interpolated_defects(defect, neighbour, blend_handling) is the one rule for which defect pixels are interpolated; prepare_postage_stamps (the central veto) and prepare_ngmix_weights (the fill) both call it. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01N4HLdzDyc3CiPLrgJ48GVo --- astra.yaml | 4 +- .../ngmix_package/defect_interpolation.py | 65 +++++++------------ src/shapepipe/modules/ngmix_package/ngmix.py | 44 ++++++++++--- tests/module/test_defect_interpolation.py | 33 +--------- 4 files changed, 63 insertions(+), 83 deletions(-) diff --git a/astra.yaml b/astra.yaml index f67c84804..9e104cae1 100644 --- a/astra.yaml +++ b/astra.yaml @@ -1459,8 +1459,8 @@ analyses: pixels whose neighbours beyond both ends are kept pixels (on the stamp, not defects, and not noisefill neighbours) are Clough-Tocher interpolated from the kept pixels within - SUPPORT_RADIUS = 4 px, averaged over the stamp's quarter turns, - in the image and the metacal noise image alike. Other defects + SUPPORT_RADIUS = 4 px, with one triangulation for the image + and the metacal noise image. Other defects (wider holes, edge bands, runs ending on a noisefill neighbour) get independent noise at the per-pixel background RMS when supplied, or the stamp's robust noise scale otherwise. The diff --git a/src/shapepipe/modules/ngmix_package/defect_interpolation.py b/src/shapepipe/modules/ngmix_package/defect_interpolation.py index 07bea48db..5e539b437 100644 --- a/src/shapepipe/modules/ngmix_package/defect_interpolation.py +++ b/src/shapepipe/modules/ngmix_package/defect_interpolation.py @@ -107,43 +107,17 @@ def fourfold(mask): return mask | np.rot90(mask) | np.rot90(mask, 2) | np.rot90(mask, 3) -def _interpolate_once(planes, excluded, target): - """One Clough-Tocher interpolant of every plane at ``target``; NaN - elsewhere and where the support cannot reach.""" - out = np.full(planes.shape, np.nan) - support = binary_dilation( - target, structure=np.ones((3, 3), dtype=bool), - iterations=SUPPORT_RADIUS, - ) & ~excluded - points = np.argwhere(support).astype(float) - if len(points) < 3: - return out - query = np.argwhere(target) - try: - interpolant = CloughTocher2DInterpolator( - points, planes[:, support].T, fill_value=np.nan, - ) - except QhullError: - return out - out[:, query[:, 0], query[:, 1]] = interpolant(query.astype(float)).T - return out - - def interpolate_defects(planes, excluded, target): """Replace the ``target`` pixels of every plane by a Clough-Tocher interpolant of the kept pixels around them. - @sc [decision:shape_measurement.defect_fill] shared-rotation-averaged-interpolant + @sc [decision:shape_measurement.defect_fill] shared-interpolant The support is the pixels within ``SUPPORT_RADIUS`` (4 px) of the target outside ``excluded``; no excluded pixel enters it, so their values are - never read. For each quarter turn of the stamp, one Delaunay triangulation of the support - serves every plane, so the science image and the metacal noise image - see the same linear operator and fixnoise mirrors the science image's - interpolated noise. A regular grid's triangulation has degenerate - diagonals, so one orientation has a preferred direction; averaging the - four quarter-turned operators makes the fill commute with rotations of - the stamp. That removes up to 6e-5 of c2 for a column or 3-px bleed - 6 px from the object. + never read. One Delaunay triangulation of the support serves every + plane, so the science image and the metacal noise image see the same + linear operator and fixnoise mirrors the science image's interpolated + noise. Parameters ---------- @@ -159,7 +133,7 @@ def interpolate_defects(planes, excluded, target): ------- numpy.ndarray A copy of ``planes`` with ``target`` pixels interpolated; NaN at a - target pixel whose support is degenerate in some orientation. + target pixel the support cannot reach. """ planes = np.asarray(planes, dtype=float) excluded = np.asarray(excluded, dtype=bool) @@ -167,15 +141,20 @@ def interpolate_defects(planes, excluded, target): out = planes.copy() if not target.any(): return out - turns = [ - np.rot90( - _interpolate_once( - np.rot90(planes, k, axes=(1, 2)), np.rot90(excluded, k), - np.rot90(target, k), - ), - -k, axes=(1, 2), - )[:, target] - for k in range(4) - ] - out[:, target] = np.mean(turns, axis=0) + out[:, target] = np.nan + support = binary_dilation( + target, structure=np.ones((3, 3), dtype=bool), + iterations=SUPPORT_RADIUS, + ) & ~excluded + points = np.argwhere(support).astype(float) + if len(points) < 3: + return out + query = np.argwhere(target) + try: + interpolant = CloughTocher2DInterpolator( + points, planes[:, support].T, fill_value=np.nan, + ) + except QhullError: + return out + out[:, query[:, 0], query[:, 1]] = interpolant(query.astype(float)).T return out diff --git a/src/shapepipe/modules/ngmix_package/ngmix.py b/src/shapepipe/modules/ngmix_package/ngmix.py index a0530dd27..7c1a0a337 100644 --- a/src/shapepipe/modules/ngmix_package/ngmix.py +++ b/src/shapepipe/modules/ngmix_package/ngmix.py @@ -1556,9 +1556,8 @@ def prepare_postage_stamps( if defect.mean() > EPOCH_MASKED_FRACTION_CUT: stamp.epoch_cuts["masked_fraction"] += 1 continue - removed = neighbour if blend_handling == "noisefill" else None if central_defect_vetoes( - defect, interpolable_defects(defect, removed) + defect, interpolated_defects(defect, neighbour, blend_handling) ): stamp.epoch_cuts["central_veto"] += 1 continue @@ -1992,6 +1991,35 @@ def defect_mask(weight, flag, bkg_rms=None): return defect +def interpolated_defects(defect, neighbour, blend_handling): + """The defect pixels :func:`prepare_ngmix_weights` interpolates. + + Under ``"noisefill"`` the neighbour pixels' light is replaced, so they + neither support the interpolant nor are interpolated + (:func:`interpolable_defects` with ``removed=neighbour``); under + ``"uberseg"`` their light stays and supports it. The epoch cuts + (:func:`central_defect_vetoes`) read the same set. + + Parameters + ---------- + defect : numpy.ndarray of bool + Defect mask of one epoch stamp (:func:`defect_mask`). + neighbour : numpy.ndarray of bool or None + The epoch's neighbour mask (``Postage_stamp.neighbours``); ``None`` + marks no pixel. + blend_handling : {"noisefill", "uberseg"} + Neighbour treatment. + + Returns + ------- + numpy.ndarray of bool + ``True`` on the defect pixels to interpolate; the other defect + pixels are noise-filled. + """ + removed = neighbour if blend_handling == "noisefill" else None + return interpolable_defects(defect, removed) + + def central_defect_vetoes(defect, interpolated): """Whether a defect near the stamp centre drops the epoch. @@ -2012,8 +2040,8 @@ def central_defect_vetoes(defect, interpolated): defect : numpy.ndarray of bool Defect mask of one epoch stamp (:func:`defect_mask`). interpolated : numpy.ndarray of bool - The defect pixels that are interpolated (:func:`interpolable_defects` - with the epoch's removed neighbour pixels); the rest are noise-filled. + The defect pixels that are interpolated + (:func:`interpolated_defects`); the rest are noise-filled. Returns ------- @@ -2049,7 +2077,7 @@ def prepare_ngmix_weights( @sc [decision:shape_measurement.defect_fill,decision:shape_measurement.blend_handling] defects-filled-whatever-the-blend-handling Every pixel of :func:`defect_mask` gets weight 0 and is filled the same way under every ``blend_handling``. The short runs of - :func:`interpolable_defects` take a Clough-Tocher interpolant of the + :func:`interpolated_defects` take a Clough-Tocher interpolant of the pixels whose light the image keeps (:func:`interpolate_defects`), in the image and the noise image alike; the other defects take noise at the background RMS. Metacal shears the whole image whatever the weights, so @@ -2125,7 +2153,7 @@ def prepare_ngmix_weights( If ``blend_handling`` is unknown, or ``"uberseg"`` lacks ``seg`` or ``object_number``. RuntimeError - If the interpolant does not reach a pixel :func:`interpolable_defects` + If the interpolant does not reach a pixel :func:`interpolated_defects` selected (degenerate support, or non-finite image values in it). @sc [decision:masking.pixel_mask_source,decision:shape_measurement.blend_handling,decision:shape_measurement.defect_fill,decision:shape_measurement.galaxy_pixel_weights,decision:shape_measurement.weight_symmetrization] """ @@ -2190,7 +2218,7 @@ def prepare_ngmix_weights( noise_img = rng.standard_normal(gal.shape) * sig_noise noise_img_gal = rng.standard_normal(gal.shape) * sig_noise gal_filled = np.where(clean, gal, noise_img_gal).astype(gal.dtype) - interpolated = interpolable_defects(defect, removed_neighbour) + interpolated = interpolated_defects(defect, neighbour, blend_handling) if interpolated.any(): # One operator for the image and the noise image, so fixnoise # mirrors the science image's interpolated noise. @@ -2200,7 +2228,7 @@ def prepare_ngmix_weights( if not np.all(np.isfinite(filled)): raise RuntimeError( "The defect interpolant is not finite on pixels" - + " interpolable_defects selected." + + " interpolated_defects selected." ) gal_filled[interpolated] = filled[0] noise_img[interpolated] = filled[1] diff --git a/tests/module/test_defect_interpolation.py b/tests/module/test_defect_interpolation.py index b550b7f57..791bb1c15 100644 --- a/tests/module/test_defect_interpolation.py +++ b/tests/module/test_defect_interpolation.py @@ -3,9 +3,8 @@ :func:`interpolable_defects` picks the defect pixels that lie in a short row or column run with support pixels at both ends; :func:`interpolate_defects` -fills them from the nearby clean pixels with a Clough-Tocher interpolant, -averaged over the four quarter turns of the stamp and shared by every plane -(science image and metacal noise image). +fills them from the nearby clean pixels with one Clough-Tocher interpolant +shared by every plane (science image and metacal noise image). """ import numpy as np @@ -206,39 +205,13 @@ def test_interpolation_reproduces_planes_without_reading_defects(seed): ) -def test_interpolation_commutes_with_quarter_turns(): - """Rotating the stamp and its mask rotates the fill. A regular grid's - Delaunay triangulation has degenerate diagonals, so a single-orientation - interpolant does not commute; the four-orientation average does. - - Failure mode: the average over orientations is skipped, so the fill has a - preferred direction. - """ - n = 31 - rows, cols = np.indices((n, n)) - image = np.exp(-((rows - 15.3) ** 2 + (cols - 14.6) ** 2) / 10.0) - image += 0.05 * np.random.RandomState(3).normal(size=(n, n)) - planes = image[None] - defect = _mask(n) - target = interpolable_defects(defect) - out = interpolate_defects(planes, defect, target) - for k in range(1, 4): - rotated = interpolate_defects( - np.rot90(planes, k, axes=(1, 2)), np.rot90(defect, k), - np.rot90(target, k), - ) - npt.assert_allclose( - rotated, np.rot90(out, k, axes=(1, 2)), atol=1e-12, rtol=0 - ) - - def test_every_plane_sees_the_same_operator(): """The fill is one linear operator applied to every plane: filling a * image + b * noise gives a * fill(image) + b * fill(noise), up to the Clough-Tocher gradient solver's tolerance. Failure mode: the noise image is filled differently from the science - image (another support, triangulation or orientation set), so metacal's + image (another support or triangulation), so metacal's fixnoise no longer mirrors the science image's correlated noise. """ n = 31 From 178ccf5864ae295d28af9cec03f33e68361909ce Mon Sep 17 00:00:00 2001 From: Cail Daley Date: Sat, 3 Oct 2026 01:56:41 +0200 Subject: [PATCH 21/30] masked-pixel figures: one defect treatment, off-tile as zero weight pixel_classes compares noisefill and uberseg; defect_gallery shows raw, image and weight under the default. Both take the interpolated set from interpolated_defects, and violet marks only pixels zeroed by the quarter-turn symmetrization. dr6_epochs gives off-tile pixels zero weight and stores the off_tile mask. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01N4HLdzDyc3CiPLrgJ48GVo --- .../masked_pixels/defect_gallery.py | 54 ++++++----------- .../validation/masked_pixels/dr6_epochs.py | 9 +-- scripts/validation/masked_pixels/figstyle.py | 11 ++-- .../validation/masked_pixels/pixel_classes.py | 59 +++++++++---------- 4 files changed, 57 insertions(+), 76 deletions(-) diff --git a/scripts/validation/masked_pixels/defect_gallery.py b/scripts/validation/masked_pixels/defect_gallery.py index 1aed130f4..f894d5c06 100644 --- a/scripts/validation/masked_pixels/defect_gallery.py +++ b/scripts/validation/masked_pixels/defect_gallery.py @@ -1,6 +1,5 @@ """One real DR6 epoch per defect type: the raw epoch stamp, then the image -and weights ngmix receives under DEFECT_FILL = noise and -DEFECT_FILL = interpolate (BLEND_HANDLING = noisefill throughout). +and weights ngmix receives (BLEND_HANDLING = noisefill, the default). Each epoch is an npz written by ``dr6_epochs.py cut``. Run from the shapepipe checkout root inside the shapepipe container: @@ -19,20 +18,18 @@ from figstyle import (AQUA, BLUE, INK, ORANGE, WEIGHT_LEGEND, clean, outline, show_image, show_weight) -from shapepipe.modules.ngmix_package.defect_interpolation import ( - interpolable_defects, -) from shapepipe.modules.ngmix_package.ngmix import ( defect_mask, + interpolated_defects, prepare_ngmix_weights, ) -COLS = ["raw stamp", "image", "weight", "image", "weight"] +COLS = ["raw stamp", "image ngmix sees", "weight"] def main(out_path, rows): plt.rcParams.update({"font.size": 10}) - fig, axs = plt.subplots(len(rows), 5, figsize=(11.2, 2.2 * len(rows)), + fig, axs = plt.subplots(len(rows), 3, figsize=(7.4, 2.35 * len(rows)), dpi=150, gridspec_kw=dict(hspace=0.06, wspace=0.05)) for i, (kind, path) in enumerate(rows): @@ -40,23 +37,18 @@ def main(out_path, rows): rms = float(np.median(ep["bkg_rms"])) stretch = dict(vmax=120.0 * rms, soft=3.0 * rms) defect = defect_mask(ep["weight"], ep["flag"], ep["bkg_rms"]) - interp = interpolable_defects(defect) - out = {} - for fill in ("noise", "interpolate"): - out[fill] = prepare_ngmix_weights( - ep["gal"], ep["weight"], ep["flag"], - np.random.RandomState(1), bkg_rms=ep["bkg_rms"], - neighbour=ep["neighbour"], defect_fill=fill, - )[:2] + interp = interpolated_defects(defect, ep["neighbour"], "noisefill") + img, w, _ = prepare_ngmix_weights( + ep["gal"], ep["weight"], ep["flag"], np.random.RandomState(1), + bkg_rms=ep["bkg_rms"], neighbour=ep["neighbour"], + ) show_image(axs[i, 0], ep["gal"], **stretch) outline(axs[i, 0], ep["neighbour"], BLUE, lw=0.9) outline(axs[i, 0], defect, INK, lw=0.9) - show_image(axs[i, 1], out["noise"][0], **stretch) - show_weight(axs[i, 2], out["noise"][1]) - show_image(axs[i, 3], out["interpolate"][0], **stretch) - outline(axs[i, 3], interp, AQUA, lw=0.9) - outline(axs[i, 3], defect & ~interp, ORANGE, lw=0.9) - show_weight(axs[i, 4], out["interpolate"][1], interp) + show_image(axs[i, 1], img, **stretch) + outline(axs[i, 1], interp, AQUA, lw=0.9) + outline(axs[i, 1], defect & ~interp, ORANGE, lw=0.9) + show_weight(axs[i, 2], w, interp, defect | ep["neighbour"]) axs[i, 0].set_ylabel( f"{kind}\n{ep['label']}\n{ep['source']}", color=INK, fontsize=8, linespacing=1.15, @@ -66,25 +58,15 @@ def main(out_path, rows): for ax in axs.flat: clean(ax) - top = axs[0, 1].get_position().y1 + 0.03 - for (a, b), t in (((1, 2), "DEFECT_FILL = noise"), - ((3, 4), "DEFECT_FILL = interpolate")): - x0 = axs[0, a].get_position().x0 - x1 = axs[0, b].get_position().x1 - fig.text((x0 + x1) / 2, top, t, ha="center", va="bottom", color=INK, - fontsize=11, family="monospace") - fig.add_artist(Line2D([x0 + 0.005, x1 - 0.005], [top - 0.004] * 2, - color=INK, lw=0.8)) - handles = [ - Line2D([], [], color=INK, lw=1.6, label="defect (raw stamp)"), + Line2D([], [], color=INK, lw=1.6, label="defect"), Line2D([], [], color=BLUE, lw=1.6, - label="neighbour (raw stamp; noise-filled, zero weight)"), - Line2D([], [], color=ORANGE, lw=1.6, label="defect, noise-filled"), + label="neighbour (noise-filled, zero weight)"), Line2D([], [], color=AQUA, lw=1.6, label="defect, interpolated"), + Line2D([], [], color=ORANGE, lw=1.6, label="defect, noise-filled"), ] + WEIGHT_LEGEND - fig.legend(handles=handles, loc="upper center", ncol=3, frameon=False, - fontsize=9, handlelength=1.4, + fig.legend(handles=handles, loc="upper center", ncol=2, frameon=False, + fontsize=8.5, handlelength=1.4, bbox_to_anchor=(0.5, axs[-1, 0].get_position().y0 - 0.005)) fig.savefig(out_path, bbox_inches="tight") diff --git a/scripts/validation/masked_pixels/dr6_epochs.py b/scripts/validation/masked_pixels/dr6_epochs.py index 6faf1dec4..6f1ecb068 100644 --- a/scripts/validation/masked_pixels/dr6_epochs.py +++ b/scripts/validation/masked_pixels/dr6_epochs.py @@ -13,7 +13,8 @@ pixel of the object's (XWIN_WORLD, YWIN_WORLD) in the CCD's own WCS, CCDs numbered as split_exp names them (HDU index - 1). The tile VIGNET and seg stamp are MegaCam-flipped onto the epoch and split into neighbour and -off-tile pixels (split_tile_markers). The background is a sigma-clipped +off-tile pixels (split_tile_markers); the off-tile pixels get zero weight, +as prepare_postage_stamps gives them. The background is a sigma-clipped median over the clean pixels of a 257x257 box around the object, and the background RMS is constant over the stamp; the pipeline takes both from SExtractor's maps instead. Only the survey flag and weight maps mark @@ -36,7 +37,6 @@ from scipy.ndimage import find_objects, label from shapepipe.modules.ngmix_package.ngmix import ( - OFF_TILE_FLAG, Ngmix, defect_mask, split_tile_markers, @@ -120,9 +120,10 @@ def cut(hdus, ext, pos, tile, i): _, bkg, rms = sigma_clipped_stats(img[box][good], sigma=3.0) stamps["gal"] = stamps["gal"] - bkg seg, neighbour, off_tile = tile_overlay(tile, i, ccd) - stamps["flag"][off_tile] = OFF_TILE_FLAG + stamps["weight"][off_tile] = 0 stamps.update( bkg_rms=np.full((N, N), rms), seg=seg, neighbour=neighbour, + off_tile=off_tile, object_number=int(tile["NUMBER"][i]), ccd=ccd, offset=offset[0], int_pos=int_pos[0], ) @@ -139,7 +140,7 @@ def scan(hdus, tile): flg = get_stamps(hdus[2][ext].data, pos, RAD)[0].astype(np.int32) for i, w, f in zip(rows, wgt, flg): seg, neighbour, off_tile = tile_overlay(tile, i, ext - 1) - f[off_tile] = OFF_TILE_FLAG + w[off_tile] = 0 defect = defect_mask(w, f) if not defect.any(): continue diff --git a/scripts/validation/masked_pixels/figstyle.py b/scripts/validation/masked_pixels/figstyle.py index ecbe0fcd5..977f12795 100644 --- a/scripts/validation/masked_pixels/figstyle.py +++ b/scripts/validation/masked_pixels/figstyle.py @@ -26,19 +26,22 @@ def show_image(ax, img, vmax=120.0, soft=3.0): WEIGHT_CMAP = ListedColormap([PAPER, DARK, VIOLET]) -def weight_classes(w, interpolated): +def weight_classes(w, interpolated, zeroed=None): """0 = weighted, 1 = zero weight, 2 = zero weight only as a quarter-turn - copy of an interpolated pixel.""" + copy of an interpolated pixel. ``zeroed`` marks the pixels that have + zero weight for another reason (defects, neighbours); they stay 1.""" cls = np.where(w > 0, 0, 1) copies = fourfold(interpolated) & ~interpolated + if zeroed is not None: + copies &= ~zeroed cls[(w == 0) & copies] = 2 return cls -def show_weight(ax, w, interpolated=None): +def show_weight(ax, w, interpolated=None, zeroed=None): if interpolated is None: interpolated = np.zeros(w.shape, bool) - ax.imshow(weight_classes(w, interpolated), origin="lower", + ax.imshow(weight_classes(w, interpolated, zeroed), origin="lower", cmap=WEIGHT_CMAP, vmin=-0.5, vmax=2.5, interpolation="nearest") diff --git a/scripts/validation/masked_pixels/pixel_classes.py b/scripts/validation/masked_pixels/pixel_classes.py index b12c72084..29136d44d 100644 --- a/scripts/validation/masked_pixels/pixel_classes.py +++ b/scripts/validation/masked_pixels/pixel_classes.py @@ -1,6 +1,6 @@ """Pixel classes of one epoch stamp and what ngmix receives: a synthetic -blend under noisefill + noise and uberseg + interpolate, then a real DR6 -blend under uberseg + interpolate. +blend under BLEND_HANDLING = noisefill and uberseg, then a real DR6 blend +under uberseg. Run from the shapepipe checkout root inside the shapepipe container: PYTHONPATH=src:scripts/validation/masked_pixels \ @@ -21,23 +21,15 @@ from figstyle import (AQUA, BLUE, INK, INK2, ORANGE, PAPER, TARGET, WEIGHT_LEGEND, clean, show_image, show_weight) -from shapepipe.modules.ngmix_package.defect_interpolation import ( - interpolable_defects, -) from shapepipe.modules.ngmix_package.ngmix import ( EPOCH_CENTRAL_DEFECT_RADIUS, EPOCH_INTERPOLATED_DEFECT_RADIUS, - OFF_TILE_FLAG, defect_mask, + interpolated_defects, prepare_ngmix_weights, + uberseg_mask, ) -SETTINGS = { - "noisefill_noise": dict(blend_handling="noisefill", defect_fill="noise"), - "uberseg_interpolate": dict(blend_handling="uberseg", - defect_fill="interpolate"), -} - def synthetic(): """The synthetic epoch: a target, a neighbour 15 px away, a bad column, @@ -63,26 +55,29 @@ def blob(r0, c0, sigma, flux, q=1.0, theta=0.0): flag = np.zeros((N, N), np.int32) flag[:, C + 12] = 1 # bad column, 12 px right of centre flag[C + 9, C - 9] = 1 # hot pixel, 12.7 px from centre - flag[N - 5:, :] = OFF_TILE_FLAG # off-tile band at the stamp's edge + off_tile = np.zeros((N, N), bool) + off_tile[N - 5:, :] = True # off-tile band at the stamp's edge gal[:, C + 12] = 400.0 gal[C + 9, C - 9] = 400.0 - return dict(gal=gal, weight=np.ones((N, N)), flag=flag, + return dict(gal=gal, weight=np.where(off_tile, 0.0, 1.0), flag=flag, bkg_rms=np.full((N, N), rms), seg=seg, object_number=1, - neighbour=seg == 2) + neighbour=seg == 2, off_tile=off_tile) -def run(ep, setting): - kw = dict(SETTINGS[setting]) - if kw["blend_handling"] == "uberseg": +def run(ep, blend_handling): + kw = {} + if blend_handling == "uberseg": kw.update(seg=ep["seg"], object_number=int(ep["object_number"])) img, w, _ = prepare_ngmix_weights( ep["gal"], ep["weight"], ep["flag"], np.random.RandomState(1), - bkg_rms=ep["bkg_rms"], neighbour=ep["neighbour"], **kw, + bkg_rms=ep["bkg_rms"], neighbour=ep["neighbour"], + blend_handling=blend_handling, **kw, ) defect = defect_mask(ep["weight"], ep["flag"], ep["bkg_rms"]) - interp = (interpolable_defects(defect) if kw["defect_fill"] == - "interpolate" else np.zeros_like(defect)) - return img, w, interp + interp = interpolated_defects(defect, ep["neighbour"], blend_handling) + neighbour = (ep["neighbour"] if blend_handling == "noisefill" else + uberseg_mask(ep["seg"], int(ep["object_number"]))) + return img, w, interp, defect | neighbour CLASS_CMAP = ListedColormap([PAPER, BLUE, ORANGE, AQUA, TARGET]) @@ -95,7 +90,7 @@ def classes(ep): cls[ep["neighbour"]] = 1 defect = defect_mask(ep["weight"], ep["flag"], ep["bkg_rms"]) cls[defect] = 2 - cls[ep["flag"] == OFF_TILE_FLAG] = 3 + cls[ep["off_tile"]] = 3 return cls @@ -142,23 +137,23 @@ def main(real_path, out_path): axs[1, 0].text(c, c - 6.3, "7 px", ha="center", va="bottom", fontsize=8, color=INK) - for j, (setting, title) in enumerate( - (("noisefill_noise", "(b) noisefill + noise\n(the defaults)"), - ("uberseg_interpolate", "(c) uberseg + interpolate")), start=1): - img, w, interp = run(syn, setting) + for j, (blend, title) in enumerate( + (("noisefill", "(b) noisefill (default)"), + ("uberseg", "(c) uberseg")), start=1): + img, w, interp, zeroed = run(syn, blend) show_image(axs[0, j], img) - show_weight(axs[1, j], w, interp) + show_weight(axs[1, j], w, interp, zeroed) axs[0, j].set_title(title, color=INK, fontsize=10) - # (d) the real DR6 epoch: its classes, then uberseg + interpolate + # (d) the real DR6 epoch and its classes, (e) under uberseg show_image(axs[0, 3], real["gal"], vmax=vmax_real, soft=soft_real) class_panel(axs[1, 3], real) axs[0, 3].set_title(f"(d) {real['label']}\n{real['source']}", color=INK, fontsize=10) - img, w, interp = run(real, "uberseg_interpolate") + img, w, interp, zeroed = run(real, "uberseg") show_image(axs[0, 4], img, vmax=vmax_real, soft=soft_real) - show_weight(axs[1, 4], w, interp) - axs[0, 4].set_title("uberseg + interpolate", color=INK, fontsize=10) + show_weight(axs[1, 4], w, interp, zeroed) + axs[0, 4].set_title("(e) uberseg", color=INK, fontsize=10) for ax in axs.flat: clean(ax) From b82c77eb823cb83939c4d7cecc9759bc8da26815 Mon Sep 17 00:00:00 2001 From: Cail Daley Date: Sat, 3 Oct 2026 02:18:11 +0200 Subject: [PATCH 22/30] Bias-vs-distance figure for one defect: production treatment per defect type, noise fill as counterfactual Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01N4HLdzDyc3CiPLrgJ48GVo --- .../masked_pixels/bias_vs_distance.py | 217 ++++++++++++++++++ 1 file changed, 217 insertions(+) create mode 100644 scripts/validation/masked_pixels/bias_vs_distance.py diff --git a/scripts/validation/masked_pixels/bias_vs_distance.py b/scripts/validation/masked_pixels/bias_vs_distance.py new file mode 100644 index 000000000..25aaeaed9 --- /dev/null +++ b/scripts/validation/masked_pixels/bias_vs_distance.py @@ -0,0 +1,217 @@ +"""Shear bias from one detector defect against its distance from the object, +under the production defect treatment and under noise fill. + +Production interpolates the defects ``interpolable_defects`` selects +(columns, 3-px bleeds, finite bleeds, single pixels) and noise-fills the +rest (edge bands). The counterfactual noise-fills every defect, by making +``interpolated_defects`` select none. The recovery is the full-matrix +metacal measurement of ``tests/helpers/defect_response``; the figure plots +the worse axis of |m| and |c| with the veto radii and the bounds +|m| < 1%, |c| < 5e-4. + +Run from the shapepipe checkout root inside the shapepipe container: + PYTHONPATH=src:. python scripts/validation/masked_pixels/bias_vs_distance.py \ + scan OUT.json NPROC + PYTHONPATH=src:scripts/validation/masked_pixels \ + python scripts/validation/masked_pixels/bias_vs_distance.py \ + plot IN.json OUT.png +""" +import json +import sys + +import numpy as np + +N, CENTRE = 51, 25 +DISTANCES = range(3, 17) +SEEDS = range(6) +GALAXIES = [(0.3, 0.7), (0.5, 0.7), (0.7, 0.9)] # (hlr, PSF FWHM) arcsec +INTERPOLATED = ("column", "bleed", "finite_bleed", "pixel") +KINDS = INTERPOLATED + ("edge",) +BOUND_M, BOUND_C = 0.01, 5e-4 + + +def geometry(kind, distance): + """A defect whose nearest pixel is ``distance`` px from the stamp + centre (the geometries of tests/science/test_defect_recovery.py).""" + bad = np.zeros((N, N), dtype=bool) + near = CENTRE + distance + if kind == "pixel": + bad[CENTRE, near] = True + elif kind == "column": + bad[:, near] = True + elif kind == "bleed": + bad[:, near:near + 3] = True + elif kind == "finite_bleed": + bad[CENTRE - 5:CENTRE + 6, near:near + 3] = True + elif kind == "edge": + bad[:, near:] = True + return bad + + +def one(job): + kind, fill, distance, hlr, psf = job + from shapepipe.modules.ngmix_package import ngmix as ngm + from tests.helpers.defect_response import defect_response + + if fill == "noise": + ngm.interpolated_defects = ( + lambda defect, neighbour, blend: np.zeros_like(defect)) + row = dict(kind=kind, fill=fill, d=distance, hlr=hlr, psf=psf) + try: + r = defect_response(geometry(kind, distance), hlr=hlr, psf=psf, + seeds=SEEDS) + row.update(m=r["m"], m_err=r["m_err"], c=r["c"], c_err=r["c_err"]) + except Exception as e: # noqa: BLE001 + row["error"] = repr(e) + return row + + +def scan(out_path, nproc): + from multiprocessing import get_context + + jobs = [(k, f, d, h, p) for h, p in GALAXIES for d in DISTANCES + for k in KINDS + for f in (("production", "noise") if k in INTERPOLATED + else ("production",))] + # spawn: each worker imports ngmix afresh, so the noise-fill patch + # cannot leak into a production job. + with get_context("spawn").Pool(nproc, maxtasksperchild=1) as pool: + rows = pool.map(one, jobs, chunksize=1) + with open(out_path, "w") as f: + json.dump(rows, f, indent=1) + + +def worst(row, q): + return max(abs(x) for x in row[q]) + + +def smallest_passing(rows): + """Smallest distance from which every larger distance passes both + bounds (None if the largest fails).""" + rows = sorted(rows, key=lambda r: r["d"]) + passing = None + for r in reversed(rows): + if "error" in r or worst(r, "m") >= BOUND_M or worst(r, "c") >= BOUND_C: + break + passing = r["d"] + return passing + + +def plot(in_path, out_path): + import matplotlib + + matplotlib.use("Agg") + import matplotlib.pyplot as plt + from matplotlib.lines import Line2D + + from figstyle import AQUA, BLUE, INK, INK2, ORANGE, VIOLET + from shapepipe.modules.ngmix_package.ngmix import ( + EPOCH_CENTRAL_DEFECT_RADIUS as RN, + EPOCH_INTERPOLATED_DEFECT_RADIUS as RI, + EPOCH_MASKED_FRACTION_CUT as FRAC_CUT, + ) + + rows = [r for r in json.load(open(in_path))] + grid, fail, bad_ink = "#e4e3df", "#f6e3dc", "#a8462a" + series = { + "column": ("bad column", BLUE), + "bleed": ("3-px bleed", ORANGE), + "finite_bleed": ("finite 3-px bleed (11 rows)", VIOLET), + "pixel": ("single pixel", AQUA), + "edge": ("edge band (noise-filled)", INK), + } + plt.rcParams.update({"font.size": 10.5, "axes.edgecolor": INK2, + "xtick.color": INK2, "ytick.color": INK2, + "axes.labelcolor": INK}) + fig, axs = plt.subplots( + 2, len(GALAXIES), figsize=(13, 7.4), dpi=150, sharex=True, + sharey="row", gridspec_kw=dict(hspace=0.1, wspace=0.05)) + for j, (hlr, psf) in enumerate(GALAXIES): + for i, (q, bound, scale, ylabel) in enumerate([ + ("m", BOUND_M, 100, "worse-axis |m| [%]"), + ("c", BOUND_C, 1, "worse-axis |c|")]): + ax = axs[i, j] + b = bound * scale + ax.axhspan(b, 1e4, color=fail, zorder=0, lw=0) + ax.axhline(b, color=bad_ink, lw=0.9, zorder=1) + for r, txt, ha, dx in ( + (RI, f"interpolated\nveto {RI:g} px", "right", -0.2), + (RN, f"noise-fill\nveto {RN:g} px", "left", 0.2)): + ax.axvline(r, color=INK2, ls=":", lw=1.3, zorder=1) + if i == 0: + ax.text(r + dx, 8e2, txt, color=INK2, fontsize=8.5, + va="top", ha=ha, linespacing=1.1) + for kind, (label, col) in series.items(): + for fill, ls, lw, alpha in (("production", "-", 2.0, 1.0), + ("noise", (0, (3, 2)), 1.4, 0.8)): + sel = sorted( + (r for r in rows if r["kind"] == kind + and r["fill"] == fill and r["hlr"] == hlr + and r["psf"] == psf and "error" not in r), + key=lambda r: r["d"]) + if not sel: + continue + d = np.array([r["d"] for r in sel]) + v = np.array([worst(r, q) * scale for r in sel]) + z = 3 if fill == "production" else 2 + ax.plot(d, v, color=col, ls=ls, lw=lw, alpha=alpha, + zorder=z) + if fill != "production": + continue + cut = np.array([geometry(kind, x).mean() > FRAC_CUT + for x in d]) + ax.plot(d[~cut], v[~cut], "o", color=col, ms=3.5, + zorder=z) + ax.plot(d[cut], v[cut], "o", mfc="white", mec=col, + mew=1.1, ms=4, zorder=z) + ax.set_yscale("log") + ax.set_ylim((1e-3, 1e3) if q == "m" else (1e-6, 0.5)) + ax.grid(axis="y", color=grid, lw=0.6, which="major") + for s in ("top", "right"): + ax.spines[s].set_visible(False) + ax.set_xticks(range(4, 17, 2)) + if j == 0: + ax.set_ylabel(ylabel) + ax.text(16.5, b * 1.3, "|m| > 1%" if q == "m" else + "|c| > 5e-4", color=bad_ink, fontsize=9, + ha="right", va="bottom") + if i == 0: + ax.set_title(f'{hlr}″ galaxy, {psf}″ PSF', color=INK, + fontsize=11, loc="left") + fig.supxlabel("distance of the defect's nearest pixel from the object " + "centre [px]", fontsize=10.5, color=INK, y=0.045) + fig.suptitle("Shear bias from one defect: production treatment (solid) " + "vs noise fill (dashed); 6 seeds, worse of axes 1, 2", + color=INK2, fontsize=10, y=0.965) + handles = [Line2D([], [], color=c, lw=2, marker="o", ms=3.5, label=l) + for l, c in series.values()] + handles += [Line2D([], [], lw=0, label=" ")] # keeps styles in column 3 + handles += [ + Line2D([], [], color=INK2, lw=2, marker="o", ms=3.5, + label="production (interpolated; edge band noise-filled)"), + Line2D([], [], color=INK2, lw=1.4, ls=(0, (3, 2)), + label="noise fill (not used for narrow defects)"), + Line2D([], [], color=INK2, lw=0, marker="o", mfc="white", mec=INK2, + ms=4, label="epoch dropped by the 1/3 masked-fraction cut"), + ] + fig.legend(handles=handles, frameon=False, fontsize=9.5, ncol=3, + loc="lower center", bbox_to_anchor=(0.5, -0.075)) + fig.savefig(out_path, bbox_inches="tight") + + +if __name__ == "__main__": + if sys.argv[1] == "scan": + scan(sys.argv[2], int(sys.argv[3])) + elif sys.argv[1] == "plot": + plot(sys.argv[2], sys.argv[3]) + elif sys.argv[1] == "summary": + rows = json.load(open(sys.argv[2])) + for hlr, psf in GALAXIES: + for kind in KINDS: + for fill in ("production", "noise"): + sel = [r for r in rows if r["kind"] == kind + and r["fill"] == fill and r["hlr"] == hlr + and r["psf"] == psf] + if sel: + print(f"{hlr}/{psf} {kind:13s} {fill:10s} " + f"passes from {smallest_passing(sel)} px") From 3bd49d3e72d1f096321dfb8ff953a781bc752e54 Mon Sep 17 00:00:00 2001 From: Cail Daley Date: Sat, 3 Oct 2026 02:25:00 +0200 Subject: [PATCH 23/30] ngmix: epoch masked-fraction cut at 10%, matching DES EPOCH_MASKED_FRACTION_CUT = 0.10 (was 1/3). The central veto is the bias control; far-field defects bias nothing measured up to 58% of the stamp, so the fraction cut is set for DES Y3/Y6 comparability and as a margin against real-data effects the sims do not model. On tile 202.301 it drops 5.08% of epochs (3.38% at 1/3) and empties 0.55% of objects (0.35%); the extra drops are all CCD-edge or off-tile bands 10-20 px from the object. astra.yaml: ten_percent is the default, one_third excluded with the reason; universes/committed.yaml follows. Tests: the defect-set cut test uses 9.8%/11.8% epochs; corner and edge-neighbour classification tests use geometries under 10%; the off-tile central-veto test lifts the fraction cut to isolate the veto; the far-field recovery cases no longer assert against the cut and are documented as the evidence it is not a bias control. Co-Authored-By: Claude Opus 5.5 --- astra.yaml | 69 +++++++++++--------- src/shapepipe/modules/ngmix_package/ngmix.py | 7 +- tests/module/test_ngmix_defect_fill.py | 69 ++++++++++++-------- tests/science/test_defect_recovery.py | 12 ++-- universes/committed.yaml | 2 +- 5 files changed, 94 insertions(+), 65 deletions(-) diff --git a/astra.yaml b/astra.yaml index 9e104cae1..ae64cdc31 100644 --- a/astra.yaml +++ b/astra.yaml @@ -239,7 +239,7 @@ analyses: On exposures SExtractor reads the split flag image, producing IMAFLAGS_ISO, which the PSF star selection requires to be zero. The multi-epoch vignet run cuts flag stamps from the same image; ngmix - gives flagged pixels weight 0 and drops epochs more than a third + gives flagged pixels weight 0 and drops epochs more than 10% flagged (shape_measurement.defect_fill, shape_measurement.epoch_masked_fraction_cut). Tiles have no flag image, and ShapePipe's tile SExtractor run requests no IMAFLAGS_ISO @@ -1676,46 +1676,55 @@ analyses: label: Per-epoch masked-fraction cut rationale: >- [HARDCODED] An epoch whose stamp has more than - EPOCH_MASKED_FRACTION_CUT = 1/3 of its pixels in the raw, + EPOCH_MASKED_FRACTION_CUT = 10% of its pixels in the raw, unsymmetrized defect set is dropped from the multi-epoch fit: flagged pixels (any nonzero exposure flag bit), zero-weight pixels (off-tile pixels included) and invalid-RMS pixels, the set - defect_fill fills. On a 51-px stamp an object within about 8.5 px - of the tile edge fails the 1/3 cut. Neighbour footprints are not + defect_fill fills. On a 51-px stamp an object within about 20 px + of the tile edge fails the cut. Neighbour footprints are not counted: every epoch shares the tile VIGNET, so a large neighbour would drop them all. An object with no surviving epoch has no shape. Before the cut, an epoch is dropped silently if its galaxy stamp is all zeros or its background-subtracted sigma_mad is not - positive. The cut guards no shear bias that was measured: defects - the central veto keeps, covering up to 58% of the stamp (two-sided - bands, corner Ls and frames 12 px or more from the centre), give - |m11|, |m22| <= 0.05% and |c| < 7e-5 on 0.3 and 0.5 arcsec galaxies - through a round and an elliptical 0.7 arcsec PSF (PR #922). The one - biased far-field case, a one-sided edge band reaching 10 px (31%), - is the veto radius's own bound (m11 = -0.83%). So 1/3 is a - robustness margin, and what it costs is the number of epochs the - real-tile tally shows it dropping. DES was stricter: Y1 rejected any - epoch with a masked or zero-weight pixel, or with its central - 4-pixel region masked; Y3 cut at 10% raw zero-weight pixels; Y6 - drops images more than 10% missing and keeps objects at - mfrac < 0.1 (mask_multi_epoch_drop). A defect near the centre is - handled separately (central_defect_veto). - default: one_third + positive. The cut is not a bias control; the central veto is + (central_defect_veto). Defects the veto keeps, covering up to 58% + of the stamp (two-sided bands, corner Ls and frames 12 px or more + from the centre), give |m11|, |m22| <= 0.05% and |c| < 7e-5 on 0.3 + and 0.5 arcsec galaxies through a round and an elliptical 0.7 + arcsec PSF (PR #922). The one biased far-field case, a one-sided + edge band reaching 10 px (31%), is the veto radius's own bound + (m11 = -0.83%). The 10% cut is chosen for comparability with DES + and as a margin against real-data effects the simulations do not + model, and its measured cost is small. On real tile 202.301 + (interior shape-proxy objects), it drops 5.08% of epochs, leaves + 0.55% of objects with no epoch and costs 13.7% of objects at least + one epoch. Every epoch it removes beyond a 1/3 cut is a CCD-edge or + off-tile band whose nearest defect is 10-20 px from the object; + bad-column clusters cover 2-8% of a stamp, and only 4 epochs + exceed 10% because of them. DES Y1 rejected any epoch with a + masked or zero-weight pixel, or with its central 4-pixel region + masked; Y3 cut at 10% raw zero-weight pixels; Y6 drops images more + than 10% missing and keeps objects at mfrac < 0.1 + (mask_multi_epoch_drop). + default: ten_percent options: - one_third: - label: 1/3 of the stamp in the defect set - description: >- - Drop an epoch only if more than 1/3 of the stamp pixels are - defects. ten_percent: label: 10% (DES Y3 / Y6) + description: >- + Drop an epoch if more than 10% of the stamp pixels are defects, + matching DES Y3 max_zero_weight_frac and Y6 + max_masked_fraction. + insights: [mask_multi_epoch_drop] + one_third: + label: 1/3 of the stamp in the defect set excluded: true excluded_reason: >- - Matches DES Y3 max_zero_weight_frac and Y6 max_masked_fraction, - but far-field defects bias nothing measured up to 58% of the - stamp, so 10% would only discard epochs, among them every edge - band more than 5 px wide. - insights: [mask_multi_epoch_drop] + Equally unbiased in simulations, since far-field defects bias + nothing measured up to 58% of the stamp, and it keeps more + epochs (3.38% dropped on tile 202.301 against 5.08% at 10%). + But the extra epochs it keeps are CCD-edge and off-tile bands + 10-20 px from the object, where effects the simulations do not + model are likeliest, and it departs from the DES convention. any_masked: label: Any masked pixel (DES Y1) description: >- @@ -1976,7 +1985,7 @@ analyses: (Sheldon & Huff 2017). DES Y6 drops input images more than 10% missing and cuts objects at masked fraction mfrac < 0.1, which its image simulations show avoids shear calibration bias. ShapePipe's - cut is 1/3. + cut is also 10%. created_at: "2026-09-26T00:00:00Z" derived: true evidence: diff --git a/src/shapepipe/modules/ngmix_package/ngmix.py b/src/shapepipe/modules/ngmix_package/ngmix.py index 7c1a0a337..a7109ad1c 100644 --- a/src/shapepipe/modules/ngmix_package/ngmix.py +++ b/src/shapepipe/modules/ngmix_package/ngmix.py @@ -37,9 +37,12 @@ # of its stamp is in :func:`defect_mask`, or when a noise-filled # (interpolated) defect pixel lies closer than EPOCH_CENTRAL_DEFECT_RADIUS # (EPOCH_INTERPOLATED_DEFECT_RADIUS) pixels to the stamp centre (see -# :func:`central_defect_vetoes`). +# :func:`central_defect_vetoes`). The central veto is the bias control: defects +# outside it bias nothing measured up to 58% of the stamp. The 10% fraction +# cut matches DES Y3/Y6 and guards against real-data effects the simulations +# do not model, at the cost of about 5% of epochs on a real tile. # @sc [decision:shape_measurement.epoch_masked_fraction_cut] -EPOCH_MASKED_FRACTION_CUT = 1 / 3 +EPOCH_MASKED_FRACTION_CUT = 0.10 # @sc [decision:shape_measurement.central_defect_veto] EPOCH_CENTRAL_DEFECT_RADIUS = 10 # @sc [decision:shape_measurement.central_defect_veto] diff --git a/tests/module/test_ngmix_defect_fill.py b/tests/module/test_ngmix_defect_fill.py index 29d534e46..b0e3819fe 100644 --- a/tests/module/test_ngmix_defect_fill.py +++ b/tests/module/test_ngmix_defect_fill.py @@ -37,7 +37,9 @@ from shapepipe.modules.ngmix_package.ngmix import ( EPOCH_CENTRAL_DEFECT_RADIUS, EPOCH_INTERPOLATED_DEFECT_RADIUS, + EPOCH_MASKED_FRACTION_CUT, Ngmix, + defect_mask, make_ngmix_observation, prepare_ngmix_weights, prepare_postage_stamps, @@ -214,7 +216,7 @@ def per_epoch(make): def _two_sided_band(width): """Mask of ``width`` columns on each side of the stamp, far from the - centre (at least 17 px for width 9).""" + centre (at least 23 px for width 3).""" band = np.zeros((N_STAMP, N_STAMP), dtype=bool) band[:, :width] = True band[:, -width:] = True @@ -224,23 +226,23 @@ def _two_sided_band(width): def _epochs(): """A clean epoch and four masked ones, all defects far from the centre. - * band10: 10 flagged columns on one side, 19.6% (39% if symmetrized); - * flag18: 2 x 9 flagged columns, 35.3%; - * dead18: the same columns at zero weight, no flags; - * rms18: the same columns with an invalid background RMS. + * band5: 5 flagged columns on one side, 9.8% (35% with its quarter turns); + * flag6: 2 x 3 flagged columns, 11.8%; + * dead6: the same columns at zero weight, no flags; + * rms6: the same columns with an invalid background RMS. """ clean = np.zeros((N_STAMP, N_STAMP), dtype=np.int32) ones = np.ones((N_STAMP, N_STAMP)) - band10 = clean.copy() - band10[:, -10:] = 1 - two = _two_sided_band(9) + band5 = clean.copy() + band5[:, -5:] = 1 + two = _two_sided_band(3) dead = ones.copy() dead[two] = 0.0 rms = ones.copy() rms[two] = np.nan return { "2100001-10": (clean, ones), - "2100002-11": (band10, ones), + "2100002-11": (band5, ones), "2100003-12": (two.astype(np.int32), ones), "2100004-13": (clean.copy(), dead), "2100005-14": (clean.copy(), ones, rms), @@ -258,15 +260,21 @@ def _surviving(epochs, **kwargs): def test_epoch_cut_counts_the_defect_set(): - """At the default 1/3 cut, the three 35% epochs are dropped, whichever - defect source masks them, and the 20% band survives. + """At the 10% cut, the three 11.8% epochs are dropped, whichever defect + source masks them, and the 9.8% band survives. - Failure modes: the cut counts flags only (keeps dead18 and rms18), omits - one defect source, or counts a symmetrized set (drops band10); in each + Failure modes: the cut counts flags only (keeps dead6 and rms6), omits + one defect source, or counts a symmetrized set (drops band5); in each case the cut disagrees with the set that is zero-weighted and filled (epoch-cut-on-defect-mask). """ - assert _surviving(_epochs()) == ["2100001-10", "2100002-11"] + epochs = _epochs() + fractions = [ + defect_mask(weight, flag, *rms).mean() + for flag, weight, *rms in epochs.values() + ] + assert fractions[1] <= EPOCH_MASKED_FRACTION_CUT < min(fractions[2:]) + assert _surviving(epochs) == ["2100001-10", "2100002-11"] # --- prepare_postage_stamps: the central-defect veto ----------------------- @@ -600,7 +608,7 @@ def test_neighbour_markers_near_the_centre_keep_every_epoch(): # marked, so it is a neighbour, not off-tile. large = _tile_with_neighbour() large[1:-1, _CENTRE + 3:-1] = _MARKER - assert (large == _MARKER).mean() > 1 / 3 + assert (large == _MARKER).mean() > EPOCH_MASKED_FRACTION_CUT for tile in (small, large): stamp, _, _ = _marker_stamp(tile) assert len(stamp.gals) == len(_MARKER_EPOCH_NAMES) @@ -772,18 +780,23 @@ def test_object_three_px_from_the_tile_edge_is_dropped(): ) -def test_the_central_veto_sees_off_tile_pixels(): - """An off-tile band 12 px from the object passes both cuts; one 9 px - away (17 columns, exactly 1/3 of the stamp, which the fraction cut - keeps) is noise-filled inside the 10-px radius and vetoed. +def test_the_central_veto_sees_off_tile_pixels(monkeypatch): + """An off-tile band 12 px from the object passes the central veto; one + 9 px away is noise-filled inside the 10-px radius and vetoed. + + Every off-tile band within reach of the veto also exceeds the 10% + fraction cut (this one is 17 columns, 1/3 of the stamp), so the fraction + cut is lifted here to isolate the veto, which is what keeps an edge + object's epochs safe if the fraction cut is ever relaxed. Failure mode: the central veto does not read the off-tile set. """ + monkeypatch.setattr(ngmix_module, "EPOCH_MASKED_FRACTION_CUT", 1.0) sky = np.random.default_rng(5).normal(0.0, 1.0, (N_STAMP, N_STAMP)) far, near = sky.copy(), sky.copy() far[:, :_CENTRE - 11] = _MARKER near[:, :_CENTRE - 8] = _MARKER - assert (near == _MARKER).mean() == 1 / 3 + assert (near == _MARKER).mean() > EPOCH_MASKED_FRACTION_CUT kept, _, _ = _marker_stamp(far) assert len(kept.gals) == len(_MARKER_EPOCH_NAMES) vetoed, _, _ = _marker_stamp(near) @@ -801,8 +814,8 @@ def test_corner_off_tile_region_and_border_neighbour_are_classified(): neighbour is swallowed); the classification ignores the MegaCam flip. """ tile = np.random.default_rng(5).normal(0.0, 1.0, (N_STAMP, N_STAMP)) - tile[:5, :] = _MARKER - tile[:, -5:] = _MARKER + tile[:2, :] = _MARKER + tile[:, -2:] = _MARKER tile[40:, :6] = _MARKER # neighbour on the bottom-left border stamp, epochs, _ = _marker_stamp(tile) names = {id(v[0]): name for name, v in epochs.items()} @@ -812,7 +825,7 @@ def test_corner_off_tile_region_and_border_neighbour_are_classified(): ): name = names[id(flag)] off_tile = _off_tile_expected(tile, name) - assert off_tile.sum() == 5 * N_STAMP * 2 - 25 + assert off_tile.sum() == 2 * N_STAMP * 2 - 4 npt.assert_array_equal(weight == 0, off_tile) npt.assert_array_equal(flag, 0) npt.assert_array_equal( @@ -914,7 +927,7 @@ def test_dr6_segmap_stamps_split_into_off_image_and_neighbours(): def test_a_neighbour_completing_rows_beside_the_tile_edge_stays_a_neighbour(): - """An object 14 px from the tile's left edge, with a wide neighbour + """An object 20 px from the tile's left edge, with a wide neighbour footprint that runs from the tile edge across the stamp: in the rows of that footprint every stamp pixel is -1e30, off the image or on the neighbour. Only the off-image columns are off-tile; the footprint is the @@ -925,13 +938,13 @@ def test_a_neighbour_completing_rows_beside_the_tile_edge_stays_a_neighbour(): fill interpolates (off-tile-is-marked-border-rows-and-columns). """ seg = np.zeros((80, 80), np.int32) - seg[20:24, 0:45] = 5 - seg[27:32, 13:18] = 1 - x, y = np.array([15.0, 30.0]), np.array([30.0, 22.0]) + seg[20:24, 0:50] = 5 + seg[27:32, 19:24] = 1 + x, y = np.array([21.0, 30.0]), np.array([30.0, 22.0]) vignets, off_image = _segmap_stamps(seg, x, y) tile, off = vignets[0], off_image[0] footprint = (tile == _MARKER) & ~off - assert footprint.sum() == 4 * (N_STAMP - 11) + assert footprint.sum() == 4 * (N_STAMP - 5) assert ((tile == _MARKER).all(axis=1) & ~off.all(axis=1)).sum() == 4 stamp, epochs, _ = _marker_stamp(tile) diff --git a/tests/science/test_defect_recovery.py b/tests/science/test_defect_recovery.py index 539a6834d..ce12f52a5 100644 --- a/tests/science/test_defect_recovery.py +++ b/tests/science/test_defect_recovery.py @@ -1,6 +1,6 @@ -"""Shear recovery for the defects the epoch cuts keep. +"""Shear recovery for the defects the central veto keeps. -Physics invariant: every defect the cuts keep leaves both additive terms +Physics invariant: every defect the central veto keeps leaves both additive terms |c1|, |c2| < 5e-4 and both diagonal multiplicative terms |m11|, |m22| < 1%, from the full 2x2 response matrix. A filled defect biases m anisotropically (m11 and m22 can differ tenfold), so a scalar m would hide it. @@ -15,6 +15,12 @@ cases sit at the radii themselves, so lowering either below its calibrated value turns this red. +The bound is checked against the veto alone, not the masked-fraction cut: +the edge bands at the noise-fill radius cover about 30% of the stamp, which +``EPOCH_MASKED_FRACTION_CUT`` (10%) drops in production. That they stay +within the bound is the evidence that the fraction cut is not a bias +control, so it can be chosen for DES comparability and robustness alone. + Positive control: a 3-px bleed three pixels inside the interpolated-defect radius breaks the bound. """ @@ -30,7 +36,6 @@ from shapepipe.modules.ngmix_package.ngmix import ( EPOCH_CENTRAL_DEFECT_RADIUS, EPOCH_INTERPOLATED_DEFECT_RADIUS, - EPOCH_MASKED_FRACTION_CUT, central_defect_vetoes, defect_mask, ) @@ -109,7 +114,6 @@ def test_kept_defects_recover_shear_on_both_axes(kind, distance, hlr, fwhm, np.testing.assert_array_equal(masked, bad) interpolated = interpolable_defects(masked) assert interpolated.any() == (kind in INTERPOLATED) - assert masked.mean() <= EPOCH_MASKED_FRACTION_CUT assert not central_defect_vetoes(masked, interpolated) m, c, result = recover(bad, hlr, fwhm, psf_shear, tmp_path) assert m < 0.01, result diff --git a/universes/committed.yaml b/universes/committed.yaml index 9a49ba6ec..44002be0b 100644 --- a/universes/committed.yaml +++ b/universes/committed.yaml @@ -55,7 +55,7 @@ analyses: defect_fill: interpolate blend_handling: noisefill weight_symmetrization: interpolated - epoch_masked_fraction_cut: one_third + epoch_masked_fraction_cut: ten_percent central_defect_veto: fixed_radii catalogue_assembly: decisions: From b0ce5af9719db07b11e950ae4a1c4dc7f611e01d Mon Sep 17 00:00:00 2001 From: Cail Daley Date: Sat, 3 Oct 2026 02:26:52 +0200 Subject: [PATCH 24/30] bias_vs_distance: legend names the masked-fraction cut from its constant Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01N4HLdzDyc3CiPLrgJ48GVo --- scripts/validation/masked_pixels/bias_vs_distance.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/validation/masked_pixels/bias_vs_distance.py b/scripts/validation/masked_pixels/bias_vs_distance.py index 25aaeaed9..5737c7483 100644 --- a/scripts/validation/masked_pixels/bias_vs_distance.py +++ b/scripts/validation/masked_pixels/bias_vs_distance.py @@ -192,7 +192,7 @@ def plot(in_path, out_path): Line2D([], [], color=INK2, lw=1.4, ls=(0, (3, 2)), label="noise fill (not used for narrow defects)"), Line2D([], [], color=INK2, lw=0, marker="o", mfc="white", mec=INK2, - ms=4, label="epoch dropped by the 1/3 masked-fraction cut"), + ms=4, label=f"epoch dropped by the {FRAC_CUT:.0%} masked-fraction cut"), ] fig.legend(handles=handles, frameon=False, fontsize=9.5, ncol=3, loc="lower center", bbox_to_anchor=(0.5, -0.075)) From 8bc7946a1d3b4fa47a5150d75d81dd0bc9b83403 Mon Sep 17 00:00:00 2001 From: Cail Daley Date: Sat, 3 Oct 2026 02:42:12 +0200 Subject: [PATCH 25/30] ngmix: noise-fill central veto at 13 px, covering the 0.7" galaxy through a 0.9" PSF A noise-filled 4-column cluster (8% of the stamp, kept by the 10% cut) on that galaxy gives m11 = -6.6% at 10 px and first passes at 13 px. The science test checks the cluster and the edge band there; the bias-vs-distance script gains the cluster. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01N4HLdzDyc3CiPLrgJ48GVo --- astra.yaml | 44 ++++++++++++------- .../masked_pixels/bias_vs_distance.py | 30 +++++++------ .../validation/masked_pixels/pixel_classes.py | 5 ++- src/shapepipe/modules/ngmix_package/ngmix.py | 6 ++- tests/module/test_ngmix_defect_fill.py | 9 ++-- tests/science/test_defect_recovery.py | 21 +++++---- 6 files changed, 68 insertions(+), 47 deletions(-) diff --git a/astra.yaml b/astra.yaml index ae64cdc31..ebde883fc 100644 --- a/astra.yaml +++ b/astra.yaml @@ -1445,8 +1445,8 @@ analyses: (symmetrized_4fold_noise). The cost of not symmetrizing, a hole in the galaxy light, is bounded by central_defect_veto. Interpolation is the only fill because it lets interpolated defects come to 7 px - of the object, against a size-blind 10 px for noise fill, which a - 0.7 arcsec galaxy through a 0.9 arcsec PSF needs widened to 14 px. + of the object, against 13 px for noise fill, which a 0.7 arcsec + galaxy through a 0.9 arcsec PSF needs. Values: VIGNETMAKER_RUNNER_RUN_1.MASKING = False; VIGNETMAKER_RUNNER_RUN_2.MASKING = False. @@ -1478,10 +1478,10 @@ analyses: excluded_reason: >- A noise-filled defect leaves a hole in the object's light that metacal shears and the sky does not. Columns, 3-px bleeds and - single pixels then need the 10 px veto on 0.3 and 0.5 arcsec - galaxies through a 0.7 arcsec PSF, and 14 px on a 0.7 arcsec + single pixels then need a 10 px veto on 0.3 and 0.5 arcsec + galaxies through a 0.7 arcsec PSF, and 13 px on a 0.7 arcsec galaxy through a 0.9 arcsec PSF (a 3-px bleed at 10 px gives - m11 = -6.8%). Interpolated, the same defects pass from 7 px on + m11 = -6.5%). Interpolated, the same defects pass from 7 px on all three (PR #922 bias scan). insights: [mask_metacal_acts_on_whole_stamp, mask_bad_column_symmetrize] symmetrized_4fold_noise: @@ -1632,7 +1632,7 @@ analyses: label: Per-epoch veto on a defect near the stamp centre rationale: >- [HARDCODED] An epoch is dropped when a defect pixel lies strictly - closer to the stamp centre than the radius of its fill: 10 px for + closer to the stamp centre than the radius of its fill: 13 px for noise-filled pixels (EPOCH_CENTRAL_DEFECT_RADIUS) and 7 px for interpolated ones (EPOCH_INTERPOLATED_DEFECT_RADIUS). The veto reads the same interpolated set as the fill, including the runs @@ -1646,24 +1646,34 @@ analyses: the defects it keeps give |m11|, |m22| < 1% and |c1|, |c2| < 5e-4, on 51-px stamps at known RMS and high S/N, through a round and a (0.05, 0.02) elliptical 0.7 arcsec PSF, on galaxies of half-light - radius 0.3 and 0.5 arcsec, and for interpolated defects also 0.7 - arcsec through a 0.9 arcsec PSF. Known limit: at 10 px the widest - edge band the veto keeps sits at the 1% bound through the - elliptical PSF (m11 = -0.94% +/- 0.06%). Guarded by + radius 0.3 and 0.5 arcsec, and 0.7 arcsec through a 0.9 arcsec + PSF. That largest galaxy sets both radii: 3-px bleeds the 7 px, + noise-filled holes the 13 px (a 4-column cluster at 13 px gives + m11 = -0.67% +/- 0.05%, c1 = 7e-5). Guarded by tests/science/test_defect_recovery.py. Values: - EPOCH_CENTRAL_DEFECT_RADIUS = 10; + EPOCH_CENTRAL_DEFECT_RADIUS = 13; EPOCH_INTERPOLATED_DEFECT_RADIUS = 7. default: fixed_radii options: fixed_radii: - label: Fixed radii, 10 px for noise-filled and 7 px for interpolated defects + label: Fixed radii, 13 px for noise-filled and 7 px for interpolated defects + noise_fill_10px: + label: 10 px for noise-filled defects + excluded: true + excluded_reason: >- + Enough for 0.3 and 0.5 arcsec galaxies through a 0.7 arcsec + PSF, not for a 0.7 arcsec galaxy through a 0.9 arcsec PSF. A + noise-filled 4-column cluster (8% of the stamp, so the 10% + masked-fraction cut keeps it) gives m11 = -6.6% +/- 0.3% and + c1 = -4.3e-3 at 10 px, -3.6% and -1.6e-3 at 11 px, -1.7% and + -3.7e-4 at 12 px, and first passes at 13 px (PR #922, 6 seeds). disabled: label: No central veto excluded: true excluded_reason: >- - A column two pixels inside the 10 px radius, noise-filled, gives - |m| > 2% and |c| > 1e-3 on a 0.5 arcsec galaxy; a 3-px bleed + A noise-filled column 8 px from a 0.5 arcsec galaxy gives + m11 = -3.7% and c1 = -3.6e-3; a 3-px bleed three pixels inside the 7 px radius, interpolated, gives |c| > 1e-3 on a 0.3 arcsec galaxy. size_scaled_radius: @@ -1691,9 +1701,9 @@ analyses: of the stamp (two-sided bands, corner Ls and frames 12 px or more from the centre), give |m11|, |m22| <= 0.05% and |c| < 7e-5 on 0.3 and 0.5 arcsec galaxies through a round and an elliptical 0.7 - arcsec PSF (PR #922). The one biased far-field case, a one-sided - edge band reaching 10 px (31%), is the veto radius's own bound - (m11 = -0.83%). The 10% cut is chosen for comparability with DES + arcsec PSF (PR #922). A one-sided edge band reaching the 13 px + radius (25%) gives |m| <= 0.01% on those galaxies through the + round PSF and m11 = -0.64% on a 0.7 arcsec galaxy through a 0.9 arcsec PSF. The 10% cut is chosen for comparability with DES and as a margin against real-data effects the simulations do not model, and its measured cost is small. On real tile 202.301 (interior shape-proxy objects), it drops 5.08% of epochs, leaves diff --git a/scripts/validation/masked_pixels/bias_vs_distance.py b/scripts/validation/masked_pixels/bias_vs_distance.py index 5737c7483..9d64abc39 100644 --- a/scripts/validation/masked_pixels/bias_vs_distance.py +++ b/scripts/validation/masked_pixels/bias_vs_distance.py @@ -3,7 +3,7 @@ Production interpolates the defects ``interpolable_defects`` selects (columns, 3-px bleeds, finite bleeds, single pixels) and noise-fills the -rest (edge bands). The counterfactual noise-fills every defect, by making +rest (4-column clusters, edge bands). The counterfactual noise-fills every defect, by making ``interpolated_defects`` select none. The recovery is the full-matrix metacal measurement of ``tests/helpers/defect_response``; the figure plots the worse axis of |m| and |c| with the veto radii and the bounds @@ -11,10 +11,10 @@ Run from the shapepipe checkout root inside the shapepipe container: PYTHONPATH=src:. python scripts/validation/masked_pixels/bias_vs_distance.py \ - scan OUT.json NPROC + scan OUT.json NPROC [KIND,KIND...] PYTHONPATH=src:scripts/validation/masked_pixels \ python scripts/validation/masked_pixels/bias_vs_distance.py \ - plot IN.json OUT.png + plot OUT.png IN.json [IN.json ...] """ import json import sys @@ -26,7 +26,7 @@ SEEDS = range(6) GALAXIES = [(0.3, 0.7), (0.5, 0.7), (0.7, 0.9)] # (hlr, PSF FWHM) arcsec INTERPOLATED = ("column", "bleed", "finite_bleed", "pixel") -KINDS = INTERPOLATED + ("edge",) +KINDS = INTERPOLATED + ("cluster4", "edge") BOUND_M, BOUND_C = 0.01, 5e-4 @@ -43,6 +43,8 @@ def geometry(kind, distance): bad[:, near:near + 3] = True elif kind == "finite_bleed": bad[CENTRE - 5:CENTRE + 6, near:near + 3] = True + elif kind == "cluster4": + bad[:, near:near + 4] = True elif kind == "edge": bad[:, near:] = True return bad @@ -66,11 +68,11 @@ def one(job): return row -def scan(out_path, nproc): +def scan(out_path, nproc, kinds=KINDS): from multiprocessing import get_context jobs = [(k, f, d, h, p) for h, p in GALAXIES for d in DISTANCES - for k in KINDS + for k in kinds for f in (("production", "noise") if k in INTERPOLATED else ("production",))] # spawn: each worker imports ngmix afresh, so the noise-fill patch @@ -97,7 +99,7 @@ def smallest_passing(rows): return passing -def plot(in_path, out_path): +def plot(out_path, *in_paths): import matplotlib matplotlib.use("Agg") @@ -111,14 +113,16 @@ def plot(in_path, out_path): EPOCH_MASKED_FRACTION_CUT as FRAC_CUT, ) - rows = [r for r in json.load(open(in_path))] + rows = [r for path in in_paths for r in json.load(open(path))] grid, fail, bad_ink = "#e4e3df", "#f6e3dc", "#a8462a" + GOLD = "#c49a1c" series = { "column": ("bad column", BLUE), "bleed": ("3-px bleed", ORANGE), "finite_bleed": ("finite 3-px bleed (11 rows)", VIOLET), "pixel": ("single pixel", AQUA), "edge": ("edge band (noise-filled)", INK), + "cluster4": ("4-column cluster (noise-filled)", GOLD), } plt.rcParams.update({"font.size": 10.5, "axes.edgecolor": INK2, "xtick.color": INK2, "ytick.color": INK2, @@ -185,10 +189,9 @@ def plot(in_path, out_path): color=INK2, fontsize=10, y=0.965) handles = [Line2D([], [], color=c, lw=2, marker="o", ms=3.5, label=l) for l, c in series.values()] - handles += [Line2D([], [], lw=0, label=" ")] # keeps styles in column 3 handles += [ Line2D([], [], color=INK2, lw=2, marker="o", ms=3.5, - label="production (interpolated; edge band noise-filled)"), + label="production (narrow defects interpolated)"), Line2D([], [], color=INK2, lw=1.4, ls=(0, (3, 2)), label="noise fill (not used for narrow defects)"), Line2D([], [], color=INK2, lw=0, marker="o", mfc="white", mec=INK2, @@ -201,11 +204,12 @@ def plot(in_path, out_path): if __name__ == "__main__": if sys.argv[1] == "scan": - scan(sys.argv[2], int(sys.argv[3])) + scan(sys.argv[2], int(sys.argv[3]), + *([tuple(sys.argv[4].split(","))] if len(sys.argv) > 4 else [])) elif sys.argv[1] == "plot": - plot(sys.argv[2], sys.argv[3]) + plot(sys.argv[2], *sys.argv[3:]) elif sys.argv[1] == "summary": - rows = json.load(open(sys.argv[2])) + rows = [r for path in sys.argv[2:] for r in json.load(open(path))] for hlr, psf in GALAXIES: for kind in KINDS: for fill in ("production", "noise"): diff --git a/scripts/validation/masked_pixels/pixel_classes.py b/scripts/validation/masked_pixels/pixel_classes.py index 29136d44d..0d4d377d3 100644 --- a/scripts/validation/masked_pixels/pixel_classes.py +++ b/scripts/validation/masked_pixels/pixel_classes.py @@ -132,7 +132,8 @@ def main(real_path, out_path): axs[0, 0].set_ylabel("image ngmix sees", color=INK, fontsize=11) axs[1, 0].set_ylabel("pixel class / weight", color=INK, fontsize=11) c = 25 - axs[1, 0].text(c, c + 10.8, "10 px", ha="center", va="bottom", + rn = EPOCH_CENTRAL_DEFECT_RADIUS + axs[1, 0].text(c, c + rn + 0.8, f"{rn:g} px", ha="center", va="bottom", fontsize=8, color=INK) axs[1, 0].text(c, c - 6.3, "7 px", ha="center", va="bottom", fontsize=8, color=INK) @@ -172,7 +173,7 @@ def main(real_path, out_path): Patch(color=BLUE, label="neighbour (−1e30 in tile)"), Patch(color=ORANGE, label="defect (flag / weight / RMS)"), Patch(color=AQUA, label="off-tile (a defect)"), - Line2D([], [], color=INK, ls="--", lw=1, label="10 px veto"), + Line2D([], [], color=INK, ls="--", lw=1, label=f"{EPOCH_CENTRAL_DEFECT_RADIUS:g} px veto"), Line2D([], [], color=INK, ls=":", lw=1, label="7 px veto (interpolated)"), ] + WEIGHT_LEGEND diff --git a/src/shapepipe/modules/ngmix_package/ngmix.py b/src/shapepipe/modules/ngmix_package/ngmix.py index a7109ad1c..29ad18615 100644 --- a/src/shapepipe/modules/ngmix_package/ngmix.py +++ b/src/shapepipe/modules/ngmix_package/ngmix.py @@ -38,13 +38,15 @@ # (interpolated) defect pixel lies closer than EPOCH_CENTRAL_DEFECT_RADIUS # (EPOCH_INTERPOLATED_DEFECT_RADIUS) pixels to the stamp centre (see # :func:`central_defect_vetoes`). The central veto is the bias control: defects -# outside it bias nothing measured up to 58% of the stamp. The 10% fraction +# outside it bias nothing measured up to 58% of the stamp. The noise-fill +# radius is set by the largest galaxy tested (0.7" through a 0.9" PSF), the +# interpolated radius by 3-px bleeds on the same galaxy. The 10% fraction # cut matches DES Y3/Y6 and guards against real-data effects the simulations # do not model, at the cost of about 5% of epochs on a real tile. # @sc [decision:shape_measurement.epoch_masked_fraction_cut] EPOCH_MASKED_FRACTION_CUT = 0.10 # @sc [decision:shape_measurement.central_defect_veto] -EPOCH_CENTRAL_DEFECT_RADIUS = 10 +EPOCH_CENTRAL_DEFECT_RADIUS = 13 # @sc [decision:shape_measurement.central_defect_veto] EPOCH_INTERPOLATED_DEFECT_RADIUS = 7 diff --git a/tests/module/test_ngmix_defect_fill.py b/tests/module/test_ngmix_defect_fill.py index b0e3819fe..e1e564524 100644 --- a/tests/module/test_ngmix_defect_fill.py +++ b/tests/module/test_ngmix_defect_fill.py @@ -781,8 +781,8 @@ def test_object_three_px_from_the_tile_edge_is_dropped(): def test_the_central_veto_sees_off_tile_pixels(monkeypatch): - """An off-tile band 12 px from the object passes the central veto; one - 9 px away is noise-filled inside the 10-px radius and vetoed. + """An off-tile band at EPOCH_CENTRAL_DEFECT_RADIUS from the object passes + the central veto; one 9 px away is noise-filled inside it and vetoed. Every off-tile band within reach of the veto also exceeds the 10% fraction cut (this one is 17 columns, 1/3 of the stamp), so the fraction @@ -794,7 +794,8 @@ def test_the_central_veto_sees_off_tile_pixels(monkeypatch): monkeypatch.setattr(ngmix_module, "EPOCH_MASKED_FRACTION_CUT", 1.0) sky = np.random.default_rng(5).normal(0.0, 1.0, (N_STAMP, N_STAMP)) far, near = sky.copy(), sky.copy() - far[:, :_CENTRE - 11] = _MARKER + rn = int(EPOCH_CENTRAL_DEFECT_RADIUS) + far[:, :_CENTRE - rn + 1] = _MARKER near[:, :_CENTRE - 8] = _MARKER assert (near == _MARKER).mean() > EPOCH_MASKED_FRACTION_CUT kept, _, _ = _marker_stamp(far) @@ -1019,7 +1020,7 @@ def test_a_column_beside_a_noisefill_neighbour_is_vetoed_as_noise_filled(): Failure mode: the veto decides which pixels are interpolated from the defect mask alone, so it keeps the epoch at the 7-px radius while the - fill noise-fills pixels 8 px from the object, inside the 10 px that + fill noise-fills pixels 8 px from the object, inside the radius that noise fill needs. """ column = np.zeros((N_STAMP, N_STAMP), dtype=np.int32) diff --git a/tests/science/test_defect_recovery.py b/tests/science/test_defect_recovery.py index ce12f52a5..eb582d26d 100644 --- a/tests/science/test_defect_recovery.py +++ b/tests/science/test_defect_recovery.py @@ -10,13 +10,15 @@ round and with ellipticity (0.05, 0.02), and on a 0.7" galaxy through a 0.9" PSF. These pass only because the interpolated pixels' quarter turns also lose their weight: an unsymmetrized column at 8 px gives -c1 = -1.3e-3. Defects too wide to interpolate (a 5-px bleed, edge bands) -are noise-filled, and are checked at ``EPOCH_CENTRAL_DEFECT_RADIUS``. The -cases sit at the radii themselves, so lowering either below its calibrated -value turns this red. +c1 = -1.3e-3. Defects too wide to interpolate (a 4-column cluster, a 5-px +bleed, edge bands) are noise-filled, and are checked at +``EPOCH_CENTRAL_DEFECT_RADIUS`` on the same three galaxies; the 0.7" +galaxy through the 0.9" PSF sets that radius (a 4-column cluster at 10 px +gives m11 = -6.6%). The cases sit at the radii themselves, so lowering +either below its calibrated value turns this red. The bound is checked against the veto alone, not the masked-fraction cut: -the edge bands at the noise-fill radius cover about 30% of the stamp, which +the edge bands at the noise-fill radius cover about 25% of the stamp, which ``EPOCH_MASKED_FRACTION_CUT`` (10%) drops in production. That they stay within the bound is the evidence that the fraction cut is not a bias control, so it can be chosen for DES comparability and robustness alone. @@ -63,6 +65,8 @@ def geometry(kind, distance): bad[:, near:near + 3] = True elif kind == "finite_bleed": bad[CENTRE - 5:CENTRE + 6, near:near + 3] = True + elif kind == "cluster4": + bad[:, near:near + 4] = True elif kind == "wide_bleed": bad[:, near:near + 5] = True elif kind == "edge": @@ -79,10 +83,9 @@ def geometry(kind, distance): for psf in (ROUND, ELLIPTICAL)] + [(k, RN, h, 0.7, ROUND) for k in ("wide_bleed", "edge") for h in (0.3, 0.5)] - # The widest edge band the veto keeps, on the 0.5" galaxy through the - # elliptical PSF, sits at the bound at the radius itself - # (m11 = -0.94% +/- 0.06%), so it is checked one pixel out. - + [("edge", RN + 1, 0.5, 0.7, ELLIPTICAL)] + + [("edge", RN, 0.5, 0.7, ELLIPTICAL)] + + [(k, RN, 0.7, 0.9, psf) for k in ("cluster4", "edge") + for psf in (ROUND, ELLIPTICAL)] + [("edge", N - CENTRE - 5, 0.5, 0.7, psf) for psf in (ROUND, ELLIPTICAL)] ) From ef083a2a725d0f6bdaa46c6d06dd23fa99cb429d Mon Sep 17 00:00:00 2001 From: Cail Daley Date: Sat, 3 Oct 2026 18:41:48 +0200 Subject: [PATCH 26/30] ngmix: DEFECT_WEIGHTING option; noise image interpolated in fixnoise's frame One function, defect_weighting_masks, decides how interpolated defects enter the image and the weight map: fourfold_zero (the PR's design, still the default), des_y6 (pizza-cutter: mask OR its k=1 quarter turn, interpolated at full weight), hole and full. The option reaches ngmix from the run config: `defect_weighting` -> SP_DEFECT_WEIGHTING (tile_ngmix) -> DEFECT_WEIGHTING in config_tile_Ng_template.ini, validated at parse time against the module's DEFECT_WEIGHTINGS; tile_vignets carries it too so a change reruns the whole shape group. Params pin updated (tile_ngmix only): a campaign boundary. Under every option the metacal noise image is now interpolated in the frame ngmix's fixnoise shears it in (rot90 k=1, back with k=3), by the science image's operator. Interpolated in the detector frame, its interpolated pixels sat a quarter turn from the science image's and left an S/N-dependent additive bias (c1 = 2.7e-3 for a 3-px bleed at 7 px on the 0.7"/0.9" galaxy at S/N 30). A test pins ngmix's k=1/k=3 behaviourally. astra: weight_symmetrization becomes shape_measurement.defect_weighting. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01N4HLdzDyc3CiPLrgJ48GVo --- astra.yaml | 107 +++++----- src/shapepipe/modules/ngmix_package/ngmix.py | 194 ++++++++++++++---- src/shapepipe/modules/ngmix_runner.py | 16 +- tests/module/test_ngmix.py | 2 + tests/module/test_ngmix_defect_fill.py | 105 ++++++++-- tests/unit/test_workflow_defect_weighting.py | 71 +++++++ tests/workflow/params_pin.json | 4 +- universes/committed.yaml | 2 +- workflow/Snakefile | 19 ++ workflow/config.yaml | 4 + .../config/cfis/config_tile_Ng_template.ini | 7 + workflow/rules/tile.smk | 10 +- 12 files changed, 434 insertions(+), 107 deletions(-) create mode 100644 tests/unit/test_workflow_defect_weighting.py diff --git a/astra.yaml b/astra.yaml index ebde883fc..f1a6225ba 100644 --- a/astra.yaml +++ b/astra.yaml @@ -1223,7 +1223,7 @@ analyses: metacal_scheme, centroid_source, epoch_flux_rescaling, psf_epoch_averaging, galaxy_pixel_weights, psf_likelihood_noise, megacam_ccd_flip, defect_fill, blend_handling, - weight_symmetrization, epoch_masked_fraction_cut, + defect_weighting, epoch_masked_fraction_cut, central_defect_veto] decisions: ngmix_seed_mode: @@ -1464,7 +1464,7 @@ analyses: (wider holes, edge bands, runs ending on a noisefill neighbour) get independent noise at the per-pixel background RMS when supplied, or the stamp's robust noise scale otherwise. The - interpolated pixels keep weight 0 (weight_symmetrization). For a + interpolated pixels are weighted by defect_weighting. For a 3-px bleed 6 px from a 0.5 arcsec galaxy, PR #916 measures m11 = +0.89% when the fill is symmetrized, against +0.19% otherwise. @@ -1578,56 +1578,69 @@ analyses: the ~2% uberseg-only bias in DES Y1 simulations, although Sheldon et al. 2020 found similar blend biases with and without MOF. insights: [mask_des_y1_uberseg_only, mask_blend_bias_detection] - weight_symmetrization: - label: Four-fold symmetric weight holes of interpolated defects + defect_weighting: + label: Weighting of interpolated defects (DEFECT_WEIGHTING) rationale: >- ngmix fits a Gaussian to a profile that is not one, so a zero-weight - hole on one side of the object pulls the fit: a spin-2 term. - [HARDCODED] prepare_ngmix_weights cancels it for the interpolated - defects, whose light stays: the weight is also zeroed on their three - quarter-turn copies about the stamp centre, which keep their light. - Pixels whose light is replaced (noise-filled defects, noisefill - neighbours) are not symmetrized: the hole is then in the image - metacal shears, and a weight orbit leaves m unchanged (-0.53% - against -0.54% for a noise-filled column 10 px from a 0.5 arcsec - galaxy) while moving c by about 1e-4 either way. The uberseg - neighbour side is not symmetrized either - (interpolated_and_neighbours). The rotation is about the stamp - centre, where the extractor places the object to within half a - pixel per axis; the measurements draw the object uniformly within - that half pixel. Measured with tests/helpers/defect_response.py (6 - seeds, full response matrix, galaxy half-light radius 0.5 arcsec, - 0.7 arcsec PSF). - default: interpolated + hole on one side of the object pulls the fit (a spin-2 term), while + interpolated light at full weight is a smooth guess the fit trusts + as data. DEFECT_WEIGHTING (prepare_ngmix_weights, + defect_weighting_masks; the workflow's `defect_weighting`) chooses + between them for the interpolated defects only: noise-filled pixels + and noisefill neighbours have no light to weight and keep weight 0, + and the uberseg neighbour side keeps weight 0 under every option. + Under every option the metacal noise image is interpolated in + fixnoise's quarter-turned frame (ngmix turns it k=1 before shearing + and k=3 after), by the operator that interpolates the science image. + Interpolated in the detector frame, its interpolated pixels sit a + quarter turn from the science image's, and the correlated noise of + the interpolant survives fixnoise as an additive bias that grows as + S/N falls: a 3-px bleed 7 px from a 0.7 arcsec galaxy through a 0.9 + arcsec PSF gives c1 = 2.7e-3 +/- 0.5e-3 at S/N 30 under + fourfold_zero, and 4.6e-4 or less on every galaxy with the noise + image turned. Measured with paired, antithetic-noise seeds on 51-px + stamps, galaxies of half-light radius 0.3 and 0.5 arcsec through a + 0.7 arcsec PSF and 0.7 arcsec through a 0.9 arcsec PSF, round and + (0.05, 0.02) PSFs, defects 7-12 px from the centre (sym-verify + harness, PR #922). The rotation is about the stamp centre, where the + extractor places the object to within half a pixel per axis. + Values: + ngmix.py::DEFECT_WEIGHTING = "fourfold_zero". + default: fourfold_zero options: - interpolated: - label: Symmetrize the interpolated defects + fourfold_zero: + label: Interpolate; zero the weight on the defect and its three quarter turns description: >- - An interpolated column 8 px from the galaxy gives c1 = -1.3e-3 - unsymmetrized and +2e-6 symmetrized (PR #916). + The quarter-turn copies keep their light. An interpolated column + 8 px from the galaxy gives c1 = -1.3e-3 with a one-sided hole and + +2e-6 symmetrized (PR #916, noiseless). insights: [mask_bad_column_symmetrize] - interpolated_and_neighbours: - label: Also symmetrize the uberseg neighbour side (#815) - excluded: true - excluded_reason: >- - Alone, the neighbour-side hole is a strong pull: a neighbour - footprint with no light 10 px away gives c1 = -7.8e-3 - unsymmetrized and 2e-6 symmetrized. But a real neighbour leaves - light in the target's uberseg cell, which pulls the other way, - and the one-sided hole partly offsets it: for a neighbour of a - tenth of the target's flux at 10 px, c1 = +1.1e-3 unsymmetrized - and +1.2e-2 symmetrized; at equal flux, +0.13 and +0.21. The - symmetrized hole also keeps 117 of 1580 weighted pixels and 97% - of the target's S/N at 10 px (99% unsymmetrized), 25 pixels and - 71% at 7 px, none at 5 px. Worth revisiting only on a - neighbour-subtracted image (blend_handling.mof_subtract). - insights: [mask_bad_column_symmetrize, mask_uberseg_neighbour_bias] - none: - label: No symmetrization - excluded: true - excluded_reason: >- - Every zero-weight hole stays one-sided: the interpolated column - above gives c1 = -1.3e-3. + des_y6: + label: "DES Y6 (pizza-cutter): mask OR its quarter turn, interpolated at full weight" + description: >- + The defect mask is ORed with its np.rot90 (k=1) copy about the + stamp centre and the union is interpolated, as pizza-cutter + symmetrizes the bit mask and weight of each SE slice before + interpolating; every interpolated pixel keeps full weight. + Departures forced by per-object stamps: the rotation is about + the object, not a cell centre; there is no mfrac cut, the epoch + cuts (10% masked fraction, central veto) stand in for it; the + noise image is interpolated in fixnoise's turned frame rather + than in the science frame; and copies that land on a + noise-filled pixel stay noise-filled. Without the turned noise + image it gives c1 = -5.5e-3 +/- 1.8e-3 for the bleed above at + S/N 30. + insights: [mask_des_defect_practice, mask_bad_column_symmetrize] + hole: + label: Interpolate; zero the weight on the defect only + description: >- + The one-sided hole pulls the fit: c1 = -5.7e-3 for the bleed + above, noiseless. + full: + label: Interpolate at full weight, no symmetrization + description: >- + c1 = 6.8e-4 for the bleed above through the elliptical PSF, + noiseless; 9.5e-3 at S/N 30 without the turned noise image. central_defect_veto: label: Per-epoch veto on a defect near the stamp centre rationale: >- diff --git a/src/shapepipe/modules/ngmix_package/ngmix.py b/src/shapepipe/modules/ngmix_package/ngmix.py index 29ad18615..f7580782e 100644 --- a/src/shapepipe/modules/ngmix_package/ngmix.py +++ b/src/shapepipe/modules/ngmix_package/ngmix.py @@ -32,6 +32,13 @@ # Neighbour treatments selectable with the BLEND_HANDLING option. BLEND_HANDLINGS = ("noisefill", "uberseg") +# How interpolated defects enter the image and the weight map, selectable +# with the DEFECT_WEIGHTING option (see :func:`defect_weighting_masks`), and +# the value an absent or empty option takes. +DEFECT_WEIGHTINGS = ("des_y6", "fourfold_zero", "hole", "full") +# @sc [decision:shape_measurement.defect_weighting] +DEFECT_WEIGHTING = "fourfold_zero" + # The epoch cuts (see :func:`prepare_postage_stamps`). Calibration outputs, # not options: an epoch is dropped when more than EPOCH_MASKED_FRACTION_CUT # of its stamp is in :func:`defect_mask`, or when a noise-filled @@ -753,6 +760,12 @@ class Ngmix(object): dilate_neighbour : int, optional Neighbour-mask dilation iterations for ``"uberseg"`` (see :func:`uberseg_mask`); the default is ``1``. + metacal_psf : str, optional + Metacal reconvolution-kernel scheme (see :func:`do_ngmix_metacal`). + defect_weighting : str, optional + How interpolated defects are weighted, one of ``DEFECT_WEIGHTINGS`` + (see :func:`defect_weighting_masks`); the default is + ``DEFECT_WEIGHTING``. Notes ----- @@ -764,8 +777,8 @@ class Ngmix(object): IndexError If the length of the input file list is incorrect ValueError - If ``blend_handling`` is unknown, or ``"uberseg"`` is selected - without ``seg_cat_path``. + If ``blend_handling`` or ``defect_weighting`` is unknown, or + ``"uberseg"`` is selected without ``seg_cat_path``. """ @@ -786,6 +799,7 @@ def __init__( seg_cat_path=None, dilate_neighbour=1, metacal_psf="fitgauss", + defect_weighting=DEFECT_WEIGHTING, ): # Base count = catalogue + vignets, excluding the f_wcs headers (passed @@ -805,6 +819,12 @@ def __init__( + f" {BLEND_HANDLINGS}" ) + if defect_weighting not in DEFECT_WEIGHTINGS: + raise ValueError( + f"Unknown DEFECT_WEIGHTING '{defect_weighting}'; expected one" + + f" of {DEFECT_WEIGHTINGS}" + ) + # Fail fast at construction (not deep in the per-epoch loop) when # uberseg is requested without its segmentation input (shapepipe#776). if blend_handling == "uberseg" and seg_cat_path is None: @@ -855,6 +875,7 @@ def __init__( self._seg_cat_path = seg_cat_path self._dilate_neighbour = dilate_neighbour self._metacal_psf = metacal_psf + self._defect_weighting = defect_weighting self._w_log = w_log @@ -1337,6 +1358,7 @@ def process(self): object_number=obj_id, dilate_neighbour=self._dilate_neighbour, metacal_psf=self._metacal_psf, + defect_weighting=self._defect_weighting, ) except Exception as ee: self._w_log.info( @@ -1441,8 +1463,8 @@ def prepare_postage_stamps( An epoch is dropped when more than ``EPOCH_MASKED_FRACTION_CUT`` of its stamp lies in :func:`defect_mask`, the set :func:`prepare_ngmix_weights` zero-weights and fills, or when :func:`central_defect_vetoes` finds a - defect near the object. The quarter-turn weight copies of interpolated - pixels keep their light and are not counted. + defect near the object. The quarter-turn copies of interpolated pixels + that ``DEFECT_WEIGHTING`` zero-weights or interpolates are not counted. @sc [decision:shape_measurement.central_defect_veto,decision:shape_measurement.epoch_masked_fraction_cut,decision:shape_measurement.blend_handling] neighbour-markers-are-not-defects The tile VIGNET's -1e30 markers on other detections' footprints @@ -2053,31 +2075,93 @@ def central_defect_vetoes(defect, interpolated): bool ``True`` if the epoch should be dropped. """ - rows, cols = np.indices(defect.shape) - distance = np.hypot( - rows - (defect.shape[0] - 1) / 2, cols - (defect.shape[1] - 1) / 2 - ) radius = np.where( interpolated, EPOCH_INTERPOLATED_DEFECT_RADIUS, EPOCH_CENTRAL_DEFECT_RADIUS, ) - return bool(np.any(defect & (distance < radius))) + return bool(np.any(defect & (centre_distance(defect.shape) < radius))) + + +def centre_distance(shape): + """Distance of every stamp pixel from the stamp centre, in pixels.""" + rows, cols = np.indices(shape) + return np.hypot(rows - (shape[0] - 1) / 2, cols - (shape[1] - 1) / 2) + + +def defect_weighting_masks(interpolated, clean, defect_weighting): + """The pixels each DEFECT_WEIGHTING interpolates, re-weights and zeroes. + + @sc [decision:shape_measurement.defect_weighting,label:physics] one-place-decides-defect-weights + ``interpolated`` (:func:`interpolated_defects`) starts at weight 0, like + every defect. The options: + + * ``"fourfold_zero"``: interpolate it; the weight is also zeroed on its + three quarter-turn copies about the stamp centre, which keep their + light, so the likelihood hole has no spin-2 part. + * ``"des_y6"``: DES Y6's pizza-cutter symmetrization on a per-object + stamp. The mask is ORed with its quarter turn (``np.rot90``, k=1) + about the stamp centre, the union is interpolated, and every + interpolated pixel keeps its full weight. Copies that land on a pixel + whose light is already replaced (a noise-filled defect, a noisefill + neighbour) stay as they are. + * ``"hole"``: interpolate it and leave it at weight 0. + * ``"full"``: interpolate it at full weight. + + Parameters + ---------- + interpolated : numpy.ndarray of bool + The interpolable defect pixels of a square stamp. + clean : numpy.ndarray of bool + Pixels whose light the image keeps raw. + defect_weighting : str + One of ``DEFECT_WEIGHTINGS``. + + Returns + ------- + fill : numpy.ndarray of bool + Pixels to replace by the interpolant. + restore : numpy.ndarray of bool + Defect pixels that get their full inverse-variance weight back. + zero : numpy.ndarray of bool + Pixels whose weight is zeroed though their light stays. + + Raises + ------ + ValueError + If ``defect_weighting`` is unknown. + """ + none = np.zeros_like(interpolated) + if defect_weighting == "des_y6": + fill = interpolated | (np.rot90(interpolated) & clean) + return fill, interpolated, none + if defect_weighting == "fourfold_zero": + return interpolated, none, fourfold(interpolated) + if defect_weighting == "hole": + return interpolated, none, none + if defect_weighting == "full": + return interpolated, interpolated, none + raise ValueError( + f"Unknown DEFECT_WEIGHTING '{defect_weighting}'; expected one of" + + f" {DEFECT_WEIGHTINGS}" + ) def prepare_ngmix_weights( gal, weight, flag, rng, bkg_rms=None, blend_handling="noisefill", seg=None, object_number=None, - dilate_neighbour=0, neighbour=None, + dilate_neighbour=0, neighbour=None, defect_weighting=DEFECT_WEIGHTING, ): """Build one epoch's image, weight map and noise image for ngmix. - Every stamp pixel falls in one of three classes. A clean pixel keeps its + Every stamp pixel falls in one of four classes. A clean pixel keeps its light and its weight. A pixel whose light is replaced by noise keeps neither: a defect that is not interpolated, or under ``"noisefill"`` a - marked neighbour pixel. A pixel whose light stays at zero weight is a - hole in the likelihood only: an interpolated defect and its quarter-turn - copies, or under ``"uberseg"`` a neighbour-side pixel. + marked neighbour pixel. An interpolated pixel takes the interpolant, at + the weight ``defect_weighting`` gives it. A pixel whose light stays at + zero weight is a hole in the likelihood only: under ``"fourfold_zero"`` + the quarter-turn copies of the interpolated defects, under ``"uberseg"`` + a neighbour-side pixel. @sc [decision:shape_measurement.defect_fill,decision:shape_measurement.blend_handling] defects-filled-whatever-the-blend-handling Every pixel of :func:`defect_mask` gets weight 0 and is filled the same @@ -2099,13 +2183,26 @@ def prepare_ngmix_weights( sky that metacal shears with the target and which supports the interpolant. - @sc [decision:shape_measurement.weight_symmetrization,label:physics] symmetrized-weight-holes - A one-sided zero-weight hole pulls the Gaussian fit toward its side, so - the weight is also zeroed on the three quarter-turn copies, about the - stamp centre, of every interpolated pixel, and their light stays. - Noise-filled pixels and the uberseg neighbour side are not symmetrized. - Measurements: astra decision ``shape_measurement.weight_symmetrization``; - guarded by ``tests/science/test_defect_recovery.py``. + @sc [decision:shape_measurement.defect_weighting] defect-weighting-option + ``defect_weighting`` decides which pixels are interpolated and how they + and their quarter-turn copies are weighted + (:func:`defect_weighting_masks`). Noise-filled pixels and the uberseg + neighbour side are never symmetrized, and an interpolated pixel on the + uberseg neighbour side stays at weight 0. Measurements: astra decision + ``shape_measurement.defect_weighting``; guarded by + ``tests/science/test_defect_recovery.py``. + + @sc [decision:shape_measurement.defect_weighting,label:physics] noise-image-in-the-fixnoise-frame + Metacal's ``fixnoise`` turns the noise image a quarter turn + (``np.rot90``, k=1) before shearing it and turns the result back (k=3) + before adding it to the sheared image. The noise image is therefore + interpolated in that turned frame, by the operator that interpolates the + science image: in the frame where both are sheared, the interpolated + pixels and their correlated noise coincide. Interpolating it in the + detector frame instead puts its interpolated pixels a quarter turn away + from the science image's, and the correlated noise of an interpolated + column then survives the cancellation as an additive bias. + ``tests/module/test_ngmix_defect_fill.py`` pins ngmix's k=1/k=3. Parameters ---------- @@ -2140,6 +2237,8 @@ def prepare_ngmix_weights( neighbour : numpy.ndarray of bool, optional The epoch's neighbour mask (``Postage_stamp.neighbours``); read only under ``blend_handling="noisefill"``. ``None`` marks no pixel. + defect_weighting : str, optional + One of ``DEFECT_WEIGHTINGS``; the default is ``DEFECT_WEIGHTING``. Returns ------- @@ -2150,23 +2249,29 @@ def prepare_ngmix_weights( Inverse-variance weight map for ngmix. numpy.ndarray Noise image: an independent realisation over the whole stamp, for - metacal's ``fixnoise``, interpolated where the galaxy image is. + metacal's ``fixnoise``, interpolated in fixnoise's quarter-turned + frame where the galaxy image is interpolated. Raises ------ ValueError - If ``blend_handling`` is unknown, or ``"uberseg"`` lacks ``seg`` or - ``object_number``. + If ``blend_handling`` or ``defect_weighting`` is unknown, or + ``"uberseg"`` lacks ``seg`` or ``object_number``. RuntimeError If the interpolant does not reach a pixel :func:`interpolated_defects` selected (degenerate support, or non-finite image values in it). - @sc [decision:masking.pixel_mask_source,decision:shape_measurement.blend_handling,decision:shape_measurement.defect_fill,decision:shape_measurement.galaxy_pixel_weights,decision:shape_measurement.weight_symmetrization] + @sc [decision:masking.pixel_mask_source,decision:shape_measurement.blend_handling,decision:shape_measurement.defect_fill,decision:shape_measurement.galaxy_pixel_weights,decision:shape_measurement.defect_weighting] """ if blend_handling not in BLEND_HANDLINGS: raise ValueError( f"Unknown blend_handling '{blend_handling}'; expected one of" + f" {BLEND_HANDLINGS}" ) + if defect_weighting not in DEFECT_WEIGHTINGS: + raise ValueError( + f"Unknown defect_weighting '{defect_weighting}'; expected one of" + + f" {DEFECT_WEIGHTINGS}" + ) if blend_handling == "uberseg" and (seg is None or object_number is None): raise ValueError( "blend_handling='uberseg' requires a segmentation map and the" @@ -2225,19 +2330,26 @@ def prepare_ngmix_weights( gal_filled = np.where(clean, gal, noise_img_gal).astype(gal.dtype) interpolated = interpolated_defects(defect, neighbour, blend_handling) if interpolated.any(): - # One operator for the image and the noise image, so fixnoise - # mirrors the science image's interpolated noise. + fill, restore, zero = defect_weighting_masks( + interpolated, clean, defect_weighting + ) + # One operator for the image and the noise image, the latter in + # fixnoise's quarter-turned frame (noise-image-in-the-fixnoise-frame). filled = interpolate_defects( - [gal, noise_img], ~clean, interpolated - )[:, interpolated] - if not np.all(np.isfinite(filled)): + [gal, np.rot90(noise_img)], ~clean | fill, fill + ) + if not np.all(np.isfinite(filled[:, fill])): raise RuntimeError( "The defect interpolant is not finite on pixels" - + " interpolated_defects selected." + + " defect_weighting_masks selected." ) - gal_filled[interpolated] = filled[0] - noise_img[interpolated] = filled[1] - weight_map[fourfold(interpolated)] = 0.0 + gal_filled[fill] = filled[0, fill] + noise_img = np.rot90(filled[1], -1) + restore = restore & ~neighbour_side + weight_map[restore] = np.broadcast_to( + 1.0 / np.asarray(sig_noise) ** 2, gal.shape + )[restore] + weight_map[zero] = 0.0 return gal_filled, weight_map, noise_img @@ -2246,7 +2358,7 @@ def make_ngmix_observation( gal, weight, flag, psf, wcs, rng, bkg_rms=None, centroid_source="wcs", offset=None, blend_handling="noisefill", seg=None, object_number=None, - dilate_neighbour=0, neighbour=None, + dilate_neighbour=0, neighbour=None, defect_weighting=DEFECT_WEIGHTING, ): """Build an ngmix Observation for a single galaxy epoch. @@ -2303,6 +2415,9 @@ def make_ngmix_observation( :func:`prepare_ngmix_weights` under ``blend_handling="uberseg"``. neighbour : numpy.ndarray of bool, optional Neighbour mask passed through to :func:`prepare_ngmix_weights`. + defect_weighting : str, optional + Interpolated-defect weighting passed through to + :func:`prepare_ngmix_weights`. Returns ------- @@ -2332,6 +2447,7 @@ def make_ngmix_observation( gal, weight, flag, rng, bkg_rms=bkg_rms, blend_handling=blend_handling, seg=seg, object_number=object_number, dilate_neighbour=dilate_neighbour, neighbour=neighbour, + defect_weighting=defect_weighting, ) if centroid_source == "hsm": @@ -2562,7 +2678,7 @@ def make_runners(prior, flux_guess, rng): def do_ngmix_metacal( stamp, prior, flux_guess, rng, centroid_source="wcs", blend_handling="noisefill", object_number=None, dilate_neighbour=0, - metacal_psf="fitgauss", + metacal_psf="fitgauss", defect_weighting=DEFECT_WEIGHTING, ): """Do Ngmix Metacal. @@ -2609,6 +2725,9 @@ def do_ngmix_metacal( round PSF that metacal reconvolves with after shearing, so it moves the metacal *response* (and therefore the recovered shear) but never reaches the deconvolution, which is by the PSF image. + defect_weighting : str, optional + Interpolated-defect weighting (one of ``DEFECT_WEIGHTINGS``) passed + through to :func:`make_ngmix_observation`. Returns ------- @@ -2619,7 +2738,7 @@ def do_ngmix_metacal( dict (:func:`average_original_psf`). The two PSF dicts share keys but describe different PSFs; the named fields guard against transposing them. Unpacks positionally as ``resdict, psf_res, psf_orig_res``. - @sc [decision:shape_measurement.defect_fill,decision:shape_measurement.metacal_scheme,decision:shape_measurement.weight_symmetrization] + @sc [decision:shape_measurement.defect_fill,decision:shape_measurement.metacal_scheme,decision:shape_measurement.defect_weighting] """ n_epoch = len(stamp.gals) if n_epoch == 0: @@ -2645,6 +2764,7 @@ def do_ngmix_metacal( neighbour=( stamp.neighbours[n_e] if n_e < len(stamp.neighbours) else None ), + defect_weighting=defect_weighting, ) gal_obs_list.append(gal_obs) diff --git a/src/shapepipe/modules/ngmix_runner.py b/src/shapepipe/modules/ngmix_runner.py index ebe4f670c..2b1ed8bde 100644 --- a/src/shapepipe/modules/ngmix_runner.py +++ b/src/shapepipe/modules/ngmix_runner.py @@ -12,6 +12,7 @@ from shapepipe.modules.module_decorator import module_runner from shapepipe.modules.ngmix_package.ngmix import ( + DEFECT_WEIGHTING, Ngmix, write_empty_tile_output, ) @@ -47,7 +48,7 @@ def ngmix_runner( ): """Define The Ngmix Runner. - @sc [decision:shape_measurement.blend_handling,decision:shape_measurement.centroid_source,decision:shape_measurement.defect_fill,decision:shape_measurement.metacal_scheme,decision:shape_measurement.galaxy_pixel_weights] + @sc [decision:shape_measurement.blend_handling,decision:shape_measurement.centroid_source,decision:shape_measurement.defect_fill,decision:shape_measurement.metacal_scheme,decision:shape_measurement.galaxy_pixel_weights,decision:shape_measurement.defect_weighting] @sc [label:operations] empty-tile-product A tile whose PSF or galaxy vignette store is entirely empty never @@ -146,6 +147,18 @@ def ngmix_runner( else: blend_handling = "noisefill" + # DEFECT_WEIGHTING (optional, environment-expanded): how interpolated + # defects are weighted, one of DEFECT_WEIGHTINGS (see + # defect_weighting_masks). Absent or empty takes DEFECT_WEIGHTING; the + # workflow sets it from the run config's `defect_weighting`. + defect_weighting = DEFECT_WEIGHTING + if config.has_option(module_config_sec, "DEFECT_WEIGHTING"): + defect_weighting = ( + config.getexpanded(module_config_sec, "DEFECT_WEIGHTING").strip() + or DEFECT_WEIGHTING + ) + w_log.info(f"DEFECT_WEIGHTING = {defect_weighting}") + # DILATE_NEIGHBOUR (optional): binary-dilation iterations enlarging the # uberseg neighbour mask, to absorb the few-pixel coadd-vs-epoch seg-overlay # offset. Ignored unless BLEND_HANDLING = uberseg. Default 1 (~one pixel). @@ -214,6 +227,7 @@ def ngmix_runner( seg_cat_path=seg_vignet_path, dilate_neighbour=dilate_neighbour, metacal_psf=metacal_psf, + defect_weighting=defect_weighting, ) # Process ngmix shape measurement and metacalibration diff --git a/tests/module/test_ngmix.py b/tests/module/test_ngmix.py index 89acb58ea..8c322eb41 100644 --- a/tests/module/test_ngmix.py +++ b/tests/module/test_ngmix.py @@ -663,6 +663,7 @@ def test_process_counts_flagged_fits_across_batches(tmp_path, monkeypatch, flags inst._blend_handling = "noisefill" inst._dilate_neighbour = 1 inst._metacal_psf = "fitgauss" + inst._defect_weighting = module.DEFECT_WEIGHTING inst._save_batch = 1 inst._zero_point = 30. inst._output_dir = str(tmp_path) @@ -760,6 +761,7 @@ def capture(stamp, prior, flux_guess, rng, **kwargs): inst._blend_handling = "noisefill" inst._dilate_neighbour = 1 inst._metacal_psf = "fitgauss" + inst._defect_weighting = module.DEFECT_WEIGHTING inst._save_batch = -1 inst._w_log = _RecordingLogger() diff --git a/tests/module/test_ngmix_defect_fill.py b/tests/module/test_ngmix_defect_fill.py index e1e564524..bd54101ac 100644 --- a/tests/module/test_ngmix_defect_fill.py +++ b/tests/module/test_ngmix_defect_fill.py @@ -35,6 +35,7 @@ interpolate_defects, ) from shapepipe.modules.ngmix_package.ngmix import ( + DEFECT_WEIGHTINGS, EPOCH_CENTRAL_DEFECT_RADIUS, EPOCH_INTERPOLATED_DEFECT_RADIUS, EPOCH_MASKED_FRACTION_CUT, @@ -508,18 +509,40 @@ def _hot_stamp(): return gal, np.ones((N_STAMP, N_STAMP)), flag +def _expected_masks(target, defect, defect_weighting): + """The fill and the zero-weight set each DEFECT_WEIGHTING promises, + written out independently of ``defect_weighting_masks``.""" + noisefilled = defect & ~target + if defect_weighting == "des_y6": + return target | (np.rot90(target) & ~defect), noisefilled + zero = { + "fourfold_zero": defect | fourfold(target), + "hole": defect, + "full": noisefilled, + }[defect_weighting] + return target, zero + + +@pytest.mark.parametrize("defect_weighting", DEFECT_WEIGHTINGS) @pytest.mark.parametrize("blend_handling", ["noisefill", "uberseg"]) -def test_interpolated_fill_and_its_weights(blend_handling): +def test_interpolated_fill_and_its_weights(blend_handling, defect_weighting): """Short defect runs take the interpolant of the clean image; other - defects take noise; the weight is zero on the defects and on the - quarter-turn orbit of the interpolated pixels, whose light stays; the - metacal noise image is interpolated with the same operator. - - Failure modes: the orbit is not zero-weighted (a one-sided hole in the - likelihood biases c), or its light is replaced; the fill mask is - symmetrized; wide defects or edge bands are extrapolated; raw defect - values leak; the noise image keeps independent noise where the science - image is smooth. + defects take noise. Under fourfold_zero the weight is zero on the + defects and on the quarter-turn orbit of the interpolated pixels, whose + light stays; under hole on the defects; under full and des_y6 only on + the noise-filled ones. des_y6 also interpolates the clean quarter-turn + (k=1) copies of the interpolated pixels, at full weight. The uberseg + neighbour side stays at weight 0 under every option. The metacal noise + image is interpolated by the same operator in fixnoise's quarter-turned + frame. + + Failure modes: the orbit is not zero-weighted under fourfold_zero (a + one-sided hole in the likelihood biases c), or its light is replaced; + the copies are not interpolated under des_y6, or a noise-filled pixel + is; an interpolated pixel on the neighbour side gets weight; wide + defects or edge bands are extrapolated; raw defect values leak; the + noise image is interpolated in the detector frame + (noise-image-in-the-fixnoise-frame). """ gal, weight, flag = _hot_stamp() defect = flag != 0 @@ -533,20 +556,68 @@ def test_interpolated_fill_and_its_weights(blend_handling): gal_out, w_out, noise_out = prepare_ngmix_weights( gal, weight, flag, np.random.RandomState(4), bkg_rms=np.ones((N_STAMP, N_STAMP)), blend_handling=blend_handling, - **kwargs, + defect_weighting=defect_weighting, **kwargs, ) neighbour = ( uberseg_mask(kwargs["seg"], 1, dilate_neighbour=1) if kwargs else np.zeros_like(defect) ) - npt.assert_array_equal(w_out == 0.0, defect | fourfold(target) | neighbour) - npt.assert_array_equal(gal_out[~defect], gal[~defect]) - expected = interpolate_defects(gal[None], defect, target)[0] - npt.assert_allclose(gal_out[target], expected[target], rtol=1e-5) + fill, zero = _expected_masks(target, defect, defect_weighting) + if defect_weighting == "des_y6": + assert (fill & ~defect).any() + npt.assert_array_equal(w_out == 0.0, zero | neighbour) + npt.assert_array_equal(w_out[~(zero | neighbour)], 1.0) + kept = ~defect & ~fill + npt.assert_array_equal(gal_out[kept], gal[kept]) + expected = interpolate_defects(gal[None], defect | fill, fill)[0] + npt.assert_allclose(gal_out[fill], expected[fill], rtol=1e-5) assert np.all(np.abs(gal_out[defect & ~target]) < 10.0) - refilled = interpolate_defects(noise_out[None], defect, target)[0] - npt.assert_allclose(noise_out[target], refilled[target], atol=1e-5) + turned = np.rot90(noise_out) + refilled = interpolate_defects(turned[None], defect | fill, fill)[0] + npt.assert_allclose(turned[fill], refilled[fill], atol=1e-5) + + +def test_fixnoise_turns_the_noise_image_k1_then_k3(): + """ngmix's fixnoise adds np.rot90(sheared(np.rot90(noise, 1)), 3) to + each sheared image: with an asymmetric noise image, the fixnoise output + minus the plain metacal output is exactly that. + + Failure mode: an ngmix release changes the turn (k=3 first, or none), + and the noise image prepare_ngmix_weights interpolates in the k=1 frame + no longer lines up with the science image's interpolated pixels + (noise-image-in-the-fixnoise-frame). + """ + from ngmix import DiagonalJacobian, Observation + from ngmix.metacal import get_all_metacal + + n = 33 + rows, cols = np.indices((n, n)) - (n - 1) / 2 + psf_image = np.exp(-(rows ** 2 + cols ** 2) / (2 * 2.0 ** 2)) + psf_image /= psf_image.sum() + galaxy = 100 * np.exp(-(rows ** 2 + cols ** 2) / (2 * 3.0 ** 2)) + jacobian = DiagonalJacobian(row=(n - 1) / 2, col=(n - 1) / 2, scale=0.2) + noise = np.random.RandomState(3).normal(size=(n, n)) + noise[:, 20] += 5.0 + + def metacal(image, noise_image=None): + obs = Observation( + image, weight=np.ones((n, n)), jacobian=jacobian, + psf=Observation(psf_image, jacobian=jacobian), noise=noise_image, + ) + return get_all_metacal( + obs, psf="gauss", types=["noshear", "1p"], + fixnoise=noise_image is not None, use_noise_image=True, + rng=np.random.RandomState(0), + ) + + fixed = metacal(galaxy, noise) + plain = metacal(galaxy) + turned = metacal(np.rot90(noise, 1).copy()) + for t in ("noshear", "1p"): + added = fixed[t].image - plain[t].image + npt.assert_allclose(added, np.rot90(turned[t].image, 3), atol=1e-9) + assert not np.allclose(added, np.rot90(turned[t].image, 1)) # --- The tile VIGNET's -1e30 neighbour markers are not defects ------------- diff --git a/tests/unit/test_workflow_defect_weighting.py b/tests/unit/test_workflow_defect_weighting.py new file mode 100644 index 000000000..e3580db7b --- /dev/null +++ b/tests/unit/test_workflow_defect_weighting.py @@ -0,0 +1,71 @@ +"""The run config's ``defect_weighting`` reaches ngmix's DEFECT_WEIGHTING. + +The workflow reads ``defect_weighting`` (Snakefile), tile_ngmix exports it as +``SP_DEFECT_WEIGHTING``, and config_tile_Ng_template.ini expands it into +``[NGMIX_RUNNER] DEFECT_WEIGHTING``; empty takes ngmix's default. These +files never see each other at run time, so their agreement is asserted here, +statically, plus the expansion through ShapePipe's own config parser. +""" + +import ast +import re +from pathlib import Path + +import pytest + +from shapepipe.modules.ngmix_package.ngmix import ( + DEFECT_WEIGHTING, + DEFECT_WEIGHTINGS, +) +from shapepipe.pipeline.config import CustomParser + +REPO_ROOT = Path(__file__).resolve().parents[2] +WORKFLOW = REPO_ROOT / "workflow" +TEMPLATE = WORKFLOW / "config" / "cfis" / "config_tile_Ng_template.ini" + + +def test_the_default_is_an_option(): + assert DEFECT_WEIGHTING in DEFECT_WEIGHTINGS + assert set(DEFECT_WEIGHTINGS) == {"des_y6", "fourfold_zero", "hole", + "full"} + + +def test_the_snakefile_reads_the_options_from_the_module_it_runs(): + """The Snakefile validates ``defect_weighting`` against the tuple it + parses out of ngmix.py; parse it the same way here.""" + snakefile = (WORKFLOW / "Snakefile").read_text() + assert 'config.get("defect_weighting")' in snakefile + assert "DEFECT_WEIGHTINGS" in snakefile + tree = ast.parse( + (REPO_ROOT / "src" / "shapepipe" / "modules" / "ngmix_package" + / "ngmix.py").read_text() + ) + parsed = next( + ast.literal_eval(node.value) for node in tree.body + if isinstance(node, ast.Assign) + and [getattr(t, "id", None) for t in node.targets] + == ["DEFECT_WEIGHTINGS"] + ) + assert parsed == DEFECT_WEIGHTINGS + + +def test_tile_ngmix_exports_it_and_tile_vignets_carries_it(): + """tile_ngmix exports SP_DEFECT_WEIGHTING; tile_vignets carries the + value as a param, so a change reruns the whole group, never the chunks + alone.""" + rules = (WORKFLOW / "rules" / "tile.smk").read_text() + ngmix = rules[rules.index("rule tile_ngmix:"):] + vignets = rules[rules.index("rule tile_vignets:"): + rules.index("rule tile_ngmix:")] + assert re.search(r'"SP_DEFECT_WEIGHTING":\s*DEFECT_WEIGHTING', ngmix) + assert re.search(r"defect_weighting\s*=\s*DEFECT_WEIGHTING", vignets) + + +@pytest.mark.parametrize("exported", ["", "des_y6"]) +def test_the_template_expands_the_export(monkeypatch, exported): + parser = CustomParser() + parser.read(TEMPLATE) + monkeypatch.setenv("SP_DEFECT_WEIGHTING", exported) + assert parser.getexpanded("NGMIX_RUNNER", "DEFECT_WEIGHTING") == exported + monkeypatch.delenv("SP_DEFECT_WEIGHTING") + assert parser.getexpanded("NGMIX_RUNNER", "DEFECT_WEIGHTING") == "" diff --git a/tests/workflow/params_pin.json b/tests/workflow/params_pin.json index 6de487277..70f9eaade 100644 --- a/tests/workflow/params_pin.json +++ b/tests/workflow/params_pin.json @@ -19,12 +19,12 @@ "tile_make_cat": "a57518b04c11f70bf41b320532fddffdd1115fdac604d7dd3928eb61cca28f23", "tile_merge_cats": "ff21216ea804dccc2d2c290d2b2499d5d05f0c34c0a56993c233f43fe3c06bdb", "tile_merge_headers": "7a344849d62936e2f5598dc8731a2c4947eff2c4c7218b1a731dd2a9577e7111", - "tile_ngmix": "6ed10a3a4d3658ba0303fb0c23ad5100ec8cbdc638ca36c3b25875087b5c3415", + "tile_ngmix": "488c26f957cbd5e9200d72660f9fa68b6961208bf1b5ce712788cf5f0164b30b", "tile_uncompress": "1e2b01acbf9708e0371070fb01c5b9568d7efb5f89d1835fc6bf91e2c8b60cb3", "tile_vignets": "9d4ae0d99c18217f2185f245281312454c8a219ec1628176e08c71a5efc4dc91" }, "schema": 1, - "sha256": "d67ee5da728e8fee513e1f01a522b386eb6a86d21c9a36250afd6013a06301d5", + "sha256": "4ce19f7da13c4820179b9278097f3ed57e4d2357d5c118290edb7eb774a0fc9f", "unit_pre": { "exp_get_images": "8dec850af212879f225fcf27a5f1281e1a075264c7b97d38c2214395d360168c", "exp_psf": "f2358ddf7385918dc5033d10b37f6dc97a15d02b071a3ea0a4619a5f7e6f5bec", diff --git a/universes/committed.yaml b/universes/committed.yaml index 44002be0b..ffee1686b 100644 --- a/universes/committed.yaml +++ b/universes/committed.yaml @@ -54,7 +54,7 @@ analyses: megacam_ccd_flip: megapipe_flip defect_fill: interpolate blend_handling: noisefill - weight_symmetrization: interpolated + defect_weighting: fourfold_zero epoch_masked_fraction_cut: ten_percent central_defect_veto: fixed_radii catalogue_assembly: diff --git a/workflow/Snakefile b/workflow/Snakefile index ac9d0760c..188c8c5fe 100644 --- a/workflow/Snakefile +++ b/workflow/Snakefile @@ -31,6 +31,7 @@ the manifest says "this stage succeeded", the log says "here is what happened" (the contract is argued in completeness.py's docstring). """ +import ast import fnmatch import functools import hashlib @@ -231,6 +232,24 @@ scattergather: NGMIX_CHUNKS = workflow._scatter["ngmix"] +# How ngmix weights interpolated defects (DEFECT_WEIGHTING in +# config_tile_Ng_template.ini, exported by tile_ngmix as SP_DEFECT_WEIGHTING). +# Unset: empty, and ngmix takes its own default. The allowed values are read +# from the ngmix module this campaign runs, so a typo fails here, at parse time, +# rather than in every tile's ngmix job. +DEFECT_WEIGHTING = config.get("defect_weighting") or "" +_NGMIX_PY = (Path(workflow.basedir).parent / "src" / "shapepipe" / "modules" + / "ngmix_package" / "ngmix.py") +_ngmix_tree = ast.parse(_NGMIX_PY.read_text()) +DEFECT_WEIGHTINGS = next( + ast.literal_eval(node.value) for node in _ngmix_tree.body + if isinstance(node, ast.Assign) + and [getattr(t, "id", None) for t in node.targets] == ["DEFECT_WEIGHTINGS"]) +if DEFECT_WEIGHTING and DEFECT_WEIGHTING not in DEFECT_WEIGHTINGS: + raise WorkflowError( + f"Invalid defect_weighting={DEFECT_WEIGHTING!r}; expected one of " + f"{list(DEFECT_WEIGHTINGS)}.") + # --- parse-time index build + load (D1) ------------------------------------ # The COMPUTE invocation's parse IS the index build, and it runs UNCONDITIONALLY # there — no "some Fe output exists" guard. That guard used to make a diff --git a/workflow/config.yaml b/workflow/config.yaml index f144a8a0e..46d0eb214 100644 --- a/workflow/config.yaml +++ b/workflow/config.yaml @@ -211,6 +211,10 @@ max_mem_mb: 750000 # ngmix chunks per tile. ngmix_chunks: 8 +# How ngmix weights interpolated defects: des_y6, fourfold_zero, hole or full +# (astra decision shape_measurement.defect_weighting). Unset: ngmix's default. +# defect_weighting: des_y6 + # Node-local tile store (vignets, WCS). Unset: the profile's /local/scratch # bind (nibi NVMe, candide node /tmp). Dense image-sim campaigns on candide # want a shared directory instead. diff --git a/workflow/config/cfis/config_tile_Ng_template.ini b/workflow/config/cfis/config_tile_Ng_template.ini index 5d4c510e5..0ba92e7cc 100644 --- a/workflow/config/cfis/config_tile_Ng_template.ini +++ b/workflow/config/cfis/config_tile_Ng_template.ini @@ -113,6 +113,13 @@ BKG_RMS_VIGNET_PATH = $NGMIX_VIGNET_DIR/vignetmaker_runner_run_2/output/backgrou # width, this bounds per-worker memory). SAVE_BATCH = 250 +# DEFECT_WEIGHTING: how interpolated defects are weighted (des_y6, +# fourfold_zero, hole or full; see ngmix.defect_weighting_masks). The workflow +# exports SP_DEFECT_WEIGHTING from the run config's `defect_weighting`; empty +# takes ngmix's default. +# @sc [decision:shape_measurement.defect_weighting] +DEFECT_WEIGHTING = ${SP_DEFECT_WEIGHTING:-} + # Magnitude zero-point # @sc [decision:photometric_zeropoint] MAG_ZP = 30.0 diff --git a/workflow/rules/tile.smk b/workflow/rules/tile.smk index 107bce4f9..bc747979b 100644 --- a/workflow/rules/tile.smk +++ b/workflow/rules/tile.smk @@ -591,7 +591,12 @@ rule tile_vignets: # it carries the splitter's fingerprint too. Same hash, same constant — # the Snakefile's NGMIX_RANGE_HASH argues what it guards, and # tile_ngmix's copy below carries the mid-campaign-edit warning. - range_hash = NGMIX_RANGE_HASH + range_hash = NGMIX_RANGE_HASH, + # tile_ngmix exports it, so it is a params change there; carried here + # too so that changing it reruns the whole group from tile_vignets, + # never the chunks alone (see the mid-campaign deletion note on + # tile_ngmix's range_hash). + defect_weighting = DEFECT_WEIGHTING # 8, not 16, for the same reason tile_ngmix is 1: `-b {threads}` is SMP # batch size over input FILE SETS, and a tile is one set -- this run's own # log says "Batch size: 16 / Total number of processes: 1". 16 was the @@ -644,7 +649,8 @@ rule tile_ngmix: f"{TILE_DIR}/logs/tile_ngmix_{{chunk}}.json" params: pre = lambda wc: unit_pre("tile_ngmix", wc.tile, - env={"SP_NGMIX_CHUNK": wc.chunk, "NGMIX_N_CHUNKS": NGMIX_CHUNKS}, + env={"SP_NGMIX_CHUNK": wc.chunk, "NGMIX_N_CHUNKS": NGMIX_CHUNKS, + "SP_DEFECT_WEIGHTING": DEFECT_WEIGHTING}, # Two steps, not `eval "$(...)"`: a command substitution inside eval # discards the script's exit status, so a missing sexcat would fall # through to shapepipe_run with an unset range and fail as something From cce34cee4fa8772047efa13d1da0e13559c438bd Mon Sep 17 00:00:00 2001 From: Cail Daley Date: Sat, 3 Oct 2026 18:41:55 +0200 Subject: [PATCH 27/30] ngmix: per-object defect diagnostics in the shape catalogue Over the epochs handed to the fit, each object now carries n_epoch_interp (epochs with an interpolated defect pixel), min_dist_interp and min_dist_noisefill (smallest distance in pixels from the stamp centre to an interpolated / noise-filled defect pixel; -1 when there is none). They read only masks, so distance-split null tests are catalogue queries. make_cat writes them as NGMIX_N_EPOCH_INTERP, NGMIX_MIN_DIST_INTERP and NGMIX_MIN_DIST_NOISEFILL (never-fit rows: 0, -1, -1), and both final_cat.param files list them. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01N4HLdzDyc3CiPLrgJ48GVo --- .../modules/make_cat_package/make_cat.py | 24 ++++++- src/shapepipe/modules/ngmix_package/ngmix.py | 67 ++++++++++++++++++- tests/module/test_make_cat.py | 21 +++++- tests/module/test_ngmix.py | 6 ++ tests/module/test_ngmix_defect_fill.py | 32 +++++++++ tests/module/test_psf_averaging_properties.py | 10 ++- tests/module/test_psf_grammar_properties.py | 23 ++++--- workflow/config/cfis/final_cat.param | 7 ++ .../config/cfis_image_sims/final_cat.param | 7 ++ 9 files changed, 177 insertions(+), 20 deletions(-) diff --git a/src/shapepipe/modules/make_cat_package/make_cat.py b/src/shapepipe/modules/make_cat_package/make_cat.py index 8a6f9548a..b4e7721ee 100644 --- a/src/shapepipe/modules/make_cat_package/make_cat.py +++ b/src/shapepipe/modules/make_cat_package/make_cat.py @@ -334,11 +334,14 @@ def _save_ngmix_data(self, ngmix_cat_path, moments=False): Save the NGMIX catalogue into the final one. Column grammar: ``NGMIX[m]_[_ERR][_]_``, - plus four OBJECT/SHEAR-less per-object metadata columns + plus OBJECT/SHEAR-less per-object metadata columns (``NGMIX[m]_MCAL_FLAGS``, ``NGMIX_N_EPOCH``, - ``NGMIX_MCAL_TYPES_FAIL``, ``NGMIX_NEIGHBOUR_FLAG`` — the last a blend + ``NGMIX_MCAL_TYPES_FAIL``, ``NGMIX_NEIGHBOUR_FLAG`` — a blend flag set when the coadd seg stamp held a non-central footprint, - shapepipe#776). The galaxy is the implicit default object + shapepipe#776 — and the defect diagnostics ``NGMIX_N_EPOCH_INTERP``, + ``NGMIX_MIN_DIST_INTERP``, ``NGMIX_MIN_DIST_NOISEFILL``, described + by ``Postage_stamp.record_defects``; -1 is their absent distance, + and a never-fit row carries 0 and -1). The galaxy is the implicit default object and carries NO ``OBJECT`` token (``NGMIX_G1_NOSHEAR``, dropping the ``GAL`` segment carried by the pre-#761 names). The explicit PSF objects are ``PSF_ORIG`` @@ -391,6 +394,16 @@ def _save_ngmix_data(self, ngmix_cat_path, moments=False): # Per-object blend flag (shapepipe#776): the seg stamp held a # non-central footprint. Galaxy-only (non-moments), like N_EPOCH. ngmix_neighbour_flag = ngmix_cat_file.get_data()["neighbour_flag"] + # Defect diagnostics (ngmix Postage_stamp.record_defects). + defect_columns = { + "NGMIX_N_EPOCH_INTERP": "n_epoch_interp", + "NGMIX_MIN_DIST_INTERP": "min_dist_interp", + "NGMIX_MIN_DIST_NOISEFILL": "min_dist_noisefill", + } + ngmix_defects = { + out: ngmix_cat_file.get_data()[col] + for out, col in defect_columns.items() + } # Needed in both moments and non-moments modes (used unconditionally # below), so read them outside the branch. ngmix_mcal_flags = ngmix_cat_file.get_data()["mcal_flags"] @@ -413,6 +426,9 @@ def _save_ngmix_data(self, ngmix_cat_path, moments=False): np.full(n_obj, get_mcal_types_fail(never_fit), dtype=float), ) self._add2dict("NGMIX_NEIGHBOUR_FLAG", np.zeros(n_obj)) + self._add2dict("NGMIX_N_EPOCH_INTERP", np.zeros(n_obj)) + self._add2dict("NGMIX_MIN_DIST_INTERP", np.full(n_obj, -1.0)) + self._add2dict("NGMIX_MIN_DIST_NOISEFILL", np.full(n_obj, -1.0)) prefix = f"NGMIX{m}" @@ -576,6 +592,8 @@ def _save_ngmix_data(self, ngmix_cat_path, moments=False): ngmix_neighbour_flag[ind[0]], idx, ) + for out, values in ngmix_defects.items(): + self._add2dict(out, values[ind[0]], idx) ngmix_cat_file.close() diff --git a/src/shapepipe/modules/ngmix_package/ngmix.py b/src/shapepipe/modules/ngmix_package/ngmix.py index f7580782e..8fb33db27 100644 --- a/src/shapepipe/modules/ngmix_package/ngmix.py +++ b/src/shapepipe/modules/ngmix_package/ngmix.py @@ -259,6 +259,10 @@ def empty_metacal_output(): 'n_epoch_model', 'mcal_types_fail', 'neighbour_flag', + # defect diagnostics (Postage_stamp.record_defects) + 'n_epoch_interp', + 'min_dist_interp', + 'min_dist_noisefill', 'nfev_fit', # galaxy 'g1', @@ -661,9 +665,59 @@ def __init__( self.epoch_cuts = Counter( considered=0, masked_fraction=0, central_veto=0 ) + # Defect diagnostics over the epochs kept (see + # :meth:`record_defects`). + self.n_epoch_interp = 0 + self.min_dist_interp = np.inf + self.min_dist_noisefill = np.inf self.bkg_sub = bkg_sub self.megacam_flip = megacam_flip + def record_defects(self, defect, interpolated): + """Add one kept epoch to the defect diagnostics. + + @sc [label:schema] defect-diagnostic-columns + Per object, over the epochs handed to the fit: ``n_epoch_interp`` + counts those with an interpolated defect pixel, and + ``min_dist_interp`` (``min_dist_noisefill``) is the smallest distance + in pixels from the stamp centre to an interpolated (noise-filled) + defect pixel (:func:`centre_distance`). A distance is -1 when no + kept epoch has such a pixel. They read only masks, so splitting a + catalogue on them selects on nothing shear-responsive. + + Parameters + ---------- + defect : numpy.ndarray of bool + The epoch's defect mask (:func:`defect_mask`). + interpolated : numpy.ndarray of bool + Its interpolated defect pixels (:func:`interpolated_defects`). + """ + distance = centre_distance(defect.shape) + noisefill = defect & ~interpolated + if interpolated.any(): + self.n_epoch_interp += 1 + self.min_dist_interp = min( + self.min_dist_interp, distance[interpolated].min() + ) + if noisefill.any(): + self.min_dist_noisefill = min( + self.min_dist_noisefill, distance[noisefill].min() + ) + + def defect_diagnostics(self): + """The defect diagnostic columns, -1 for an absent distance.""" + return { + "n_epoch_interp": self.n_epoch_interp, + "min_dist_interp": ( + float(self.min_dist_interp) + if np.isfinite(self.min_dist_interp) else -1.0 + ), + "min_dist_noisefill": ( + float(self.min_dist_noisefill) + if np.isfinite(self.min_dist_noisefill) else -1.0 + ), + } + class Vignet(): """Vignet. @@ -985,6 +1039,12 @@ def compile_results(self, results): output_dict[name]["neighbour_flag"].append( results[idx]["neighbour_flag"] ) + # Defect diagnostics (Postage_stamp.record_defects), likewise + # object-level and replicated. + for key in ( + "n_epoch_interp", "min_dist_interp", "min_dist_noisefill" + ): + output_dict[name][key].append(results[idx][key]) # ngmix 2.x reports the solver's function-evaluation count # (nfev, ~tens-hundreds; -1 on some failures), not the v1 # 1-5 retry count, so the column is named accordingly. Fits @@ -1379,6 +1439,7 @@ def process(self): # epochs that survived the PSF fit and entered the model, # not the number of epochs submitted (v1 contract) res['n_epoch_model'] = psf_res['n_epoch'] + res.update(stamp.defect_diagnostics()) # The mcal flag columns are derived from the per-type results in # compile_results; here they only feed the run-health count. if get_mcal_flags(res) != 0: @@ -1583,11 +1644,11 @@ def prepare_postage_stamps( if defect.mean() > EPOCH_MASKED_FRACTION_CUT: stamp.epoch_cuts["masked_fraction"] += 1 continue - if central_defect_vetoes( - defect, interpolated_defects(defect, neighbour, blend_handling) - ): + interpolated = interpolated_defects(defect, neighbour, blend_handling) + if central_defect_vetoes(defect, interpolated): stamp.epoch_cuts["central_veto"] += 1 continue + stamp.record_defects(defect, interpolated) # One unpickle per exposure (all CCDs), reused across this object's # epochs; the cache is per-call, so bounded by the object's exposures diff --git a/tests/module/test_make_cat.py b/tests/module/test_make_cat.py index e29109d19..a14de667a 100644 --- a/tests/module/test_make_cat.py +++ b/tests/module/test_make_cat.py @@ -58,6 +58,7 @@ def warning(self, message, *_args, **_kwargs): "n_epoch_model", "mcal_types_fail", "neighbour_flag", + "n_epoch_interp", "min_dist_interp", "min_dist_noisefill", "nfev_fit", "g1", "g1_err", "g2", "g2_err", "T", "T_err", @@ -87,6 +88,8 @@ def _ngmix_row(obj_id): "n_epoch_model": 3, "mcal_types_fail": 0, "neighbour_flag": 1, + "n_epoch_interp": 2, "min_dist_interp": 7.5, + "min_dist_noisefill": 14.25, "nfev_fit": 7, "g1": 0.10, "g1_err": 0.011, "g2": -0.20, "g2_err": 0.022, "T": 0.30, "T_err": 0.033, @@ -118,7 +121,8 @@ def _write_ngmix_cat(path, obj_ids): fits.Column( name=key, format="K" if key in ("id", "n_epoch_model", "mcal_types_fail", - "nfev_fit", "flags", "mcal_flags") else "D", + "n_epoch_interp", "nfev_fit", "flags", + "mcal_flags") else "D", array=np.array([row[key] for row in rows]), ) for key in NGMIX_KEYS @@ -176,7 +180,8 @@ def test_save_ngmix_data_uses_new_grammar_and_no_old_names(tmp_path): # Object-level metadata columns carry no OBJECT/SHEAR token. for col in ( "NGMIX_MCAL_FLAGS", "NGMIX_N_EPOCH", "NGMIX_MCAL_TYPES_FAIL", - "NGMIX_NEIGHBOUR_FLAG", + "NGMIX_NEIGHBOUR_FLAG", "NGMIX_N_EPOCH_INTERP", + "NGMIX_MIN_DIST_INTERP", "NGMIX_MIN_DIST_NOISEFILL", ): assert col in out, f"missing {col}" @@ -275,6 +280,15 @@ def test_save_ngmix_data_fills_sentinels_for_absent_objects(tmp_path): npt.assert_allclose(n_epoch[present], row["n_epoch_model"]) npt.assert_allclose(n_epoch[absent], [0.0, 0.0]) + for col, key, never_fit in ( + ("NGMIX_N_EPOCH_INTERP", "n_epoch_interp", 0.0), + ("NGMIX_MIN_DIST_INTERP", "min_dist_interp", -1.0), + ("NGMIX_MIN_DIST_NOISEFILL", "min_dist_noisefill", -1.0), + ): + values = np.asarray(out[col]) + npt.assert_allclose(values[present], row[key]) + npt.assert_allclose(values[absent], [never_fit, never_fit]) + def test_low_match_fraction_warns_and_continues_with_sentinels(tmp_path): """A low match count warns while unmatched detections stay in the output.""" @@ -314,6 +328,9 @@ def _metacal_result(obj_id): "obj_id": obj_id, "n_epoch_model": row["n_epoch_model"], "neighbour_flag": row["neighbour_flag"], + "n_epoch_interp": row["n_epoch_interp"], + "min_dist_interp": row["min_dist_interp"], + "min_dist_noisefill": row["min_dist_noisefill"], } for key in NGMIX_KEYS: if key.endswith("_psf_orig") or key.endswith("_psf_reconv"): diff --git a/tests/module/test_ngmix.py b/tests/module/test_ngmix.py index 8c322eb41..1435ec3d1 100644 --- a/tests/module/test_ngmix.py +++ b/tests/module/test_ngmix.py @@ -330,6 +330,9 @@ def _fake_metacal_result(T, T_err, T_psf, T_psf_err): "n_epoch_model": 1, "mcal_types_fail": 0, "neighbour_flag": 0, + "n_epoch_interp": 0, + "min_dist_interp": -1.0, + "min_dist_noisefill": -1.0, # original image PSF (psfex/mccd) family "g1_psf_orig": ORIG_PSF_G[0], "g2_psf_orig": ORIG_PSF_G[1], @@ -633,6 +636,9 @@ def test_process_counts_flagged_fits_across_batches(tmp_path, monkeypatch, flags gals=[np.ones((5, 5))], ra=[42.], dec=[30.], ccd=20, epoch_cuts=Counter(considered=1), jacobs=[galsim.JacobianWCS(.186, 0., 0., .186)], + defect_diagnostics=lambda: dict( + n_epoch_interp=0, min_dist_interp=-1.0, min_dist_noisefill=-1.0, + ), ) psf = dict( n_epoch=1, g_psf=[.01, -.01], g_psf_err=[.001, .001], diff --git a/tests/module/test_ngmix_defect_fill.py b/tests/module/test_ngmix_defect_fill.py index bd54101ac..96e4ae52a 100644 --- a/tests/module/test_ngmix_defect_fill.py +++ b/tests/module/test_ngmix_defect_fill.py @@ -620,6 +620,38 @@ def metacal(image, noise_image=None): assert not np.allclose(added, np.rot90(turned[t].image, 1)) +def test_defect_diagnostics_summarise_the_kept_epochs(): + """Over the epochs the veto keeps, ``n_epoch_interp`` counts those with + an interpolated pixel and the two distances are the nearest + interpolated and noise-filled defect pixels to the stamp centre; an + object without either has -1. + + Failure modes: a vetoed epoch is counted; noise-filled and interpolated + pixels are swapped; the sentinel is 0, a valid distance + (defect-diagnostic-columns). + """ + vignet, tile_cat, psf_obj, gal_obj = _fake_inputs(_veto_epochs()) + stamp = prepare_postage_stamps( + vignet, 1, 0, tile_cat, bkg_sub=False, + psf_obj=psf_obj, gal_obj=gal_obj, + ) + assert stamp.defect_diagnostics() == { + "n_epoch_interp": 2, + "min_dist_interp": float(EPOCH_INTERPOLATED_DEFECT_RADIUS), + "min_dist_noisefill": float(EPOCH_CENTRAL_DEFECT_RADIUS), + } + clean = {"2100001-10": _veto_epochs()["2100001-10"]} + vignet, tile_cat, psf_obj, gal_obj = _fake_inputs(clean) + stamp = prepare_postage_stamps( + vignet, 1, 0, tile_cat, bkg_sub=False, + psf_obj=psf_obj, gal_obj=gal_obj, + ) + assert stamp.defect_diagnostics() == { + "n_epoch_interp": 0, "min_dist_interp": -1.0, + "min_dist_noisefill": -1.0, + } + + # --- The tile VIGNET's -1e30 neighbour markers are not defects ------------- # # The tile VIGNET carries -1e30 on the footprints of other detections. Every diff --git a/tests/module/test_psf_averaging_properties.py b/tests/module/test_psf_averaging_properties.py index d22435719..e4b1f38b8 100644 --- a/tests/module/test_psf_averaging_properties.py +++ b/tests/module/test_psf_averaging_properties.py @@ -258,12 +258,16 @@ def test_all_epochs_failed_raises_zero_division(flagged_specs): "NGMIX_MCAL_FLAGS": LM_FUNC_NOTFINITE, "NGMIX_MCAL_TYPES_FAIL": len(METACAL_TYPES), "NGMIX_NEIGHBOUR_FLAG": 0.0, + "NGMIX_N_EPOCH_INTERP": 0.0, + "NGMIX_MIN_DIST_INTERP": -1.0, + "NGMIX_MIN_DIST_NOISEFILL": -1.0, } # Per-key write format and a measured value distinct from every sentinel, so a # matched row is unmistakably "overwritten" and an absent row unmistakably not. _NGMIX_KEYS = [ - "id", "n_epoch_model", "mcal_types_fail", "neighbour_flag", "nfev_fit", + "id", "n_epoch_model", "mcal_types_fail", "neighbour_flag", + "n_epoch_interp", "min_dist_interp", "min_dist_noisefill", "nfev_fit", "g1", "g1_err", "g2", "g2_err", "T", "T_err", "flux", "flux_err", "s2n", "mag", "mag_err", "flags", "mcal_flags", "g1_psf_orig", "g2_psf_orig", "g1_err_psf_orig", "g2_err_psf_orig", @@ -272,8 +276,8 @@ def test_all_epochs_failed_raises_zero_division(flagged_specs): "T_psf_reconv", "T_err_psf_reconv", ] _INT_KEYS = { - "id", "n_epoch_model", "mcal_types_fail", "neighbour_flag", "nfev_fit", - "flags", "mcal_flags" + "id", "n_epoch_model", "mcal_types_fail", "neighbour_flag", + "n_epoch_interp", "nfev_fit", "flags", "mcal_flags" } _SHEAR_EXTS = ["1M", "1P", "2M", "2P", "NOSHEAR"] diff --git a/tests/module/test_psf_grammar_properties.py b/tests/module/test_psf_grammar_properties.py index 461597557..43081165f 100644 --- a/tests/module/test_psf_grammar_properties.py +++ b/tests/module/test_psf_grammar_properties.py @@ -4,9 +4,10 @@ hypothesis to pin three invariants of the renamed grammar ``NGMIX[m]_[_ERR][_]_`` (shapepipe#749, #761): the galaxy is the implicit default object and carries NO ``OBJECT`` token -(``NGMIX_G1_NOSHEAR``, never ``NGMIX_G1_GAL_NOSHEAR``), plus the four +(``NGMIX_G1_NOSHEAR``, never ``NGMIX_G1_GAL_NOSHEAR``), plus the OBJECT/SHEAR-less metadata columns (``NGMIX[m]_MCAL_FLAGS``, ``NGMIX_N_EPOCH``, -``NGMIX_MCAL_TYPES_FAIL``, ``NGMIX_NEIGHBOUR_FLAG``), where the ORIGINAL image +``NGMIX_MCAL_TYPES_FAIL``, ``NGMIX_NEIGHBOUR_FLAG`` and the defect +diagnostics), where the ORIGINAL image PSF (``PSF_ORIG``) and the metacal RECONVOLUTION kernel (``PSF_RECONV``) are independent fits of *different* PSFs: @@ -63,11 +64,12 @@ def info(self, *_args, **_kwargs): # shear-type extension. Integer-typed keys carry FITS format "K", the rest # "D"; everything else mirrors test_make_cat._ngmix_row. INT_KEYS = { - "id", "n_epoch_model", "mcal_types_fail", "neighbour_flag", "nfev_fit", - "flags", "mcal_flags", + "id", "n_epoch_model", "mcal_types_fail", "neighbour_flag", + "n_epoch_interp", "nfev_fit", "flags", "mcal_flags", } NGMIX_KEYS = [ - "id", "n_epoch_model", "mcal_types_fail", "neighbour_flag", "nfev_fit", + "id", "n_epoch_model", "mcal_types_fail", "neighbour_flag", + "n_epoch_interp", "min_dist_interp", "min_dist_noisefill", "nfev_fit", "g1", "g1_err", "g2", "g2_err", "T", "T_err", "flux", "flux_err", "s2n", "mag", "mag_err", @@ -109,7 +111,8 @@ def _base_row(obj_id): return { "id": obj_id, "n_epoch_model": 3, "mcal_types_fail": 0, "neighbour_flag": 1, - "nfev_fit": 7, + "n_epoch_interp": 2, "min_dist_interp": 7.5, + "min_dist_noisefill": 14.25, "nfev_fit": 7, "g1": 0.10, "g1_err": 0.011, "g2": -0.20, "g2_err": 0.022, "T": 0.30, "T_err": 0.033, "flux": 100.0, "flux_err": 1.0, "s2n": 55.0, @@ -178,7 +181,8 @@ def _run_save_ngmix(ngmix_path, obj_ids, cat_size_target=None): _SHEAR = "NOSHEAR|1P|1M|2P|2M" GRAMMAR_RE = re.compile( rf"^NGMIXm?_(?:{_COMPONENT})(?:_ERR)?(?:_(?:{_OBJECT}))?_(?:{_SHEAR})$" - rf"|^NGMIXm?_(?:MCAL_FLAGS|MCAL_TYPES_FAIL|N_EPOCH|NEIGHBOUR_FLAG)$" + rf"|^NGMIXm?_(?:MCAL_FLAGS|MCAL_TYPES_FAIL|N_EPOCH|NEIGHBOUR_FLAG" + rf"|N_EPOCH_INTERP|MIN_DIST_INTERP|MIN_DIST_NOISEFILL)$" ) # The shipped final-catalogue param files, two levels up from tests/module/. @@ -328,7 +332,7 @@ def test_emitted_column_names_match_grammar(obj_ids, tmp_path_factory): """Every column _save_ngmix_data emits matches the grammar regex. The grammar is ``NGMIX[m]_[_ERR]__`` plus the - four metadata columns. The regex is anchored, so a legacy token (``_PSFo``, + metadata columns. The regex is anchored, so a legacy token (``_PSFo``, ``_Tpsf``, ``NGMIX_ELL_*``) or a malformed name would fail to match. Run over hypothesis-varied object sets to confirm the emitted name set is input-independent and always well-formed. @@ -417,7 +421,8 @@ def test_param_file_ngmix_tokens_are_producible(param_path, obj_ids): # ngmix: galaxy (no object token) or explicit PSF_ORIG/PSF_RECONV. r"NGMIXm?_(?:G1|G2|T|SNR|FLUX|MAG|FLAGS)(?:_ERR)?" r"(?:_(?:PSF_ORIG|PSF_RECONV))?_(?:NOSHEAR|1P|1M|2P|2M)" - r"|NGMIXm?_(?:MCAL_FLAGS|MCAL_TYPES_FAIL|N_EPOCH|NEIGHBOUR_FLAG)" + r"|NGMIXm?_(?:MCAL_FLAGS|MCAL_TYPES_FAIL|N_EPOCH|NEIGHBOUR_FLAG" + r"|N_EPOCH_INTERP|MIN_DIST_INTERP|MIN_DIST_NOISEFILL)" # HSM: g-type, explicit PSF/STAR object, singular FLAG; the multi-epoch # sink in make_cat._save_psf_data appends a bare epoch index. r"|HSM_(?:G1|G2|T|M4_1|M4_2|RHO4)_(?:PSF|STAR)(?:_\d+)?" diff --git a/workflow/config/cfis/final_cat.param b/workflow/config/cfis/final_cat.param index fb8b3f3f7..850d7c0ac 100644 --- a/workflow/config/cfis/final_cat.param +++ b/workflow/config/cfis/final_cat.param @@ -165,6 +165,13 @@ HSM_RHO4_PSF_12 # Blend flag: coadd seg stamp held a non-central footprint (shapepipe#776) NGMIX_NEIGHBOUR_FLAG +# Defect diagnostics over the kept epochs: epochs with interpolated pixels, +# min distance (px) from the centre to an interpolated / noise-filled defect +# pixel, -1 when none (ngmix Postage_stamp.record_defects) +NGMIX_N_EPOCH_INTERP +NGMIX_MIN_DIST_INTERP +NGMIX_MIN_DIST_NOISEFILL + ## Shape measurement outputs ## Ngmix: model fitting diff --git a/workflow/config/cfis_image_sims/final_cat.param b/workflow/config/cfis_image_sims/final_cat.param index 9db095b66..08ce89967 100644 --- a/workflow/config/cfis_image_sims/final_cat.param +++ b/workflow/config/cfis_image_sims/final_cat.param @@ -31,6 +31,13 @@ NGMIX_MCAL_FLAGS NGMIX_MCAL_TYPES_FAIL NGMIX_NEIGHBOUR_FLAG +# Defect diagnostics over the kept epochs: epochs with interpolated pixels, +# min distance (px) from the centre to an interpolated / noise-filled defect +# pixel, -1 when none (ngmix Postage_stamp.record_defects) +NGMIX_N_EPOCH_INTERP +NGMIX_MIN_DIST_INTERP +NGMIX_MIN_DIST_NOISEFILL + # PSF ellipticity (original image PSF) NGMIX_G1_PSF_ORIG_NOSHEAR NGMIX_G2_PSF_ORIG_NOSHEAR From 6ae3f9db101e2663271efd357a5f110a9865d74f Mon Sep 17 00:00:00 2001 From: Cail Daley Date: Mon, 5 Oct 2026 14:33:32 +0200 Subject: [PATCH 28/30] Plain-language defect_fill rationale; name DEFECT_WEIGHTING's default The defect_fill rationale in astra.yaml now says what a defect is, why its image value matters under metacal, what the fill does, and why. The ngmix template and config.yaml name the default defect weighting, fourfold_zero. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_012yfkBvkzPv33iJxXZ6jAxd --- astra.yaml | 46 +++++++++---------- tests/unit/test_workflow_defect_weighting.py | 2 +- workflow/config.yaml | 2 +- .../config/cfis/config_tile_Ng_template.ini | 3 +- 4 files changed, 25 insertions(+), 28 deletions(-) diff --git a/astra.yaml b/astra.yaml index 65bcf8274..2f965d0d3 100644 --- a/astra.yaml +++ b/astra.yaml @@ -1450,31 +1450,27 @@ analyses: defect_fill: label: Image content of defect pixels before metacal rationale: >- - A defect pixel (nonzero instrument flag, zero exposure weight, - invalid background RMS, or off-tile) gets weight 0 in - prepare_ngmix_weights. Its image value still matters: ngmix's - metacal deconvolves, shears and reconvolves an InterpolatedImage of - the whole image and copies the weights through, so a zero-weight - pixel's content spreads into the weighted pixels within about a PSF - width. DES's ngmixer fills for that reason: "it may be important for - codes that take moments or use FFTs". [HARDCODED] Under every - BLEND_HANDLING, short defect runs are interpolated and the rest are - noise-filled (interpolate). The tile VIGNET holds -1e30 on other - detections' footprints and beyond the tile's edge (written by - SExtractor). The runs of entirely -1e30 stamp rows and columns that - start at a stamp border (the off-image part of a rectangle clip) are - off-tile: the epoch holds the object's own light there, so they are - defects, at zero exposure weight. The other markers, including a - neighbour footprint that completes an interior row beside an - off-tile band, are neighbour pixels, not defects; blend_handling - decides their treatment. The fill uses the unsymmetrized defect - set: DES symmetrized its masks, but four-fold symmetrization - quadruples m and still leaves an additive c1 - (symmetrized_4fold_noise). The cost of not symmetrizing, a hole in - the galaxy light, is bounded by central_defect_veto. Interpolation - is the only fill because it lets interpolated defects come to 7 px - of the object, against 13 px for noise fill, which a 0.7 arcsec - galaxy through a 0.9 arcsec PSF needs. + [HARDCODED] A defect is a pixel with no usable data: a nonzero + instrument flag, zero exposure weight, an invalid background RMS, + or off the tile. prepare_ngmix_weights gives it weight 0, and this + decision sets the image value it holds. That value matters because + metacal deconvolves, shears and reconvolves the whole stamp, which + spreads a zero-weight pixel's content into its weighted neighbours + within about a PSF width; DES fills defects for the same reason. + Short defect runs are interpolated from the surrounding good + pixels and the rest are filled with noise, under every + BLEND_HANDLING. Interpolation is used because an epoch can then + keep a defect as close as 7 px to the object, whereas a 0.7 arcsec + galaxy through a 0.9 arcsec PSF needs noise-filled defects at least + 13 px away. Only the defect pixels are filled, not a four-fold + symmetrized mask as in DES, since symmetrizing quadruples m and + leaves an additive c1 (symmetrized_4fold_noise); the hole this + leaves near the galaxy is bounded by central_defect_veto. In the + tile VIGNET, SExtractor writes -1e30 over other detections and + beyond the tile edge. Rows and columns that are entirely -1e30 and + start at the stamp border are off-tile, hence defects (the epoch + has the object's light there); every other -1e30 pixel is a + neighbour, handled by blend_handling. Values: VIGNETMAKER_RUNNER_RUN_1.MASKING = False; VIGNETMAKER_RUNNER_RUN_2.MASKING = False. diff --git a/tests/unit/test_workflow_defect_weighting.py b/tests/unit/test_workflow_defect_weighting.py index e3580db7b..c7312f4de 100644 --- a/tests/unit/test_workflow_defect_weighting.py +++ b/tests/unit/test_workflow_defect_weighting.py @@ -2,7 +2,7 @@ The workflow reads ``defect_weighting`` (Snakefile), tile_ngmix exports it as ``SP_DEFECT_WEIGHTING``, and config_tile_Ng_template.ini expands it into -``[NGMIX_RUNNER] DEFECT_WEIGHTING``; empty takes ngmix's default. These +``[NGMIX_RUNNER] DEFECT_WEIGHTING``; empty takes the module default. These files never see each other at run time, so their agreement is asserted here, statically, plus the expansion through ShapePipe's own config parser. """ diff --git a/workflow/config.yaml b/workflow/config.yaml index 46d0eb214..716d71449 100644 --- a/workflow/config.yaml +++ b/workflow/config.yaml @@ -212,7 +212,7 @@ max_mem_mb: 750000 ngmix_chunks: 8 # How ngmix weights interpolated defects: des_y6, fourfold_zero, hole or full -# (astra decision shape_measurement.defect_weighting). Unset: ngmix's default. +# (astra decision shape_measurement.defect_weighting). Unset: fourfold_zero. # defect_weighting: des_y6 # Node-local tile store (vignets, WCS). Unset: the profile's /local/scratch diff --git a/workflow/config/cfis/config_tile_Ng_template.ini b/workflow/config/cfis/config_tile_Ng_template.ini index 0ba92e7cc..42f968828 100644 --- a/workflow/config/cfis/config_tile_Ng_template.ini +++ b/workflow/config/cfis/config_tile_Ng_template.ini @@ -116,7 +116,8 @@ SAVE_BATCH = 250 # DEFECT_WEIGHTING: how interpolated defects are weighted (des_y6, # fourfold_zero, hole or full; see ngmix.defect_weighting_masks). The workflow # exports SP_DEFECT_WEIGHTING from the run config's `defect_weighting`; empty -# takes ngmix's default. +# takes the default, fourfold_zero (DEFECT_WEIGHTING in +# shapepipe/modules/ngmix_package/ngmix.py). # @sc [decision:shape_measurement.defect_weighting] DEFECT_WEIGHTING = ${SP_DEFECT_WEIGHTING:-} From 88ed8cde6baba40702c408248679f5d57e6cef3d Mon Sep 17 00:00:00 2001 From: Cail Daley Date: Mon, 5 Oct 2026 15:34:11 +0200 Subject: [PATCH 29/30] des_y6: interpolate a quarter-turn copy only where the interpolant reaches it A copy wedged in a corner of the clean region (an edge band beside the stamp edge) lies outside the support's hull, so the fill came back NaN and the finiteness guard dropped every epoch of the object. Such copies now keep their light, like copies on already-replaced pixels. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_012yfkBvkzPv33iJxXZ6jAxd --- src/shapepipe/modules/ngmix_package/ngmix.py | 11 +++++-- tests/module/test_ngmix_defect_fill.py | 31 +++++++++++++++++++- 2 files changed, 38 insertions(+), 4 deletions(-) diff --git a/src/shapepipe/modules/ngmix_package/ngmix.py b/src/shapepipe/modules/ngmix_package/ngmix.py index 8fb33db27..4031992cf 100644 --- a/src/shapepipe/modules/ngmix_package/ngmix.py +++ b/src/shapepipe/modules/ngmix_package/ngmix.py @@ -2165,7 +2165,12 @@ def defect_weighting_masks(interpolated, clean, defect_weighting): about the stamp centre, the union is interpolated, and every interpolated pixel keeps its full weight. Copies that land on a pixel whose light is already replaced (a noise-filled defect, a noisefill - neighbour) stay as they are. + neighbour) stay as they are, and so do copies the interpolant cannot + reach: a copy is interpolated only where it passes + :func:`interpolable_defects` among the pixels the image keeps, so a + copy wedged into a corner of the clean region (an edge band beside + the stamp edge) keeps its light rather than leaving the fill + non-finite. * ``"hole"``: interpolate it and leave it at weight 0. * ``"full"``: interpolate it at full weight. @@ -2194,8 +2199,8 @@ def defect_weighting_masks(interpolated, clean, defect_weighting): """ none = np.zeros_like(interpolated) if defect_weighting == "des_y6": - fill = interpolated | (np.rot90(interpolated) & clean) - return fill, interpolated, none + copy = interpolable_defects(np.rot90(interpolated) & clean, ~clean) + return interpolated | copy, interpolated, none if defect_weighting == "fourfold_zero": return interpolated, none, fourfold(interpolated) if defect_weighting == "hole": diff --git a/tests/module/test_ngmix_defect_fill.py b/tests/module/test_ngmix_defect_fill.py index 96e4ae52a..bf4f7f056 100644 --- a/tests/module/test_ngmix_defect_fill.py +++ b/tests/module/test_ngmix_defect_fill.py @@ -514,7 +514,8 @@ def _expected_masks(target, defect, defect_weighting): written out independently of ``defect_weighting_masks``.""" noisefilled = defect & ~target if defect_weighting == "des_y6": - return target | (np.rot90(target) & ~defect), noisefilled + copy = interpolable_defects(np.rot90(target) & ~defect, defect) + return target | copy, noisefilled zero = { "fourfold_zero": defect | fourfold(target), "hole": defect, @@ -578,6 +579,34 @@ def test_interpolated_fill_and_its_weights(blend_handling, defect_weighting): npt.assert_allclose(turned[fill], refilled[fill], atol=1e-5) +@pytest.mark.parametrize("band", [1, 2, 4]) +def test_des_y6_copy_in_a_corner_of_the_clean_region_keeps_its_light(band): + """An isolated defect whose quarter-turn copy lands where the top edge + band meets the stamp's right edge: under des_y6 the copy has no clean + pixel beyond the band or the stamp edge, so the interpolant cannot + reach it and it keeps its light; the defect itself is interpolated. + + Failure mode: the copy is interpolated anyway, the interpolant is NaN + there and prepare_ngmix_weights raises, dropping every epoch of the + object. + """ + n = N_STAMP + flag = np.zeros((n, n), dtype=np.int32) + flag[:band] = 1 + flag[n - 1, n - 1 - band] = 1 + gal = np.random.RandomState(1).normal(size=(n, n)) + gal_out, w_out, _ = prepare_ngmix_weights( + gal, np.ones((n, n)), flag, np.random.RandomState(4), + bkg_rms=np.ones((n, n)), defect_weighting="des_y6", + ) + corner = (band, n - 1) + assert np.rot90(flag != 0)[corner] and flag[corner] == 0 + assert gal_out[corner] == gal[corner] + assert w_out[corner] == 1.0 + assert np.all(np.isfinite(gal_out)) + assert gal_out[n - 1, n - 1 - band] != gal[n - 1, n - 1 - band] + + def test_fixnoise_turns_the_noise_image_k1_then_k3(): """ngmix's fixnoise adds np.rot90(sheared(np.rot90(noise, 1)), 3) to each sheared image: with an asymmetric noise image, the fixnoise output From 4c06ab535c8b3eb2c38c0da8e1f3b55ba0cfe8db Mon Sep 17 00:00:00 2001 From: Cail Daley Date: Mon, 5 Oct 2026 15:59:35 +0200 Subject: [PATCH 30/30] ngmix: a failed epoch drops that epoch, not the object An exception while building one epoch's ngmix observation (for example the non-finite interpolant guard in prepare_ngmix_weights) used to reach the per-object except, losing every epoch of the object. do_ngmix_metacal now catches it per epoch, drops that epoch and fits the object on the rest; the existing epoch logic (N_EPOCH, masked-fraction cut) decides the rest. An object whose every epoch fails still raises. The drop is kept: Postage_stamp.epoch_failures holds (epoch name, exception), process() logs each at warning level with tile, object and epoch, adds them to the "epoch cuts" tally as failed=, and writes the count as a new per-object column n_epoch_failed -> NGMIX_N_EPOCH_FAILED (final_cat.param for cfis and cfis_image_sims; never-fit rows 0). astra.yaml epoch_masked_fraction_cut records the rule; contract failed-epoch-dropped. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_012yfkBvkzPv33iJxXZ6jAxd --- astra.yaml | 5 +- .../modules/make_cat_package/make_cat.py | 7 +- src/shapepipe/modules/ngmix_package/ngmix.py | 94 +++++++++++---- tests/module/test_make_cat.py | 9 +- tests/module/test_ngmix.py | 112 +++++++++++++++++- tests/module/test_ngmix_defect_fill.py | 12 +- tests/module/test_psf_averaging_properties.py | 7 +- tests/module/test_psf_grammar_properties.py | 12 +- workflow/config/cfis/final_cat.param | 2 + .../config/cfis_image_sims/final_cat.param | 2 + 10 files changed, 225 insertions(+), 37 deletions(-) diff --git a/astra.yaml b/astra.yaml index 2f965d0d3..a3ee09139 100644 --- a/astra.yaml +++ b/astra.yaml @@ -1733,7 +1733,10 @@ analyses: would drop them all. An object with no surviving epoch has no shape. Before the cut, an epoch is dropped silently if its galaxy stamp is all zeros or its background-subtracted sigma_mad is not - positive. The cut is not a bias control; the central veto is + positive. After it, an epoch whose ngmix observation raises while + being built (for example a non-finite defect interpolant) is + dropped, logged with its tile, object and epoch, and counted in + NGMIX_N_EPOCH_FAILED; the object is fitted on its other epochs. The cut is not a bias control; the central veto is (central_defect_veto). Defects the veto keeps, covering up to 58% of the stamp (two-sided bands, corner Ls and frames 12 px or more from the centre), give |m11|, |m22| <= 0.05% and |c| < 7e-5 on 0.3 diff --git a/src/shapepipe/modules/make_cat_package/make_cat.py b/src/shapepipe/modules/make_cat_package/make_cat.py index b4e7721ee..388dab2fd 100644 --- a/src/shapepipe/modules/make_cat_package/make_cat.py +++ b/src/shapepipe/modules/make_cat_package/make_cat.py @@ -336,6 +336,8 @@ def _save_ngmix_data(self, ngmix_cat_path, moments=False): Column grammar: ``NGMIX[m]_[_ERR][_]_``, plus OBJECT/SHEAR-less per-object metadata columns (``NGMIX[m]_MCAL_FLAGS``, ``NGMIX_N_EPOCH``, + ``NGMIX_N_EPOCH_FAILED`` — epochs dropped because building their + observation raised, see ``ngmix.do_ngmix_metacal`` — ``NGMIX_MCAL_TYPES_FAIL``, ``NGMIX_NEIGHBOUR_FLAG`` — a blend flag set when the coadd seg stamp held a non-central footprint, shapepipe#776 — and the defect diagnostics ``NGMIX_N_EPOCH_INTERP``, @@ -394,8 +396,10 @@ def _save_ngmix_data(self, ngmix_cat_path, moments=False): # Per-object blend flag (shapepipe#776): the seg stamp held a # non-central footprint. Galaxy-only (non-moments), like N_EPOCH. ngmix_neighbour_flag = ngmix_cat_file.get_data()["neighbour_flag"] - # Defect diagnostics (ngmix Postage_stamp.record_defects). + # Failed-epoch count (ngmix.do_ngmix_metacal) and defect diagnostics + # (ngmix Postage_stamp.record_defects). defect_columns = { + "NGMIX_N_EPOCH_FAILED": "n_epoch_failed", "NGMIX_N_EPOCH_INTERP": "n_epoch_interp", "NGMIX_MIN_DIST_INTERP": "min_dist_interp", "NGMIX_MIN_DIST_NOISEFILL": "min_dist_noisefill", @@ -426,6 +430,7 @@ def _save_ngmix_data(self, ngmix_cat_path, moments=False): np.full(n_obj, get_mcal_types_fail(never_fit), dtype=float), ) self._add2dict("NGMIX_NEIGHBOUR_FLAG", np.zeros(n_obj)) + self._add2dict("NGMIX_N_EPOCH_FAILED", np.zeros(n_obj)) self._add2dict("NGMIX_N_EPOCH_INTERP", np.zeros(n_obj)) self._add2dict("NGMIX_MIN_DIST_INTERP", np.full(n_obj, -1.0)) self._add2dict("NGMIX_MIN_DIST_NOISEFILL", np.full(n_obj, -1.0)) diff --git a/src/shapepipe/modules/ngmix_package/ngmix.py b/src/shapepipe/modules/ngmix_package/ngmix.py index 4031992cf..4e3c0e541 100644 --- a/src/shapepipe/modules/ngmix_package/ngmix.py +++ b/src/shapepipe/modules/ngmix_package/ngmix.py @@ -257,6 +257,9 @@ def empty_metacal_output(): names2 = [ 'id', 'n_epoch_model', + # epochs dropped because building their observation raised + # (failed-epoch-dropped) + 'n_epoch_failed', 'mcal_types_fail', 'neighbour_flag', # defect diagnostics (Postage_stamp.record_defects) @@ -665,6 +668,12 @@ def __init__( self.epoch_cuts = Counter( considered=0, masked_fraction=0, central_veto=0 ) + # The exposure-CCD name of each kept epoch, for the log. + self.epoch_names = [] + # ``(epoch name, exception)`` for each kept epoch whose observation + # raised while being built; :func:`do_ngmix_metacal` drops the epoch + # and records it here (failed-epoch-dropped). + self.epoch_failures = [] # Defect diagnostics over the epochs kept (see # :meth:`record_defects`). self.n_epoch_interp = 0 @@ -1033,6 +1042,9 @@ def compile_results(self, results): output_dict[name]["n_epoch_model"].append( results[idx]["n_epoch_model"] ) + output_dict[name]["n_epoch_failed"].append( + results[idx]["n_epoch_failed"] + ) output_dict[name]["mcal_types_fail"].append(mcal_types_fail) # Per-object blend flag (see process()); replicated across all # shear types like id / n_epoch_model / mcal_types_fail. @@ -1335,7 +1347,9 @@ def process(self): id_first = -1 id_last = -1 count_batch = 0 - epoch_cuts = Counter(considered=0, masked_fraction=0, central_veto=0) + epoch_cuts = Counter( + considered=0, masked_fraction=0, central_veto=0, failed=0 + ) n_emptied = 0 saved_batch_cumul = 0 @@ -1426,6 +1440,15 @@ def process(self): ) n_ngmix_fail += 1 continue + finally: + # Epochs do_ngmix_metacal dropped (failed-epoch-dropped). + epoch_cuts["failed"] += len(stamp.epoch_failures) + for epoch_name, err in stamp.epoch_failures: + self._w_log.warning( + f"ngmix: tile {self._file_number_string} object" + + f" {obj_id} epoch {epoch_name} dropped:" + + f" {type(err).__name__}: {err}" + ) res['obj_id'] = obj_id # Neighbour flag: does the coadd seg stamp hold any non-central, @@ -1439,6 +1462,7 @@ def process(self): # epochs that survived the PSF fit and entered the model, # not the number of epochs submitted (v1 contract) res['n_epoch_model'] = psf_res['n_epoch'] + res['n_epoch_failed'] = len(stamp.epoch_failures) res.update(stamp.defect_diagnostics()) # The mcal flag columns are derived from the per-type results in # compile_results; here they only feed the run-health count. @@ -1493,6 +1517,7 @@ def process(self): + f" considered={epoch_cuts['considered']}" + f" masked_fraction={epoch_cuts['masked_fraction']}" + f" central_veto={epoch_cuts['central_veto']}" + + f" failed={epoch_cuts['failed']}" + f" objects_emptied={n_emptied}" ) log_run_health(self._w_log, count, n_fitted, n_flagged) @@ -1684,6 +1709,7 @@ def prepare_postage_stamps( stamp.flags.append(flag_vign) stamp.neighbours.append(neighbour) stamp.bkg_rms.append(bkg_rms_vign_scaled) + stamp.epoch_names.append(expccd_name) if tile_seg is not None: stamp.segs.append(tile_seg) stamp.jacobs.append(jacob) @@ -2751,6 +2777,14 @@ def do_ngmix_metacal( Performs metacalibration on a single multi-epoch object and returns the joint shape measurement with NGMIX. + @sc [decision:shape_measurement.epoch_masked_fraction_cut,label:operations] failed-epoch-dropped + An epoch whose observation raises while being built (for example the + non-finite interpolant guard in :func:`prepare_ngmix_weights`) is + dropped and the object is fitted on its other epochs. The epoch's name + and exception are appended to ``stamp.epoch_failures``; the caller logs + them and writes their count as ``n_epoch_failed``. Errors outside + building the observations propagate. + Parameters ---------- stamp : Postage_stamp @@ -2804,6 +2838,11 @@ def do_ngmix_metacal( dict (:func:`average_original_psf`). The two PSF dicts share keys but describe different PSFs; the named fields guard against transposing them. Unpacks positionally as ``resdict, psf_res, psf_orig_res``. + + Raises + ------ + ValueError + If the stamp has no epoch, or every epoch failed to build. @sc [decision:shape_measurement.defect_fill,decision:shape_measurement.metacal_scheme,decision:shape_measurement.defect_weighting] """ n_epoch = len(stamp.gals) @@ -2813,26 +2852,41 @@ def do_ngmix_metacal( gal_obs_list = ObsList() for n_e in range(n_epoch): bkg_rms = stamp.bkg_rms[n_e] if len(stamp.bkg_rms) > n_e else None - gal_obs = make_ngmix_observation( - stamp.gals[n_e], - stamp.weights[n_e], - stamp.flags[n_e], - stamp.psfs[n_e], - stamp.jacobs[n_e], - rng, - bkg_rms=bkg_rms, - centroid_source=centroid_source, - offset=stamp.offsets[n_e] if n_e < len(stamp.offsets) else None, - blend_handling=blend_handling, - seg=stamp.segs[n_e] if n_e < len(stamp.segs) else None, - object_number=object_number, - dilate_neighbour=dilate_neighbour, - neighbour=( - stamp.neighbours[n_e] if n_e < len(stamp.neighbours) else None - ), - defect_weighting=defect_weighting, - ) + try: + gal_obs = make_ngmix_observation( + stamp.gals[n_e], + stamp.weights[n_e], + stamp.flags[n_e], + stamp.psfs[n_e], + stamp.jacobs[n_e], + rng, + bkg_rms=bkg_rms, + centroid_source=centroid_source, + offset=( + stamp.offsets[n_e] if n_e < len(stamp.offsets) else None + ), + blend_handling=blend_handling, + seg=stamp.segs[n_e] if n_e < len(stamp.segs) else None, + object_number=object_number, + dilate_neighbour=dilate_neighbour, + neighbour=( + stamp.neighbours[n_e] + if n_e < len(stamp.neighbours) else None + ), + defect_weighting=defect_weighting, + ) + except Exception as err: + stamp.epoch_failures.append(( + stamp.epoch_names[n_e] + if n_e < len(stamp.epoch_names) else str(n_e), + err, + )) + continue gal_obs_list.append(gal_obs) + if len(gal_obs_list) == 0: + raise ValueError( + f"all {n_epoch} epochs failed to build an observation" + ) runner, psf_runner = make_runners(prior, flux_guess, rng) diff --git a/tests/module/test_make_cat.py b/tests/module/test_make_cat.py index a14de667a..857f743eb 100644 --- a/tests/module/test_make_cat.py +++ b/tests/module/test_make_cat.py @@ -56,6 +56,7 @@ def warning(self, message, *_args, **_kwargs): NGMIX_KEYS = [ "id", "n_epoch_model", + "n_epoch_failed", "mcal_types_fail", "neighbour_flag", "n_epoch_interp", "min_dist_interp", "min_dist_noisefill", @@ -86,6 +87,7 @@ def _ngmix_row(obj_id): return { "id": obj_id, "n_epoch_model": 3, + "n_epoch_failed": 1, "mcal_types_fail": 0, "neighbour_flag": 1, "n_epoch_interp": 2, "min_dist_interp": 7.5, @@ -120,7 +122,8 @@ def _write_ngmix_cat(path, obj_ids): cols = [ fits.Column( name=key, - format="K" if key in ("id", "n_epoch_model", "mcal_types_fail", + format="K" if key in ("id", "n_epoch_model", "n_epoch_failed", + "mcal_types_fail", "n_epoch_interp", "nfev_fit", "flags", "mcal_flags") else "D", array=np.array([row[key] for row in rows]), @@ -180,7 +183,7 @@ def test_save_ngmix_data_uses_new_grammar_and_no_old_names(tmp_path): # Object-level metadata columns carry no OBJECT/SHEAR token. for col in ( "NGMIX_MCAL_FLAGS", "NGMIX_N_EPOCH", "NGMIX_MCAL_TYPES_FAIL", - "NGMIX_NEIGHBOUR_FLAG", "NGMIX_N_EPOCH_INTERP", + "NGMIX_NEIGHBOUR_FLAG", "NGMIX_N_EPOCH_FAILED", "NGMIX_N_EPOCH_INTERP", "NGMIX_MIN_DIST_INTERP", "NGMIX_MIN_DIST_NOISEFILL", ): assert col in out, f"missing {col}" @@ -281,6 +284,7 @@ def test_save_ngmix_data_fills_sentinels_for_absent_objects(tmp_path): npt.assert_allclose(n_epoch[absent], [0.0, 0.0]) for col, key, never_fit in ( + ("NGMIX_N_EPOCH_FAILED", "n_epoch_failed", 0.0), ("NGMIX_N_EPOCH_INTERP", "n_epoch_interp", 0.0), ("NGMIX_MIN_DIST_INTERP", "min_dist_interp", -1.0), ("NGMIX_MIN_DIST_NOISEFILL", "min_dist_noisefill", -1.0), @@ -327,6 +331,7 @@ def _metacal_result(obj_id): res = { "obj_id": obj_id, "n_epoch_model": row["n_epoch_model"], + "n_epoch_failed": row["n_epoch_failed"], "neighbour_flag": row["neighbour_flag"], "n_epoch_interp": row["n_epoch_interp"], "min_dist_interp": row["min_dist_interp"], diff --git a/tests/module/test_ngmix.py b/tests/module/test_ngmix.py index 1435ec3d1..cfa5ccf93 100644 --- a/tests/module/test_ngmix.py +++ b/tests/module/test_ngmix.py @@ -328,6 +328,7 @@ def _fake_metacal_result(T, T_err, T_psf, T_psf_err): res = { "obj_id": 1, "n_epoch_model": 1, + "n_epoch_failed": 0, "mcal_types_fail": 0, "neighbour_flag": 0, "n_epoch_interp": 0, @@ -634,7 +635,7 @@ def test_process_counts_flagged_fits_across_batches(tmp_path, monkeypatch, flags galaxies = {str(i): {"exp-1": {"OFFSET": [0., 0.]}} for i in tile.obj_id} stamp = SimpleNamespace( gals=[np.ones((5, 5))], ra=[42.], dec=[30.], ccd=20, - epoch_cuts=Counter(considered=1), + epoch_cuts=Counter(considered=1), epoch_failures=[], jacobs=[galsim.JacobianWCS(.186, 0., 0., .186)], defect_diagnostics=lambda: dict( n_epoch_interp=0, min_dist_interp=-1.0, min_dist_noisefill=-1.0, @@ -737,7 +738,7 @@ def test_process_centroid_prior_is_each_objects_own_pixel_scale(monkeypatch): obj_id: SimpleNamespace( gals=[np.ones((5, 5))] * len(jacobs), jacobs=jacobs, ra=[10. * obj_id], dec=[30.], ccd=obj_id, - epoch_cuts=Counter(considered=len(jacobs)), + epoch_cuts=Counter(considered=len(jacobs)), epoch_failures=[], ) for obj_id, jacobs in epochs.items() } @@ -1274,3 +1275,110 @@ def test_wcs_centroid_requires_offset(): gal, weight, flag, psf, jacob, np.random.RandomState(7), centroid_source="wcs", offset=None, ) + + +def test_failed_epoch_is_dropped_not_the_object(): + """Contract failed-epoch-dropped: one epoch whose observation raises is + dropped, and the object is fitted on its other epochs. + + The poisoned epoch has one flagged pixel, so it is interpolated, beside + NaN pixels that support the interpolant: the non-finite-interpolant guard + in prepare_ngmix_weights raises a RuntimeError for that epoch alone. + """ + from shapepipe.modules.ngmix_package.ngmix import ( + Postage_stamp, + do_ngmix_metacal, + get_prior, + ) + from shapepipe.testing.simulate import make_data + + rng = np.random.RandomState(5) + gals, psfs, _, weights, flags, jacobs = make_data( + rng=np.random.RandomState(123), shear=(0.02, 0.0), noise=1e-4, + n_epochs=3, img_size=51, + ) + gals = [np.array(g, dtype=float) for g in gals] + flags = [np.array(f) for f in flags] + flags[1][5, 5] = 1 + gals[1][4:7, 4:7] = np.nan + gals[1][5, 5] = 0.0 + stamp = Postage_stamp(bkg_sub=False, megacam_flip=False) + stamp.gals, stamp.psfs, stamp.weights, stamp.flags, stamp.jacobs = ( + gals, psfs, weights, flags, jacobs, + ) + stamp.bkg_rms = [np.full(g.shape, 1e-4) for g in gals] + stamp.epoch_names = ["2000001-1", "2000002-1", "2000003-1"] + + res, psf_res, _ = do_ngmix_metacal( + stamp, get_prior(0.1857, rng), 1.0, rng, centroid_source="hsm", + ) + + assert [name for name, _ in stamp.epoch_failures] == ["2000002-1"] + assert isinstance(stamp.epoch_failures[0][1], RuntimeError) + assert psf_res["n_epoch"] == 2 + assert res["noshear"]["flags"] == 0 + assert np.all(np.isfinite(res["noshear"]["g"])) + + +def test_process_logs_and_counts_failed_epochs(tmp_path, monkeypatch): + """process logs each dropped epoch with tile, object and epoch, and + writes the count as n_epoch_failed (contract failed-epoch-dropped).""" + from types import SimpleNamespace + from shapepipe.modules.ngmix_package import ngmix as module + + tile = SimpleNamespace(obj_id=[7], flux=None, seg=None) + galaxies = {"7": {"exp-1": {"OFFSET": [0., 0.]}}} + stamp = SimpleNamespace( + gals=[np.ones((5, 5))] * 2, ra=[42.], dec=[30.], ccd=20, + epoch_cuts=Counter(considered=3), + epoch_failures=[], + jacobs=[galsim.JacobianWCS(.186, 0., 0., .186)] * 2, + defect_diagnostics=lambda: dict( + n_epoch_interp=0, min_dist_interp=-1.0, min_dist_noisefill=-1.0, + ), + ) + psf = dict( + n_epoch=2, g_psf=[.01, -.01], g_psf_err=[.001, .001], + T_psf=.1, T_psf_err=.01, + ) + + def fit(stamp, *args, **kwargs): + stamp.epoch_failures.append(("2000002-1", RuntimeError("poisoned"))) + return _fake_metacal_result(.18, .02, .09, .001), psf, psf + + monkeypatch.setattr(module, "Tile_cat", lambda *args: tile) + monkeypatch.setattr( + module, "prepare_postage_stamps", lambda *args, **kwargs: stamp, + ) + monkeypatch.setattr(module, "do_ngmix_metacal", fit) + warnings = [] + log = _RecordingLogger() + log.warning = lambda msg, *a, **k: warnings.append(msg) + inst = object.__new__(Ngmix) + inst._tile_cat_path = "in-memory-tile" + inst._seg_cat_path = None + inst._vignet_cat = SimpleNamespace( + gal_vign_cat=galaxies, psf_vign_cat=galaxies, close=lambda: None, + ) + inst._centroid_source = "wcs" + inst._id_obj_min = inst._id_obj_max = -1 + inst._bkg_sub = True + inst._blend_handling = "noisefill" + inst._dilate_neighbour = 1 + inst._metacal_psf = "fitgauss" + inst._defect_weighting = module.DEFECT_WEIGHTING + inst._save_batch = -1 + inst._zero_point = 30. + inst._output_dir = str(tmp_path) + inst._file_number_string = "-001-001" + inst._w_log = log + + inst.process() + + dropped = [msg for msg in warnings if "dropped" in msg] + assert len(dropped) == 1 + for part in ("-001-001", "object 7", "epoch 2000002-1", "poisoned"): + assert part in dropped[0] + with fits.open(inst.get_output_path(str(tmp_path))) as hdul: + npt.assert_array_equal(hdul["NOSHEAR"].data["n_epoch_failed"], [1]) + npt.assert_array_equal(hdul["NOSHEAR"].data["n_epoch_model"], [2]) diff --git a/tests/module/test_ngmix_defect_fill.py b/tests/module/test_ngmix_defect_fill.py index bf4f7f056..2e5392f3a 100644 --- a/tests/module/test_ngmix_defect_fill.py +++ b/tests/module/test_ngmix_defect_fill.py @@ -451,7 +451,8 @@ def no_fit(*_args, **_kwargs): (k, int(v)) for k, v in re.findall(r"(\w+)=(\d+)", lines[0]) ) assert tally == dict( - considered=5, masked_fraction=2, central_veto=2, objects_emptied=1 + considered=5, masked_fraction=2, central_veto=2, failed=0, + objects_emptied=1, ), lines[0] @@ -863,18 +864,21 @@ class _Stop(Exception): def fake_observation(*args, **kwargs): seen.append(kwargs["neighbour"]) - if len(seen) == len(stamp.gals): - raise _Stop - return None + return object() + + def stop(*args, **kwargs): + raise _Stop monkeypatch.setattr( ngmix_module, "make_ngmix_observation", fake_observation, ) + monkeypatch.setattr(ngmix_module, "make_runners", stop) monkeypatch.setattr(ngmix_module, "ObsList", list) with pytest.raises(_Stop): ngmix_module.do_ngmix_metacal( stamp, None, 1.0, np.random.RandomState(0), ) + assert len(seen) == len(stamp.gals) for got, want in zip(seen, stamp.neighbours): assert got is want diff --git a/tests/module/test_psf_averaging_properties.py b/tests/module/test_psf_averaging_properties.py index e4b1f38b8..d6799bc29 100644 --- a/tests/module/test_psf_averaging_properties.py +++ b/tests/module/test_psf_averaging_properties.py @@ -258,6 +258,7 @@ def test_all_epochs_failed_raises_zero_division(flagged_specs): "NGMIX_MCAL_FLAGS": LM_FUNC_NOTFINITE, "NGMIX_MCAL_TYPES_FAIL": len(METACAL_TYPES), "NGMIX_NEIGHBOUR_FLAG": 0.0, + "NGMIX_N_EPOCH_FAILED": 0.0, "NGMIX_N_EPOCH_INTERP": 0.0, "NGMIX_MIN_DIST_INTERP": -1.0, "NGMIX_MIN_DIST_NOISEFILL": -1.0, @@ -266,7 +267,8 @@ def test_all_epochs_failed_raises_zero_division(flagged_specs): # Per-key write format and a measured value distinct from every sentinel, so a # matched row is unmistakably "overwritten" and an absent row unmistakably not. _NGMIX_KEYS = [ - "id", "n_epoch_model", "mcal_types_fail", "neighbour_flag", + "id", "n_epoch_model", "n_epoch_failed", "mcal_types_fail", + "neighbour_flag", "n_epoch_interp", "min_dist_interp", "min_dist_noisefill", "nfev_fit", "g1", "g1_err", "g2", "g2_err", "T", "T_err", "flux", "flux_err", "s2n", "mag", "mag_err", "flags", "mcal_flags", @@ -276,7 +278,8 @@ def test_all_epochs_failed_raises_zero_division(flagged_specs): "T_psf_reconv", "T_err_psf_reconv", ] _INT_KEYS = { - "id", "n_epoch_model", "mcal_types_fail", "neighbour_flag", + "id", "n_epoch_model", "n_epoch_failed", "mcal_types_fail", + "neighbour_flag", "n_epoch_interp", "nfev_fit", "flags", "mcal_flags" } _SHEAR_EXTS = ["1M", "1P", "2M", "2P", "NOSHEAR"] diff --git a/tests/module/test_psf_grammar_properties.py b/tests/module/test_psf_grammar_properties.py index 43081165f..4a6442f13 100644 --- a/tests/module/test_psf_grammar_properties.py +++ b/tests/module/test_psf_grammar_properties.py @@ -64,11 +64,13 @@ def info(self, *_args, **_kwargs): # shear-type extension. Integer-typed keys carry FITS format "K", the rest # "D"; everything else mirrors test_make_cat._ngmix_row. INT_KEYS = { - "id", "n_epoch_model", "mcal_types_fail", "neighbour_flag", + "id", "n_epoch_model", "n_epoch_failed", "mcal_types_fail", + "neighbour_flag", "n_epoch_interp", "nfev_fit", "flags", "mcal_flags", } NGMIX_KEYS = [ - "id", "n_epoch_model", "mcal_types_fail", "neighbour_flag", + "id", "n_epoch_model", "n_epoch_failed", "mcal_types_fail", + "neighbour_flag", "n_epoch_interp", "min_dist_interp", "min_dist_noisefill", "nfev_fit", "g1", "g1_err", "g2", "g2_err", "T", "T_err", @@ -110,7 +112,7 @@ def _base_row(obj_id): """One object's per-key values; PSF keys overwritten by the caller.""" return { "id": obj_id, - "n_epoch_model": 3, "mcal_types_fail": 0, "neighbour_flag": 1, + "n_epoch_model": 3, "n_epoch_failed": 1, "mcal_types_fail": 0, "neighbour_flag": 1, "n_epoch_interp": 2, "min_dist_interp": 7.5, "min_dist_noisefill": 14.25, "nfev_fit": 7, "g1": 0.10, "g1_err": 0.011, "g2": -0.20, "g2_err": 0.022, @@ -182,7 +184,7 @@ def _run_save_ngmix(ngmix_path, obj_ids, cat_size_target=None): GRAMMAR_RE = re.compile( rf"^NGMIXm?_(?:{_COMPONENT})(?:_ERR)?(?:_(?:{_OBJECT}))?_(?:{_SHEAR})$" rf"|^NGMIXm?_(?:MCAL_FLAGS|MCAL_TYPES_FAIL|N_EPOCH|NEIGHBOUR_FLAG" - rf"|N_EPOCH_INTERP|MIN_DIST_INTERP|MIN_DIST_NOISEFILL)$" + rf"|N_EPOCH_FAILED|N_EPOCH_INTERP|MIN_DIST_INTERP|MIN_DIST_NOISEFILL)$" ) # The shipped final-catalogue param files, two levels up from tests/module/. @@ -422,7 +424,7 @@ def test_param_file_ngmix_tokens_are_producible(param_path, obj_ids): r"NGMIXm?_(?:G1|G2|T|SNR|FLUX|MAG|FLAGS)(?:_ERR)?" r"(?:_(?:PSF_ORIG|PSF_RECONV))?_(?:NOSHEAR|1P|1M|2P|2M)" r"|NGMIXm?_(?:MCAL_FLAGS|MCAL_TYPES_FAIL|N_EPOCH|NEIGHBOUR_FLAG" - r"|N_EPOCH_INTERP|MIN_DIST_INTERP|MIN_DIST_NOISEFILL)" + r"|N_EPOCH_FAILED|N_EPOCH_INTERP|MIN_DIST_INTERP|MIN_DIST_NOISEFILL)" # HSM: g-type, explicit PSF/STAR object, singular FLAG; the multi-epoch # sink in make_cat._save_psf_data appends a bare epoch index. r"|HSM_(?:G1|G2|T|M4_1|M4_2|RHO4)_(?:PSF|STAR)(?:_\d+)?" diff --git a/workflow/config/cfis/final_cat.param b/workflow/config/cfis/final_cat.param index 850d7c0ac..aeab8b412 100644 --- a/workflow/config/cfis/final_cat.param +++ b/workflow/config/cfis/final_cat.param @@ -48,6 +48,8 @@ NGMIX_G2_PSF_ORIG_NOSHEAR # Number of epochs (exposures) N_EPOCH NGMIX_N_EPOCH +# epochs dropped because building their ngmix observation raised +NGMIX_N_EPOCH_FAILED # Per-epoch identity and PSF shape, slot n = 1..12 (config_tile_Mc.ini's # N_EPOCH_SLOTS): EXP_ID_n/CCD_n name the exposure and CCD of epoch n, aligned diff --git a/workflow/config/cfis_image_sims/final_cat.param b/workflow/config/cfis_image_sims/final_cat.param index 08ce89967..f90495bfe 100644 --- a/workflow/config/cfis_image_sims/final_cat.param +++ b/workflow/config/cfis_image_sims/final_cat.param @@ -45,6 +45,8 @@ NGMIX_G2_PSF_ORIG_NOSHEAR # Number of epochs (exposures) N_EPOCH NGMIX_N_EPOCH +# epochs dropped because building their ngmix observation raised +NGMIX_N_EPOCH_FAILED ## Shape measurement outputs ## Ngmix: model fitting