From 6b699e6a8518568a4a8d5974b7f96786d6b0a79d Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Sat, 10 Oct 2026 19:58:50 -0400 Subject: [PATCH 1/5] Transfer the ESI employer premium onto the ACS local lane's ACS spine (#454) The lane pools a dense donor release with the ACS spine. The donor carries employer_sponsored_insurance_premiums from the meps_esi_premiums stage; ACS rows did not, so fill_reviewed_nulls refused the frame. A lane-local one-family transfer fits the premium on the donor's ASEC observation role, whose wages are measured, and the pooled frame is anchored with the stacked pool's kernel: an ACS person without wages carries no premium and the rest are scaled so both spines carry the same weighted premium per unit of household mass. The lane-wide weighted column equals the donor rows' total divided by their share of household mass. The declared ACS transfer plan and the stacked pool's plan are unchanged. Co-Authored-By: Claude Opus 5.5 --- .../us_runtime/acs_local_esi_premiums.py | 468 ++++++++ .../build/us_runtime/acs_multispine.py | 50 +- .../us/test_us_acs_local_esi_premiums.py | 1042 +++++++++++++++++ .../engine_free/us/test_us_spine_blindness.py | 6 + .../us_acs_local_esi_premiums.py | 325 +++++ 5 files changed, 1889 insertions(+), 2 deletions(-) create mode 100644 packages/microcosm-build/src/microcosm/build/us_runtime/acs_local_esi_premiums.py create mode 100644 packages/microcosm-build/tests/engine_free/us/test_us_acs_local_esi_premiums.py create mode 100644 test_support/microcosm_build/us_acs_local_esi_premiums.py diff --git a/packages/microcosm-build/src/microcosm/build/us_runtime/acs_local_esi_premiums.py b/packages/microcosm-build/src/microcosm/build/us_runtime/acs_local_esi_premiums.py new file mode 100644 index 000000000..7782fb548 --- /dev/null +++ b/packages/microcosm-build/src/microcosm/build/us_runtime/acs_local_esi_premiums.py @@ -0,0 +1,468 @@ +"""Employer premiums on the ACS spine of the retained ACS local lane (#454). + +The lane's staging frame pools a dense ASEC-by-PUF donor release with the ACS +spine (:func:`~.base_pool.with_optional_acs_spine`). The donor carries +``employer_sponsored_insurance_premiums`` from the ``meps_esi_premiums`` stage. +ACS has no policyholder flag, payment status, coverage tier or firm size, so +the stage cannot run on its rows, and the declared ACS transfer plan does not +name the column. Three steps put it on every row, on the stage's scale: + +1. **Qualify the donor** (:func:`prepare_acs_local_esi_premium_donor`). The + donor must carry the column and the raw ASEC fields the stage read, and + pass the stage's own signal gate. The transfer then fits on its ASEC + observation role only (support clone 0), whose wages are the measured CPS + wages. The lane's other transfers fit on the PUF-detail role, whose wages + are PUF-imputed while its premium is a copy of the source person's: on + those rows the premium no longer follows the wage predictor. +2. **Transfer** the column onto the ACS rows with the lane's own one-family + plan (:func:`acs_local_esi_premium_transfer_target_families`). The declared + ACS plan and the stacked pool's plan are untouched. +3. **Anchor** the pooled frame (:func:`with_acs_local_esi_premium_anchor`) + with the stacked pool's kernel, + :func:`~.esi_premiums.with_us_esi_premium_pool_anchor`: it clears the + premium where the ACS person reports zero wages and scales the rest so both + spines carry the same weighted premium per unit of household mass. The + donor rows are never rewritten. + +The kernel tells the two kinds of row apart by the CPS record id +(``PERIDNUM``) and the raw ASEC fields, which the donor rows carry and the ACS +rows do not. It refuses a transferred row whose wage is missing. ACS PUMS +leaves ``WAGP`` blank below age 15, outside its earnings universe, and the +lane keeps that blank in the staging frame. The anchor therefore reads those +people's wages as zero under the named universe rule of +:mod:`.acs_income_universe`, in its view of the frame only. A blank wage at +age 15 or older stays missing and is refused. + +Invariant: the lane-wide weighted column equals the donor rows' weighted +column divided by their share of household mass. Pooling multiplies every +donor household weight by that share, so the lane-wide column is the donor +release's own. + +The kernel and both gates read a handful of columns, but the kernel copies +every table it is given and each gate copies the person table. This module +hands them a narrow view of the pooled frame: the staging frame holds two full +spines, and the lane's memory budget has no room for further copies of them. +""" + +from __future__ import annotations + +import math +from collections.abc import Mapping +from typing import Any + +import numpy as np +import pandas as pd + +from microcosm.build.gates import GateResult +from microcosm.build.source_runtime import SourceRuntimeError +from microcosm.build.us_runtime.acs_income_universe import ( + ACS_PUMS_EARNINGS_MINIMUM_AGE, + ACS_PUMS_EARNINGS_SOURCE_COLUMNS, +) +from microcosm.build.us_runtime.acs_transfer import ( + TargetFamilies, + resolve_acs_donor_channel, +) +from microcosm.build.us_runtime.esi_premiums import ( + US_ESI_EMPLOYER_PREMIUM_COLUMN, + US_ESI_PREMIUMS_OUTPUT_COLUMNS, + US_ESI_PREMIUMS_POOL_ANCHOR_PERSON_INPUTS, + US_ESI_PREMIUMS_REQUIRED_SOURCE_COLUMNS, + US_ESI_PREMIUMS_WAGE_COLUMN, + us_esi_premiums_anchor_gate, + us_esi_premiums_signal_gate, + with_us_esi_premium_pool_anchor, +) +from microcosm.build.us_runtime.support_provenance import ( + BASE_ASEC_SUPPORT_CHANNEL, + has_assembled_support_metadata, + has_support_role_metadata, + support_clone_index_column, + support_source_id_column, +) +from microcosm.frame import Frame +from microcosm.frame.units import US_SCHEMA + +__all__ = [ + "ACS_LOCAL_ESI_PREMIUM_FAMILY", + "acs_local_esi_premium_gate_payload", + "acs_local_esi_premium_gates", + "acs_local_esi_premium_transfer_target_families", + "prepare_acs_local_esi_premium_donor", + "with_acs_local_esi_premium_anchor", +] + +#: The stacked pool files the same column under the same family name +#: (``multispine_pool.pool_transfer_target_families``). +ACS_LOCAL_ESI_PREMIUM_FAMILY = "source_operator_esi_premiums" + +_PERSON = US_SCHEMA.person_entity +_STATE_COLUMN = "state_fips" +#: Raw ASEC person fields whose presence marks a row the stage derived itself. +_RAW_EVIDENCE_COLUMNS: tuple[str, ...] = tuple( + column + for column in US_ESI_PREMIUMS_REQUIRED_SOURCE_COLUMNS + if column != _STATE_COLUMN +) +#: The CPS record id. The kernel requires it beside the raw fields on every +#: row the stage derived, and blank on every transferred row. +_CPS_RECORD_ID_COLUMN = "PERIDNUM" +_CLONE_INDEX_COLUMN = support_clone_index_column(_PERSON) +_SOURCE_ID_COLUMN = support_source_id_column(_PERSON) +_AGE_COLUMN = "age" +#: The raw PUMS field the ACS wage is mapped from, and the named rule that +#: makes its blank a zero: the field applies to people aged 15 and older. +_ACS_WAGE_SOURCE_COLUMN = ACS_PUMS_EARNINGS_SOURCE_COLUMNS[US_ESI_PREMIUMS_WAGE_COLUMN] +_ACS_WAGE_UNIVERSE_RULE = ( + f"acs_2024_pums_{_ACS_WAGE_SOURCE_COLUMN.lower()}" + f"_age_{ACS_PUMS_EARNINGS_MINIMUM_AGE}_plus" +) +#: Person columns the pool anchor kernel reads, without the two support +#: provenance columns. The kernel looks a clone's wages up by +#: ``person_source_id`` across the whole frame. A stacked pool makes that id +#: unique at assembly; this lane does not: ACS rows keep their pre-remap ids +#: as source ids, which can repeat a donor's. Every ACS row here is its own +#: source record, so the view omits the lookup's columns and the kernel reads +#: each row's own wages. +_ANCHOR_PERSON_COLUMNS: tuple[str, ...] = ( + *US_ESI_PREMIUMS_OUTPUT_COLUMNS, + *( + column + for column in US_ESI_PREMIUMS_POOL_ANCHOR_PERSON_INPUTS + if column not in (_CLONE_INDEX_COLUMN, _SOURCE_ID_COLUMN) + ), +) +#: Person columns the two ESI gates read: the output, the raw fields and +#: State the cells are recomputed from, the record id that settles each +#: row's kind, wages and support provenance for the transferred-row summary, +#: and the stable source identity the donor's clones are compared by. +_GATE_PERSON_COLUMNS: tuple[str, ...] = tuple( + dict.fromkeys( + ( + *US_ESI_PREMIUMS_OUTPUT_COLUMNS, + *US_ESI_PREMIUMS_REQUIRED_SOURCE_COLUMNS, + *US_ESI_PREMIUMS_POOL_ANCHOR_PERSON_INPUTS, + "source_year", + "source_household_id", + "source_person_id", + ) + ) +) +_GATE_HOUSEHOLD_COLUMNS: tuple[str, ...] = (_STATE_COLUMN,) + + +def acs_local_esi_premium_transfer_target_families() -> TargetFamilies: + """The lane's separate ASEC-only pass for the employer premium.""" + + return {_PERSON: {ACS_LOCAL_ESI_PREMIUM_FAMILY: US_ESI_PREMIUMS_OUTPUT_COLUMNS}} + + +def _narrow_view( + frame: Frame, + *, + person_columns: tuple[str, ...], + household_columns: tuple[str, ...] = (), + person_values: Mapping[str, np.ndarray] | None = None, +) -> Frame: + """The frame's structure and weights with only the named value columns. + + ``person_values`` replaces a person column in the view; the frame's own + column is not touched. + """ + + schema = frame.schema + tables: dict[str, pd.DataFrame] = {} + for entity in frame.entities: + table = frame.table(entity) + if entity == schema.person_entity: + keep = [ + schema.entity_id_column(entity), + *(schema.membership_column(group) for group in schema.group_entities), + *person_columns, + ] + else: + keep = [schema.entity_id_column(entity)] + if entity == "household": + keep.extend(household_columns) + present = list(dict.fromkeys(column for column in keep if column in table)) + tables[entity] = table.loc[:, present] + if person_values: + person = tables[schema.person_entity].copy(deep=False) + for column, values in person_values.items(): + person[column] = values + tables[schema.person_entity] = person + return Frame( + tables, + schema, + {entity: frame.weights_for(entity) for entity in frame.weighted_entities}, + frame.strata, + mass_log=frame.mass_log, + metadata=frame.metadata, + ) + + +def prepare_acs_local_esi_premium_donor( + frame: Frame, +) -> tuple[Frame, dict[str, object]]: + """Qualify the dense donor and select the rows the lane's transfer fits on. + + Refuses a donor that lacks the stage's output or the raw ASEC fields it + read, that fails the stage's signal gate, or whose support roles cannot be + told apart. Returns the ASEC observation role (support clone 0) and a + receipt. The donor frame itself is not changed. + """ + + if frame.schema != US_SCHEMA: + raise ValueError("Local ESI premium donor requires the US schema.") + person = frame.table(_PERSON) + missing = [ + column + for column in (*US_ESI_PREMIUMS_OUTPUT_COLUMNS, *_RAW_EVIDENCE_COLUMNS) + if column not in person + ] + if missing: + raise ValueError( + f"Local ESI premium donor lacks {missing}: the donor release was " + "built without the meps_esi_premiums stage (microcosm#454). The " + "lane transfers the employer premium from a donor that carries it " + "and never defaults it." + ) + if _CPS_RECORD_ID_COLUMN not in person: + raise ValueError( + f"Local ESI premium donor lacks {_CPS_RECORD_ID_COLUMN!r}, the CPS " + "record id that marks its rows as the ones the stage derived. " + "Without it the pooled frame cannot tell them from the ACS rows." + ) + if US_ESI_PREMIUMS_WAGE_COLUMN not in person: + raise ValueError( + f"Local ESI premium donor lacks {US_ESI_PREMIUMS_WAGE_COLUMN!r}, the " + "wage the transfer conditions the premium on." + ) + if not has_support_role_metadata( + person, entity=_PERSON + ) or has_assembled_support_metadata(person, entity=_PERSON): + raise ValueError( + "Local ESI premium donor requires explicit legacy ASEC observation " + "roles; unclassified or multispine records need their own source " + "lineage." + ) + gate = us_esi_premiums_signal_gate( + _narrow_view( + frame, + person_columns=_GATE_PERSON_COLUMNS, + household_columns=_GATE_HOUSEHOLD_COLUMNS, + ) + ) + if not gate.passed: + raise ValueError( + "Local ESI premium donor fails the meps_esi_premiums signal gate: " + + "; ".join(gate.failures) + ) + donor, role = resolve_acs_donor_channel(frame, BASE_ASEC_SUPPORT_CHANNEL) + donor_person = donor.table(_PERSON) + employer = donor_person[US_ESI_EMPLOYER_PREMIUM_COLUMN].to_numpy(dtype=np.float64) + wages = pd.to_numeric( + donor_person[US_ESI_PREMIUMS_WAGE_COLUMN], errors="coerce" + ).to_numpy(dtype=np.float64, na_value=np.nan) + if not np.isfinite(wages).all(): + raise ValueError( + "Local ESI premium donor has ASEC observations with no " + f"{US_ESI_PREMIUMS_WAGE_COLUMN!r}; the transfer never treats a " + "missing wage as zero." + ) + weights = np.asarray(frame.resolve_weights(_PERSON).values, dtype=np.float64) + donor_weights = np.asarray(donor.resolve_weights(_PERSON).values, dtype=np.float64) + positive = employer > 0 + return donor, { + "producer": "meps_esi_premiums", + "donor_channel": role, + "donor_wage_column": US_ESI_PREMIUMS_WAGE_COLUMN, + "donor_wage_basis": "measured CPS ASEC wages of the observation role", + "donor_rows": int(frame.n(_PERSON)), + "asec_observation_rows": int(donor.n(_PERSON)), + "donor_employer_premium_total": float( + weights @ person[US_ESI_EMPLOYER_PREMIUM_COLUMN].to_numpy(dtype=np.float64) + ), + "asec_observation_positive_share": ( + float(donor_weights[positive].sum() / donor_weights.sum()) + if donor_weights.sum() + else 0.0 + ), + # The structural zero the transfer has to learn: how much of the + # observation role's premium mass sits on people with no measured + # wages (employed at interview, no wages in the income year). + "asec_observation_premium_share_without_wages": ( + float( + donor_weights[positive & ~(wages > 0)] + @ employer[positive & ~(wages > 0)] + ) + / float(donor_weights @ employer) + if float(donor_weights @ employer) + else 0.0 + ), + "donor_signal_gate": {"passed": True, "failures": []}, + } + + +def _wages_under_the_acs_universe( + person: pd.DataFrame, +) -> tuple[np.ndarray, dict[str, object]]: + """Wages for the anchor's view, with ACS universe blanks read as zero. + + ACS PUMS leaves ``WAGP`` blank for people below age 15: the field does + not apply to them. The lane keeps that blank in the staging frame, and the + anchor refuses a missing wage. Only a transferred row below the age floor + whose mapped wage is null, and whose raw field is blank where the frame + carries it, takes a zero here. Any other missing wage stays missing. + """ + + wages = pd.to_numeric( + person[US_ESI_PREMIUMS_WAGE_COLUMN], errors="coerce" + ).to_numpy(dtype=np.float64, na_value=np.nan) + transferred = person[list(_RAW_EVIDENCE_COLUMNS)].isna().all(axis=1).to_numpy() + blank = transferred & np.isnan(wages) + outside = np.zeros(len(person), dtype=bool) + if blank.any() and _AGE_COLUMN in person: + age = pd.to_numeric(person[_AGE_COLUMN], errors="coerce").to_numpy( + dtype=np.float64, na_value=np.nan + ) + outside = blank & (age < ACS_PUMS_EARNINGS_MINIMUM_AGE) + if _ACS_WAGE_SOURCE_COLUMN in person: + outside &= person[_ACS_WAGE_SOURCE_COLUMN].isna().to_numpy() + wages = np.where(outside, 0.0, wages) + return wages, { + "rule_id": _ACS_WAGE_UNIVERSE_RULE, + "minimum_age": ACS_PUMS_EARNINGS_MINIMUM_AGE, + "raw_source_column": _ACS_WAGE_SOURCE_COLUMN, + "transferred_rows_with_blank_wage": int(blank.sum()), + "transferred_rows_outside_universe_read_as_zero": int(outside.sum()), + # The staging frame keeps the source's blanks; the zeros exist in the + # anchor's view only. + "frame_cells_written": 0, + } + + +def with_acs_local_esi_premium_anchor(frame: Frame) -> tuple[Frame, dict[str, object]]: + """Hold the pooled lane frame's transferred premiums to the donor's scale. + + Runs :func:`~.esi_premiums.with_us_esi_premium_pool_anchor` on a narrow + view of ``frame`` and writes the anchored column back. Rows with the raw + ASEC fields (the donor spine) are never rewritten. On the other rows the + premium is cleared where the person reports zero wages and the rest is + scaled by one factor, so both spines carry the same weighted premium per + unit of household mass. A blank ACS wage below age 15 is a zero wage + (:func:`_wages_under_the_acs_universe`); any other missing wage is + refused. A frame already on that scale, or with no transferred row, is + returned unchanged. Returns the frame and the kernel's receipt. + """ + + if frame.schema != US_SCHEMA: + raise ValueError("Local ESI premium anchor requires the US schema.") + person = frame.table(_PERSON) + absent = [column for column in _ANCHOR_PERSON_COLUMNS if column not in person] + if absent: + raise SourceRuntimeError( + f"Local ESI premium anchor: the person table lacks {absent}; it " + "needs the transferred premium on every row, the raw ASEC fields " + "that mark the donor rows and the wages that place the structural " + "zeros." + ) + if _CLONE_INDEX_COLUMN in person: + transferred = person[list(_RAW_EVIDENCE_COLUMNS)].isna().all(axis=1) + clone_index = pd.to_numeric(person[_CLONE_INDEX_COLUMN], errors="coerce") + clones = transferred & ~clone_index.eq(0) + if clones.any(): + raise SourceRuntimeError( + f"Local ESI premium anchor: {int(clones.sum())} row(s) without " + "raw ASEC fields are not clone-0 source records. The lane reads " + "each transferred person's own wages; support clones of " + "transferred rows need the stacked pool's source-record lookup." + ) + wages, wage_universe = _wages_under_the_acs_universe(person) + view = _narrow_view( + frame, + person_columns=_ANCHOR_PERSON_COLUMNS, + person_values={US_ESI_PREMIUMS_WAGE_COLUMN: wages}, + ) + anchored, receipt = with_us_esi_premium_pool_anchor(view) + receipt = dict(receipt) | { + "kernel": "with_us_esi_premium_pool_anchor", + "transferred_source_record": "each transferred row is its own source record", + "wage_universe": wage_universe, + } + if anchored is view: + return frame, receipt + id_column = US_SCHEMA.entity_id_column(_PERSON) + anchored_person = anchored.table(_PERSON) + if not np.array_equal( + anchored_person[id_column].to_numpy(), person[id_column].to_numpy() + ): + raise SourceRuntimeError( + "Local ESI premium anchor: the kernel reordered the person table." + ) + tables = { + entity: (person.copy(deep=False) if entity == _PERSON else frame.table(entity)) + for entity in frame.entities + } + tables.update({name: frame.link(name) for name in frame.links}) + for column in US_ESI_PREMIUMS_OUTPUT_COLUMNS: + tables[_PERSON][column] = anchored_person[column].to_numpy(dtype=np.float64) + result = Frame( + tables, + frame.schema, + {entity: frame.weights_for(entity) for entity in frame.weighted_entities}, + frame.strata, + mass_log=frame.mass_log, + metadata=frame.metadata, + ) + return result, receipt + + +def acs_local_esi_premium_gates( + frame: Frame, *, time_period: int +) -> tuple[GateResult, GateResult]: + """Both ESI premium gates on a lane frame: ``(signal, anchor)``. + + The gates grade a pooled frame by row kind: the donor rows, which carry + the raw ASEC fields, get the full MEPS-IC recomputation, and the ACS rows + are compared with them. They run on a narrow view; the verdicts are those + of the full frame. + """ + + view = _narrow_view( + frame, + person_columns=_GATE_PERSON_COLUMNS, + household_columns=_GATE_HOUSEHOLD_COLUMNS, + ) + return ( + us_esi_premiums_signal_gate(view), + us_esi_premiums_anchor_gate(view, time_period=time_period), + ) + + +def _strict_json(value: Any) -> Any: + if isinstance(value, Mapping): + return {str(key): _strict_json(item) for key, item in value.items()} + if isinstance(value, (list, tuple)): + return [_strict_json(item) for item in value] + if isinstance(value, np.generic): + return _strict_json(value.item()) + if isinstance(value, float) and not math.isfinite(value): + return None + return value + + +def acs_local_esi_premium_gate_payload(gate: GateResult) -> dict[str, object]: + """One gate verdict for a lane summary or gate report. + + Failures are kept verbatim, so a diagnostic build that waived them still + records what was red. Non-finite detail values become ``null``: the + staging summary is written as strict JSON. + """ + + return { + "name": gate.name, + "passed": bool(gate.passed), + "failures": list(gate.failures), + "details": _strict_json(dict(gate.details)), + } diff --git a/packages/microcosm-build/src/microcosm/build/us_runtime/acs_multispine.py b/packages/microcosm-build/src/microcosm/build/us_runtime/acs_multispine.py index e34c4b806..830138da6 100644 --- a/packages/microcosm-build/src/microcosm/build/us_runtime/acs_multispine.py +++ b/packages/microcosm-build/src/microcosm/build/us_runtime/acs_multispine.py @@ -27,6 +27,10 @@ with_acs_spm_independence_role, without_acs_group_quarters_children, ) +from microcosm.build.us_runtime.acs_local_esi_premiums import ( + acs_local_esi_premium_transfer_target_families, + with_acs_local_esi_premium_anchor, +) from microcosm.build.us_runtime.acs_local_hours import ( acs_local_hours_transfer_target_families, complete_acs_local_under15_hours, @@ -92,6 +96,8 @@ def build_optional_acs_multispine( hours_donor_factory: Callable[[Frame], tuple[Frame, Frame, dict[str, object]]] | None = None, hours_under15_policy: str | None = None, + esi_premium_donor_factory: Callable[[Frame], tuple[Frame, dict[str, object]]] + | None = None, donor_spine: str = ASEC_PUF_DONOR_SPINE, donor_channel: str | None = ACS_DONOR_CHANNEL_AUTO, seed: int = 0, @@ -113,6 +119,12 @@ def build_optional_acs_multispine( Alternatively, ``hours_donor_factory`` qualifies that donor lazily: a source-complete ACS spine never needs its raw ASEC fields opened. + ``esi_premium_donor_factory`` qualifies the base as the donor of the + employer premium (microcosm#454) and returns its ASEC observation role. + The premium then reaches the ACS rows through its own ASEC-only pass, and + the pooled frame is held to the donor rows' premium per unit of household + mass (:func:`~.acs_local_esi_premiums.with_acs_local_esi_premium_anchor`). + Large intermediate frames are released as soon as the next stage has materialized its own frame. This cannot make the final dense pool small, but it avoids retaining the raw, mapped, transferred, and pooled versions @@ -209,6 +221,30 @@ def build_optional_acs_multispine( hours_imputed_inputs = tuple(hours_transfer.imputed_inputs) hours_donor_channel = hours_transfer.resolved_donor_channel del hours_transfer + esi_fit_records = () + esi_imputed_inputs = () + esi_donor_channel = None + esi_source = None + if esi_premium_donor_factory is not None: + # The premium follows measured wages, so it fits on the ASEC + # observation role like usual hours, not on the PUF-detail role the + # tax details use: that role's wages are PUF-imputed. + esi_donor, esi_source = esi_premium_donor_factory(base) + esi_transfer = transfer_acs_inputs( + mapped_frame, + esi_donor, + target_families=acs_local_esi_premium_transfer_target_families(), + donor_spine=donor_spine, + donor_channel=BASE_ASEC_SUPPORT_CHANNEL, + seed=seed, + n_estimators=n_estimators, + max_targets_per_fit=max_targets_per_fit, + ) + mapped_frame = esi_transfer.frame + esi_fit_records = tuple(esi_transfer.fit_records) + esi_imputed_inputs = tuple(esi_transfer.imputed_inputs) + esi_donor_channel = esi_transfer.resolved_donor_channel + del esi_transfer, esi_donor transferred = transfer_acs_inputs( mapped_frame, base, @@ -221,9 +257,9 @@ def build_optional_acs_multispine( ) del mapped_frame adult_care_gate = _require_recipient_adult_care_structure(transferred.frame) - fit_records = hours_fit_records + tuple(transferred.fit_records) + fit_records = hours_fit_records + esi_fit_records + tuple(transferred.fit_records) imputed_provenance = _json_ready_sequence( - hours_imputed_inputs + tuple(transferred.imputed_inputs) + hours_imputed_inputs + esi_imputed_inputs + tuple(transferred.imputed_inputs) ) deferred_inputs = tuple(transferred.deferred_inputs) if puma_ladder is not None: @@ -251,6 +287,11 @@ def build_optional_acs_multispine( ) pooled = with_optional_acs_spine(base, transferred_frame, **pool_options) del transferred_frame + esi_anchor = None + if esi_premium_donor_factory is not None: + # Pooling has set each spine's share of household mass; the anchor + # reads it from the live frame. + pooled, esi_anchor = with_acs_local_esi_premium_anchor(pooled) provenance = { "enabled": True, @@ -279,6 +320,11 @@ def build_optional_acs_multispine( provenance["local_hours_source"] = hours_source if modeled_hours is not None: provenance["hours_modeled_completion"] = modeled_hours + if esi_premium_donor_factory is not None: + provenance["fit_configuration"]["esi_premium_donor_channel"] = esi_donor_channel + provenance["local_esi_premiums"] = _json_ready_mapping( + {"source": esi_source, "anchor": esi_anchor} + ) if puma_ladder is not None: geography = us_puma_ladder_assignment_summary( pooled.table("household"), diff --git a/packages/microcosm-build/tests/engine_free/us/test_us_acs_local_esi_premiums.py b/packages/microcosm-build/tests/engine_free/us/test_us_acs_local_esi_premiums.py new file mode 100644 index 000000000..fa60bd06c --- /dev/null +++ b/packages/microcosm-build/tests/engine_free/us/test_us_acs_local_esi_premiums.py @@ -0,0 +1,1042 @@ +"""ESI employer premiums on the ACS local lane's ACS spine (microcosm #454). + +The lane pools a dense ASEC-by-PUF donor release with the ACS spine +(``with_optional_acs_spine``): the donor keeps ``1 - acs_share`` of its +household mass and ACS takes the rest. The donor carries the premium from the +``meps_esi_premiums`` stage; the lane transfers it onto the ACS rows from the +donor's ASEC observation role and anchors the pooled frame. + +Invariants of :func:`with_acs_local_esi_premium_anchor`, for every lane frame: + +* **Conservation**: the lane-wide weighted employer premium equals the donor + rows' total divided by their share of household mass. Pooling multiplies + every donor household weight by that share, so this is the donor release's + own weighted column. +* **Equal intensity**: both spines carry the same weighted premium per unit of + household mass. +* **Donor rows are immutable**: no donor cell changes by a byte. +* **Structural zeros**: an ACS person who reports zero wages carries no + premium, and neither does one below age 15, whose wage ACS leaves blank. +* **No invented zeros**: a blank wage at age 15 or older is refused, and the + frame's own blanks are never overwritten. +* **Proportionality**: every other ACS premium is one common multiple of the + value the transfer drew. +* **Idempotence**: an anchored frame is returned unchanged. +* **Identity independence**: ACS ids that repeat the donor's change nothing. +""" + +from __future__ import annotations + +import importlib.util +import json +from functools import cache +from pathlib import Path + +import numpy as np +import pandas as pd +import pytest +from hypothesis import HealthCheck, assume, given, settings +from hypothesis import strategies as st + +from microcosm.build.source_runtime import SourceRuntimeError +from microcosm.build.us_runtime import acs_multispine +from microcosm.build.us_runtime import esi_premiums as esi +from microcosm.build.us_runtime.acs_inputs import map_acs_native_inputs +from microcosm.build.us_runtime.acs_local_esi_premiums import ( + ACS_LOCAL_ESI_PREMIUM_FAMILY, + acs_local_esi_premium_gate_payload, + acs_local_esi_premium_gates, + acs_local_esi_premium_transfer_target_families, + prepare_acs_local_esi_premium_donor, + with_acs_local_esi_premium_anchor, +) +from microcosm.build.us_runtime.acs_local_hours import ( + acs_local_transfer_target_families, +) +from microcosm.build.us_runtime.acs_pums import AcsPumsSource +from microcosm.build.us_runtime.acs_transfer import ( + declared_acs_transfer_target_families, + transfer_acs_inputs, +) +from microcosm.build.us_runtime.esi_premiums import ( + US_ESI_PREMIUMS_OUTPUT_COLUMNS, + refuse_unassigned_us_esi_premiums, + us_esi_premiums_anchor_gate, + us_esi_premiums_signal_gate, + with_us_esi_premium_pool_anchor, +) +from microcosm.build.us_runtime.h5_io import ( + LEGACY_NULLABLE_STAGING_ARTIFACT_KIND, + load_legacy_calibrated_us_h5, + write_nullable_us_h5, +) +from microcosm.build.us_runtime.multispine_pool import pool_transfer_target_families +from microcosm.frame import US_SCHEMA, Frame, WeightKind, Weights +from test_support.microcosm_build.us_acs_local_esi_premiums import ( + ACS, + ASEC_PUF, + EMPLOYER, + WAGES, + build_lane, + dense_donor, + donor_rows, + employer_total, + household_mass_share, + person_weights, + pooled, + raw_acs, + reweighted, + transferred_acs, +) + +#: CMS NHE Table 24 employer contribution, CY2024: the stage's anchor. +ANCHOR = float(esi.EMPLOYER_PREMIUM_ANCHOR["values"]["2024"]) +RAW = tuple( + column + for column in esi.US_ESI_PREMIUMS_REQUIRED_SOURCE_COLUMNS + if column != "state_fips" +) +#: Above every donor id, so ACS ids do not repeat the donor's. +_DISJOINT_FIRST_ID = 5_000_000 + +requires_pytables = pytest.mark.skipif( + importlib.util.find_spec("tables") is None, + reason="requires pytables (the build environment)", +) + + +def _column(frame: Frame, column: str) -> np.ndarray: + return frame.table("person")[column].to_numpy(dtype=float) + + +def _replace_person(frame: Frame, person: pd.DataFrame) -> Frame: + return Frame( + { + entity: person if entity == "person" else frame.table(entity) + for entity in frame.entities + }, + frame.schema, + {"household": frame.weights_for("household")}, + frame.strata, + ) + + +# --- The lane's plan leaves the shared plans alone ---------------------------- + + +def test_the_lane_plan_names_the_premium_and_changes_no_shared_plan() -> None: + declared_before = declared_acs_transfer_target_families() + pool_before = pool_transfer_target_families() + + local = acs_local_esi_premium_transfer_target_families() + + assert local == {"person": {ACS_LOCAL_ESI_PREMIUM_FAMILY: (EMPLOYER,)}} + assert local["person"][ACS_LOCAL_ESI_PREMIUM_FAMILY] == ( + US_ESI_PREMIUMS_OUTPUT_COLUMNS + ) + assert declared_acs_transfer_target_families() == declared_before + assert pool_transfer_target_families() == pool_before + # The declared plan still does not own the column, so the stacked pool + # keeps it under its own pool-local family, where the early fill reads it. + declared_targets = { + target + for families in declared_before.values() + for targets in families.values() + for target in targets + } + assert EMPLOYER not in declared_targets + assert pool_before["person"][ACS_LOCAL_ESI_PREMIUM_FAMILY] == (EMPLOYER,) + # The hours plan the staging tool composes with is untouched too. + assert EMPLOYER not in { + target + for families in acs_local_transfer_target_families().values() + for targets in families.values() + for target in targets + } + + +# --- Donor qualification ------------------------------------------------------ + + +def test_the_donor_is_its_asec_observation_role_with_measured_wages() -> None: + donor = dense_donor() + before = {entity: donor.table(entity).copy() for entity in donor.entities} + + selected, receipt = prepare_acs_local_esi_premium_donor(donor) + + person = selected.table("person") + assert len(person) == 320 + assert set(person["person_support_channel"]) == {"asec"} + assert set(person["person_support_clone_index"]) == {0} + # Measured wages: a premium only where the person has wages. + assert not ((person[EMPLOYER] > 0) & ~(person[WAGES] > 0)).any() + assert receipt["donor_channel"] == "asec" + assert receipt["donor_rows"] == 640 + assert receipt["asec_observation_rows"] == 320 + assert receipt["asec_observation_premium_share_without_wages"] == 0.0 + assert receipt["donor_employer_premium_total"] == pytest.approx( + employer_total(donor), rel=1e-12 + ) + assert receipt["donor_signal_gate"] == {"passed": True, "failures": []} + for entity in donor.entities: + pd.testing.assert_frame_equal(donor.table(entity), before[entity]) + + +def _donor_without(*columns: str) -> Frame: + donor = dense_donor(80) + return _replace_person(donor, donor.table("person").drop(columns=list(columns))) + + +@pytest.mark.parametrize("column", [EMPLOYER, "NOW_OWNGRP", "PEIO1COW"]) +def test_a_donor_built_without_the_stage_is_refused(column: str) -> None: + with pytest.raises(ValueError, match="meps_esi_premiums stage") as error: + prepare_acs_local_esi_premium_donor(_donor_without(column)) + assert column in str(error.value) + assert "never defaults" in str(error.value) + + +def test_a_donor_without_the_cps_record_id_is_refused() -> None: + with pytest.raises(ValueError, match="lacks 'PERIDNUM'"): + prepare_acs_local_esi_premium_donor(_donor_without("PERIDNUM")) + + +def test_a_donor_row_with_raw_fields_and_no_record_id_is_refused() -> None: + # The record id and the raw fields must agree on every row. + donor = dense_donor(80) + person = donor.table("person").copy() + person["PERIDNUM"] = person["PERIDNUM"].astype(object) + person.loc[person.index[3], "PERIDNUM"] = None + with pytest.raises(ValueError, match="without a CPS record id"): + prepare_acs_local_esi_premium_donor(_replace_person(donor, person)) + + +def test_a_donor_without_wages_is_refused() -> None: + with pytest.raises(ValueError, match="conditions the premium on"): + prepare_acs_local_esi_premium_donor(_donor_without(WAGES)) + + +def test_a_donor_whose_observation_role_has_missing_wages_is_refused() -> None: + donor = dense_donor(80) + person = donor.table("person").copy() + person.loc[person["person_support_clone_index"].eq(0).idxmax(), WAGES] = np.nan + with pytest.raises(ValueError, match="never treats a missing wage as zero"): + prepare_acs_local_esi_premium_donor(_replace_person(donor, person)) + + +def test_a_donor_without_support_roles_is_refused() -> None: + donor = dense_donor(80) + tables = { + entity: donor.table(entity).drop( + columns=[ + f"{entity}_source_id", + f"{entity}_support_channel", + f"{entity}_support_clone_index", + ] + ) + for entity in donor.entities + } + unclassified = Frame( + tables, US_SCHEMA, {"household": donor.weights_for("household")} + ) + with pytest.raises(ValueError, match="explicit legacy ASEC observation roles"): + prepare_acs_local_esi_premium_donor(unclassified) + + +def test_an_assembled_multispine_donor_is_refused() -> None: + donor = dense_donor(80) + person = donor.table("person").copy() + person["person_spine_source_id"] = person["person_source_id"] + with pytest.raises(ValueError, match="multispine records need their own"): + prepare_acs_local_esi_premium_donor(_replace_person(donor, person)) + + +def test_a_donor_failing_the_stage_signal_gate_is_refused() -> None: + # A premium outside its MEPS-IC cell: no longer one common multiple. + donor = dense_donor(80) + person = donor.table("person").copy() + positive = person[EMPLOYER] > 0 + person.loc[positive.idxmax(), EMPLOYER] *= 3.0 + with pytest.raises(ValueError, match="fails the meps_esi_premiums signal gate"): + prepare_acs_local_esi_premium_donor(_replace_person(donor, person)) + + +# --- The anchor on every lane frame ------------------------------------------- + +_amount = st.one_of( + st.just(0.0), + st.floats(min_value=1.0, max_value=40_000.0, allow_nan=False), +) +_weight = st.floats(min_value=0.5, max_value=5_000.0, allow_nan=False) + + +@cache +def _small_donor() -> Frame: + return dense_donor(40) + + +@st.composite +def _lane_cases(draw) -> dict[str, object]: + """Any donor weights, any share, and any ACS households and draws. + + The donor's household weights are rescaled one by one, so the donor rows + do not share one weight profile; ACS households hold one to several + people. + """ + + people = draw(st.integers(min_value=2, max_value=10)) + premiums = np.asarray(draw(st.lists(_amount, min_size=people, max_size=people))) + wages = np.asarray(draw(st.lists(_amount, min_size=people, max_size=people))) + # Some ACS people are children: PUMS leaves their wage blank. + child = np.asarray(draw(st.lists(st.booleans(), min_size=people, max_size=people))) + wages = np.where(child, np.nan, wages) + ages = np.where(child, 9.0, 30.0 + np.arange(people)) + joins = draw(st.lists(st.booleans(), min_size=people - 1, max_size=people - 1)) + households = np.concatenate([[0], np.cumsum(~np.asarray(joins, dtype=bool))]) + count = int(households.max()) + 1 + weights = np.asarray(draw(st.lists(_weight, min_size=count, max_size=count))) + assume(((premiums > 0) & (wages > 0)).any()) + donor = _small_donor() + factors = np.asarray( + draw( + st.lists( + st.floats(min_value=0.05, max_value=20.0), + min_size=donor.n("household"), + max_size=donor.n("household"), + ) + ) + ) + scale = draw(st.floats(min_value=0.01, max_value=100.0)) + return { + "premiums": premiums, + "wages": wages, + "ages": ages, + "weights": weights, + "households": households, + "acs_share": draw(st.floats(min_value=0.05, max_value=0.95)), + "donor": reweighted(donor, factors * scale), + } + + +def _lane_frame(case: dict[str, object], *, first_id: int = 1) -> Frame: + acs = transferred_acs( + case["premiums"], + case["wages"], + case["weights"], + households=case["households"], + ages=case["ages"], + first_id=first_id, + ) + return pooled(case["donor"], acs, acs_share=case["acs_share"]) + + +_SETTINGS = settings( + max_examples=60, + deadline=None, + suppress_health_check=[HealthCheck.too_slow, HealthCheck.filter_too_much], +) + + +@given(_lane_cases()) +@_SETTINGS +def test_lane_total_is_the_donor_rows_total_over_their_mass_share(case) -> None: + frame = _lane_frame(case) + donor = donor_rows(frame) + share = household_mass_share(frame, donor) + + anchored, receipt = with_acs_local_esi_premium_anchor(frame) + + # Pooling leaves the donor spine 1 - acs_share of the household mass. + assert share == pytest.approx(1.0 - case["acs_share"], rel=1e-9) + assert receipt["source_household_mass_share"] == pytest.approx(share, rel=1e-12) + expected = employer_total(frame, donor) / share + assert employer_total(anchored) == pytest.approx(expected, rel=1e-9) + assert receipt["employer_premium_total"] == pytest.approx(expected, rel=1e-9) + # Differential: that is the donor release's column at its own weights. + assert employer_total(anchored) == pytest.approx( + employer_total(case["donor"]), rel=1e-9 + ) + + +@given(_lane_cases()) +@_SETTINGS +def test_both_spines_carry_the_same_premium_per_unit_of_household_mass(case) -> None: + frame = _lane_frame(case) + donor = donor_rows(frame) + share = household_mass_share(frame, donor) + + anchored, _receipt = with_acs_local_esi_premium_anchor(frame) + + assert employer_total(anchored, ~donor) / (1.0 - share) == pytest.approx( + employer_total(anchored, donor) / share, rel=1e-9 + ) + + +@given(_lane_cases()) +@_SETTINGS +def test_donor_rows_and_every_other_column_are_untouched(case) -> None: + frame = _lane_frame(case) + donor = donor_rows(frame) + + anchored, _receipt = with_acs_local_esi_premium_anchor(frame) + + before, after = frame.table("person"), anchored.table("person") + assert list(after.columns) == list(before.columns) + pd.testing.assert_frame_equal(after.loc[donor], before.loc[donor]) + untouched = [column for column in before.columns if column != EMPLOYER] + pd.testing.assert_frame_equal(after[untouched], before[untouched]) + for entity in frame.entities: + if entity != "person": + pd.testing.assert_frame_equal(anchored.table(entity), frame.table(entity)) + np.testing.assert_array_equal(person_weights(anchored), person_weights(frame)) + assert anchored.weights_for("household").kind is WeightKind.IMPORTANCE + assert list(anchored.strata) == list(frame.strata) + assert anchored.mass_log == frame.mass_log + + +@given(_lane_cases()) +@_SETTINGS +def test_acs_people_without_wages_carry_nothing_and_the_rest_one_multiple( + case, +) -> None: + frame = _lane_frame(case) + donor = donor_rows(frame) + workers = _column(frame, WAGES) > 0 + + anchored, receipt = with_acs_local_esi_premium_anchor(frame) + + cleared = ~donor & ~workers + assert not _column(anchored, EMPLOYER)[cleared].any() + assert receipt["cleared_rows"][EMPLOYER] == int( + (cleared & (_column(frame, EMPLOYER) > 0)).sum() + ) + scaled = ~donor & workers + np.testing.assert_allclose( + _column(anchored, EMPLOYER)[scaled], + _column(frame, EMPLOYER)[scaled] * receipt["scale_factor"], + rtol=1e-12, + ) + values = _column(anchored, EMPLOYER) + assert np.isfinite(values).all() and (values >= 0).all() + # Every row carries the premium: nothing for a null fill to refuse. + refuse_unassigned_us_esi_premiums(anchored, consumer="lane property") + + +@given(_lane_cases()) +@_SETTINGS +def test_an_anchored_lane_frame_is_returned_unchanged(case) -> None: + anchored, first = with_acs_local_esi_premium_anchor(_lane_frame(case)) + + again, second = with_acs_local_esi_premium_anchor(anchored) + + assert first["status"] in {"scaled", "already_on_anchor"} + assert again is anchored + assert second["status"] == "already_on_anchor" + assert second["employer_premium_total"] == pytest.approx( + first["employer_premium_total"], rel=1e-12 + ) + + +@given(_lane_cases()) +@_SETTINGS +def test_acs_ids_that_repeat_the_donors_change_nothing(case) -> None: + # The pool kernel finds a clone's wages by person_source_id across the + # whole frame, and refuses a frame where source records repeat one. In + # this lane ACS rows keep their pre-remap ids as source ids. + colliding = _lane_frame(case, first_id=1) + disjoint = _lane_frame(case, first_id=_DISJOINT_FIRST_ID) + acs = ~donor_rows(colliding) + assert set(colliding.table("person").loc[acs, "person_source_id"]) & set( + colliding.table("person").loc[~acs, "person_source_id"] + ) + with pytest.raises(SourceRuntimeError, match="repeat a 'person_source_id'"): + with_us_esi_premium_pool_anchor(colliding) + + anchored, receipt = with_acs_local_esi_premium_anchor(colliding) + reference, reference_receipt = with_acs_local_esi_premium_anchor(disjoint) + + np.testing.assert_array_equal( + _column(anchored, EMPLOYER), _column(reference, EMPLOYER) + ) + assert receipt["scale_factor"] == reference_receipt["scale_factor"] + if np.isnan(case["wages"]).any(): + # The kernel alone has no universe rule for a blank ACS wage. + with pytest.raises(SourceRuntimeError, match="missing wage is not a zero"): + with_us_esi_premium_pool_anchor(disjoint) + return + # Differential: where the ids are unique the kernel runs on the whole + # frame, and the lane's narrow view gives the same bytes. + whole, whole_receipt = with_us_esi_premium_pool_anchor(disjoint) + np.testing.assert_array_equal( + _column(reference, EMPLOYER), _column(whole, EMPLOYER) + ) + assert reference_receipt["scale_factor"] == whole_receipt["scale_factor"] + assert reference_receipt["cleared_rows"] == whole_receipt["cleared_rows"] + + +def test_the_worked_lane_frame() -> None: + # Donor release: 40 one-person households cloned once. ACS: three people, + # one without wages. acs_share 0.25 leaves the donor 0.75 of the mass. + donor = _small_donor() + release_total = employer_total(donor) + acs = transferred_acs( + np.asarray([10_000.0, 5_000.0, 8_000.0]), + np.asarray([50_000.0, 0.0, 20_000.0]), + np.asarray([1.0, 1.0, 2.0]), + ) + frame = pooled(donor, acs, acs_share=0.25) + rows = donor_rows(frame) + + anchored, receipt = with_acs_local_esi_premium_anchor(frame) + + assert receipt["source_household_mass_share"] == pytest.approx(0.75, rel=1e-12) + assert employer_total(frame, rows) == pytest.approx(0.75 * release_total, rel=1e-9) + assert receipt["transferred_employer_premium_target"] == pytest.approx( + 0.25 * release_total, rel=1e-9 + ) + assert receipt["cleared_rows"] == {EMPLOYER: 1} + assert employer_total(anchored) == pytest.approx(release_total, rel=1e-9) + premiums = _column(anchored, EMPLOYER)[~rows] + # One factor on the two workers: 10,000 and 8,000 keep their ratio. + assert premiums[1] == 0.0 + assert premiums[0] / premiums[2] == pytest.approx(10_000.0 / 8_000.0, rel=1e-12) + + +def test_a_transferred_support_clone_is_refused() -> None: + frame = pooled( + _small_donor(), + transferred_acs(np.asarray([9_000.0]), np.asarray([30_000.0]), np.ones(1)), + ) + person = frame.table("person").copy() + person.loc[~donor_rows(frame), "person_support_clone_index"] = 1 + with pytest.raises(SourceRuntimeError, match="not clone-0 source records"): + with_acs_local_esi_premium_anchor(_replace_person(frame, person)) + + +@pytest.mark.parametrize("column", [EMPLOYER, "NOW_HIPAID", "PERIDNUM", WAGES]) +def test_a_lane_frame_missing_an_anchor_input_is_refused(column: str) -> None: + frame = pooled( + _small_donor(), + transferred_acs(np.asarray([9_000.0]), np.asarray([30_000.0]), np.ones(1)), + ) + person = frame.table("person").drop(columns=[column]) + with pytest.raises(SourceRuntimeError, match="the person table lacks") as error: + with_acs_local_esi_premium_anchor(_replace_person(frame, person)) + assert column in str(error.value) + + +def test_acs_rows_with_no_premium_on_any_worker_are_refused() -> None: + # Nothing to scale: the anchor cannot put the ACS half on the donor's + # scale, so staging fails rather than shipping half the column. + frame = pooled( + _small_donor(), + transferred_acs( + np.asarray([0.0, 7_000.0]), np.asarray([30_000.0, 0.0]), np.ones(2) + ), + ) + with pytest.raises(SourceRuntimeError, match="nothing to scale"): + with_acs_local_esi_premium_anchor(frame) + + +def test_a_multi_person_household_scales_as_one_unit_of_mass() -> None: + # Three ACS people in one household and one alone: the household's mass + # counts once, and its non-worker is cleared. + donor = _small_donor() + acs = transferred_acs( + np.asarray([6_000.0, 4_000.0, 5_000.0, 3_000.0]), + np.asarray([40_000.0, 25_000.0, 0.0, 30_000.0]), + np.asarray([3.0, 1.0]), + households=np.asarray([0, 0, 0, 1]), + ) + frame = pooled(donor, acs, acs_share=0.4) + rows = donor_rows(frame) + + anchored, receipt = with_acs_local_esi_premium_anchor(frame) + + assert household_mass_share(frame, rows) == pytest.approx(0.6, rel=1e-12) + assert receipt["cleared_rows"] == {EMPLOYER: 1} + assert employer_total(anchored) == pytest.approx(employer_total(donor), rel=1e-9) + premiums = _column(anchored, EMPLOYER)[~rows] + assert premiums[2] == 0.0 + np.testing.assert_allclose( + premiums[[0, 1, 3]] / receipt["scale_factor"], [6_000.0, 4_000.0, 3_000.0] + ) + + +def _lane_with_a_child(**changes) -> Frame: + """Two ACS adults and one nine-year-old whose wage PUMS left blank.""" + + columns = { + "premiums": np.asarray([8_000.0, 5_000.0, 6_000.0]), + "wages": np.asarray([40_000.0, np.nan, 30_000.0]), + "ages": np.asarray([41.0, 9.0, 52.0]), + } | changes + acs = transferred_acs( + columns["premiums"], columns["wages"], np.ones(3), ages=columns["ages"] + ) + return pooled(_small_donor(), acs) + + +def test_a_blank_acs_wage_below_fifteen_is_a_zero_wage_in_the_anchor_only() -> None: + frame = _lane_with_a_child() + acs = ~donor_rows(frame) + + anchored, receipt = with_acs_local_esi_premium_anchor(frame) + + premiums = _column(anchored, EMPLOYER)[acs] + assert premiums[1] == 0.0 and (premiums[[0, 2]] > 0).all() + assert receipt["cleared_rows"] == {EMPLOYER: 1} + assert receipt["wage_universe"] == { + "rule_id": "acs_2024_pums_wagp_age_15_plus", + "minimum_age": 15, + "raw_source_column": "WAGP", + "transferred_rows_with_blank_wage": 1, + "transferred_rows_outside_universe_read_as_zero": 1, + "frame_cells_written": 0, + } + # The staging frame keeps the source's blank: no zero is written to it. + np.testing.assert_array_equal( + _column(anchored, WAGES)[acs], [40_000.0, np.nan, 30_000.0] + ) + assert employer_total(anchored) == pytest.approx( + employer_total(_small_donor()), rel=1e-9 + ) + + +@pytest.mark.parametrize("age", [15.0, 40.0, np.nan]) +def test_a_blank_wage_at_fifteen_or_older_is_refused(age: float) -> None: + # Fifteen is inside the PUMS earnings universe, and an unknown age is not + # evidence of being outside it. + frame = _lane_with_a_child(ages=np.asarray([41.0, age, 52.0])) + with pytest.raises(SourceRuntimeError, match="missing wage is not a zero wage"): + with_acs_local_esi_premium_anchor(frame) + + +def test_a_child_whose_raw_wage_is_not_blank_is_not_read_as_zero() -> None: + # The mapped wage is null but PUMS reported one: that is a mapping fault, + # not the universe rule. + frame = _lane_with_a_child() + person = frame.table("person").copy() + acs = ~donor_rows(frame) + person["WAGP"] = np.where(acs, 1_200.0, np.nan) + with pytest.raises(SourceRuntimeError, match="missing wage is not a zero wage"): + with_acs_local_esi_premium_anchor(_replace_person(frame, person)) + + +def test_a_blank_raw_wage_below_fifteen_corroborates_the_universe_rule() -> None: + frame = _lane_with_a_child() + person = frame.table("person").copy() + acs = ~donor_rows(frame) + person["WAGP"] = np.nan + person.loc[acs, "WAGP"] = [40_000.0, np.nan, 30_000.0] + + _anchored, receipt = with_acs_local_esi_premium_anchor( + _replace_person(frame, person) + ) + + assert ( + receipt["wage_universe"]["transferred_rows_outside_universe_read_as_zero"] == 1 + ) + + +def test_an_untransferred_acs_spine_is_refused_by_the_anchor() -> None: + # A lane frame whose ACS rows never got the premium: null, never zero. + frame = pooled( + _small_donor(), + transferred_acs(np.asarray([np.nan]), np.asarray([30_000.0]), np.ones(1)), + ) + with pytest.raises(SourceRuntimeError, match="the transfer left a gap"): + with_acs_local_esi_premium_anchor(frame) + with pytest.raises(SourceRuntimeError, match="never filled"): + refuse_unassigned_us_esi_premiums(frame, consumer="ACS local release") + + +# --- The staging orchestration, with the real transfer ------------------------ + + +@pytest.fixture(scope="module") +def lane() -> dict[str, object]: + donor = dense_donor() + result = build_lane(donor, raw_acs()) + return {"donor": donor, "result": result, "frame": result.frame} + + +def test_every_lane_row_carries_the_premium(lane) -> None: + frame = lane["frame"] + person = frame.table("person") + acs = person["person_spine"].eq(ACS).to_numpy() + + assert acs.sum() == 280 and (~acs).sum() == 640 + np.testing.assert_array_equal(acs, ~donor_rows(frame)) + assert set(person.loc[~acs, "person_spine"]) == {ASEC_PUF} + values = person[EMPLOYER].to_numpy(dtype=float) + assert np.isfinite(values).all() and (values >= 0).all() + assert (values[acs] > 0).any() and np.unique(values[acs]).size > 2 + # The raw ASEC fields stay null on the ACS rows: they mark the row kind. + assert person.loc[acs, list(RAW)].isna().all().all() + assert person.loc[~acs, list(RAW)].notna().all().all() + # fill_reviewed_nulls' refusal has nothing to refuse. + refuse_unassigned_us_esi_premiums( + frame, consumer="ACS local release (fill_reviewed_nulls)" + ) + + +def test_the_lane_total_is_the_donor_releases_total(lane) -> None: + frame, donor = lane["frame"], lane["donor"] + rows = donor_rows(frame) + anchor = lane["result"].provenance["local_esi_premiums"]["anchor"] + + assert household_mass_share(frame, rows) == pytest.approx(0.5, rel=1e-12) + assert employer_total(frame) == pytest.approx(employer_total(donor), rel=1e-9) + assert employer_total(frame, rows) == pytest.approx( + 0.5 * employer_total(donor), rel=1e-9 + ) + assert employer_total(frame, ~rows) == pytest.approx( + 0.5 * employer_total(donor), rel=1e-9 + ) + assert anchor["status"] == "scaled" + assert anchor["kernel"] == "with_us_esi_premium_pool_anchor" + assert anchor["source_household_mass_share"] == pytest.approx(0.5, rel=1e-12) + assert anchor["employer_premium_total"] == pytest.approx( + employer_total(donor), rel=1e-9 + ) + + +def test_the_donor_spine_is_the_release_byte_for_byte(lane) -> None: + frame, donor = lane["frame"], lane["donor"] + person = frame.table("person") + rows = donor_rows(frame) + + np.testing.assert_array_equal( + person.loc[rows, EMPLOYER].to_numpy(), _column(donor, EMPLOYER) + ) + np.testing.assert_array_equal( + person.loc[rows, WAGES].to_numpy(), _column(donor, WAGES) + ) + + +def test_acs_people_without_wages_carry_no_premium(lane) -> None: + person = lane["frame"].table("person") + acs = ~donor_rows(lane["frame"]) + workers = (person[WAGES] > 0).to_numpy() + anchor = lane["result"].provenance["local_esi_premiums"]["anchor"] + + assert (acs & ~workers).sum() == 112 + assert not person.loc[acs & ~workers, EMPLOYER].any() + assert (person.loc[acs & workers, EMPLOYER] > 0).any() + # The fit saw measured wages, so no draw needed clearing. + assert anchor["transferred_rows_without_source_wages"] == 112 + assert anchor["cleared_rows"] == {EMPLOYER: 0} + assert 0.5 <= anchor["scale_factor"] <= 2.0 + + +def test_the_premium_transfers_from_asec_observations_in_its_own_pass(lane) -> None: + provenance = lane["result"].provenance + entries = [ + item for item in provenance["imputed_inputs"] if item["column"] == EMPLOYER + ] + + assert provenance["fit_configuration"]["esi_premium_donor_channel"] == "asec" + assert provenance["local_esi_premiums"]["source"]["donor_channel"] == "asec" + assert len(entries) == 1 + assert entries[0]["family"] == ACS_LOCAL_ESI_PREMIUM_FAMILY + assert entries[0]["donor_channel"] == "asec" + assert entries[0]["imputed_recipient_rows"] == 280 + assert entries[0]["unmodeled_recipient_rows"] == 0 + assert "__acs_transfer_employment_income" in entries[0]["predictors"] + json.dumps(provenance, allow_nan=False) + + +def test_the_tax_detail_transfer_keeps_its_puf_donor() -> None: + # The lane's other families still fit on the PUF-detail role. + donor = dense_donor(120) + person = donor.table("person").copy() + puf = person["person_support_clone_index"].eq(1).to_numpy() + person["taxable_interest_income"] = np.where(puf, 500.0, 5.0) + np.arange( + len(person) + ) + result = build_lane( + _replace_person(donor, person), + raw_acs(60), + n_estimators=5, + target_families={"person": {"tax_detail": ("taxable_interest_income",)}}, + ) + + configuration = result.provenance["fit_configuration"] + assert configuration["resolved_donor_channel"] == "puf_tax_detail" + assert configuration["esi_premium_donor_channel"] == "asec" + channels = { + item["column"]: item["donor_channel"] + for item in result.provenance["imputed_inputs"] + } + assert channels == { + EMPLOYER: "asec", + "taxable_interest_income": "puf_tax_detail", + } + acs = ~donor_rows(result.frame) + assert ( + result.frame.table("person").loc[acs, "taxable_interest_income"] >= 500 + ).all() + + +def test_acs_children_get_no_premium_and_keep_their_blank_wage() -> None: + # The real orchestration on an ACS spine with children: PUMS leaves their + # WAGP blank, the native mapping keeps the blank, and the anchor reads it + # as zero wages without writing a zero into the staging frame. + donor = dense_donor(120) + result = build_lane(donor, raw_acs(100, children=20), n_estimators=5) + person = result.frame.table("person") + acs = ~donor_rows(result.frame) + child = acs & (person["age"] < 15).to_numpy() + anchor = result.provenance["local_esi_premiums"]["anchor"] + + assert child.sum() == 20 + assert person.loc[child, WAGES].isna().all() + assert person.loc[child, "WAGP"].isna().all() + assert not person.loc[child, EMPLOYER].any() + assert anchor["wage_universe"]["transferred_rows_with_blank_wage"] == 20 + assert ( + anchor["wage_universe"]["transferred_rows_outside_universe_read_as_zero"] == 20 + ) + assert employer_total(result.frame) == pytest.approx( + employer_total(donor), rel=1e-9 + ) + signal, anchor_gate = acs_local_esi_premium_gates(result.frame, time_period=2024) + assert signal.passed, signal.failures + assert anchor_gate.passed, anchor_gate.failures + refuse_unassigned_us_esi_premiums(result.frame, consumer="ACS local release") + + +def test_no_factory_leaves_the_orchestration_as_it_was() -> None: + # A caller that does not ask for the premium gets none: no pass, no anchor. + donor = dense_donor(80) + with pytest.MonkeyPatch.context() as patch: + patch.setattr( + acs_multispine, + "build_acs_pums_unit_frame", + lambda *a, **k: (raw_acs(40), {}), + ) + result = acs_multispine.build_optional_acs_multispine( + donor, + AcsPumsSource(Path("unused-hus.zip"), Path("unused-pus.zip")), + target_families={}, + ) + + assert "local_esi_premiums" not in result.provenance + assert "esi_premium_donor_channel" not in result.provenance["fit_configuration"] + acs = ~donor_rows(result.frame) + assert result.frame.table("person").loc[acs, EMPLOYER].isna().all() + + +# --- Which donor role, and what it does to the structural zero ---------------- + + +def _acs_draws(donor: Frame, channel: str) -> tuple[pd.Series, pd.Series]: + mapped = map_acs_native_inputs(raw_acs()).frame + person = transfer_acs_inputs( + mapped, + donor, + target_families=acs_local_esi_premium_transfer_target_families(), + donor_channel=channel, + seed=0, + n_estimators=20, + ).frame.table("person") + return person[EMPLOYER], person[WAGES] + + +def test_the_puf_detail_role_would_put_premiums_on_people_without_wages() -> None: + # On PUF-detail rows the premium is a copy of the source person's and the + # wages are PUF-imputed, so wages no longer say who holds a job. + donor = dense_donor() + donor_person = donor.table("person") + puf = donor_person["person_support_clone_index"].eq(1).to_numpy() + premium, wages = _column(donor, EMPLOYER), _column(donor, WAGES) + assert not ((premium > 0) & ~(wages > 0))[~puf].any() + assert ((premium > 0) & ~(wages > 0))[puf].any() + + asec_premium, asec_wages = _acs_draws(donor, "asec") + puf_premium, puf_wages = _acs_draws(donor, "puf_tax_detail") + + def share_without_wages_drawing_a_premium(drawn, measured) -> float: + idle = ~(measured > 0) + return float((drawn[idle] > 0).mean()) + + def share_of_workers_drawing_a_premium(drawn, measured) -> float: + return float((drawn[measured > 0] > 0).mean()) + + # ASEC observations: the fit learns the structural zero from measured wages. + assert share_without_wages_drawing_a_premium(asec_premium, asec_wages) == 0.0 + # PUF-detail role: people without wages draw premiums, and workers draw + # fewer, because the fit cannot tell the two apart. + assert share_without_wages_drawing_a_premium(puf_premium, puf_wages) > 0.1 + assert share_of_workers_drawing_a_premium( + puf_premium, puf_wages + ) < share_of_workers_drawing_a_premium(asec_premium, asec_wages) + + +# --- Both gates on the lane frame --------------------------------------------- + + +def test_both_gates_pass_on_the_lane_frame(lane) -> None: + frame, donor = lane["frame"], lane["donor"] + + signal, anchor = acs_local_esi_premium_gates(frame, time_period=2024) + + assert signal.passed, signal.failures + assert anchor.passed, anchor.failures + assert signal.details["source_rows"] == 640 + assert signal.details["transferred_rows"] == 280 + assert signal.details["employer_premium_outside_universe_rows"] == 0 + assert signal.details["clone_disagreement_source_persons"] == 0 + assert signal.details["transferred_source_records_with_premium_and_no_wages"] == 0 + # The donor ran the stage at the whole anchor, and the lane keeps its + # anchor-universe total and its scale factor. + alone = us_esi_premiums_anchor_gate(donor, time_period=2024) + assert alone.passed, alone.failures + assert anchor.details["anchor_universe_employer_total"] == pytest.approx( + ANCHOR, rel=1e-9 + ) + assert anchor.details["relative_error"] == pytest.approx( + alone.details["relative_error"], abs=1e-9 + ) + assert anchor.details["scale_factor"] == pytest.approx( + alone.details["scale_factor"], rel=1e-9 + ) + assert anchor.details["source_household_mass_share"] == pytest.approx( + 0.5, rel=1e-12 + ) + assert anchor.details[ + "transferred_to_source_per_household_mass_ratio" + ] == pytest.approx(1.0, rel=1e-9) + assert "employed share" in anchor.details["anchor_universe_assumption"] + + +def test_the_narrow_gate_view_gives_the_whole_frame_verdicts(lane) -> None: + # Differential: the gates read a few columns but copy every table, so the + # lane hands them a narrow view. It must not change a verdict or a detail. + frame = lane["frame"] + + signal, anchor = acs_local_esi_premium_gates(frame, time_period=2024) + whole_signal = us_esi_premiums_signal_gate(frame) + whole_anchor = us_esi_premiums_anchor_gate(frame, time_period=2024) + + for narrow, whole in ((signal, whole_signal), (anchor, whole_anchor)): + assert narrow.name == whole.name + assert narrow.passed is whole.passed + assert narrow.failures == whole.failures + assert json.dumps(dict(narrow.details), sort_keys=True, default=str) == ( + json.dumps(dict(whole.details), sort_keys=True, default=str) + ) + + +def _with_acs_premium(frame: Frame, transform) -> Frame: + person = frame.table("person").copy() + acs = ~donor_rows(frame) + person.loc[acs, EMPLOYER] = transform(person.loc[acs, EMPLOYER]) + return _replace_person(frame, person) + + +def test_an_acs_spine_without_the_premium_fails_the_signal_gate(lane) -> None: + zeroed = _with_acs_premium(lane["frame"], lambda values: values * 0.0) + + signal, _anchor = acs_local_esi_premium_gates(zeroed, time_period=2024) + + assert not signal.passed + assert any( + "rows without raw ASEC columns" in failure for failure in signal.failures + ) + + +@pytest.mark.parametrize("factor", [0.7, 1.4]) +def test_an_acs_spine_off_the_donor_rows_scale_fails_the_anchor_gate( + lane, factor: float +) -> None: + scaled = _with_acs_premium(lane["frame"], lambda values: values * factor) + + _signal, anchor = acs_local_esi_premium_gates(scaled, time_period=2024) + + assert not anchor.passed + assert any("per unit of household mass" in failure for failure in anchor.failures) + assert anchor.details[ + "transferred_to_source_per_household_mass_ratio" + ] == pytest.approx(factor, rel=1e-9) + + +def test_a_lane_frame_from_a_donor_without_the_stage_fails_both_gates() -> None: + # A waived diagnostic build: no row carries the column at all. + acs = transferred_acs(np.asarray([9_000.0]), np.asarray([30_000.0]), np.ones(1)) + frame = pooled( + _donor_without(EMPLOYER), + _replace_person(acs, acs.table("person").drop(columns=[EMPLOYER])), + ) + assert EMPLOYER not in frame.table("person") + + signal, anchor = acs_local_esi_premium_gates(frame, time_period=2024) + + assert not signal.passed and not anchor.passed + assert signal.failures == (f"person column missing: {EMPLOYER}.",) + assert anchor.failures == (f"person column missing: {EMPLOYER}.",) + + +def test_moving_mass_between_spines_moves_neither_the_total_nor_the_ratio( + lane, +) -> None: + # Calibration moves mass between spines. Both carry the same premium per + # unit of household mass, so a shift that conserves the frame's mass + # leaves the lane total where it was. + frame = lane["frame"] + household = frame.table("household") + acs_household = household["household_spine"].eq(ACS).to_numpy() + weights = np.asarray(frame.weights_for("household").values, dtype=float).copy() + weights[acs_household] *= 1.3 + weights *= frame.weights_for("household").total / weights.sum() + moved = Frame( + {entity: frame.table(entity) for entity in frame.entities}, + frame.schema, + {"household": Weights(weights, WeightKind.CALIBRATED)}, + frame.strata, + ) + + signal, anchor = acs_local_esi_premium_gates(moved, time_period=2024) + + assert signal.passed, signal.failures + assert anchor.passed, anchor.failures + assert anchor.details[ + "transferred_to_source_per_household_mass_ratio" + ] == pytest.approx(1.0, rel=1e-9) + assert employer_total(moved) == pytest.approx(employer_total(frame), rel=1e-9) + + +def test_a_gate_payload_is_strict_json_and_keeps_failures_verbatim(lane) -> None: + signal, anchor = acs_local_esi_premium_gates( + _with_acs_premium(lane["frame"], lambda values: values * 0.0), + time_period=2024, + ) + + payload = acs_local_esi_premium_gate_payload(signal) + + assert payload["name"] == "esi_premiums_signal" + assert payload["passed"] is False + assert payload["failures"] == list(signal.failures) + json.dumps(payload, allow_nan=False) + json.dumps(acs_local_esi_premium_gate_payload(anchor), allow_nan=False) + + +@requires_pytables +def test_the_staging_h5_round_trip_keeps_the_verdicts(lane, tmp_path) -> None: + # Finalize grades the calibrated H5 the lane wrote, not the frame in memory. + frame = lane["frame"] + path = tmp_path / "acs_multispine_staging.h5" + + write_nullable_us_h5( + frame, path, period=2024, artifact_kind=LEGACY_NULLABLE_STAGING_ARTIFACT_KIND + ) + loaded = load_legacy_calibrated_us_h5(path) + + signal, anchor = acs_local_esi_premium_gates(loaded, time_period=2024) + assert signal.passed, signal.failures + assert anchor.passed, anchor.failures + assert employer_total(loaded) == pytest.approx(employer_total(frame), rel=1e-12) + np.testing.assert_array_equal(donor_rows(loaded), donor_rows(frame)) + refuse_unassigned_us_esi_premiums(loaded, consumer="ACS local release") + again, receipt = with_acs_local_esi_premium_anchor(loaded) + assert again is loaded and receipt["status"] == "already_on_anchor" diff --git a/packages/microcosm-build/tests/engine_free/us/test_us_spine_blindness.py b/packages/microcosm-build/tests/engine_free/us/test_us_spine_blindness.py index 2ee2cc8bb..ad9c53b07 100644 --- a/packages/microcosm-build/tests/engine_free/us/test_us_spine_blindness.py +++ b/packages/microcosm-build/tests/engine_free/us/test_us_spine_blindness.py @@ -235,6 +235,12 @@ # Exact source-universe validator/receipt owner; no population treatment. "acs_income_universe.py", "acs_inputs.py", + # Legacy local-lane employer premium (#454): donor qualification, a + # one-family transfer plan and narrow views for the pool anchor and + # the two ESI gates. It selects donors by support role and tells the + # rows apart by whether the raw ASEC fields are present, never by a + # spine or support-channel column, so it owns no provenance boundary. + "acs_local_esi_premiums.py", # Legacy source-hours completion and release gate; outside the registry. "acs_local_hours.py", "acs_multispine.py", diff --git a/test_support/microcosm_build/us_acs_local_esi_premiums.py b/test_support/microcosm_build/us_acs_local_esi_premiums.py new file mode 100644 index 000000000..d71b26618 --- /dev/null +++ b/test_support/microcosm_build/us_acs_local_esi_premiums.py @@ -0,0 +1,325 @@ +"""Fixtures for the ACS local lane's ESI employer premium (microcosm #454). + +The lane pools a dense ASEC-by-PUF donor release with the ACS spine. These +builders make both sides small: a donor the ``meps_esi_premiums`` stage ran on +before the PUF clone, and an ACS frame as the PUMS loader hands it over. +""" + +from __future__ import annotations + +from pathlib import Path + +import numpy as np +import pandas as pd +import pytest + +from microcosm.build.us_runtime import acs_multispine +from microcosm.build.us_runtime.acs_local_esi_premiums import ( + prepare_acs_local_esi_premium_donor, +) +from microcosm.build.us_runtime.acs_multispine import AcsMultispineResult +from microcosm.build.us_runtime.acs_pums import AcsPumsSource +from microcosm.build.us_runtime.base_pool import with_optional_acs_spine +from microcosm.build.us_runtime.esi_premiums import ( + US_ESI_EMPLOYER_PREMIUM_COLUMN, + US_ESI_PREMIUMS_WAGE_COLUMN, + with_us_esi_premium_inputs, +) +from microcosm.build.us_runtime.puf_support import clone_us_frame_for_puf_support +from microcosm.frame import US_SCHEMA, Frame, WeightKind, Weights + +EMPLOYER = US_ESI_EMPLOYER_PREMIUM_COLUMN +WAGES = US_ESI_PREMIUMS_WAGE_COLUMN +ASEC_PUF = "asec_puf" +ACS = "acs_2024_1yr" +_STATES = (6, 36, 48, 12, 17, 39, 53, 13) +_GROUP_OFFSETS = { + "tax_unit": 100_000, + "spm_unit": 200_000, + "family": 300_000, + "marital_unit": 400_000, +} + + +def _one_person_households( + person: pd.DataFrame, + household: pd.DataFrame, + weights: np.ndarray, + *, + stratum: str, + kind: WeightKind, +) -> Frame: + tables = {"person": person, "household": household} + for group, offset in _GROUP_OFFSETS.items(): + person[f"person_{group}_id"] = person["person_household_id"] + offset + tables[group] = pd.DataFrame( + {f"{group}_id": household["household_id"] + offset} + ) + return Frame( + tables, + US_SCHEMA, + {"household": Weights(np.asarray(weights, dtype=np.float64), kind)}, + pd.Series([stratum] * len(person), dtype=object), + ) + + +def asec_observations(households: int = 320) -> Frame: + """One-person ASEC households with the raw fields the stage reads. + + Of every 20 people, 7 are employed policyholders, 2 are policyholders + without a job, 5 are employed without a policy and 6 are neither. Wages + are the measured wages: positive exactly for the employed. Age and sex do + not tell the employed from the rest, so only wages can. + """ + + index = np.arange(households) + kind = index % 20 + holder = kind < 9 + employed = (kind < 7) | ((kind >= 9) & (kind < 14)) + tier = np.where(holder, np.asarray([1, 2, 3])[index % 3], 0) + person = pd.DataFrame( + { + "person_id": index + 1, + "person_household_id": index + 1, + # The CPS record id: with the raw fields, it marks a row the + # stage derived. + "PERIDNUM": [f"asec-{position:05d}" for position in index], + "age": 24.0 + index % 41, + "is_female": index % 2 == 0, + WAGES: np.where(employed, 28_000.0 + 850.0 * (index % 97), 0.0), + "NOW_OWNGRP": np.where(holder, 1, 2), + "NOW_HIPAID": np.where(holder, np.asarray([1, 2, 2, 3])[index % 4], 0), + "NOW_GRPFTYP2": tier, + "NOW_GRPFTYP": np.asarray([0, 1, 1, 2])[tier], + "PEMLR": np.where(employed, 1, 5), + "NOEMP": np.where(employed, np.asarray([1, 3, 6, 0])[index % 4], 0), + "PEIO1COW": np.where( + employed, np.asarray([4, 4, 5, 2, 3, 1])[index % 6], 0 + ), + "source_year": 2024, + "source_household_id": index + 1, + "source_person_id": 1, + } + ) + household = pd.DataFrame( + { + "household_id": index + 1, + "state_fips": np.asarray(_STATES, dtype=np.int64)[index % len(_STATES)], + } + ) + return _one_person_households( + person, + household, + 900.0 + 37.0 * (index % 11), + stratum=ASEC_PUF, + kind=WeightKind.DESIGN, + ) + + +def dense_donor(households: int = 320, *, puf_wage_seed: int = 5) -> Frame: + """A dense ASEC-by-PUF donor release that ran the ``meps_esi_premiums`` stage. + + The stage assigns one premium per source person; the PUF clone copies it. + The clone's wages are then replaced, as PUF imputation replaces them: a + seeded permutation, so a PUF-detail row's wages no longer say whether its + source person holds a job. Weights are calibrated, as a release's are. + """ + + staged = with_us_esi_premium_inputs( + asec_observations(households), seed=0, time_period=2024 + ) + cloned = clone_us_frame_for_puf_support(staged) + person = cloned.table("person").copy() + puf = person["person_support_clone_index"].to_numpy() == 1 + rng = np.random.default_rng(puf_wage_seed) + person.loc[puf, WAGES] = rng.permutation(person.loc[puf, WAGES].to_numpy()) + return Frame( + { + entity: person if entity == "person" else cloned.table(entity) + for entity in cloned.entities + }, + US_SCHEMA, + { + "household": Weights( + np.asarray(cloned.weights_for("household").values, dtype=np.float64), + WeightKind.CALIBRATED, + ) + }, + cloned.strata, + ) + + +def raw_acs(households: int = 280, *, first_id: int = 1, children: int = 0) -> Frame: + """One-person ACS households as the PUMS loader returns them. + + Three in five adults report wages, at any age. ``first_id`` defaults to 1 + so the ACS ids repeat the donor's, as the two real id spaces may. + ``children`` appends that many people aged 9, whose ``WAGP`` is blank: + PUMS asks earnings of people aged 15 and older. + """ + + index = np.arange(households + children) + child = index >= households + employed = (index % 5 < 3) & ~child + person = pd.DataFrame( + { + "person_id": index + first_id, + "person_household_id": index + first_id, + "AGEP": np.where(child, 9, 23 + index % 43), + "SEX": np.where(index % 2 == 1, 2, 1), + "ADJINC": 1_000_000, + "WAGP": np.where( + child, + np.nan, + np.where(employed, 26_000.0 + 1_100.0 * (index % 83), 0.0), + ), + } + ) + household = pd.DataFrame( + { + "household_id": index + first_id, + "state_fips": np.asarray(_STATES, dtype=np.int64)[ + (index + 3) % len(_STATES) + ], + "TYPEHUGQ": 1, + } + ) + return _one_person_households( + person, + household, + 40.0 + 3.0 * (index % 13), + stratum=ACS, + kind=WeightKind.DESIGN, + ) + + +def transferred_acs( + premiums: np.ndarray, + wages: np.ndarray, + weights: np.ndarray, + *, + households: np.ndarray | None = None, + ages: np.ndarray | None = None, + first_id: int = 1, +) -> Frame: + """An ACS frame after the lane's transfers. + + ``premiums`` stands for whatever the ESI transfer drew, so a property can + range over every draw without fitting a forest. ``households`` gives each + person's household position (nondecreasing, starting at 0; one person per + household by default) and ``weights`` one weight per household. ``ages`` + defaults to adults; a NaN wage is a blank the source left. + """ + + index = np.arange(len(premiums)) + members = index if households is None else np.asarray(households, dtype=np.int64) + count = int(members.max()) + 1 + person = pd.DataFrame( + { + "person_id": index + first_id, + "person_household_id": members + first_id, + "age": ( + 30.0 + index % 40 if ages is None else np.asarray(ages, dtype=float) + ), + "is_female": index % 2 == 0, + WAGES: np.asarray(wages, dtype=np.float64), + EMPLOYER: np.asarray(premiums, dtype=np.float64), + } + ) + household_index = np.arange(count) + household = pd.DataFrame( + { + "household_id": household_index + first_id, + "state_fips": np.asarray(_STATES, dtype=np.int64)[ + household_index % len(_STATES) + ], + } + ) + return _one_person_households( + person, household, weights, stratum=ACS, kind=WeightKind.DESIGN + ) + + +def reweighted(frame: Frame, factor: float | np.ndarray) -> Frame: + """The frame with its household weights multiplied by ``factor``. + + ``factor`` is one number for every household or one per household. + """ + + weights = frame.weights_for("household") + return Frame( + {entity: frame.table(entity) for entity in frame.entities}, + frame.schema, + { + "household": Weights( + np.asarray(weights.values, dtype=np.float64) * factor, weights.kind + ) + }, + frame.strata, + metadata=frame.metadata, + ) + + +def pooled(donor: Frame, acs: Frame, *, acs_share: float = 0.5) -> Frame: + """The lane's pooled frame before the anchor.""" + + return with_optional_acs_spine(donor, acs, acs_share=acs_share) + + +def donor_rows(frame: Frame) -> np.ndarray: + """People whose premium the stage derived: they carry the CPS record id.""" + + return frame.table("person")["PERIDNUM"].notna().to_numpy() + + +def person_weights(frame: Frame) -> np.ndarray: + return np.asarray(frame.resolve_weights("person").values, dtype=np.float64) + + +def employer_total(frame: Frame, rows: np.ndarray | None = None) -> float: + """Weighted employer premium over ``rows`` (every person by default).""" + + weights = person_weights(frame) + values = frame.table("person")[EMPLOYER].to_numpy(dtype=np.float64) + rows = np.ones(len(values), dtype=bool) if rows is None else rows + return float(weights[rows] @ values[rows]) + + +def household_mass_share(frame: Frame, rows: np.ndarray) -> float: + """Share of household mass in the households of ``rows``.""" + + household = frame.table("household") + members = frame.table("person").loc[rows, "person_household_id"] + selected = household["household_id"].isin(members).to_numpy() + weights = np.asarray(frame.weights_for("household").values, dtype=np.float64) + return float(weights[selected].sum() / weights.sum()) + + +def build_lane( + donor: Frame, + acs: Frame, + *, + acs_share: float = 0.5, + n_estimators: int = 20, + seed: int = 0, + target_families: dict | None = None, +) -> AcsMultispineResult: + """Run the lane's staging orchestration on fixture frames. + + Only the PUMS loader is replaced; native mapping, the ESI transfer, the + pooling and the anchor are the production code. + """ + + with pytest.MonkeyPatch.context() as patch: + patch.setattr( + acs_multispine, "build_acs_pums_unit_frame", lambda *a, **k: (acs, {}) + ) + return acs_multispine.build_optional_acs_multispine( + donor, + AcsPumsSource(Path("unused-hus.zip"), Path("unused-pus.zip")), + acs_share=acs_share, + target_families=target_families or {}, + esi_premium_donor_factory=prepare_acs_local_esi_premium_donor, + seed=seed, + n_estimators=n_estimators, + ) From 5b4d434fe187dbc4afa2007a6a276e3ad4c4cf7c Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Sat, 10 Oct 2026 19:58:50 -0400 Subject: [PATCH 2/5] Grade the ESI employer premium at staging, finalize and package in the ACS local lane (#454) Staging qualifies the donor before the ACS sources are fetched and runs both ESI premium gates on the pooled frame. Finalize runs them on the calibrated artifact and records esi_premiums_signal and esi_premiums_anchor bound to the artifact bytes; input_coverage passes only if both do, where it was a fixed pass. Package refuses a report without a passing verdict for the bytes it packages. --allow-esi-premium-gaps records a red verdict for a diagnostic build, and the Modal stage plan forwards it. Co-Authored-By: Claude Opus 5.5 --- .../us/test_us_acs_local_release_tool.py | 110 ++++++ .../us/test_us_acs_local_release_tool.py | 330 ++++++++++++++++-- .../test_us_acs_multispine_legacy_builder.py | 180 +++++++++- .../us/test_us_modal_stage_plan_tool.py | 18 + .../us_acs_local_release_tool.py | 34 ++ tools/_legacy/build_us_acs_multispine_base.py | 109 +++++- tools/build_us_acs_local_release.py | 130 ++++++- tools/modal_us_stage_plan.py | 3 + 8 files changed, 872 insertions(+), 42 deletions(-) create mode 100644 packages/microcosm-build/tests/engine_contract/us/test_us_acs_local_release_tool.py create mode 100644 test_support/microcosm_build/us_acs_local_release_tool.py diff --git a/packages/microcosm-build/tests/engine_contract/us/test_us_acs_local_release_tool.py b/packages/microcosm-build/tests/engine_contract/us/test_us_acs_local_release_tool.py new file mode 100644 index 000000000..f9488bd36 --- /dev/null +++ b/packages/microcosm-build/tests/engine_contract/us/test_us_acs_local_release_tool.py @@ -0,0 +1,110 @@ +"""The ACS local release tool's engine pass on a lane frame (microcosm #454). + +``fill_reviewed_nulls`` applies the nullable-artifact contract with the +installed engine's input names and defaults. It refuses a frame whose ESI +employer premium is null on some rows, which is what an ACS spine pooled +beside a donor used to carry. The lane now transfers and anchors the column +at staging, so the engine pass must run on its frame. +""" + +from __future__ import annotations + +import json + +import numpy as np +import pytest + +from microcosm.build.source_runtime import SourceRuntimeError +from microcosm.frame import Frame +from test_support.microcosm_build.us_acs_local_esi_premiums import ( + EMPLOYER, + build_lane, + dense_donor, + donor_rows, + raw_acs, +) +from test_support.microcosm_build.us_acs_local_release_tool import ( + load_staging_builder_module, + load_tool_module, +) + +#: An input the donor release carries and no transfer in this fixture gives +#: the ACS rows: the kind of reviewed null the engine pass exists to fill. +_REVIEWED_NULL = "pre_subsidy_rent" + + +def _with_person(frame: Frame, person) -> Frame: + return Frame( + { + entity: person if entity == "person" else frame.table(entity) + for entity in frame.entities + }, + frame.schema, + {"household": frame.weights_for("household")}, + frame.strata, + ) + + +@pytest.fixture(scope="module") +def staged() -> Frame: + """A staged lane frame with one reviewed engine-input null on ACS rows.""" + + frame = build_lane(dense_donor(120), raw_acs(100), n_estimators=5).frame + person = frame.table("person").copy() + person[_REVIEWED_NULL] = np.where(donor_rows(frame), 9_600.0, np.nan) + return _with_person(frame, person) + + +def _summary(tmp_path, frame: Frame): + """The staging summary's null register, as the staging tool audits it.""" + + audit = load_staging_builder_module()._legacy._engine_input_null_audit(frame) + path = tmp_path / "acs_multispine_staging.summary.json" + path.write_text(json.dumps({"reviewed_engine_input_nulls": audit})) + return path, audit + + +def test_the_engine_pass_runs_on_the_lane_frame_and_leaves_the_premium( + tmp_path, staged +) -> None: + tool = load_tool_module() + summary_path, audit = _summary(tmp_path, staged) + # Every row carries the premium, so the staging audit registers no null + # for it: there is nothing for the engine pass to fill or to refuse. + assert EMPLOYER not in {entry["column"] for entry in audit} + assert {entry["column"] for entry in audit} == {_REVIEWED_NULL} + before = staged.table("person")[EMPLOYER].to_numpy(copy=True) + + projected, _dropped = tool.project_input_only(staged) + fills, drift = tool.fill_reviewed_nulls(projected, summary_path) + + person = projected.table("person") + assert [fill["column"] for fill in fills] == [_REVIEWED_NULL] + assert fills[0]["filled_rows"] == 100 + assert fills[0]["missing_rows_by_spine"] == {"acs_2024_1yr": 100} + assert drift == [] + assert person[_REVIEWED_NULL].notna().all() + # The engine knows the premium as an input and the pass did not touch it. + assert EMPLOYER in person.columns + np.testing.assert_array_equal(person[EMPLOYER].to_numpy(), before) + assert (person.loc[~donor_rows(staged), EMPLOYER] > 0).any() + + +def test_the_engine_pass_still_refuses_an_untransferred_acs_spine( + tmp_path, staged +) -> None: + # A staging H5 built before the lane transferred the column: null on ACS + # rows, and registered as a reviewed null by that build's audit. + tool = load_tool_module() + person = staged.table("person").copy() + person.loc[~donor_rows(staged), EMPLOYER] = np.nan + stale = _with_person(staged, person) + summary_path, audit = _summary(tmp_path, stale) + assert EMPLOYER in {entry["column"] for entry in audit} + + with pytest.raises(SourceRuntimeError, match="never filled") as error: + tool.fill_reviewed_nulls(stale, summary_path) + + assert f"'{EMPLOYER}': 100" in str(error.value) + # Refused before any fill: the registered null was not defaulted to zero. + assert stale.table("person")[EMPLOYER].isna().sum() == 100 diff --git a/packages/microcosm-build/tests/engine_free/us/test_us_acs_local_release_tool.py b/packages/microcosm-build/tests/engine_free/us/test_us_acs_local_release_tool.py index b2fd944d7..f0d037f60 100644 --- a/packages/microcosm-build/tests/engine_free/us/test_us_acs_local_release_tool.py +++ b/packages/microcosm-build/tests/engine_free/us/test_us_acs_local_release_tool.py @@ -20,6 +20,10 @@ US_ORG_OCCUPATION_AND_WORK_EXPERIENCE_OUTPUT_COLUMNS, ) from microcosm.data import stored_inputs +from test_support.microcosm_build.us_acs_local_release_tool import ( + load_staging_builder_module, + load_tool_module, +) from test_support.paths import paths_for _TEST_PATHS = paths_for("microcosm-build") @@ -126,30 +130,42 @@ def _spm_measurement_gate_passes(monkeypatch): ) -def _load_tool_module(): - root = _TEST_PATHS.repository - path = root / "tools" / "build_us_acs_local_release.py" - spec = importlib.util.spec_from_file_location( - "build_us_acs_local_release", - path, - ) - module = importlib.util.module_from_spec(spec) - assert spec.loader is not None - spec.loader.exec_module(module) - return module +_load_tool_module = load_tool_module +_load_staging_builder_module = load_staging_builder_module + +def _passing_esi_premium_gates(artifact_sha: str) -> dict[str, dict]: + """The finalize report's ESI premium entries (microcosm#454), green and + bound to ``artifact_sha``: what the package stage requires before it + reaches the checks the other package tests are about.""" -def _load_staging_builder_module(): - root = _TEST_PATHS.repository - path = root / "tools" / "build_us_acs_multispine_base.py" - spec = importlib.util.spec_from_file_location( - "build_us_acs_multispine_base", - path, + return { + name: { + "passed": True, + "failures": [], + "detail": {}, + "artifact_sha256": artifact_sha, + "enforced": True, + } + for name in ("esi_premiums_signal", "esi_premiums_anchor") + } + + +def _stub_esi_premium_gates( + module, monkeypatch, *, signal: tuple = (), anchor: tuple = () +) -> None: + """Fix the lane's ESI premium verdict; its own tests cover the grading.""" + + from microcosm.build.gates import GateResult + + monkeypatch.setattr( + module, + "acs_local_esi_premium_gates", + lambda frame, *, time_period: ( + GateResult(name="esi_premiums_signal", passed=not signal, failures=signal), + GateResult(name="esi_premiums_anchor", passed=not anchor, failures=anchor), + ), ) - module = importlib.util.module_from_spec(spec) - assert spec.loader is not None - spec.loader.exec_module(module) - return module def test_household_chunks_refuse_population_aggregates(monkeypatch, tmp_path) -> None: @@ -628,6 +644,7 @@ def _package_evidence_args(module, tmp_path: Path, monkeypatch, *, hours_report) "detail": {}, "artifact_sha256": artifact_sha, }, + **_passing_esi_premium_gates(artifact_sha), } if hours_report is not None: gates["acs_local_hours_signal"] = hours_report @@ -1392,8 +1409,13 @@ def _stub_local_hours_gate(module, monkeypatch) -> None: ) -def _patch_finalize_collaborators(module, monkeypatch, frame=None, *, identity=True): - """Stub the ladder gate and composition; ``frame=None`` loads real bytes.""" +def _patch_finalize_collaborators( + module, monkeypatch, frame=None, *, identity=True, esi_premiums=True +): + """Stub the ladder gate and composition; ``frame=None`` loads real bytes. + + ``esi_premiums=False`` leaves the real ESI premium gates in place. + """ import microcosm.build.us_runtime.puma_ladder as puma from microcosm.build.gates import GateResult @@ -1408,6 +1430,8 @@ def _patch_finalize_collaborators(module, monkeypatch, frame=None, *, identity=T ) monkeypatch.setattr(module, "spine_composition", lambda *a, **k: {}) _stub_local_hours_gate(module, monkeypatch) + if esi_premiums: + _stub_esi_premium_gates(module, monkeypatch) if frame is not None: monkeypatch.setattr(module, "_load_staging_frame", lambda *a, **k: frame) if identity: @@ -1418,8 +1442,8 @@ def _patch_finalize_collaborators(module, monkeypatch, frame=None, *, identity=T ) -def _run_finalize(module, monkeypatch, args, frame=None): - _patch_finalize_collaborators(module, monkeypatch, frame) +def _run_finalize(module, monkeypatch, args, frame=None, *, esi_premiums=True): + _patch_finalize_collaborators(module, monkeypatch, frame, esi_premiums=esi_premiums) with pytest.raises(SystemExit) as exc: module.do_finalize(args) report = ( @@ -1717,11 +1741,9 @@ def _package_args_with_hours(module, tmp_path, monkeypatch, *, gate_state): "detail": {}, "artifact_sha256": artifact_sha, } - gates = ( - {} - if gate_state == "missing" - else {"hours_worked_signal": gate, "acs_local_hours_signal": local_gate} - ) + gates = _passing_esi_premium_gates(artifact_sha) + if gate_state != "missing": + gates |= {"hours_worked_signal": gate, "acs_local_hours_signal": local_gate} args.gate_report.write_text(json.dumps({"gates": gates})) args.out_summary.write_text(json.dumps({"simulation_ready": True})) evidence = { @@ -2917,3 +2939,251 @@ def test_a_null_esi_premium_input_is_refused_before_any_default_fill(tmp_path) - module.fill_reviewed_nulls(frame, absent_register) assert "ACS local release" in str(error.value) assert "'employer_sponsored_insurance_premiums': 2" in str(error.value) + + +# --- microcosm#454: the lane's own verdict on the ESI employer premium -------- + + +@pytest.fixture(scope="module") +def esi_lane_frame(): + """A staged lane frame: the real ESI transfer, pooling and anchor.""" + + from test_support.microcosm_build.us_acs_local_esi_premiums import ( + build_lane, + dense_donor, + raw_acs, + ) + + return build_lane(dense_donor(), raw_acs()).frame + + +def _with_acs_premium_scaled(frame, factor: float): + """The lane frame with every ACS row's premium multiplied by ``factor``.""" + + from microcosm.frame import Frame + + person = frame.table("person").copy() + acs = person["person_spine"].eq("acs_2024_1yr") + person.loc[acs, "employer_sponsored_insurance_premiums"] *= factor + return Frame( + { + entity: person if entity == "person" else frame.table(entity) + for entity in frame.entities + }, + frame.schema, + {"household": frame.weights_for("household")}, + frame.strata, + ) + + +def test_finalize_grades_the_esi_premium_on_the_calibrated_artifact( + tmp_path, monkeypatch, esi_lane_frame +) -> None: + """Both gates run on the artifact and their verdict replaces the + hard-coded input-coverage pass.""" + + module = _load_tool_module() + args = _finalize_args(module, tmp_path) + artifact_sha = module._sha256(args.out_h5) + + message, report = _run_finalize( + module, monkeypatch, args, esi_lane_frame, esi_premiums=False + ) + + gates = report["gates"] + for name in module.ESI_PREMIUM_GATE_NAMES: + assert gates[name]["passed"] is True, gates[name]["failures"] + assert gates[name]["failures"] == [] + assert gates[name]["artifact_sha256"] == artifact_sha + assert gates[name]["enforced"] is True + assert name not in message + signal, anchor = (gates[name]["detail"] for name in module.ESI_PREMIUM_GATE_NAMES) + assert signal["source_rows"] == 640 and signal["transferred_rows"] == 280 + assert anchor["time_period"] == module.PERIOD + assert anchor["source_household_mass_share"] == pytest.approx(0.5, rel=1e-12) + assert anchor["transferred_to_source_per_household_mass_ratio"] == pytest.approx( + 1.0, rel=1e-9 + ) + assert abs(anchor["relative_error"]) < 1e-9 + assert gates["input_coverage"]["passed"] is True + assert gates["input_coverage"]["binds"] == "donor_certified_release" + assert gates["input_coverage"]["graded_on_artifact"] == list( + module.ESI_PREMIUM_GATE_NAMES + ) + # The report is strict JSON even where a detail is not finite. + json.loads(args.gate_report.read_text(), parse_constant=pytest.fail) + + +@pytest.mark.parametrize("factor", [0.0, 1.4]) +def test_finalize_hard_fails_on_a_red_esi_premium_gate( + tmp_path, monkeypatch, esi_lane_frame, factor +) -> None: + module = _load_tool_module() + args = _finalize_args(module, tmp_path) + frame = _with_acs_premium_scaled(esi_lane_frame, factor) + + message, report = _run_finalize( + module, monkeypatch, args, frame, esi_premiums=False + ) + + red = "esi_premiums_signal" if factor == 0.0 else "esi_premiums_anchor" + assert red in message + assert report["gates"][red]["passed"] is False + assert report["gates"][red]["failures"] + assert report["gates"][red]["enforced"] is True + # The input-coverage entry no longer reads green on its own authority. + assert report["gates"]["input_coverage"]["passed"] is False + + +def test_finalize_records_a_waived_esi_premium_gate_without_failing_on_it( + tmp_path, monkeypatch, esi_lane_frame +) -> None: + module = _load_tool_module() + args = _finalize_args(module, tmp_path) + args.allow_esi_premium_gaps = True + frame = _with_acs_premium_scaled(esi_lane_frame, 1.4) + + message, report = _run_finalize( + module, monkeypatch, args, frame, esi_premiums=False + ) + + # Finalize still fails here on the fixture's other evidence, not on this. + assert "esi_premiums" not in message + anchor = report["gates"]["esi_premiums_anchor"] + assert anchor["passed"] is False + assert anchor["enforced"] is False + assert any( + "per unit of household mass" in failure for failure in anchor["failures"] + ) + assert report["gates"]["input_coverage"]["passed"] is False + + +def test_finalize_on_an_artifact_without_the_premium_is_red_on_both_gates( + tmp_path, monkeypatch +) -> None: + """A staging H5 from a donor that predates the stage carries no column.""" + + module = _load_tool_module() + args = _finalize_args(module, tmp_path) + + message, report = _run_finalize( + module, monkeypatch, args, _plausible_hours_frame(), esi_premiums=False + ) + + for name in module.ESI_PREMIUM_GATE_NAMES: + assert name in message + assert report["gates"][name]["failures"] == [ + "person column missing: employer_sponsored_insurance_premiums." + ] + + +def _rewrite_package_gates(args, edit) -> None: + report = json.loads(args.gate_report.read_text()) + edit(report["gates"]) + args.gate_report.write_text(json.dumps(report)) + + +@pytest.mark.parametrize( + ("state", "match"), + [ + ("missing", "carries no esi_premiums_signal verdict"), + ("truthy", "carries no esi_premiums_anchor verdict"), + ("unbound", "esi_premiums_anchor gate is missing its artifact binding"), + ("stale", "esi_premiums_signal gate is missing its artifact binding"), + ("failed", "Packaging requires a passing esi_premiums_anchor gate"), + ], +) +def test_package_requires_the_esi_premium_verdict_bound_to_the_packaged_bytes( + tmp_path: Path, monkeypatch, state, match +) -> None: + module = _load_tool_module() + args = _package_evidence_args( + module, + tmp_path, + monkeypatch, + hours_report={"passed": True, "failures": [], "detail": {}}, + ) + + def edit(gates) -> None: + if state == "missing": + # A report finalized before the gates existed. + del gates["esi_premiums_signal"], gates["esi_premiums_anchor"] + elif state == "truthy": + gates["esi_premiums_anchor"]["passed"] = "true" + elif state == "unbound": + del gates["esi_premiums_anchor"]["artifact_sha256"] + elif state == "stale": + gates["esi_premiums_signal"]["artifact_sha256"] = "0" * 64 + else: + gates["esi_premiums_anchor"].update( + passed=False, failures=["invented: off the NHE anchor"] + ) + + _rewrite_package_gates(args, edit) + + with pytest.raises(SystemExit, match=match) as error: + module.do_package(args) + + if state == "failed": + assert "invented: off the NHE anchor" in str(error.value) + # Refused before any release directory exists. + assert not (args.out / "releases").exists() + assert not (args.out / "package_result.json").exists() + + +def test_package_ships_a_red_esi_premium_gate_only_as_a_recorded_waiver( + tmp_path: Path, monkeypatch +) -> None: + module = _load_tool_module() + args = _package_evidence_args( + module, + tmp_path, + monkeypatch, + hours_report={"passed": True, "failures": [], "detail": {}}, + ) + _rewrite_package_gates( + args, + lambda gates: gates["esi_premiums_signal"].update( + passed=False, + failures=["person column missing: employer_sponsored_insurance_premiums."], + enforced=False, + ), + ) + with pytest.raises(SystemExit, match="--allow-esi-premium-gaps"): + module.do_package(args) + + args.allow_esi_premium_gaps = True + result = module.do_package(args) + + shipped = json.loads( + (Path(result["release_dir"]) / "gate_summary.json").read_text() + )["gates"] + assert shipped["esi_premiums_signal"]["passed"] is False + assert shipped["esi_premiums_signal"]["enforced"] is False + assert shipped["esi_premiums_signal"]["failures"] == [ + "person column missing: employer_sponsored_insurance_premiums." + ] + assert shipped["esi_premiums_anchor"]["passed"] is True + + +def test_the_esi_premium_waiver_is_off_by_default(tmp_path: Path) -> None: + module = _load_tool_module() + + assert _finalize_args(module, tmp_path).allow_esi_premium_gaps is False + assert "--allow-esi-premium-gaps" not in module.LEGACY_STAGING_REFRESH_RECIPE + + +def test_the_staged_lane_frame_passes_the_null_refusal(esi_lane_frame) -> None: + """``fill_reviewed_nulls`` no longer refuses the lane: every row carries + the premium. The refusal runs before the engine is imported.""" + + from microcosm.build.us_runtime.esi_premiums import ( + refuse_unassigned_us_esi_premiums, + ) + + person = esi_lane_frame.table("person") + assert set(person["person_spine"]) == {"asec_puf", "acs_2024_1yr"} + assert person["employer_sponsored_insurance_premiums"].notna().all() + refuse_unassigned_us_esi_premiums( + esi_lane_frame, consumer="ACS local release (fill_reviewed_nulls)" + ) diff --git a/packages/microcosm-build/tests/engine_free/us/test_us_acs_multispine_legacy_builder.py b/packages/microcosm-build/tests/engine_free/us/test_us_acs_multispine_legacy_builder.py index 50c5757cb..03fc37cf3 100644 --- a/packages/microcosm-build/tests/engine_free/us/test_us_acs_multispine_legacy_builder.py +++ b/packages/microcosm-build/tests/engine_free/us/test_us_acs_multispine_legacy_builder.py @@ -153,6 +153,7 @@ def test_parser_exposes_production_defaults_and_transfer_controls() -> None: assert args.seed == 0 assert args.geography_seed == 0 assert args.donor_channel == builder.ACS_DONOR_CHANNEL_AUTO + assert args.allow_esi_premium_gaps is False custom = builder._parse_args( [ @@ -211,6 +212,7 @@ def test_main_wires_verified_sources_transfer_audit_export_and_summary( combined.table("person")["sstb_self_employment_income_before_lsr"] = [0.0, 2_042.0] combined.table("person")["farm_operations_income"] = [0.0, 0.0] combined.table("person")["partnership_self_employment_net_earnings"] = [0.0, 0.0] + combined.table("person")["employer_sponsored_insurance_premiums"] = [9_000.0, 0.0] base_h5 = tmp_path / "dense.h5" base_h5.write_bytes(b"dense-base") manifest_path = tmp_path / "acs_sources.json" @@ -302,7 +304,13 @@ def fake_build(actual_base, actual_source, **kwargs): "column": "takes_up_snap_if_eligible", "family": "benefit_participation", "unmodeled_recipient_rows": 0, - } + }, + # The lane owes the ACS spine the employer premium too. + { + "column": "employer_sponsored_insurance_premiums", + "family": "source_operator_esi_premiums", + "unmodeled_recipient_rows": 0, + }, ], "spm_independence_role": _spm_role_receipt(persons=1), }, @@ -386,6 +394,29 @@ def staging_hours_gate(frame, *, source_null_audit): monkeypatch.setattr(builder, "_write_dataset", fake_write) monkeypatch.setattr(builder, "acs_local_hours_signal_gate", staging_hours_gate) + def qualify_esi_premium_donor(frame): + captured.setdefault("esi_premium_donors", []).append(frame) + return frame, {"donor_channel": "asec"} + + def staging_esi_premium_gates(frame, *, time_period): + captured["staging_esi_premium_gates"] = (frame, time_period) + return ( + GateResult(name="esi_premiums_signal", passed=True, failures=()), + GateResult( + name="esi_premiums_anchor", + passed=True, + failures=(), + details={"relative_error": 0.0, "cross_check_ratio": float("nan")}, + ), + ) + + monkeypatch.setattr( + builder, "prepare_acs_local_esi_premium_donor", qualify_esi_premium_donor + ) + monkeypatch.setattr( + builder, "acs_local_esi_premium_gates", staging_esi_premium_gates + ) + arguments = [ "--base-h5", str(base_h5), @@ -432,6 +463,11 @@ def staging_hours_gate(frame, *, source_null_audit): assert build_options.pop("puma_ladder") is puma_ladder assert callable(build_options.pop("hours_donor_factory")) assert build_options.pop("drop_group_quarters_children") is True + # The donor is qualified before the ACS sources are fetched, and the same + # qualifier selects its rows when the transfer runs. + assert build_options.pop("esi_premium_donor_factory") is qualify_esi_premium_donor + assert captured["esi_premium_donors"] == [base] + assert captured["staging_esi_premium_gates"] == (combined, 2024) assert build_options == { "chunksize": 2000, "acs_share": 0.4, @@ -473,6 +509,26 @@ def staging_hours_gate(frame, *, source_null_audit): assert self_employment["details"]["mismatched_rows"] == 0 assert self_employment["details"]["non_acs_component_rows"] == 0 assert self_employment["details"]["base_and_sstb_rows"] == 0 + assert summary["local_esi_premiums"] == { + "enforced": True, + "transferred": True, + "donor_qualification_waived": None, + "provenance": None, + "signal_gate": { + "name": "esi_premiums_signal", + "passed": True, + "failures": [], + "details": {}, + }, + "anchor_gate": { + "name": "esi_premiums_anchor", + "passed": True, + "failures": [], + # Strict JSON: a non-finite detail is written as null. + "details": {"relative_error": 0.0, "cross_check_ratio": None}, + }, + } + assert summary["orchestration"]["allow_esi_premium_gaps"] is False assert "pending_engine_input_nulls" not in summary assert summary["staging_export_peak_estimate_bytes"] == 123_456 assert summary["geography_ladder"] == { @@ -531,6 +587,128 @@ def staging_hours_gate(frame, *, source_null_audit): assert summary["output"]["sha256"] == hashlib.sha256(b"combined-output").hexdigest() +def test_esi_premium_donor_is_qualified_before_fetch( + monkeypatch: pytest.MonkeyPatch, +) -> None: + """microcosm#454: a donor without the stage's premium stops the build early.""" + + builder = _load_builder_module() + monkeypatch.setattr( + builder.acs_sources, "load_acs_source_manifest", lambda path: _manifest() + ) + monkeypatch.setattr(builder, "_load_base_frame", lambda path: _frame()) + monkeypatch.setattr(builder, "_sha256", lambda path: "0" * 64) + monkeypatch.setattr( + builder, "_require_dense_donor_coverage", lambda frame, **kwargs: None + ) + + def must_not_fetch(*_args, **_kwargs): + raise AssertionError("source fetch must not run for an unqualified donor") + + monkeypatch.setattr(builder.acs_sources, "fetch_acs_pums_sources", must_not_fetch) + + with pytest.raises(SystemExit) as exc: + builder.main(["--base-h5", "prestage.h5", "--out-h5", "combined.h5"]) + + message = str(exc.value) + assert "failed the ESI employer premium qualification" in message + assert "employer_sponsored_insurance_premiums" in message + assert "meps_esi_premiums stage" in message + assert "--allow-esi-premium-gaps" in message + + +def test_esi_premium_donor_refusal_is_a_recorded_waiver_for_a_diagnostic_build() -> ( + None +): + builder = _load_builder_module() + + waiver = builder._qualify_esi_premium_donor(_frame(), allow_gaps=True) + + assert waiver is not None + assert "built without the meps_esi_premiums stage" in waiver + + +def test_local_coverage_plan_owes_the_premium_only_when_it_was_transferred() -> None: + builder = _load_builder_module() + hours_only = builder.acs_local_transfer_target_families() + + with_premium = builder._local_coverage_plan(esi_premiums=True) + without_premium = builder._local_coverage_plan(esi_premiums=False) + + assert with_premium["person"]["source_operator_esi_premiums"] == ( + "employer_sponsored_insurance_premiums", + ) + assert without_premium == hours_only + assert { + family: targets + for family, targets in with_premium["person"].items() + if family != "source_operator_esi_premiums" + } == dict(hours_only["person"]) + # The composition copies: the shared plans are not edited in place. + assert builder.acs_local_transfer_target_families() == hours_only + assert ( + "source_operator_esi_premiums" + not in (builder.declared_acs_transfer_target_families()["person"]) + ) + + +def _esi_premium_gates(*, signal: tuple[str, ...] = (), anchor: tuple[str, ...] = ()): + return lambda frame, *, time_period: ( + GateResult(name="esi_premiums_signal", passed=not signal, failures=signal), + GateResult(name="esi_premiums_anchor", passed=not anchor, failures=anchor), + ) + + +def test_red_staging_esi_premium_gates_abort_before_export( + monkeypatch: pytest.MonkeyPatch, +) -> None: + builder = _load_builder_module() + result = builder.AcsMultispineResult(frame=_frame(), provenance={}) + monkeypatch.setattr( + builder, + "acs_local_esi_premium_gates", + _esi_premium_gates(anchor=("off the NHE anchor",)), + ) + + with pytest.raises(SystemExit) as exc: + builder._local_esi_premium_evidence( + result, period=2024, waiver=None, allow_gaps=False + ) + + assert "Local staging ESI employer premium gates failed" in str(exc.value) + assert "esi_premiums_anchor: off the NHE anchor" in str(exc.value) + + +def test_waived_staging_esi_premium_gates_are_recorded_red( + monkeypatch: pytest.MonkeyPatch, +) -> None: + builder = _load_builder_module() + result = builder.AcsMultispineResult(frame=_frame(), provenance={}) + missing = ("person column missing: employer_sponsored_insurance_premiums.",) + monkeypatch.setattr( + builder, + "acs_local_esi_premium_gates", + _esi_premium_gates(signal=missing, anchor=missing), + ) + + evidence = builder._local_esi_premium_evidence( + result, + period=2024, + waiver="Local ESI premium donor lacks the column.", + allow_gaps=True, + ) + + assert evidence["enforced"] is False + assert evidence["transferred"] is False + assert evidence["donor_qualification_waived"] == ( + "Local ESI premium donor lacks the column." + ) + assert evidence["signal_gate"]["passed"] is False + assert evidence["signal_gate"]["failures"] == list(missing) + assert evidence["anchor_gate"]["failures"] == list(missing) + json.dumps(evidence, allow_nan=False) + + def test_weights_audit_failure_aborts_before_export() -> None: builder = _load_builder_module() result = builder.AcsMultispineResult( diff --git a/packages/microcosm-build/tests/engine_free/us/test_us_modal_stage_plan_tool.py b/packages/microcosm-build/tests/engine_free/us/test_us_modal_stage_plan_tool.py index fc41c2b19..bfca144fd 100644 --- a/packages/microcosm-build/tests/engine_free/us/test_us_modal_stage_plan_tool.py +++ b/packages/microcosm-build/tests/engine_free/us/test_us_modal_stage_plan_tool.py @@ -316,6 +316,24 @@ def test_package_argv_adds_release_root_and_bool_flags() -> None: assert "--allow-dirty" not in argv +@pytest.mark.parametrize("stage", ["finalize", "package"]) +def test_the_esi_premium_waiver_reaches_the_stages_that_grade_it(stage) -> None: + # microcosm#454: a diagnostic run on a checkpoint without the premium + # needs the waiver forwarded; a release plan leaves it unset. + data = _plan_data(stage) + data["options"] = {"allow_esi_premium_gaps": True} + assert "--allow-esi-premium-gaps" in plan_lib.planned_argv( + plan_lib.parse_plan(data) + ) + data["options"] = {"allow_esi_premium_gaps": False} + assert "--allow-esi-premium-gaps" not in plan_lib.planned_argv( + plan_lib.parse_plan(data) + ) + assert "--allow-esi-premium-gaps" not in plan_lib.planned_argv( + plan_lib.parse_plan(_plan_data(stage)) + ) + + def test_argv_paths_are_stable_across_stages() -> None: # run_identity.json records the staging path and every later stage # re-verifies the staging digest, so staged paths must not move. diff --git a/test_support/microcosm_build/us_acs_local_release_tool.py b/test_support/microcosm_build/us_acs_local_release_tool.py new file mode 100644 index 000000000..5a39d5d01 --- /dev/null +++ b/test_support/microcosm_build/us_acs_local_release_tool.py @@ -0,0 +1,34 @@ +"""Loaders for the ACS local release chain's two tools. + +``tools/`` is not a package, so tests in more than one category load the +tools by path through these helpers. +""" + +from __future__ import annotations + +import importlib.util +from types import ModuleType + +from test_support.paths import paths_for + +_TOOLS = paths_for("microcosm-build").repository / "tools" + + +def _load(name: str) -> ModuleType: + spec = importlib.util.spec_from_file_location(name, _TOOLS / f"{name}.py") + assert spec is not None and spec.loader is not None + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + +def load_tool_module() -> ModuleType: + """``tools/build_us_acs_local_release.py``, freshly executed.""" + + return _load("build_us_acs_local_release") + + +def load_staging_builder_module() -> ModuleType: + """``tools/build_us_acs_multispine_base.py``, the staging shim.""" + + return _load("build_us_acs_multispine_base") diff --git a/tools/_legacy/build_us_acs_multispine_base.py b/tools/_legacy/build_us_acs_multispine_base.py index 184aead8e..3dd2dcb5c 100644 --- a/tools/_legacy/build_us_acs_multispine_base.py +++ b/tools/_legacy/build_us_acs_multispine_base.py @@ -30,6 +30,12 @@ from microcosm.build.gates import GateResult from microcosm.build.us_runtime import acs_sources from microcosm.build.us_runtime.acs_inputs import acs_self_employment_identity_gate +from microcosm.build.us_runtime.acs_local_esi_premiums import ( + acs_local_esi_premium_gate_payload, + acs_local_esi_premium_gates, + acs_local_esi_premium_transfer_target_families, + prepare_acs_local_esi_premium_donor, +) from microcosm.build.us_runtime.acs_local_hours import ( ACS_UNDER15_ZERO_POLICY, acs_local_hours_signal_gate, @@ -175,6 +181,19 @@ def _parse_args(argv: list[str] | None = None) -> argparse.Namespace: "the appropriate support channel for each input family." ), ) + parser.add_argument( + "--allow-esi-premium-gaps", + action="store_true", + help=( + "Diagnostic escape hatch (microcosm#454): build from a donor " + "release that predates the meps_esi_premiums stage. No row then " + "carries employer_sponsored_insurance_premiums and both ESI gates " + "are red; what was red is written to the staging summary. A donor " + "that carries the column but fails qualification leaves it null " + "on the ACS spine, which the release tool refuses. Release builds " + "must leave this unset." + ), + ) parser.add_argument( "--donor-release-manifest", type=Path, @@ -258,6 +277,10 @@ def main(argv: list[str] | None = None) -> int: donor_channel=args.donor_channel, target_families=transfer_plan, ) + esi_premium_waiver = _qualify_esi_premium_donor( + base, allow_gaps=args.allow_esi_premium_gaps + ) + transfer_esi_premiums = esi_premium_waiver is None base_rows = _row_counts(base) base_mass = float(base.weights_for("household").total) puma_ladder_sha256 = _sha256(args.puma_ladder) @@ -281,6 +304,9 @@ def main(argv: list[str] | None = None) -> int: donor, seed=args.seed, period=args.period ), hours_under15_policy=args.hours_under15_policy, + esi_premium_donor_factory=( + prepare_acs_local_esi_premium_donor if transfer_esi_premiums else None + ), donor_channel=args.donor_channel, seed=args.seed, n_estimators=args.n_estimators, @@ -296,7 +322,7 @@ def main(argv: list[str] | None = None) -> int: transfer_coverage = _require_default_transfer_coverage( result, base, - target_families=acs_local_transfer_target_families(), + target_families=_local_coverage_plan(esi_premiums=transfer_esi_premiums), ) weights_audit = _audit_fits(result) @@ -315,6 +341,13 @@ def main(argv: list[str] | None = None) -> int: "Local staging hours gate failed: " + "; ".join(hours_gate.failures) ) + local_esi_premiums = _local_esi_premium_evidence( + result, + period=args.period, + waiver=esi_premium_waiver, + allow_gaps=args.allow_esi_premium_gaps, + ) + args.out_h5.parent.mkdir(parents=True, exist_ok=True) staging_export_peak_bytes = _preflight_staging_export(result.frame) _write_dataset(result.frame, args.out_h5, period=args.period) @@ -350,6 +383,7 @@ def main(argv: list[str] | None = None) -> int: "failures": list(hours_gate.failures), "details": dict(hours_gate.details), } + summary["local_esi_premiums"] = local_esi_premiums summary_path.parent.mkdir(parents=True, exist_ok=True) rendered = json.dumps(summary, indent=2, sort_keys=True, allow_nan=False) + "\n" summary_path.write_text(rendered, encoding="utf-8") @@ -357,6 +391,78 @@ def main(argv: list[str] | None = None) -> int: return 0 +def _qualify_esi_premium_donor(base: Frame, *, allow_gaps: bool) -> str | None: + """Qualify the donor of the employer premium before ACS acquisition. + + Returns ``None`` for a qualified donor, or the refusal a diagnostic build + waived (``--allow-esi-premium-gaps``). The transfer qualifies the donor + again when it selects its rows, so no selection is held through the ACS + load. + """ + + try: + prepare_acs_local_esi_premium_donor(base) + except ValueError as exc: + if not allow_gaps: + raise SystemExit( + "Dense ASEC-by-PUF donor failed the ESI employer premium " + f"qualification: {exc} Pass --allow-esi-premium-gaps only for " + "a diagnostic build." + ) from exc + return str(exc) + return None + + +def _local_coverage_plan(*, esi_premiums: bool) -> TargetFamilies: + """Every input the lane's transfers owe the ACS spine.""" + + plan = { + entity: dict(families) + for entity, families in acs_local_transfer_target_families().items() + } + if esi_premiums: + local = acs_local_esi_premium_transfer_target_families() + for entity, families in local.items(): + plan.setdefault(entity, {}).update(families) + return plan + + +def _local_esi_premium_evidence( + result: AcsMultispineResult, + *, + period: int, + waiver: str | None, + allow_gaps: bool, +) -> dict[str, object]: + """Grade the employer premium on the pooled staging frame (microcosm#454). + + Both gates run on the frame the staging H5 is written from: the donor + rows get the MEPS-IC recomputation and the ACS rows are compared with + them. A red gate fails the build unless a diagnostic build waived it; the + verdicts are recorded either way. + """ + + signal, anchor = acs_local_esi_premium_gates(result.frame, time_period=period) + evidence: dict[str, object] = { + "enforced": not allow_gaps, + "transferred": waiver is None, + "donor_qualification_waived": waiver, + "provenance": result.provenance.get("local_esi_premiums"), + "signal_gate": acs_local_esi_premium_gate_payload(signal), + "anchor_gate": acs_local_esi_premium_gate_payload(anchor), + } + failures = [ + f"{gate.name}: {failure}" + for gate in (signal, anchor) + for failure in gate.failures + ] + if failures and not allow_gaps: + raise SystemExit( + "Local staging ESI employer premium gates failed: " + "; ".join(failures) + ) + return evidence + + def _audit_fits(result: AcsMultispineResult) -> dict[str, object]: records = result.fit_records if not records: @@ -1184,6 +1290,7 @@ def _build_summary( "n_estimators": args.n_estimators, "max_targets_per_fit": args.max_targets_per_fit, "donor_channel": args.donor_channel, + "allow_esi_premium_gaps": bool(args.allow_esi_premium_gaps), "provenance": result.provenance, }, "weights_audit": weights_audit, diff --git a/tools/build_us_acs_local_release.py b/tools/build_us_acs_local_release.py index 36677bfcd..3dc8b8bf3 100644 --- a/tools/build_us_acs_local_release.py +++ b/tools/build_us_acs_local_release.py @@ -42,7 +42,8 @@ per-spine SSI incidence and intensity (the microcosm#403 signature, measured rather than assumed). finalize : release-style gate report (PUMA-ladder gate, calibration - gate, spine-composition evidence) + the reviewed-limitations + gate, both ESI employer-premium gates on the calibrated + artifact, spine-composition evidence) + the reviewed-limitations register for this lineage (inherited #507 SSI aged-band collapse, #393 miscellaneous-income defect, CD marginal vintage, ESS concentration, sparse-selection donor, mixed @@ -85,6 +86,10 @@ from microcosm.build.us_runtime import target_loss_weights as loss_weighting from microcosm.build.us_runtime.acs_inputs import acs_self_employment_identity_gate +from microcosm.build.us_runtime.acs_local_esi_premiums import ( + acs_local_esi_premium_gate_payload, + acs_local_esi_premium_gates, +) from microcosm.build.us_runtime.acs_local_hours import acs_local_hours_signal_gate from microcosm.build.us_runtime.acs_pums import ACS_2024_1YR_SPINE from microcosm.build.us_runtime.base_pool import spine_column @@ -562,9 +567,11 @@ def fill_reviewed_nulls( ``system`` default to the PolicyEngine-US adapter and tax-benefit system. """ - # microcosm#454: the ESI employer premium exists only where the - # meps_esi_premiums stage ran (the ASEC donor rows). Refuse before any - # fill: a registered null here would become a silent zero on the ACS spine. + # microcosm#454: the staging build transfers the ESI employer premium onto + # the ACS spine and holds it to the donor rows' scale, so every row of a + # current staging frame carries it. A null means a staging H5 built before + # that. Refuse before any fill: a registered null here would become a + # silent zero on the ACS spine. refuse_unassigned_us_esi_premiums( frame, consumer="ACS local release (fill_reviewed_nulls)" ) @@ -3069,6 +3076,56 @@ def _require_acs_self_employment_identity(frame): return gate +#: The lane's verdict on the ESI employer premium (microcosm#454), in the +#: gate report finalize writes and package requires. +ESI_PREMIUM_GATE_NAMES = ("esi_premiums_signal", "esi_premiums_anchor") + + +def _esi_premium_gate_entry(gate, *, artifact_sha256: str, enforced: bool) -> dict: + """One ESI premium verdict, bound to the artifact bytes it graded.""" + + payload = acs_local_esi_premium_gate_payload(gate) + return { + "passed": payload["passed"], + "failures": payload["failures"], + "detail": payload["details"], + "artifact_sha256": artifact_sha256, + "enforced": enforced, + } + + +def _require_esi_premium_gates(gates, *, h5_sha: str, allow_gaps: bool) -> None: + """Refuse to package without the lane's passing ESI premium verdict. + + Both gates must be present and bound to the bytes being packaged. A red + gate is refused unless this stage was also run with + ``--allow-esi-premium-gaps``; a report finalized before the gates existed + carries neither and is always refused. + """ + + for name in ESI_PREMIUM_GATE_NAMES: + gate = gates.get(name) if isinstance(gates, dict) else None + if not isinstance(gate, dict) or not isinstance(gate.get("passed"), bool): + raise SystemExit( + f"The finalize gate report carries no {name} verdict " + "(microcosm#454); an old simulation_ready summary is " + "insufficient. Re-run --stage finalize." + ) + if gate.get("artifact_sha256") != h5_sha: + raise SystemExit( + f"The {name} gate is missing its artifact binding or certifies " + "different H5 bytes. Re-run --stage finalize against the " + "current artifact." + ) + if gate["passed"] is not True and not allow_gaps: + raise SystemExit( + f"Packaging requires a passing {name} gate: " + + "; ".join(map(str, gate.get("failures") or ["no failure recorded"])) + + ". --allow-esi-premium-gaps packages a diagnostic build with " + "the red verdict recorded." + ) + + def do_finalize(args) -> None: from microcosm.build.us_runtime.hours_worked import ( US_HOURS_WORKED_POOL_OUTPUT_COLUMNS, @@ -3146,6 +3203,13 @@ def do_finalize(args) -> None: hours_gate = us_hours_worked_signal_gate( frame, required_columns=US_HOURS_WORKED_POOL_OUTPUT_COLUMNS ) + # microcosm#454: the lane's own verdict on the ESI employer premium, on + # the calibrated bytes. The donor rows get the MEPS-IC recomputation, the + # ACS rows are compared with them, and the anchor-universe total is held + # to NHE Table 24 at the calibrated weights. + esi_signal_gate, esi_anchor_gate = acs_local_esi_premium_gates( + frame, time_period=PERIOD + ) del frame gc.collect() if _sha256(args.out_h5) != hours_artifact_sha: @@ -3254,17 +3318,35 @@ def do_finalize(args) -> None: ), "detail": diagnostics.get("weight_origin"), }, + "esi_premiums_signal": _esi_premium_gate_entry( + esi_signal_gate, + artifact_sha256=hours_artifact_sha, + enforced=not args.allow_esi_premium_gaps, + ), + "esi_premiums_anchor": _esi_premium_gate_entry( + esi_anchor_gate, + artifact_sha256=hours_artifact_sha, + enforced=not args.allow_esi_premium_gaps, + ), "input_coverage": { - # Enforced upstream: the staging driver's donor-coverage gate - # hard-fails before transfer, so reaching finalize means it held. - "passed": True, + # The donor's own coverage is enforced upstream: the staging + # driver's donor-coverage gate hard-fails before transfer, so + # reaching finalize means it held. That says nothing about a + # required input on the ACS spine, so the one this lane produces + # there from a stage output, the ESI employer premium, is graded + # on this artifact by the two gates above. + "passed": bool(esi_signal_gate.passed and esi_anchor_gate.passed), "binds": "donor_certified_release", + "graded_on_artifact": ["esi_premiums_signal", "esi_premiums_anchor"], "note": ( "Input coverage is enforced on the certified donor release " "before transfer; the ACS spine inherits required inputs via " - "QRF transfer. ACS native GQ-housing / source-universe nulls " - "are reviewed_limitations (calibration_blocker: false) and " - "are NOT re-imposed on the ACS spine." + "QRF transfer. The ESI employer premium (microcosm#454) is " + "also graded on the calibrated artifact: esi_premiums_signal " + "and esi_premiums_anchor. ACS native GQ-housing / " + "source-universe nulls are reviewed_limitations " + "(calibration_blocker: false) and are NOT re-imposed on the " + "ACS spine." ), }, "spine_composition": { @@ -3320,6 +3402,12 @@ def do_finalize(args) -> None: ) if not gates[name]["passed"] ] + # --allow-esi-premium-gaps records a red ESI premium gate without failing + # a diagnostic build; the gate entries keep what was red. + if not args.allow_esi_premium_gaps: + hard_failures += [ + name for name in ESI_PREMIUM_GATE_NAMES if not gates[name]["passed"] + ] updated_summary = dict(staging_summary) updated_summary.update( { @@ -3606,6 +3694,14 @@ def do_package(args) -> dict: out_h5_sha256=h5_sha, stage="package", ) + # microcosm#454: the lane's verdict on the ESI employer premium must be + # in the report and bound to these bytes. A refusal leaves no release + # directory behind. + _require_esi_premium_gates( + gate_report.get("gates"), + h5_sha=h5_sha, + allow_gaps=args.allow_esi_premium_gaps, + ) code = _repo_code_identity(args.allow_dirty) timestamp = datetime.now(UTC).strftime("%Y%m%dT%H%M%SZ") @@ -4166,6 +4262,20 @@ def _parse_args(argv: list[str] | None = None) -> argparse.Namespace: help="Release root for --stage package (releases// lands here).", ) parser.add_argument("--allow-dirty", action="store_true") + parser.add_argument( + "--allow-esi-premium-gaps", + action="store_true", + help=( + "Diagnostic escape hatch (microcosm#454): record the ESI premium " + "gates on the calibrated artifact — " + "employer_sponsored_insurance_premiums absent, zero-mass, missing " + "on the ACS spine, missing its raw ASEC coverage columns, or off " + "the CMS NHE Table 24 anchor — without failing finalize or " + "package (for example on a staging H5 built from a donor that " + "predates the meps_esi_premiums stage). The failures are still " + "written to the gate report. Release builds must leave this unset." + ), + ) parser.add_argument( "--allow-partial-geography", action="store_true", diff --git a/tools/modal_us_stage_plan.py b/tools/modal_us_stage_plan.py index 0cbace7ff..c3b203bca 100644 --- a/tools/modal_us_stage_plan.py +++ b/tools/modal_us_stage_plan.py @@ -602,6 +602,9 @@ def _acs_local_release_argv( "batch": OptionFlag("--batch", int), "hh_chunk": OptionFlag("--hh-chunk", int), "allow_partial_geography": OptionFlag("--allow-partial-geography", bool), + # Diagnostic only (microcosm#454): finalize and package record a red + # ESI premium verdict instead of failing on it. + "allow_esi_premium_gaps": OptionFlag("--allow-esi-premium-gaps", bool), }, owned_flags=frozenset( { From 524cddba3f0e57aad9a9280c2d94b1e79ad3028f Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Sat, 10 Oct 2026 19:58:51 -0400 Subject: [PATCH 3/5] Document the ACS local lane's ESI employer premium (#454) Co-Authored-By: Claude Opus 5.5 --- README.md | 6 +- ...s-esi-premiums-acs-local-lane-454.fixed.md | 1 + docs/agent-guide.md | 11 ++ docs/us-esi-employer-premiums.md | 170 ++++++++++++++++-- 4 files changed, 175 insertions(+), 13 deletions(-) create mode 100644 changelog.d/us-esi-premiums-acs-local-lane-454.fixed.md diff --git a/README.md b/README.md index 32912b3a6..2c6d9d977 100644 --- a/README.md +++ b/README.md @@ -365,7 +365,11 @@ for what each mode contains and where the build records it. Its `--l2-basis chi_square` and `--mass-parametrization softmax` options (defaults: the historical `record` and `projection`) penalize distance from the design weights; see [penalized calibration toward the design weights](docs/calibration-l2-basis.md) -for the algebra, the evidence and the measured frontier. +for the algebra, the evidence and the measured frontier. The chain needs a +donor release that carries `employer_sponsored_insurance_premiums`: its +staging build transfers the column onto the ACS spine, and finalize grades it +on the calibrated artifact with both ESI premium gates. See +[ACS local releases](docs/us-esi-employer-premiums.md#acs-local-releases). National and ACS local-area builds now use the same typed schema-8 calibration diagnostics writer. The local builder adds its Census population marginals to a diff --git a/changelog.d/us-esi-premiums-acs-local-lane-454.fixed.md b/changelog.d/us-esi-premiums-acs-local-lane-454.fixed.md new file mode 100644 index 000000000..ecbf9d912 --- /dev/null +++ b/changelog.d/us-esi-premiums-acs-local-lane-454.fixed.md @@ -0,0 +1 @@ +The US ACS local release lane builds again and carries `employer_sponsored_insurance_premiums` on every row (microcosm #454). Its staging build transfers the column onto the ACS spine in a lane-local pass that fits on the donor release's ASEC observations, whose wages are measured, and then runs the stacked pool's anchor kernel on the pooled frame: an ACS person without wages carries no premium, and the rest are scaled so both spines carry the same weighted premium per unit of household mass. The lane-wide weighted column equals the donor rows' total divided by their share of household mass, which is the donor release's own column. The declared ACS transfer plan and the stacked pool's plan are unchanged. Both ESI premium gates run on the staging frame and, in finalize, on the calibrated artifact; the gate report records them as `esi_premiums_signal` and `esi_premiums_anchor`, bound to the artifact bytes, and its `input_coverage` entry passes only if both do. Package refuses a report without a passing verdict for the bytes it packages. A donor built before the `meps_esi_premiums` stage is refused; `--allow-esi-premium-gaps` records the red verdicts for a diagnostic build. diff --git a/docs/agent-guide.md b/docs/agent-guide.md index 455aad6cb..1fc9f4fa1 100644 --- a/docs/agent-guide.md +++ b/docs/agent-guide.md @@ -358,6 +358,17 @@ A source-enrichment publisher claim widens the range consumers accept, not what was checked: the bundle's `compatibility.claim_measurement` names the one version the probe measured. +The ACS local-area chain produces `employer_sponsored_insurance_premiums` on +its ACS spine itself (microcosm#454): the staging build +(`tools/build_us_acs_multispine_base.py`) transfers it from the donor +release's ASEC observations and holds it to the donor rows' premium per unit +of household mass, and both ESI premium gates run on the staging frame and +again in finalize on the calibrated artifact. A donor built before the +`meps_esi_premiums` stage is refused. `--allow-esi-premium-gaps`, on the +staging tool and the release tool, is for diagnostic builds only: it records a +red verdict in the staging summary and the gate report instead of failing. See +[the ESI premium note](us-esi-employer-premiums.md#acs-local-releases). + A US release or release-gate preflight that receives a multispine pool through `--base-h5` must authenticate its sibling terminal manifest. A current stacked pool whose terminal battery is red remains fail-closed unless the operator diff --git a/docs/us-esi-employer-premiums.md b/docs/us-esi-employer-premiums.md index 94f2f3aa1..594ead0ce 100644 --- a/docs/us-esi-employer-premiums.md +++ b/docs/us-esi-employer-premiums.md @@ -464,12 +464,157 @@ earnings-universe producer wrote the named zero on the 25,511 records below age 15, and Census reported `WAGP` for every one of the other 145,870. No clone needed resetting, because the stacked order fills before it clones. +## ACS local releases + +The ACS local lane (`tools/build_us_acs_multispine_base.py`, then +`tools/build_us_acs_local_release.py`) pools a dense donor release with the +ACS 2024 1-year spine. `with_optional_acs_spine` multiplies every donor +household weight by `1 - acs_share` and gives ACS the rest of the donor's +household mass (half each by default). The donor rows arrive with the column +and the raw coverage columns the release carries. ACS rows have neither. + +### The lane transfers the column in its own pass + +`acs_local_esi_premium_transfer_target_families` is a one-family plan, +`source_operator_esi_premiums`, that only the lane's staging build uses, as it +does for usual hours (#765). The declared ACS plan +(`declared_acs_transfer_target_families`) does not gain the column: + +- The lane's main transfer fits every declared family on the PUF-detail role + (`ACS_DONOR_CHANNEL_AUTO`), the wrong donor for this column (next section). +- Under one of the declared plan's existing families, the column would also + leave the stacked pool's `source_operator_esi_premiums` family: + `pool_transfer_target_families` adds a `source_operator_*` family only for + outputs the declared plan does not own. And it would enter the frozen v11 + authority reconstruction (`stacked_spine._legacy_stacked_authority_receipt`), + which leaves out post-v11 targets by family name; scoring an attested + schema-9 pool refuses an authority that differs from that reconstruction. + Declaring it under its own `source_operator_esi_premiums` family would avoid + both, but not the donor problem. +- The declared plan is also the default registry of the spine-agreement + battery (`spine_agreement`), which would gain the column as a metric. + +Taking the lane's ACS rows from the stacked pool instead would replace the +lane. Its two tools read the legacy spine tags, the legacy support roles and +the staging summary's null register, and its hours repair refuses an assembled +frame. + +### The donor rows are the ASEC observations + +The transfer fits on the donor's ASEC observation role (support clone 0). The +lane's tax-detail families fit on the PUF-detail role, which is the wrong +donor for this column: + +- On the observation role, `employment_income_before_lsr` is the measured CPS + wage, and a person has a premium only if they hold a job with an employer. +- PUF support imputation writes the clone's wages + (`PUF_TAX_DETAIL_DEFAULT_PERSON_OUTPUTS`), while the clone's premium is a + copy of its source person's. On those rows the premium no longer follows the + wage. + +The recipient side is the measured ACS wage (`WAGP`), so both sides of the fit +see a measured wage, as in the stacked pool's early fill. On the test fixture, +where only wages tell the employed from the rest, a fit on ASEC observations +gives no ACS person without wages a premium; a fit on the PUF-detail role +gives one to more than a tenth of them and to fewer workers +(`test_the_puf_detail_role_would_put_premiums_on_people_without_wages`). + +`prepare_acs_local_esi_premium_donor` qualifies the donor before the ACS +sources are fetched. It refuses a donor without the column, the raw coverage +columns, the CPS record id (`PERIDNUM`), wages or legacy support roles, and +one that fails the stage's signal gate. + +### The ACS rows are held to the donor rows' scale + +Nothing in a QRF draw keeps a total. Once the frame is pooled, +`with_acs_local_esi_premium_anchor` runs the stacked pool's kernel, +`with_us_esi_premium_pool_anchor`, on it. The kernel tells the donor rows from +the ACS rows by the CPS record id and the raw coverage columns, which only the +donor rows carry. On the ACS rows it: + +1. clears the premium where the person reports zero wages; then +2. scales the rest by one factor so the ACS rows carry the same weighted + premium per unit of household mass as the donor rows. + +The kernel refuses a transferred row whose wage is missing: a missing wage is +not a zero wage. ACS PUMS leaves `WAGP` blank below age 15, outside its +earnings universe, and the lane keeps that blank in the staging frame, where +the reviewed-null register fills it for the engine pass. The anchor reads +those people's wages as zero under the universe rule `acs_income_universe` +names (`acs_2024_pums_wagp_age_15_plus`), in its own view of the frame, and +writes no zero into the frame. A blank wage at age 15 or older, or one whose +raw `WAGP` is not blank, stays missing and stops the build. + +It never rewrites a donor row. These hold for every lane frame +(`tests/engine_free/us/test_us_acs_local_esi_premiums.py`, property-tested +over the share, both spines' weights and the transfer's draws): + +- the lane-wide weighted column equals the donor rows' total divided by their + share of household mass, `1 - acs_share`. Pooling multiplied every donor + household weight by that share, so the lane-wide column is the donor + release's own column; +- both spines carry the same weighted premium per unit of household mass, so + moving mass from one spine to the other does not move the total; +- an ACS person with zero wages, or below age 15, carries nothing; +- the frame's own wage blanks are never overwritten; +- an anchored frame is returned unchanged. + +The lane runs the kernel and both gates on a narrow view of the frame, for two +reasons: + +- **Source ids.** The kernel finds a support clone's wages and draw by + `person_source_id` across the whole frame. The stacked pool makes that id + unique at assembly. This lane does not: ACS rows keep their ids from before + the pooling remap as source ids, and those can repeat a donor's. Every ACS + row here is its own source record, so the view leaves the lookup's columns + out and the kernel reads each row's own wages. A transferred row that is not + clone 0 is refused. +- **Memory.** The kernel copies every table it is given and each gate copies + the person table. The staging frame holds two full spines, and the view + carries only the columns they read. + +**Assumption.** As in the stacked pool, ACS rows have no policyholder flag, so +the lane's anchor-universe total is the donor rows' total plus the ACS column +over the donor rows' employed share of the anchor universe. + +The donor is a calibrated release. Its anchor-universe total is within the +release tolerance of the anchor, not on it, and the staging frame starts from +the same place. + +### What the lane records + +- **Staging.** Both gates run on the pooled frame before the staging H5 is + written. A red gate fails the build. The verdicts, the donor receipt and the + anchor receipt are in the staging summary under `local_esi_premiums`. +- **Finalize.** Both gates run on the calibrated artifact. The gate report + carries them as `esi_premiums_signal` and `esi_premiums_anchor`, each bound + to the artifact's SHA-256, and either one red fails finalize. The report's + `input_coverage` entry used to be a fixed pass; it now passes only if both + gates do. +- **Package.** A report without both verdicts, with a verdict bound to other + bytes, or with a red verdict is refused before a release directory exists. +- **`fill_reviewed_nulls`** still refuses a frame whose column is null on some + rows. A current staging frame has none; a staging H5 built before this + change does. + +`--allow-esi-premium-gaps`, on both tools, records a red verdict without +failing a diagnostic build from a donor that predates the stage. No row then +carries the column. + +No certified donor release carries the column yet, so the lane's fill has not +been measured on real inputs. The stacked pool's measurement above is the +nearest evidence: the same transfer, from ASEC rows with measured wages onto +the ACS 2024 spine. It differs in its donor rows (the pooled ASEC before the +clone, at design weights, where the lane's are a calibrated release's +observation role) and in its sample. The first staging build records the +lane's own fill in `local_esi_premiums`. + ## Gates | Gate | Where | Checks | |---|---|---| -| `us_esi_premiums_signal_gate` | base build, before and after the clone; release tool, on the base and on the calibrated export | Column present, finite, nonnegative and non-constant; support clones agree; the weighted share of people with a positive value is plausible; no employer premium outside the column universe or where the employer pays none; every premium is one common multiple of its MEPS-IC share. On a pooled frame these proofs cover the rows with raw coverage columns; the other rows must carry the premium with a weighted positive share and mean within 0.5 to 2 times those rows'. | -| `us_esi_premiums_anchor_gate` | release tool, on the calibrated export, for both the dense and the sparse default | The column absent or zero-mass is red. The anchor-universe total, recomputed at the release's weights, must be within **5%** of BEA NIPA 7.8 line 17 for the release period. Employed policyholders must carry 80% to 95% of it. On a pooled frame the total adds the transferred column at the source-derived rows' employed share, and the transferred rows' column per unit of household mass must stay within 0.8 to 1.25 times the source-derived rows'. | +| `us_esi_premiums_signal_gate` | base build, before and after the clone; release tool, on the base and on the calibrated export; ACS local lane, on the staging frame and the calibrated artifact | Column present, finite, nonnegative and non-constant; support clones agree; the weighted share of people with a positive value is plausible; no employer premium outside the column universe or where the employer pays none; every premium is one common multiple of its MEPS-IC share. On a pooled frame these proofs cover the rows with raw coverage columns; the other rows must carry the premium with a weighted positive share and mean within 0.5 to 2 times those rows'. | +| `us_esi_premiums_anchor_gate` | release tool, on the calibrated export, for both the dense and the sparse default; ACS local lane, on the staging frame and the calibrated artifact | The column absent or zero-mass is red. The anchor-universe total, recomputed at the release's weights, must be within **5%** of BEA NIPA 7.8 line 17 for the release period. Employed policyholders must carry 80% to 95% of it. On a pooled frame the total adds the transferred column at the source-derived rows' employed share, and the transferred rows' column per unit of household mass must stay within 0.8 to 1.25 times the source-derived rows'. | | `us_release_input_coverage_gate` | release tool | The input is `required` with no reviewed exclusion, so an export that drops or flattens it fails. | | `assert_required_us_release_source_columns` | the L0/refit export (`l0_refit_export.export_us_l0_refit_h5`, run by `tools/export_us_l0_refit_h5.py`), unless `--allow-missing-source-columns` is passed | The column must be present and non-constant. | | reform-coverage smoke | release tool, on the written file | Neutralizing the employer premium must lower `cbo_household_market_income` by at least $500B. | @@ -537,16 +682,17 @@ file. ## What this does not do -- **ACS local releases.** An ACS spine pooled beside a donor that carries the - input has it null on its ACS rows. `fill_reviewed_nulls` refuses that frame - instead of default-filling it to zero, so this lineage does not build until - the input is assigned or transferred for every spine. The stacked pool's - fill does not reach it: that lane's ACS rows come from the legacy transfer - plan (`declared_acs_transfer_target_families`), which does not name the - column. -- **ACS-native predictors for the fill.** The stacked pool fills ACS rows from - age, sex, State and income. It does not use ACS's own employer-coverage, - class-of-worker or employment-status items. +- **An ACS local release from an older donor.** The lane transfers the column + from its donor release. A donor built before the stage has nothing to + transfer, and the lane refuses it unless a diagnostic build waives the + gates. +- **ACS-native predictors for the fill.** The stacked pool and the ACS local + lane fill ACS rows from age, sex, State and income. Neither uses ACS's own + employer-coverage, class-of-worker or employment-status items. +- **A second verdict on the transferred rows' cells.** The gates recompute the + MEPS-IC assignment on the rows that carry the raw coverage columns. They + compare the transferred rows with those rows and cannot check them against + the cells. - **A pool built from an older raw stage.** The pool reads the coverage columns from the ASEC raw-stage artifact. One written before the columns were restored lacks them, and the pool's ESI operator refuses it by name. From 70ae9dca0f8a11ebec238959017fe7840165eb4c Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Sun, 11 Oct 2026 16:42:41 -0400 Subject: [PATCH 4/5] Keep the ACS local lane's ESI code and tests anchor-agnostic (#454) Max's ruling d1317 on #1189 moves the stage's scale anchor from CMS NHE Table 24 to BEA NIPA 7.8 (B4923C). This branch reads the anchor from the stage and does not restate it: the comments, help text and test fixtures that named NHE now name the stage's anchor. Co-Authored-By: Claude Opus 5.5 --- .../tests/engine_free/us/test_us_acs_local_esi_premiums.py | 2 +- .../tests/engine_free/us/test_us_acs_local_release_tool.py | 4 ++-- .../engine_free/us/test_us_acs_multispine_legacy_builder.py | 4 ++-- tools/build_us_acs_local_release.py | 4 ++-- 4 files changed, 7 insertions(+), 7 deletions(-) diff --git a/packages/microcosm-build/tests/engine_free/us/test_us_acs_local_esi_premiums.py b/packages/microcosm-build/tests/engine_free/us/test_us_acs_local_esi_premiums.py index fa60bd06c..c6cc01419 100644 --- a/packages/microcosm-build/tests/engine_free/us/test_us_acs_local_esi_premiums.py +++ b/packages/microcosm-build/tests/engine_free/us/test_us_acs_local_esi_premiums.py @@ -89,7 +89,7 @@ transferred_acs, ) -#: CMS NHE Table 24 employer contribution, CY2024: the stage's anchor. +#: The stage's scale anchor for CY2024, read from the stage, not restated. ANCHOR = float(esi.EMPLOYER_PREMIUM_ANCHOR["values"]["2024"]) RAW = tuple( column diff --git a/packages/microcosm-build/tests/engine_free/us/test_us_acs_local_release_tool.py b/packages/microcosm-build/tests/engine_free/us/test_us_acs_local_release_tool.py index f0d037f60..b525f843b 100644 --- a/packages/microcosm-build/tests/engine_free/us/test_us_acs_local_release_tool.py +++ b/packages/microcosm-build/tests/engine_free/us/test_us_acs_local_release_tool.py @@ -3116,7 +3116,7 @@ def edit(gates) -> None: gates["esi_premiums_signal"]["artifact_sha256"] = "0" * 64 else: gates["esi_premiums_anchor"].update( - passed=False, failures=["invented: off the NHE anchor"] + passed=False, failures=["invented: off the anchor"] ) _rewrite_package_gates(args, edit) @@ -3125,7 +3125,7 @@ def edit(gates) -> None: module.do_package(args) if state == "failed": - assert "invented: off the NHE anchor" in str(error.value) + assert "invented: off the anchor" in str(error.value) # Refused before any release directory exists. assert not (args.out / "releases").exists() assert not (args.out / "package_result.json").exists() diff --git a/packages/microcosm-build/tests/engine_free/us/test_us_acs_multispine_legacy_builder.py b/packages/microcosm-build/tests/engine_free/us/test_us_acs_multispine_legacy_builder.py index 03fc37cf3..b015fce72 100644 --- a/packages/microcosm-build/tests/engine_free/us/test_us_acs_multispine_legacy_builder.py +++ b/packages/microcosm-build/tests/engine_free/us/test_us_acs_multispine_legacy_builder.py @@ -667,7 +667,7 @@ def test_red_staging_esi_premium_gates_abort_before_export( monkeypatch.setattr( builder, "acs_local_esi_premium_gates", - _esi_premium_gates(anchor=("off the NHE anchor",)), + _esi_premium_gates(anchor=("off the anchor",)), ) with pytest.raises(SystemExit) as exc: @@ -676,7 +676,7 @@ def test_red_staging_esi_premium_gates_abort_before_export( ) assert "Local staging ESI employer premium gates failed" in str(exc.value) - assert "esi_premiums_anchor: off the NHE anchor" in str(exc.value) + assert "esi_premiums_anchor: off the anchor" in str(exc.value) def test_waived_staging_esi_premium_gates_are_recorded_red( diff --git a/tools/build_us_acs_local_release.py b/tools/build_us_acs_local_release.py index 3dc8b8bf3..2393b20cb 100644 --- a/tools/build_us_acs_local_release.py +++ b/tools/build_us_acs_local_release.py @@ -3206,7 +3206,7 @@ def do_finalize(args) -> None: # microcosm#454: the lane's own verdict on the ESI employer premium, on # the calibrated bytes. The donor rows get the MEPS-IC recomputation, the # ACS rows are compared with them, and the anchor-universe total is held - # to NHE Table 24 at the calibrated weights. + # to the stage's national anchor at the calibrated weights. esi_signal_gate, esi_anchor_gate = acs_local_esi_premium_gates( frame, time_period=PERIOD ) @@ -4270,7 +4270,7 @@ def _parse_args(argv: list[str] | None = None) -> argparse.Namespace: "gates on the calibrated artifact — " "employer_sponsored_insurance_premiums absent, zero-mass, missing " "on the ACS spine, missing its raw ASEC coverage columns, or off " - "the CMS NHE Table 24 anchor — without failing finalize or " + "the stage's national anchor — without failing finalize or " "package (for example on a staging H5 built from a donor that " "predates the meps_esi_premiums stage). The failures are still " "written to the gate report. Release builds must leave this unset." From e7e85cf06f0bdb5a0d1b94b75bf3b23f6d9ac95c Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Sat, 24 Oct 2026 16:51:33 -0400 Subject: [PATCH 5/5] Fit the ACS local lane's ESI anchor to the pool kernel's wage-evidence check (#454) #1199's rounds 2 and 3 changed the kernel this lane calls, with_us_esi_premium_pool_anchor. Before trusting a transferred row's zero wage, the kernel now resolves the ACS PUMS earnings universe (resolve_acs_pums_earnings_universe), and that needs raw WAGP and the clone index beside the wage. After the rebase onto #1199's head, the lane's view dropped the clone index, so every lane build refused. - The anchor view keeps the clone index. It still drops the support source id: lane ACS rows keep pre-remap source ids that can repeat a donor's, so without that id the kernel reads each row's own wages. - The kernel's resolver scopes on the stacked pool's ACS support channel ("acs"). This lane tags its ACS rows with the late spine's channel ("acs_2024_1yr"), so on the lane that check sees no rows. _wages_under_the_acs_universe now applies the rule itself, refusing as the resolver does: - a blank raw WAGP at 15 or older, or at an unknown age, however the mapped wage was filled (round 2's zero-fill case); - below 15, a raw WAGP that is not blank, or a mapped wage that is neither null nor zero. The receipt counts the rows it checked and says why the kernel's own receipt counts none. A test pins that, so the docs are revisited if the owner module ever learns the lane's channel. - The test frames carry raw WAGP. New tests cover the zero-filled adult, the child with a nonzero mapped wage, a child already holding the named zero, and missing WAGP or clone index. Co-Authored-By: Claude Opus 5.5 --- docs/us-esi-employer-premiums.md | 21 ++++- .../us_runtime/acs_local_esi_premiums.py | 92 ++++++++++++++----- .../us/test_us_acs_local_esi_premiums.py | 50 +++++++++- .../us_acs_local_esi_premiums.py | 6 +- 4 files changed, 137 insertions(+), 32 deletions(-) diff --git a/docs/us-esi-employer-premiums.md b/docs/us-esi-employer-premiums.md index 594ead0ce..051f1532e 100644 --- a/docs/us-esi-employer-premiums.md +++ b/docs/us-esi-employer-premiums.md @@ -542,8 +542,18 @@ earnings universe, and the lane keeps that blank in the staging frame, where the reviewed-null register fills it for the engine pass. The anchor reads those people's wages as zero under the universe rule `acs_income_universe` names (`acs_2024_pums_wagp_age_15_plus`), in its own view of the frame, and -writes no zero into the frame. A blank wage at age 15 or older, or one whose -raw `WAGP` is not blank, stays missing and stops the build. +writes no zero into the frame. The rest of the rule stops the build: + +- a blank raw `WAGP` at age 15 or older, or at an unknown age, however the + mapped wage was filled; +- below 15, a raw `WAGP` that is not blank, or a mapped wage that is neither + null nor zero. + +The kernel resolves the same rule itself, on the rows tagged with the stacked +pool's ACS support channel (`acs`). This lane's ACS rows carry the late +spine's channel (`acs_2024_1yr`), so the kernel's check sees none of them and +its receipt counts no rows; the lane applies the rule before calling the +kernel, and its receipt (`wage_universe`) counts what it checked. It never rewrites a donor row. These hold for every lane frame (`tests/engine_free/us/test_us_acs_local_esi_premiums.py`, property-tested @@ -566,9 +576,10 @@ reasons: `person_source_id` across the whole frame. The stacked pool makes that id unique at assembly. This lane does not: ACS rows keep their ids from before the pooling remap as source ids, and those can repeat a donor's. Every ACS - row here is its own source record, so the view leaves the lookup's columns - out and the kernel reads each row's own wages. A transferred row that is not - clone 0 is refused. + row here is its own source record, so the view leaves the source id out and + the kernel reads each row's own wages. It keeps the clone index, which the + kernel's earnings universe requires. A transferred row that is not clone 0 + is refused. - **Memory.** The kernel copies every table it is given and each gate copies the person table. The staging frame holds two full spines, and the view carries only the columns they read. diff --git a/packages/microcosm-build/src/microcosm/build/us_runtime/acs_local_esi_premiums.py b/packages/microcosm-build/src/microcosm/build/us_runtime/acs_local_esi_premiums.py index 7782fb548..d38b86171 100644 --- a/packages/microcosm-build/src/microcosm/build/us_runtime/acs_local_esi_premiums.py +++ b/packages/microcosm-build/src/microcosm/build/us_runtime/acs_local_esi_premiums.py @@ -30,8 +30,13 @@ leaves ``WAGP`` blank below age 15, outside its earnings universe, and the lane keeps that blank in the staging frame. The anchor therefore reads those people's wages as zero under the named universe rule of -:mod:`.acs_income_universe`, in its view of the frame only. A blank wage at -age 15 or older stays missing and is refused. +:mod:`.acs_income_universe`, in its view of the frame only. A blank raw +``WAGP`` at age 15 or older is refused, however the mapped wage was filled. + +The kernel also resolves that rule itself, but only on rows tagged with the +stacked pool's ACS support channel. This lane's ACS rows carry the late +spine's channel, so the kernel's check sees none of them, and the lane +applies the rule before calling it (:func:`_wages_under_the_acs_universe`). Invariant: the lane-wide weighted column equals the donor rows' weighted column divided by their share of household mass. Pooling multiplies every @@ -117,19 +122,20 @@ f"acs_2024_pums_{_ACS_WAGE_SOURCE_COLUMN.lower()}" f"_age_{ACS_PUMS_EARNINGS_MINIMUM_AGE}_plus" ) -#: Person columns the pool anchor kernel reads, without the two support -#: provenance columns. The kernel looks a clone's wages up by -#: ``person_source_id`` across the whole frame. A stacked pool makes that id -#: unique at assembly; this lane does not: ACS rows keep their pre-remap ids -#: as source ids, which can repeat a donor's. Every ACS row here is its own -#: source record, so the view omits the lookup's columns and the kernel reads -#: each row's own wages. +#: Person columns the pool anchor kernel reads, without the support source +#: id. Given both the source id and the clone index, the kernel looks a +#: clone's wages up by ``person_source_id`` across the whole frame. A stacked +#: pool makes that id unique at assembly; this lane does not: ACS rows keep +#: their pre-remap ids as source ids, which can repeat a donor's. Every ACS +#: row here is its own source record, so the view omits the source id and the +#: kernel reads each row's own wages. The clone index stays: the kernel's +#: ACS earnings universe requires it. _ANCHOR_PERSON_COLUMNS: tuple[str, ...] = ( *US_ESI_PREMIUMS_OUTPUT_COLUMNS, *( column for column in US_ESI_PREMIUMS_POOL_ANCHOR_PERSON_INPUTS - if column not in (_CLONE_INDEX_COLUMN, _SOURCE_ID_COLUMN) + if column != _SOURCE_ID_COLUMN ), ) #: Person columns the two ESI gates read: the output, the raw fields and @@ -307,33 +313,63 @@ def prepare_acs_local_esi_premium_donor( def _wages_under_the_acs_universe( person: pd.DataFrame, ) -> tuple[np.ndarray, dict[str, object]]: - """Wages for the anchor's view, with ACS universe blanks read as zero. - - ACS PUMS leaves ``WAGP`` blank for people below age 15: the field does - not apply to them. The lane keeps that blank in the staging frame, and the - anchor refuses a missing wage. Only a transferred row below the age floor - whose mapped wage is null, and whose raw field is blank where the frame - carries it, takes a zero here. Any other missing wage stays missing. + """Wages for the anchor's view, under the ACS PUMS earnings universe. + + PUMS asks ``WAGP`` of people aged 15 and older and leaves it blank below. + The lane keeps that blank in the staging frame, and the anchor refuses a + missing wage, so a transferred row below the age floor whose raw field is + blank and whose mapped wage is null reads as a zero wage here, in the view + only. The rest of the rule is refused as + :func:`~.acs_income_universe.resolve_acs_pums_earnings_universe` refuses + it on a stacked pool: + + - a blank raw field at 15 or older, or at an unknown age, is missing + evidence however a producer filled the mapped wage; + - below 15, a raw field that is not blank or a mapped wage that is + neither null nor zero is a mapping fault. + + Any other missing wage stays missing, and the kernel refuses it. """ wages = pd.to_numeric( person[US_ESI_PREMIUMS_WAGE_COLUMN], errors="coerce" ).to_numpy(dtype=np.float64, na_value=np.nan) transferred = person[list(_RAW_EVIDENCE_COLUMNS)].isna().all(axis=1).to_numpy() - blank = transferred & np.isnan(wages) - outside = np.zeros(len(person), dtype=bool) - if blank.any() and _AGE_COLUMN in person: - age = pd.to_numeric(person[_AGE_COLUMN], errors="coerce").to_numpy( - dtype=np.float64, na_value=np.nan + age = pd.to_numeric(person[_AGE_COLUMN], errors="coerce").to_numpy( + dtype=np.float64, na_value=np.nan + ) + raw_blank = person[_ACS_WAGE_SOURCE_COLUMN].isna().to_numpy() + below = transferred & (age < ACS_PUMS_EARNINGS_MINIMUM_AGE) + # An unknown age is not evidence of being outside the universe. + eligible = transferred & ~below + unasked = eligible & raw_blank + if unasked.any(): + raise SourceRuntimeError( + f"Local ESI premium anchor: {int(unasked.sum())} transferred row(s) " + f"aged {ACS_PUMS_EARNINGS_MINIMUM_AGE} or older, or of unknown age, " + f"have a blank raw {_ACS_WAGE_SOURCE_COLUMN!r}, which PUMS asks of " + "them. A missing wage is not a zero wage, even after a producer " + "filled the mapped column." ) - outside = blank & (age < ACS_PUMS_EARNINGS_MINIMUM_AGE) - if _ACS_WAGE_SOURCE_COLUMN in person: - outside &= person[_ACS_WAGE_SOURCE_COLUMN].isna().to_numpy() + mapped_nonzero = np.isfinite(wages) & (wages != 0.0) + off_universe = below & (~raw_blank | mapped_nonzero) + if off_universe.any(): + raise SourceRuntimeError( + f"Local ESI premium anchor: {int(off_universe.sum())} transferred " + f"row(s) below age {ACS_PUMS_EARNINGS_MINIMUM_AGE} carry a raw " + f"{_ACS_WAGE_SOURCE_COLUMN!r} or a nonzero mapped wage. PUMS leaves " + "the field blank there, so this is a mapping fault, not the " + "universe rule." + ) + blank = transferred & np.isnan(wages) + outside = below & np.isnan(wages) wages = np.where(outside, 0.0, wages) return wages, { "rule_id": _ACS_WAGE_UNIVERSE_RULE, "minimum_age": ACS_PUMS_EARNINGS_MINIMUM_AGE, "raw_source_column": _ACS_WAGE_SOURCE_COLUMN, + "eligible_transferred_rows": int(eligible.sum()), + "transferred_rows_below_minimum_age": int(below.sum()), "transferred_rows_with_blank_wage": int(blank.sum()), "transferred_rows_outside_universe_read_as_zero": int(outside.sum()), # The staging frame keeps the source's blanks; the zeros exist in the @@ -389,6 +425,12 @@ def with_acs_local_esi_premium_anchor(frame: Frame) -> tuple[Frame, dict[str, ob "kernel": "with_us_esi_premium_pool_anchor", "transferred_source_record": "each transferred row is its own source record", "wage_universe": wage_universe, + # Why the kernel's own ACS earnings-universe receipt counts no rows. + "kernel_wage_evidence_scope": ( + "the kernel resolves the ACS earnings universe on the stacked " + "pool's ACS support channel; this lane's ACS rows carry the late " + "spine's, so wage_universe applies the rule" + ), } if anchored is view: return frame, receipt diff --git a/packages/microcosm-build/tests/engine_free/us/test_us_acs_local_esi_premiums.py b/packages/microcosm-build/tests/engine_free/us/test_us_acs_local_esi_premiums.py index c6cc01419..bf795e264 100644 --- a/packages/microcosm-build/tests/engine_free/us/test_us_acs_local_esi_premiums.py +++ b/packages/microcosm-build/tests/engine_free/us/test_us_acs_local_esi_premiums.py @@ -512,7 +512,10 @@ def test_a_transferred_support_clone_is_refused() -> None: with_acs_local_esi_premium_anchor(_replace_person(frame, person)) -@pytest.mark.parametrize("column", [EMPLOYER, "NOW_HIPAID", "PERIDNUM", WAGES]) +@pytest.mark.parametrize( + "column", + [EMPLOYER, "NOW_HIPAID", "PERIDNUM", WAGES, "WAGP", "person_support_clone_index"], +) def test_a_lane_frame_missing_an_anchor_input_is_refused(column: str) -> None: frame = pooled( _small_donor(), @@ -589,10 +592,16 @@ def test_a_blank_acs_wage_below_fifteen_is_a_zero_wage_in_the_anchor_only() -> N "rule_id": "acs_2024_pums_wagp_age_15_plus", "minimum_age": 15, "raw_source_column": "WAGP", + "eligible_transferred_rows": 2, + "transferred_rows_below_minimum_age": 1, "transferred_rows_with_blank_wage": 1, "transferred_rows_outside_universe_read_as_zero": 1, "frame_cells_written": 0, } + # The kernel's own earnings universe scopes on the stacked pool's ACS + # channel, which this lane's rows do not carry; the lane applied the rule. + assert receipt["transferred_wage_evidence"]["eligible_acs_person_rows"] == 0 + assert receipt["transferred_wage_evidence"]["structurally_absent_person_rows"] == 0 # The staging frame keeps the source's blank: no zero is written to it. np.testing.assert_array_equal( _column(anchored, WAGES)[acs], [40_000.0, np.nan, 30_000.0] @@ -618,10 +627,49 @@ def test_a_child_whose_raw_wage_is_not_blank_is_not_read_as_zero() -> None: person = frame.table("person").copy() acs = ~donor_rows(frame) person["WAGP"] = np.where(acs, 1_200.0, np.nan) + with pytest.raises(SourceRuntimeError, match="mapping fault"): + with_acs_local_esi_premium_anchor(_replace_person(frame, person)) + + +@pytest.mark.parametrize("mapped", [0.0, 25_000.0]) +def test_an_adult_whose_blank_raw_wage_was_filled_is_refused(mapped: float) -> None: + # PUMS asks WAGP of everyone 15 or older, so a blank there is missing + # evidence: a producer that filled the mapped wage, with zero or anything + # else, does not make it a measurement. + frame = _lane_with_a_child() + person = frame.table("person").copy() + adult = np.flatnonzero(~donor_rows(frame))[0] + person.loc[person.index[adult], ["WAGP", WAGES]] = [np.nan, mapped] with pytest.raises(SourceRuntimeError, match="missing wage is not a zero wage"): with_acs_local_esi_premium_anchor(_replace_person(frame, person)) +def test_a_child_with_a_nonzero_mapped_wage_is_refused() -> None: + frame = _lane_with_a_child() + person = frame.table("person").copy() + child = np.flatnonzero(~donor_rows(frame))[1] + person.loc[person.index[child], WAGES] = 500.0 + with pytest.raises(SourceRuntimeError, match="mapping fault"): + with_acs_local_esi_premium_anchor(_replace_person(frame, person)) + + +def test_a_child_with_the_named_zero_already_mapped_passes() -> None: + # A producer that wrote the universe zero itself leaves nothing to read. + frame = _lane_with_a_child() + person = frame.table("person").copy() + child = np.flatnonzero(~donor_rows(frame))[1] + person.loc[person.index[child], WAGES] = 0.0 + + anchored, receipt = with_acs_local_esi_premium_anchor( + _replace_person(frame, person) + ) + + assert ( + receipt["wage_universe"]["transferred_rows_outside_universe_read_as_zero"] == 0 + ) + assert _column(anchored, EMPLOYER)[~donor_rows(frame)][1] == 0.0 + + def test_a_blank_raw_wage_below_fifteen_corroborates_the_universe_rule() -> None: frame = _lane_with_a_child() person = frame.table("person").copy() diff --git a/test_support/microcosm_build/us_acs_local_esi_premiums.py b/test_support/microcosm_build/us_acs_local_esi_premiums.py index d71b26618..018d199b3 100644 --- a/test_support/microcosm_build/us_acs_local_esi_premiums.py +++ b/test_support/microcosm_build/us_acs_local_esi_premiums.py @@ -208,7 +208,8 @@ def transferred_acs( range over every draw without fitting a forest. ``households`` gives each person's household position (nondecreasing, starting at 0; one person per household by default) and ``weights`` one weight per household. ``ages`` - defaults to adults; a NaN wage is a blank the source left. + defaults to adults; a NaN wage is a blank the source left, in the raw + ``WAGP`` and the mapped wage alike. """ index = np.arange(len(premiums)) @@ -223,6 +224,9 @@ def transferred_acs( ), "is_female": index % 2 == 0, WAGES: np.asarray(wages, dtype=np.float64), + # The raw PUMS field the wage is mapped from: blank where the + # mapped wage is. + "WAGP": np.asarray(wages, dtype=np.float64), EMPLOYER: np.asarray(premiums, dtype=np.float64), } )