From 6e6b86c0dedd7fccf8d583126f9ebd5d3dd9e350 Mon Sep 17 00:00:00 2001 From: igerber Date: Thu, 3 Sep 2026 09:09:43 -0400 Subject: [PATCH] fix(results): exact fractional confidence-level labels family-wide Every text surface that names a confidence level computed the percent with int((1 - alpha) * 100) (truncation), int(round(...)) (rounding), or :.0f, so alpha=0.025 printed "97%" or "98%" over a 97.5% interval. All 34 sites now route through one shared formatter in results_base (_coverage_level / _coverage_pct, plus _alpha_pct for the dCDH "Significant at" line, which printed 2% for alpha=0.025). Default-alpha output is byte-identical. BusinessReport headline.ci_level carries the exact level: int when integral (95 unchanged), float otherwise (97.5). No schema-version bump (REPORTING.md Note). tests/test_coverage_label.py pins the formatter, the representative surfaces at alpha=0.025, and adds a source guard against reintroducing an inline percent computation. Retires the TODO.md row. --- TODO.md | 1 - .../20260903-fractional-confidence-label.md | 19 ++ diff_diff/business_report.py | 10 +- .../chaisemartin_dhaultfoeuille_results.py | 16 +- diff_diff/changes_in_changes_results.py | 4 +- diff_diff/continuous_did_results.py | 3 +- diff_diff/diagnostic_report.py | 6 +- diff_diff/diagnostics.py | 4 +- diff_diff/efficient_did_results.py | 9 +- diff_diff/had.py | 6 +- diff_diff/honest_did.py | 4 +- diff_diff/imputation_results.py | 9 +- diff_diff/lpdid_results.py | 4 +- diff_diff/lwdid_results.py | 4 +- diff_diff/lwdid_wild_bootstrap.py | 3 +- diff_diff/rdd.py | 6 +- diff_diff/results.py | 13 +- diff_diff/results_base.py | 26 ++- diff_diff/stacked_did_results.py | 9 +- diff_diff/staggered_results.py | 9 +- diff_diff/staggered_triple_diff_results.py | 4 +- diff_diff/sun_abraham.py | 4 +- diff_diff/triple_diff.py | 9 +- diff_diff/trop_results.py | 4 +- diff_diff/two_stage_results.py | 9 +- diff_diff/utils.py | 3 +- diff_diff/visualization/_continuous.py | 5 +- diff_diff/visualization/_event_study.py | 4 +- diff_diff/wooldridge_results.py | 4 +- docs/methodology/REPORTING.md | 11 + tests/test_coverage_label.py | 202 ++++++++++++++++++ 31 files changed, 362 insertions(+), 62 deletions(-) create mode 100644 changelog.d/20260903-fractional-confidence-label.md create mode 100644 tests/test_coverage_label.py diff --git a/TODO.md b/TODO.md index 2a925f445..ef6a4b10a 100644 --- a/TODO.md +++ b/TODO.md @@ -57,7 +57,6 @@ Related tracking surfaces: | `WooldridgeDiD` does not apply the W2025 Sec 5.4 `D_{G_max} x X` covariate normalization, and three sibling covariate rank deficiencies are pre-existing. Measured with the period range pinned and only the never-treated units toggled: (1) time-invariant `exovar` is absorbed by the unit FE, 4 of 26 columns, IDENTICALLY with and without never-treated units; (2) `xgvar`'s cell x covariate block, 19 of 41, identical on both panels; (3) `xtvar` under `demean_covariates=False` does exhibit the `sum_g D_g x = x` dependency that the default demeaning removes; (4) the newly-reachable case -- time-VARYING data passed through `exovar`, which its own docstring reserves for time-invariant covariates -- where the paper's `dT_i` rule would give a deterministic `D_{G_max} x X` drop instead of QR's arbitrary pick (coefficients unaffected, `1.35e-14`; `rank_deficient_action="error"` raises). REGISTRY's narrowed Sec 5.4 note cross-references this row. **Trap for whoever takes it:** `xtvar` under the DEFAULT `demean_covariates=True` is FULL RANK -- the raw block carries demeaned values while `D_g x X` carries raw ones -- and forcing the drop there moves `overall_att` 1.11903 -> 1.46269. Pinned as-is by `TestComparisonSupportFiltering::test_cells_derived_groups_did_not_leak_into_the_design`. | `diff_diff/wooldridge.py` | #729-followup | Heavy | Medium | | `WooldridgeDiD.n_control_units` counts never-treated UNITS on `control_group="never_treated"` regardless of method, but on the nonlinear paths (`logit`/`poisson`) treated units' pre-treatment rows ARE the identifying comparison -- only the OLS path absorbs them into their own cells. So the reported count under-states the comparison pool exactly where the REGISTRY control-pool asymmetry note applies. Widen to `not_yet_treated or (never_treated and method != "ols")`, or document the count as never-treated-units-by-definition. Behavior is PRE-EXISTING; documented for now in the REGISTRY control-pool Note rather than changed, because widening moves a public results field and wants its own ledger row and test matrix. | `diff_diff/wooldridge.py` | #729-followup | Mid | Low | | `WooldridgeDiD` has no opt-out for comparison-support period filtering: a user who would rather see the refusal than a reduced sample cannot ask for it. Adding one means a constructor parameter (`get_params`/`set_params` propagation, transactional validation), a ledger row, and a test matrix across both predicate branches and all three `rank_deficient_action` modes -- deliberately out of scope for the change that introduced the filter. The always-on warning is the interim answer. | `diff_diff/wooldridge.py` | #729-followup | Mid | Low | -| Fractional confidence-level display unification: every `summary()` header computes `int((1 - alpha) * 100)` (~19 truncating sites incl. the shipped M-146 staggered family), so a fit `alpha=0.025` prints "97% Confidence Interval" over a 97.5% interval; `plot_dose_response`'s band label now renders exact fractional coverage via its `_coverage_label` helper - unify the summary surfaces on the same formatter (cross-family display change: moves every summary golden/doctest that pins a header, so it wants one sweep with its own test recapture, not a per-estimator drip) | `diff_diff/results_base.py`, `diff_diff/results.py` | alpha-guard review | Quick | Low | ### Performance diff --git a/changelog.d/20260903-fractional-confidence-label.md b/changelog.d/20260903-fractional-confidence-label.md new file mode 100644 index 000000000..df5dcfbe9 --- /dev/null +++ b/changelog.d/20260903-fractional-confidence-label.md @@ -0,0 +1,19 @@ +### Fixed +- **Exact fractional confidence-level labels, family-wide**: every text surface + that names a confidence level now prints the exact coverage (`97.5%` for + `alpha=0.025`; previously truncated to `97%` by `int((1 - alpha) * 100)` or + rounded to `98%` by `int(round(...))` / `:.0f`) via one shared + `results_base._coverage_pct` formatter: the 14 `summary()` headers + (DiD/TWFE/MultiPeriod/SyntheticDiD, CallawaySantAnna, staggered and 2x2x2 + TripleDifference, StackedDiD, ImputationDiD, TwoStageDiD, EfficientDiD, + ContinuousDiD, dCDH, SunAbraham, TROP), the `EventStudyResults` / HAD / RDD / + ETWFE / LWDiD / LPDiD / ChangesInChanges table headers, the CS and dCDH sup-t + band labels, the dCDH HonestDiD block (whose "Significant at" line printed + `2%` for `alpha=0.025`; now `2.5%`), `WildBootstrapResults` and the LWDiD + wild-cluster-bootstrap summaries, `HonestDiDResults` / `PlaceboTestResults` + summaries, and BusinessReport / DiagnosticReport prose. The BusinessReport + headline `ci_level` field carries the exact level as an `int` when integral + (`95` is byte-unchanged) and a `float` otherwise (`97.5`); no schema-version + bump (REPORTING.md Note). Default-alpha output is byte-identical. A source + guard (`tests/test_coverage_label.py`) rejects any reintroduced inline + percent computation. diff --git a/diff_diff/business_report.py b/diff_diff/business_report.py index a4d29027c..42eed4b9b 100644 --- a/diff_diff/business_report.py +++ b/diff_diff/business_report.py @@ -48,7 +48,7 @@ from diff_diff._reporting_helpers import describe_target_parameter from diff_diff.diagnostic_report import DiagnosticReport, DiagnosticReportResults -from diff_diff.results_base import Diagnostic +from diff_diff.results_base import Diagnostic, _coverage_level, _coverage_pct BUSINESS_REPORT_SCHEMA_VERSION = "2.0" @@ -534,7 +534,7 @@ def _build_schema(self) -> Dict[str, Any]: "ci_upper": None, "alpha_was_honored": True, "alpha_override_caveat": None, - "ci_level": int(round((1.0 - self._context.alpha) * 100)), + "ci_level": _coverage_level(self._context.alpha), "p_value": None, "is_significant": False, "near_significance_threshold": False, @@ -683,7 +683,7 @@ def _extract_headline(self, dr_schema: Optional[Dict[str, Any]]) -> Dict[str, An f"for the confidence interval because this fit uses " f"{inference_label} inference; the displayed CI remains " f"at the fit's native level " - f"({int(round((1.0 - result_alpha) * 100))}%). The " + f"({_coverage_pct(result_alpha)}%). The " f"significance phrasing still uses the requested alpha." ) @@ -700,7 +700,7 @@ def _extract_headline(self, dr_schema: Optional[Dict[str, Any]]) -> Dict[str, An ) if att is None or not np.isfinite(att): sign = "undefined" - ci_level = int(round((1.0 - display_alpha) * 100)) + ci_level = _coverage_level(display_alpha) # bool(...) coerces away numpy bool_ — when ``p`` is a numpy NaN (e.g. # SyntheticControl, whose analytical p_value is always NaN), ``np.isfinite`` # yields a numpy bool that is NOT JSON-serializable in the schema. @@ -2073,7 +2073,7 @@ def _significance_phrase(p: Optional[float], alpha: float) -> str: """ if p is None or not np.isfinite(p): return "statistical significance cannot be assessed (p-value unavailable)" - ci_level = int(round((1.0 - alpha) * 100)) + ci_level = _coverage_pct(alpha) if p < 0.001: return "the direction of the effect is strongly supported by the data" if p < 0.01: diff --git a/diff_diff/chaisemartin_dhaultfoeuille_results.py b/diff_diff/chaisemartin_dhaultfoeuille_results.py index 74299e852..bacb688ba 100644 --- a/diff_diff/chaisemartin_dhaultfoeuille_results.py +++ b/diff_diff/chaisemartin_dhaultfoeuille_results.py @@ -34,7 +34,13 @@ from diff_diff._deprecation import deprecated_field_property from diff_diff.aggregation import AggregationMixin, AggregationResult from diff_diff.results import _get_significance_stars -from diff_diff.results_base import BaseResults, _require_fit_alpha, build_event_study_surface +from diff_diff.results_base import ( + BaseResults, + _alpha_pct, + _coverage_pct, + _require_fit_alpha, + build_event_study_surface, +) __all__ = [ "ChaisemartinDHaultfoeuilleResults", @@ -935,7 +941,7 @@ def summary(self, alpha: Optional[float] = None) -> str: decomposition diagnostic, and a footer of significance codes. """ alpha = _require_fit_alpha(alpha, self.alpha) - conf_level = int((1 - alpha) * 100) + conf_level = _coverage_pct(alpha) width = 85 sep = "=" * width thin = "-" * width @@ -1590,7 +1596,7 @@ def _render_path_effects_section( if self.path_sup_t_bands is not None and path in self.path_sup_t_bands: crit_p = self.path_sup_t_bands[path].get("crit_value", np.nan) if np.isfinite(crit_p): - conf_level = int((1 - self.alpha) * 100) + conf_level = _coverage_pct(self.alpha) lines.append( f" Sup-t critical value: {crit_p:.4f} " f"(simultaneous {conf_level}% bands)" @@ -1605,7 +1611,7 @@ def _render_honest_did_section(self, lines: List[str], width: int, thin: str) -> method_label = hd.method.replace("_", " ").title() m_val = hd.M sig_label = "Yes" if hd.is_significant else "No" - conf_pct = int((1 - hd.alpha) * 100) + conf_pct = _coverage_pct(hd.alpha) lines.extend( [ thin, @@ -1625,7 +1631,7 @@ def _render_honest_did_section(self, lines: List[str], width: int, thin: str) -> f"{'Identified set:':<35} " f"[{_fmt_float(hd.lb)}, {_fmt_float(hd.ub)}]", f"{'Robust ' + str(conf_pct) + '% CI:':<35} " f"[{_fmt_float(hd.ci_lb)}, {_fmt_float(hd.ci_ub)}]", - f"{'Significant at ' + str(int(hd.alpha * 100)) + '%:':<35} " f"{sig_label:>10}", + f"{'Significant at ' + _alpha_pct(hd.alpha) + '%:':<35} " f"{sig_label:>10}", thin, "", ] diff --git a/diff_diff/changes_in_changes_results.py b/diff_diff/changes_in_changes_results.py index 300537d1a..916aca06a 100644 --- a/diff_diff/changes_in_changes_results.py +++ b/diff_diff/changes_in_changes_results.py @@ -7,7 +7,7 @@ import pandas as pd from diff_diff._deprecation import deprecated_field_property -from diff_diff.results_base import BaseResults +from diff_diff.results_base import BaseResults, _coverage_pct _ESTIMATOR_TITLES = { "cic": "Changes-in-Changes (Athey & Imbens 2006) Results", @@ -159,7 +159,7 @@ def summary(self) -> str: """Fixed-width text summary: headline ATT block plus the quantile-effects table.""" from diff_diff.results import _get_significance_stars - ci_pct = int(round((1 - self.alpha) * 100)) + ci_pct = _coverage_pct(self.alpha) width = 88 bar = "=" * width dash = "-" * width diff --git a/diff_diff/continuous_did_results.py b/diff_diff/continuous_did_results.py index 96cad667e..76ab0665a 100644 --- a/diff_diff/continuous_did_results.py +++ b/diff_diff/continuous_did_results.py @@ -18,6 +18,7 @@ from diff_diff.results_base import ( _SUMMARY_ALPHA_MESSAGE, BaseResults, + _coverage_pct, _require_fit_alpha, build_event_study_surface, ) @@ -274,7 +275,7 @@ def summary(self, alpha: Optional[float] = None) -> str: or relabeling. Re-fit at the desired alpha instead. """ alpha = _require_fit_alpha(alpha, self.alpha, message=_SUMMARY_ALPHA_MESSAGE) - conf_level = int((1 - alpha) * 100) + conf_level = _coverage_pct(alpha) w = 85 lines = [ diff --git a/diff_diff/diagnostic_report.py b/diff_diff/diagnostic_report.py index e79863122..9f818cdd0 100644 --- a/diff_diff/diagnostic_report.py +++ b/diff_diff/diagnostic_report.py @@ -49,7 +49,7 @@ import pandas as pd from diff_diff._reporting_helpers import describe_target_parameter # noqa: E402 (top-level import) -from diff_diff.results_base import Diagnostic +from diff_diff.results_base import Diagnostic, _coverage_pct DIAGNOSTIC_REPORT_SCHEMA_VERSION = "2.0" @@ -4461,9 +4461,9 @@ def _render_overall_interpretation(schema: Dict[str, Any], labels: Dict[str, str # stays consistent with the rendered interval when alpha != 0.05. headline_alpha = headline.get("alpha") if isinstance(headline, dict) else None if isinstance(headline_alpha, (int, float)) and 0 < headline_alpha < 1: - ci_level = int(round((1.0 - headline_alpha) * 100)) + ci_level = _coverage_pct(headline_alpha) else: - ci_level = 95 + ci_level = "95" ci_finite = ( isinstance(ci, (list, tuple)) and len(ci) == 2 diff --git a/diff_diff/diagnostics.py b/diff_diff/diagnostics.py index 5cfcb4f6e..7fd329b57 100644 --- a/diff_diff/diagnostics.py +++ b/diff_diff/diagnostics.py @@ -20,7 +20,7 @@ from diff_diff._deprecation import NOT_SUPPLIED, require_arg, resolve_renamed_kwarg from diff_diff.estimators import DifferenceInDifferences from diff_diff.results import _get_significance_stars -from diff_diff.results_base import Diagnostic +from diff_diff.results_base import Diagnostic, _coverage_pct from diff_diff.utils import safe_inference, validate_binary @@ -87,7 +87,7 @@ def significance_stars(self) -> str: def summary(self) -> str: """Generate formatted summary of placebo test results.""" - conf_level = int((1 - self.alpha) * 100) + conf_level = _coverage_pct(self.alpha) lines = [ "=" * 65, diff --git a/diff_diff/efficient_did_results.py b/diff_diff/efficient_did_results.py index 5553070bd..022df1cf8 100644 --- a/diff_diff/efficient_did_results.py +++ b/diff_diff/efficient_did_results.py @@ -21,7 +21,12 @@ from diff_diff.efficient_did_aggregation import _EfficientAggregationMixin from diff_diff.efficient_did_bootstrap import EfficientDiDBootstrapMixin from diff_diff.results import _format_survey_block, _get_significance_stars -from diff_diff.results_base import BaseResults, _require_fit_alpha, build_event_study_surface +from diff_diff.results_base import ( + BaseResults, + _coverage_pct, + _require_fit_alpha, + build_event_study_surface, +) if TYPE_CHECKING: from diff_diff.efficient_did_bootstrap import EDiDBootstrapResults @@ -647,7 +652,7 @@ def summary(self, alpha: Optional[float] = None) -> str: never recomputed or relabeled - re-fit at the desired alpha). """ alpha = _require_fit_alpha(alpha, self.alpha) - conf_level = int((1 - alpha) * 100) + conf_level = _coverage_pct(alpha) lines = [ "=" * 85, diff --git a/diff_diff/had.py b/diff_diff/had.py index 0a7fd3bb7..8fb82477c 100644 --- a/diff_diff/had.py +++ b/diff_diff/had.py @@ -90,7 +90,7 @@ BiasCorrectedFit, bias_corrected_local_linear, ) -from diff_diff.results_base import BaseResults, build_event_study_surface +from diff_diff.results_base import BaseResults, _coverage_pct, build_event_study_surface from diff_diff.survey import ( SurveyMetadata, compute_survey_metadata, @@ -444,7 +444,7 @@ def __repr__(self) -> str: def summary(self) -> str: """Formatted summary table.""" width = 72 - conf_level = int((1 - self.alpha) * 100) + conf_level = _coverage_pct(self.alpha) lines = [ "=" * width, "HeterogeneousAdoptionDiD Estimation Results".center(width), @@ -884,7 +884,7 @@ def __repr__(self) -> str: def summary(self) -> str: """Formatted per-horizon summary table.""" width = 80 - conf_level = int((1 - self.alpha) * 100) + conf_level = _coverage_pct(self.alpha) lines = [ "=" * width, "HeterogeneousAdoptionDiD Event-Study Results".center(width), diff --git a/diff_diff/honest_did.py b/diff_diff/honest_did.py index 3466856ee..312207400 100644 --- a/diff_diff/honest_did.py +++ b/diff_diff/honest_did.py @@ -30,7 +30,7 @@ from diff_diff.results import ( MultiPeriodDiDResults, ) -from diff_diff.results_base import Diagnostic, _validate_vcov_subblock +from diff_diff.results_base import Diagnostic, _coverage_pct, _validate_vcov_subblock from diff_diff.utils import _get_critical_value # ============================================================================= @@ -271,7 +271,7 @@ def summary(self) -> str: str Formatted summary. """ - conf_level = int((1 - self.alpha) * 100) + conf_level = _coverage_pct(self.alpha) method_names = { "smoothness": "Smoothness (Delta^SD)", diff --git a/diff_diff/imputation_results.py b/diff_diff/imputation_results.py index 17b931e56..f9e31587e 100644 --- a/diff_diff/imputation_results.py +++ b/diff_diff/imputation_results.py @@ -15,7 +15,12 @@ from diff_diff.aggregation import AggregationMixin, AggregationResult, build_total_relay_row from diff_diff.imputation_aggregation import _ImputationAggregationMixin from diff_diff.results import _format_survey_block, _get_significance_stars -from diff_diff.results_base import BaseResults, _require_fit_alpha, build_event_study_surface +from diff_diff.results_base import ( + BaseResults, + _coverage_pct, + _require_fit_alpha, + build_event_study_surface, +) class _ImputationKitAggregator(_ImputationAggregationMixin): @@ -563,7 +568,7 @@ def summary(self, alpha: Optional[float] = None) -> str: Formatted summary. """ alpha = _require_fit_alpha(alpha, self.alpha) - conf_level = int((1 - alpha) * 100) + conf_level = _coverage_pct(alpha) lines = [ "=" * 85, diff --git a/diff_diff/lpdid_results.py b/diff_diff/lpdid_results.py index 15dc40e77..4c75b3a6a 100644 --- a/diff_diff/lpdid_results.py +++ b/diff_diff/lpdid_results.py @@ -5,7 +5,7 @@ import pandas as pd from diff_diff._deprecation import warn_deprecated_kwarg -from diff_diff.results_base import BaseResults +from diff_diff.results_base import BaseResults, _coverage_pct @dataclass @@ -227,7 +227,7 @@ def summary(self) -> str: # fit time using ``self.alpha``; the displayed level must match them, so # summary() does not accept an alpha override (it would relabel without # recomputing the intervals). - ci_pct = int(round((1 - self.alpha) * 100)) + ci_pct = _coverage_pct(self.alpha) width = 88 bar = "=" * width dash = "-" * width diff --git a/diff_diff/lwdid_results.py b/diff_diff/lwdid_results.py index cc80e1be9..86c927768 100644 --- a/diff_diff/lwdid_results.py +++ b/diff_diff/lwdid_results.py @@ -10,7 +10,7 @@ import pandas as pd from diff_diff.aggregation import AggregationMixin, AggregationResult -from diff_diff.results_base import BaseResults, EventStudyResults +from diff_diff.results_base import BaseResults, EventStudyResults, _coverage_pct # How the overall staggered standard error was obtained. Cohort effects that @@ -549,7 +549,7 @@ def summary(self) -> str: """ from diff_diff.results import _format_vcov_label, _get_significance_stars - ci_pct = int(round((1 - self.alpha) * 100)) + ci_pct = _coverage_pct(self.alpha) width = 88 bar = "=" * width dash = "-" * width diff --git a/diff_diff/lwdid_wild_bootstrap.py b/diff_diff/lwdid_wild_bootstrap.py index ce1e520ad..9f701b1cb 100644 --- a/diff_diff/lwdid_wild_bootstrap.py +++ b/diff_diff/lwdid_wild_bootstrap.py @@ -43,6 +43,7 @@ import numpy as np from diff_diff.linalg import solve_ols +from diff_diff.results_base import _coverage_pct from diff_diff.utils import wild_bootstrap_se _VALID_WEIGHT_TYPES = ("rademacher", "mammen", "webb") @@ -106,7 +107,7 @@ def summary(self) -> str: if self.p_value < 0.01 else "**" if self.p_value < 0.05 else "*" if self.p_value < 0.1 else "" ) - level = int(round((1 - self.alpha) * 100)) + level = _coverage_pct(self.alpha) return ( f"Wild Cluster Bootstrap Results\n" f"{'=' * 50}\n" diff --git a/diff_diff/rdd.py b/diff_diff/rdd.py index 2d9bdcb45..f3b35399c 100644 --- a/diff_diff/rdd.py +++ b/diff_diff/rdd.py @@ -126,7 +126,7 @@ rdbwselect, rdrobust_fit, ) -from diff_diff.results_base import BaseResults +from diff_diff.results_base import BaseResults, _coverage_pct from diff_diff.utils import safe_inference, validate_covariate_names __all__ = [ @@ -308,7 +308,7 @@ def __setstate__(self, state: Dict[str, Any]) -> None: def summary(self) -> str: """Human-readable summary with the three-row rdrobust table.""" width = 72 - conf_level = 100 * (1 - self.alpha) + conf_level = _coverage_pct(self.alpha) lines = [] lines.append("=" * width) design = "Fuzzy" if self.first_stage is not None else "Sharp" @@ -343,7 +343,7 @@ def summary(self) -> str: lines.append("-" * width) header = ( f"{'Method':<16}{'Coef.':>11}{'Std. Err.':>11}{'z':>9}" - f"{'P>|z|':>9}{'[' + f'{conf_level:g}% Conf. Int.]':>16}" + f"{'P>|z|':>9}{'[' + f'{conf_level}% Conf. Int.]':>16}" ) if self.first_stage is not None: # Fuzzy: R prints a first-stage block above the treatment diff --git a/diff_diff/results.py b/diff_diff/results.py index 87dd318ef..688bf688d 100644 --- a/diff_diff/results.py +++ b/diff_diff/results.py @@ -11,7 +11,12 @@ import numpy as np import pandas as pd -from diff_diff.results_base import _SUMMARY_ALPHA_MESSAGE, BaseResults, _require_fit_alpha +from diff_diff.results_base import ( + _SUMMARY_ALPHA_MESSAGE, + BaseResults, + _coverage_pct, + _require_fit_alpha, +) def _format_survey_block(sm, width: int) -> list: @@ -199,7 +204,7 @@ def summary(self, alpha: Optional[float] = None) -> str: Formatted summary table. """ alpha = _require_fit_alpha(alpha, self.alpha, message=_SUMMARY_ALPHA_MESSAGE) - conf_level = int((1 - alpha) * 100) + conf_level = _coverage_pct(alpha) lines = [ "=" * 70, @@ -799,7 +804,7 @@ def summary(self, alpha: Optional[float] = None) -> str: Formatted summary table. """ alpha = _require_fit_alpha(alpha, self.alpha, message=_SUMMARY_ALPHA_MESSAGE) - conf_level = int((1 - alpha) * 100) + conf_level = _coverage_pct(alpha) lines = [ "=" * 80, @@ -1299,7 +1304,7 @@ def summary(self, alpha: Optional[float] = None) -> str: Formatted summary table. """ alpha = _require_fit_alpha(alpha, self.alpha, message=_SUMMARY_ALPHA_MESSAGE) - conf_level = int((1 - alpha) * 100) + conf_level = _coverage_pct(alpha) lines = [ "=" * 75, diff --git a/diff_diff/results_base.py b/diff_diff/results_base.py index 5719273a8..c36e08b46 100644 --- a/diff_diff/results_base.py +++ b/diff_diff/results_base.py @@ -131,6 +131,30 @@ def _require_fit_alpha( return fit_alpha +def _coverage_level(alpha: float) -> Union[int, float]: + """Exact confidence level in percent: 95 for alpha=0.05, 97.5 for 0.025. + + Rounds to 6 decimals first so a float32-noised alpha (0.0500000007...) + still reads 95, then returns an int when the level is integral (so the + historical int-valued schema fields are byte-compatible) and a float + otherwise. Every text surface that names a confidence level goes through + this helper (or ``_coverage_pct``) so no summary truncates ``97.5`` to + ``97`` or rounds it to ``98``. + """ + level = round(100.0 * (1.0 - float(alpha)), 6) + return int(level) if level == int(level) else level + + +def _coverage_pct(alpha: float) -> str: + """Display form of ``_coverage_level``: ``'95'``, ``'97.5'`` (no ``%``).""" + return f"{_coverage_level(alpha):g}" + + +def _alpha_pct(alpha: float) -> str: + """Significance level in percent: ``'5'`` for alpha=0.05, ``'2.5'`` for 0.025.""" + return f"{round(100.0 * float(alpha), 6):g}" + + def _json_safe_label(value: Any) -> Any: """Convert an event-time label to a JSON-serializable form. @@ -697,7 +721,7 @@ def summary(self, alpha: Optional[float] = None) -> str: f"alpha={self.alpha}; re-aggregate to obtain alpha={alpha} " "intervals (summary() never recomputes stored inference)." ) - ci_pct = int(round((1 - self.alpha) * 100)) + ci_pct = _coverage_pct(self.alpha) lines = [ "Event-Study Effects", "=" * 78, diff --git a/diff_diff/stacked_did_results.py b/diff_diff/stacked_did_results.py index 30073c755..2a1880dbd 100644 --- a/diff_diff/stacked_did_results.py +++ b/diff_diff/stacked_did_results.py @@ -14,7 +14,12 @@ from diff_diff._deprecation import deprecated_field_property from diff_diff.aggregation import AggregationMixin, AggregationResult from diff_diff.results import _format_survey_block, _get_significance_stars -from diff_diff.results_base import BaseResults, _require_fit_alpha, build_event_study_surface +from diff_diff.results_base import ( + BaseResults, + _coverage_pct, + _require_fit_alpha, + build_event_study_surface, +) __all__ = [ "StackedDiDResults", @@ -327,7 +332,7 @@ def summary(self, alpha: Optional[float] = None) -> str: Formatted summary. """ alpha = _require_fit_alpha(alpha, self.alpha) - conf_level = int((1 - alpha) * 100) + conf_level = _coverage_pct(alpha) lines = [ "=" * 85, diff --git a/diff_diff/staggered_results.py b/diff_diff/staggered_results.py index 709a7820d..82463fea0 100644 --- a/diff_diff/staggered_results.py +++ b/diff_diff/staggered_results.py @@ -23,7 +23,12 @@ apply_bootstrap_group_overrides, ) from diff_diff.results import _format_survey_block, _get_significance_stars -from diff_diff.results_base import BaseResults, _require_fit_alpha, build_event_study_surface +from diff_diff.results_base import ( + BaseResults, + _coverage_pct, + _require_fit_alpha, + build_event_study_surface, +) from diff_diff.staggered_aggregation import ( CallawaySantAnnaAggregationMixin, fixed_cohort_agg_weights, @@ -726,7 +731,7 @@ def summary(self, alpha: Optional[float] = None) -> str: Formatted summary. """ alpha = _require_fit_alpha(alpha, self.alpha) - conf_level = int((1 - alpha) * 100) + conf_level = _coverage_pct(alpha) lines = [ "=" * 85, diff --git a/diff_diff/staggered_triple_diff_results.py b/diff_diff/staggered_triple_diff_results.py index 22c3e6be4..0f0c109f4 100644 --- a/diff_diff/staggered_triple_diff_results.py +++ b/diff_diff/staggered_triple_diff_results.py @@ -12,7 +12,7 @@ import pandas as pd from diff_diff.results import _format_survey_block, _get_significance_stars -from diff_diff.results_base import BaseResults, _require_fit_alpha +from diff_diff.results_base import BaseResults, _coverage_pct, _require_fit_alpha if TYPE_CHECKING: from diff_diff.staggered_bootstrap import CSBootstrapResults @@ -175,7 +175,7 @@ def summary(self, alpha: Optional[float] = None) -> str: Formatted summary. """ alpha = _require_fit_alpha(alpha, self.alpha) - conf_level = int((1 - alpha) * 100) + conf_level = _coverage_pct(alpha) lines = [ "=" * 85, diff --git a/diff_diff/sun_abraham.py b/diff_diff/sun_abraham.py index a83f79a00..b91570afd 100644 --- a/diff_diff/sun_abraham.py +++ b/diff_diff/sun_abraham.py @@ -34,7 +34,7 @@ from diff_diff.survey import ResolvedSurveyDesign, SurveyDesign from diff_diff.linalg import LinearRegression from diff_diff.results import _format_survey_block, _get_significance_stars -from diff_diff.results_base import BaseResults, _require_fit_alpha +from diff_diff.results_base import BaseResults, _coverage_pct, _require_fit_alpha from diff_diff.utils import ( absorbed_fe_cr1_k_increment, absorbed_fe_rank, @@ -276,7 +276,7 @@ def summary(self, alpha: Optional[float] = None) -> str: Formatted summary. """ alpha = _require_fit_alpha(alpha, self.alpha) - conf_level = int((1 - alpha) * 100) + conf_level = _coverage_pct(alpha) lines = [ "=" * 85, diff --git a/diff_diff/triple_diff.py b/diff_diff/triple_diff.py index 80d052799..64dcfd383 100644 --- a/diff_diff/triple_diff.py +++ b/diff_diff/triple_diff.py @@ -44,7 +44,12 @@ from diff_diff._staggered_triple_diff_engine import _StaggeredTripleDiffEngineMixin from diff_diff.linalg import _rank_guarded_inv, solve_logit, solve_ols from diff_diff.results import _format_survey_block, _get_significance_stars -from diff_diff.results_base import _SUMMARY_ALPHA_MESSAGE, BaseResults, _require_fit_alpha +from diff_diff.results_base import ( + _SUMMARY_ALPHA_MESSAGE, + BaseResults, + _coverage_pct, + _require_fit_alpha, +) from diff_diff.staggered_aggregation import CallawaySantAnnaAggregationMixin from diff_diff.staggered_bootstrap import CallawaySantAnnaBootstrapMixin from diff_diff.staggered_triple_diff_results import StaggeredTripleDiffResults @@ -164,7 +169,7 @@ def summary(self, alpha: Optional[float] = None) -> str: Formatted summary table. """ alpha = _require_fit_alpha(alpha, self.alpha, message=_SUMMARY_ALPHA_MESSAGE) - conf_level = int((1 - alpha) * 100) + conf_level = _coverage_pct(alpha) lines = [ "=" * 75, diff --git a/diff_diff/trop_results.py b/diff_diff/trop_results.py index 7822bb81f..e125a72d1 100644 --- a/diff_diff/trop_results.py +++ b/diff_diff/trop_results.py @@ -17,7 +17,7 @@ from typing_extensions import TypedDict from diff_diff.results import _format_survey_block, _get_significance_stars -from diff_diff.results_base import BaseResults, _require_fit_alpha +from diff_diff.results_base import BaseResults, _coverage_pct, _require_fit_alpha __all__ = [ "_LAMBDA_INF", @@ -219,7 +219,7 @@ def summary(self, alpha: Optional[float] = None) -> str: "(requested alpha={alpha}); re-fit with the desired alpha." ), ) - conf_level = int((1 - alpha) * 100) + conf_level = _coverage_pct(alpha) lines = [ "=" * 75, diff --git a/diff_diff/two_stage_results.py b/diff_diff/two_stage_results.py index dd4554173..19e56105d 100644 --- a/diff_diff/two_stage_results.py +++ b/diff_diff/two_stage_results.py @@ -14,7 +14,12 @@ from diff_diff.aggregation import AggregationMixin, AggregationResult, build_total_relay_row from diff_diff.results import _format_survey_block, _get_significance_stars -from diff_diff.results_base import BaseResults, _require_fit_alpha, build_event_study_surface +from diff_diff.results_base import ( + BaseResults, + _coverage_pct, + _require_fit_alpha, + build_event_study_surface, +) from diff_diff.two_stage_aggregation import _TwoStageAggregationMixin @@ -545,7 +550,7 @@ def summary(self, alpha: Optional[float] = None) -> str: Formatted summary. """ alpha = _require_fit_alpha(alpha, self.alpha) - conf_level = int((1 - alpha) * 100) + conf_level = _coverage_pct(alpha) lines = [ "=" * 85, diff --git a/diff_diff/utils.py b/diff_diff/utils.py index b2f5d0272..aaabc3b79 100644 --- a/diff_diff/utils.py +++ b/diff_diff/utils.py @@ -27,6 +27,7 @@ from diff_diff.linalg import _validate_cluster_k_adjustment_type from diff_diff.linalg import compute_robust_vcov as _compute_robust_vcov_linalg from diff_diff.linalg import solve_ols as _solve_ols_linalg +from diff_diff.results_base import _coverage_pct # Numerical constants for optimization algorithms _OPTIMIZATION_MAX_ITER = 1000 # Maximum iterations for weight optimization @@ -775,7 +776,7 @@ def summary(self) -> str: f"Cluster-robust SE: {self.se:.6f}", f"Bootstrap p-value: {self.p_value:.4f}", f"Studentized t-stat: {self.t_stat_original:.4f}", - f"CI ({int((1-self.alpha)*100)}%): [{self.ci_lower:.6f}, {self.ci_upper:.6f}]", + f"CI ({_coverage_pct(self.alpha)}%): [{self.ci_lower:.6f}, {self.ci_upper:.6f}]", f"Number of clusters: {self.n_clusters}", f"Bootstrap reps: {self.n_bootstrap}", f"Weight type: {self.weight_type}", diff --git a/diff_diff/visualization/_continuous.py b/diff_diff/visualization/_continuous.py index 000210724..f8d462450 100644 --- a/diff_diff/visualization/_continuous.py +++ b/diff_diff/visualization/_continuous.py @@ -7,6 +7,8 @@ import numpy as np import pandas as pd +from diff_diff.results_base import _coverage_pct + if TYPE_CHECKING: from diff_diff.continuous_did_results import ContinuousDiDResults, DoseResponseCurve @@ -17,8 +19,7 @@ def _coverage_label(alpha: float) -> str: Rounds to 6 decimals first so a float32-noised alpha (0.0500000007...) still reads 95 rather than 94.9999999255, then trims trailing zeros. """ - level = round(100.0 * (1.0 - float(alpha)), 6) - return f"{level:g}% CI" + return f"{_coverage_pct(alpha)}% CI" def plot_dose_response( diff --git a/diff_diff/visualization/_event_study.py b/diff_diff/visualization/_event_study.py index e18b48019..31fb5baef 100644 --- a/diff_diff/visualization/_event_study.py +++ b/diff_diff/visualization/_event_study.py @@ -269,7 +269,7 @@ def plot_event_study( # recomputes at the requested ``alpha``, so it never reaches here with # overrides active.) if results is not None and ci_lower_override is not None: - from diff_diff.results_base import EventStudyResults + from diff_diff.results_base import EventStudyResults, _coverage_pct if isinstance(results, EventStudyResults): stored_alpha = getattr(results, "alpha", None) @@ -278,7 +278,7 @@ def plot_event_study( f"plot_event_study(alpha={alpha}) does not apply to an " "EventStudyResults container: the stored intervals " f"drawn here are at the fit's alpha={stored_alpha} " - f"({(1 - float(stored_alpha)) * 100:g}% coverage). " + f"({_coverage_pct(stored_alpha)}% coverage). " "Re-aggregate from a fit at the desired level to " "change the plotted coverage.", UserWarning, diff --git a/diff_diff/wooldridge_results.py b/diff_diff/wooldridge_results.py index 8da445682..9612096d7 100644 --- a/diff_diff/wooldridge_results.py +++ b/diff_diff/wooldridge_results.py @@ -15,7 +15,7 @@ resolve_renamed_kwarg, warn_deprecated_kwarg, ) -from diff_diff.results_base import BaseResults +from diff_diff.results_base import BaseResults, _coverage_pct from diff_diff.utils import safe_inference @@ -774,7 +774,7 @@ def _fmt_row(label: str, att: float, se: float, t: float, p: float, ci: Tuple) - f"{p:>8.4f}{stars} [{ci_lo}, {ci_hi}]" ) - ci_pct = f"{(1 - _alpha) * 100:.0f}%" + ci_pct = f"{_coverage_pct(_alpha)}%" header = ( f"{'Parameter':<22} {'Estimate':>10} {'Std. Err.':>10} " f"{'t-stat':>8} {'P>|t|':>8} [{ci_pct} CI]" diff --git a/docs/methodology/REPORTING.md b/docs/methodology/REPORTING.md index 223ef9084..62faca605 100644 --- a/docs/methodology/REPORTING.md +++ b/docs/methodology/REPORTING.md @@ -546,6 +546,17 @@ a library setting. `"2.0"`. The schemas remain marked experimental, so the formal deprecation policy does not yet apply. +- **Note:** `headline.ci_level` carries the EXACT confidence level in + percent: an `int` for integral coverage (`95` at `alpha=0.05`, byte- + identical to the historical value) and a `float` for fractional coverage + (`97.5` at `alpha=0.025`, previously rounded to `98`). No version bump: + the key, its meaning, and every integral value are unchanged; only + previously misreported fractional levels change, and they change type + rather than enum membership. Every text surface (summary headers, + event-study table headers, sup-t band labels, report prose) derives its + percent from the same `results_base._coverage_level` / `_coverage_pct` + helpers, so display and schema cannot disagree. + ## Reference implementation(s) The phrasing rules follow the guidance in: diff --git a/tests/test_coverage_label.py b/tests/test_coverage_label.py new file mode 100644 index 000000000..e6d96c51a --- /dev/null +++ b/tests/test_coverage_label.py @@ -0,0 +1,202 @@ +"""Exact confidence-level display: no summary truncates or rounds fractional coverage. + +Every text surface that names a confidence level (``summary()`` headers, event-study +table headers, sup-t band labels, HonestDiD / placebo diagnostics, wild-bootstrap +summaries, Business/Diagnostic report prose and the serialized ``headline.ci_level`` +field) routes through ``results_base._coverage_level`` / ``_coverage_pct``. Before +this module existed, ``alpha=0.025`` printed "97% Confidence Interval" (truncation) +or "98%" (rounding) over a 97.5% interval. +""" + +import re +import warnings +from pathlib import Path + +import numpy as np +import pytest + +import diff_diff as dd +from diff_diff import prep_dgp +from diff_diff.results_base import _alpha_pct, _coverage_level, _coverage_pct + +REPO_ROOT = Path(__file__).resolve().parent.parent +_PKG_FILES = sorted( + p for p in (REPO_ROOT / "diff_diff").rglob("*.py") if "__pycache__" not in p.parts +) + +# The three shapes the sweep retired. No required identifier prefix, so bare +# ``alpha``, ``_alpha``, ``hd.alpha`` and ``self._context.alpha`` all match; an +# optional ``float(...)`` wrapper is allowed; no required ``int(`` so the +# ``:.0f`` format-string variant matches too. +_ALPHA = r"(?:float\()?[A-Za-z0-9_.]*alpha\)?" +_RETIRED_PATTERNS = [ + re.compile(r"\(\s*1(\.0)?\s*-\s*" + _ALPHA + r"\s*\)\s*\*\s*100"), + re.compile(r"100(\.0)?\s*\*\s*\(\s*1(\.0)?\s*-\s*" + _ALPHA + r"\s*\)"), + re.compile(_ALPHA + r"\s*\*\s*100\b"), +] +# (relative path, substring) pairs that are the formatter itself or prose. +_EXEMPT = [ + ("results_base.py", "level = round(100.0 * (1.0 - float(alpha)), 6)"), + ("results_base.py", 'return f"{round(100.0 * float(alpha), 6):g}"'), + ("rdd.py", "rdrobust ``level = 100*(1-alpha)``"), +] + + +class TestFormatter: + @pytest.mark.parametrize( + "alpha, level, pct", + [ + (0.05, 95, "95"), + (0.10, 90, "90"), + (0.01, 99, "99"), + (0.025, 97.5, "97.5"), + (0.001, 99.9, "99.9"), + (np.float32(0.05), 95, "95"), + ], + ) + def test_coverage_exact(self, alpha, level, pct): + got = _coverage_level(alpha) + assert got == level + # int when integral (schema byte-compatibility), float otherwise + assert isinstance(got, int) == float(level).is_integer() + assert _coverage_pct(alpha) == pct + + @pytest.mark.parametrize("alpha, pct", [(0.05, "5"), (0.025, "2.5"), (0.10, "10")]) + def test_alpha_pct(self, alpha, pct): + assert _alpha_pct(alpha) == pct + + +class TestSourceGuard: + def test_no_truncating_or_rounding_coverage_site_remains(self): + offenders = [] + for path in _PKG_FILES: + rel = path.relative_to(REPO_ROOT / "diff_diff").as_posix() + for lineno, line in enumerate(path.read_text().splitlines(), start=1): + if line.strip().startswith("#"): + continue + if any(rel == f and s in line for f, s in _EXEMPT): + continue + if any(rx.search(line) for rx in _RETIRED_PATTERNS): + offenders.append(f"{rel}:{lineno}: {line.strip()}") + assert not offenders, ( + "confidence-level percent computed inline instead of via " + "results_base._coverage_pct / _coverage_level / _alpha_pct:\n" + "\n".join(offenders) + ) + + +@pytest.fixture(scope="module") +def did_data(): + return prep_dgp.generate_did_data(seed=1) + + +@pytest.fixture(scope="module") +def staggered_data(): + return prep_dgp.generate_staggered_data(seed=1, n_units=60, n_periods=6) + + +@pytest.fixture(scope="module") +def event_study_data(): + return prep_dgp.generate_event_study_data(seed=1, n_units=80) + + +class TestFractionalAlphaSurfaces: + """alpha=0.025 renders 97.5, never 97 or 98, on one representative of each shape.""" + + def test_did_summary(self, did_data): + s = ( + dd.DifferenceInDifferences(alpha=0.025) + .fit(did_data, "outcome", "treated", "post") + .summary() + ) + assert "97.5% Confidence Interval" in s + assert "97% " not in s and "98% " not in s + + def test_callaway_santanna_summary_and_supt_band(self, staggered_data): + with warnings.catch_warnings(): + warnings.simplefilter("ignore") + r = dd.CallawaySantAnna(alpha=0.025, n_bootstrap=20, seed=1, cband=True).fit( + staggered_data, "outcome", "unit", "period", "first_treat" + ) + s = r.summary() + assert "97.5% Confidence Interval" in s + assert "97% " not in s and "98% " not in s + + def test_wooldridge_format_string_header(self, staggered_data): + with warnings.catch_warnings(): + warnings.simplefilter("ignore") + s = ( + dd.WooldridgeDiD(alpha=0.025) + .fit(staggered_data, "outcome", "unit", "period", first_treat="first_treat") + .summary() + ) + assert "97.5%" in s + assert "98%" not in s + + def test_event_study_table_header(self, event_study_data): + with warnings.catch_warnings(): + warnings.simplefilter("ignore") + s = ( + dd.TwoWayFixedEffects(alpha=0.025) + .fit( + event_study_data, + "outcome", + "treated", + unit="unit", + time="period", + event_study=True, + post_periods=[5, 6, 7, 8, 9], + ) + .summary() + ) + assert "[97.5% CI]" in s + assert "[97% CI]" not in s and "[98% CI]" not in s + + def test_dcdh_honest_significance_level(self, staggered_data): + with warnings.catch_warnings(): + warnings.simplefilter("ignore") + s = ( + dd.ChaisemartinDHaultfoeuille(alpha=0.025) + .fit( + staggered_data, + "outcome", + unit="unit", + time="period", + treatment="treated", + honest_did=True, + L_max=2, + ) + .summary() + ) + assert "Robust 97.5% CI:" in s + assert "Significant at 2.5%:" in s + assert "Significant at 2%:" not in s + + def test_business_report_headline_ci_level_and_prose(self, did_data): + r_frac = dd.DifferenceInDifferences(alpha=0.025).fit(did_data, "outcome", "treated", "post") + br = dd.BusinessReport(r_frac, outcome_label="sales", outcome_unit="usd") + h = br.to_dict()["headline"] + assert h["ci_level"] == 97.5 and isinstance(h["ci_level"], float) + assert "97.5% CI:" in br.summary() + # integral coverage stays an exact int (JSON byte-compatible with the + # historical schema value) + r_default = dd.DifferenceInDifferences().fit(did_data, "outcome", "treated", "post") + h95 = dd.BusinessReport(r_default, outcome_label="sales", outcome_unit="usd").to_dict()[ + "headline" + ] + assert h95["ci_level"] == 95 and isinstance(h95["ci_level"], int) + + def test_diagnostic_report_prose(self, did_data): + r_frac = dd.DifferenceInDifferences(alpha=0.025).fit(did_data, "outcome", "treated", "post") + with warnings.catch_warnings(): + warnings.simplefilter("ignore") + dr = dd.DiagnosticReport( + r_frac, + data=did_data, + outcome="outcome", + treatment="treated", + time="period", + unit="unit", + ) + text = dr.summary() + dr.full_report() + assert "97.5% CI:" in text + assert "98% CI:" not in text and "97% CI:" not in text