From 036eea6c2686b1e775a1e61eaa3752eb615985f0 Mon Sep 17 00:00:00 2001 From: PavelMakarchuk Date: Fri, 23 Oct 2026 15:16:44 -0400 Subject: [PATCH 1/2] Break wealth-decile ties by a fixed hash of household_id Ties in household_wealth_decile split in row order, which carries information and can move when a dataset is rebuilt. They now split by a salt-free splitmix64 hash of household_id, then by row order, so tied households keep their deciles in any file order. Every household with non-negative wealth now gets a decile from 1 to 10, also one with zero weight at the start of the order, which used to get 0. Fixes #2617 Co-Authored-By: Claude Opus 5.5 --- changelog.d/2617.fixed.md | 1 + .../tests/test_household_wealth_decile.py | 64 +++++++++++++++++-- .../wealth/household_wealth_decile.py | 31 ++++++--- 3 files changed, 84 insertions(+), 12 deletions(-) create mode 100644 changelog.d/2617.fixed.md diff --git a/changelog.d/2617.fixed.md b/changelog.d/2617.fixed.md new file mode 100644 index 0000000000..67177c9f6a --- /dev/null +++ b/changelog.d/2617.fixed.md @@ -0,0 +1 @@ +- Break wealth-decile ties by a fixed hash of `household_id` instead of row order, and give every household with non-negative wealth a decile from 1 to 10. diff --git a/policyengine_uk/tests/test_household_wealth_decile.py b/policyengine_uk/tests/test_household_wealth_decile.py index 0ef5d5b91e..ea2f5e3c0f 100644 --- a/policyengine_uk/tests/test_household_wealth_decile.py +++ b/policyengine_uk/tests/test_household_wealth_decile.py @@ -1,19 +1,33 @@ -"""Wealth ranks split ties in original household order (issue #2579).""" +"""Wealth ranks split ties by a fixed hash of household_id (#2579, #2617).""" import numpy as np import pytest from policyengine_uk.variables.household.wealth.household_wealth_decile import ( + _fixed_hash, household_wealth_decile, ) +def ids_in_hash_order(count): + """Household ids whose fixed-hash order is their row order. + + With these ids, a tie splits in row order, so a test can state the + expected deciles row by row. + """ + ids = np.arange(count * 20) + return ids[np.argsort(_fixed_hash(ids))][:count] + + class FakeHousehold: - def __init__(self, wealth, weights, sizes): + def __init__(self, wealth, weights, sizes, ids=None): self.values = { "total_wealth": np.array(wealth, dtype=float), "household_weight": np.array(weights, dtype=float), "household_count_people": np.array(sizes), + "household_id": np.array( + ids_in_hash_order(len(wealth)) if ids is None else ids, dtype=int + ), } def __call__(self, variable, period): @@ -22,7 +36,8 @@ def __call__(self, variable, period): def test_zero_wealth_ties_fill_the_bottom_deciles(): # Fifty equally weighted people: each five successive households fill - # one decile, including the fifteen households with zero wealth. + # one decile, including the fifteen households with zero wealth. The + # default ids put the tied households in hash order = row order. household = FakeHousehold([0] * 15 + list(range(1, 36)), [1] * 50, [1] * 50) result = household_wealth_decile.formula(household, 2026, None) @@ -35,7 +50,7 @@ def test_zero_wealth_ties_fill_the_bottom_deciles(): [ # Cumulative person weights 1, 2, 4, 10 out of 10 determine deciles. ([0, 0, 0, 10], [1, 1, 2, 6], [1, 1, 1, 1], [1, 2, 4, 10]), - # Sorting must restore original rows, retaining order within ties. + # Sorting must restore original rows, retaining hash order within ties. ([10, 0, 0, 0], [6, 1, 1, 2], [1, 1, 1, 1], [10, 1, 2, 4]), # The same effective weights, factored through household sizes. ([0, 0, 0, 10], [1, 1, 1, 1], [1, 1, 2, 6], [1, 2, 4, 10]), @@ -69,3 +84,44 @@ def test_zero_total_weight_still_raises(): with pytest.raises(ZeroDivisionError, match="zero total weight"): household_wealth_decile.formula(household, 2026, None) + + +def test_tied_households_keep_their_decile_when_rows_are_shuffled(): + # Issue #2617: ties split by a fixed hash of household_id, so a + # dataset rebuild that reorders households moves nobody's decile. + ids = np.arange(1000, 1050) * 7 + wealth = [0] * 15 + list(range(1, 36)) + household = FakeHousehold(wealth, [1] * 50, [1] * 50, ids) + by_id = dict(zip(ids, household_wealth_decile.formula(household, 2026, None))) + + order = np.random.default_rng(0).permutation(50) + shuffled = FakeHousehold(np.array(wealth)[order], [1] * 50, [1] * 50, ids[order]) + result = household_wealth_decile.formula(shuffled, 2026, None) + + assert dict(zip(ids[order], result)) == by_id + # The fifteen £0 households still fill deciles 1 to 3, five each. + zero_deciles = [by_id[i] for i in ids[:15]] + assert sorted(zero_deciles) == [1] * 5 + [2] * 5 + [3] * 5 + + +def test_tie_order_does_not_follow_household_id_order(): + # Row order and household_id order agree here, so a rule that + # follows either one would put the lowest ids in decile 1. + ids = np.arange(20) + household = FakeHousehold([0] * 20, [1] * 20, [1] * 20, ids) + + result = household_wealth_decile.formula(household, 2026, None) + + assert not np.array_equal(result, np.sort(result)) + + +def test_zero_weight_household_gets_a_decile_of_at_least_1(): + # Issue #2617: a zero-weight household at the start of the order + # used to get decile 0, outside the 1 to 10 range. + for ids in ([0, 1, 2], [1, 0, 2], [5, 9, 3]): + household = FakeHousehold([0, 0, 5], [0, 1, 1], [1, 1, 1], ids) + + result = household_wealth_decile.formula(household, 2026, None) + + assert result.min() >= 1 + assert result[2] == 10 diff --git a/policyengine_uk/variables/household/wealth/household_wealth_decile.py b/policyengine_uk/variables/household/wealth/household_wealth_decile.py index 35eb39e993..55b65727e4 100644 --- a/policyengine_uk/variables/household/wealth/household_wealth_decile.py +++ b/policyengine_uk/variables/household/wealth/household_wealth_decile.py @@ -1,16 +1,27 @@ from policyengine_uk.model_api import * +def _fixed_hash(values): + """A splitmix64 mix of integer ids: the same on every run, unlike hash().""" + x = np.asarray(values).astype(np.int64).astype(np.uint64) + with np.errstate(over="ignore"): + x = x + np.uint64(0x9E3779B97F4A7C15) + x = (x ^ (x >> np.uint64(30))) * np.uint64(0xBF58476D1CE4E5B9) + x = (x ^ (x >> np.uint64(27))) * np.uint64(0x94D049BB133111EB) + return x ^ (x >> np.uint64(31)) + + class household_wealth_decile(Variable): label = "household wealth decile" documentation = ( "Person-weighted decile of gross total_wealth, before debt deductions. " "Each household's weight is household_weight * household_count_people. " - "Equal wealth values are ranked in original household/entity order. " - "Changing this order, including when rebuilding a dataset, can change " - "tied households' decile assignments. Negative total_wealth participates " - "in the ranking before its output is set to -1. Household weights are " - "indivisible, so decile bins can still be empty." + "Equal wealth values are ranked by a fixed hash of household_id, then " + "by row order, so tied households keep their deciles when a dataset " + "is rebuilt in another order. Negative total_wealth participates in " + "the ranking before its output is set to -1. Every other household " + "gets a decile from 1 to 10, also one with zero weight. Household " + "weights are indivisible, so decile bins can still be empty." ) entity = Household definition_period = YEAR @@ -20,13 +31,17 @@ def formula(household, period, parameters): wealth = household("total_wealth", period) count_people = household("household_count_people", period) household_weight = household("household_weight", period) - # Split equal wealth in original household order without changing wealth. + tie_key = _fixed_hash(household("household_id", period)) + # Split equal wealth by the fixed hash, not by file order, without + # changing wealth. np.lexsort sorts by its last key first. + order = np.lexsort((np.arange(len(wealth)), tie_key, wealth)) wealth_rank = np.empty(len(wealth), dtype=int) - wealth_rank[np.argsort(wealth, kind="stable")] = np.arange(len(wealth)) + wealth_rank[order] = np.arange(len(wealth)) weighted_wealth = MicroSeries( wealth_rank, weights=household_weight * count_people ) - decile = weighted_wealth.decile_rank().values + # A zero-weight household at the start of the order would get 0. + decile = np.clip(weighted_wealth.decile_rank().values, 1, 10) # Set negatives to -1. # This avoids the bottom decile summing to a negative number, # which would flip the % change in the interface. From 47ee0515b00082cb049c0e75d545f5306e947f96 Mon Sep 17 00:00:00 2001 From: PavelMakarchuk Date: Sat, 24 Oct 2026 11:54:43 -0400 Subject: [PATCH 2/2] Give wealth deciles when every household has zero weight The formula now reads household_id, so the identifier round-trip test computes it on generated frames, one of which has no weight at all. Count each person equally there instead of raising. Co-Authored-By: Claude Opus 5.5 --- changelog.d/2617.fixed.md | 2 +- .../tests/test_household_wealth_decile.py | 11 +++++++---- .../household/wealth/household_wealth_decile.py | 12 ++++++++---- 3 files changed, 16 insertions(+), 9 deletions(-) diff --git a/changelog.d/2617.fixed.md b/changelog.d/2617.fixed.md index 67177c9f6a..88c2111180 100644 --- a/changelog.d/2617.fixed.md +++ b/changelog.d/2617.fixed.md @@ -1 +1 @@ -- Break wealth-decile ties by a fixed hash of `household_id` instead of row order, and give every household with non-negative wealth a decile from 1 to 10. +- Break wealth-decile ties by a fixed hash of `household_id` instead of row order, and give every household with non-negative wealth a decile from 1 to 10, also when every household has zero weight. diff --git a/policyengine_uk/tests/test_household_wealth_decile.py b/policyengine_uk/tests/test_household_wealth_decile.py index ea2f5e3c0f..d50558826c 100644 --- a/policyengine_uk/tests/test_household_wealth_decile.py +++ b/policyengine_uk/tests/test_household_wealth_decile.py @@ -79,11 +79,14 @@ def test_person_weighted_wealth_ranks(wealth, weights, sizes, expected): np.testing.assert_array_equal(household.values["total_wealth"], wealth) -def test_zero_total_weight_still_raises(): - household = FakeHousehold([0, 0], [0, 0], [1, 1]) +def test_zero_total_weight_counts_each_person_equally(): + # With no weight at all, the ranking falls back to one per person, so + # a household of four people takes four tenths of an eight-person frame. + household = FakeHousehold([0, 5, 10], [0, 0, 0], [2, 2, 4]) - with pytest.raises(ZeroDivisionError, match="zero total weight"): - household_wealth_decile.formula(household, 2026, None) + result = household_wealth_decile.formula(household, 2026, None) + + np.testing.assert_array_equal(result, [3, 5, 10]) def test_tied_households_keep_their_decile_when_rows_are_shuffled(): diff --git a/policyengine_uk/variables/household/wealth/household_wealth_decile.py b/policyengine_uk/variables/household/wealth/household_wealth_decile.py index 55b65727e4..7163af38be 100644 --- a/policyengine_uk/variables/household/wealth/household_wealth_decile.py +++ b/policyengine_uk/variables/household/wealth/household_wealth_decile.py @@ -20,7 +20,8 @@ class household_wealth_decile(Variable): "by row order, so tied households keep their deciles when a dataset " "is rebuilt in another order. Negative total_wealth participates in " "the ranking before its output is set to -1. Every other household " - "gets a decile from 1 to 10, also one with zero weight. Household " + "gets a decile from 1 to 10, also one with zero weight. If every " + "household has zero weight, each person counts equally. Household " "weights are indivisible, so decile bins can still be empty." ) entity = Household @@ -37,9 +38,12 @@ def formula(household, period, parameters): order = np.lexsort((np.arange(len(wealth)), tie_key, wealth)) wealth_rank = np.empty(len(wealth), dtype=int) wealth_rank[order] = np.arange(len(wealth)) - weighted_wealth = MicroSeries( - wealth_rank, weights=household_weight * count_people - ) + person_weight = household_weight * count_people + if person_weight.sum() == 0: + # A frame with no weight at all (a test or an unweighted input) + # still gets deciles: each person counts equally. + person_weight = count_people + weighted_wealth = MicroSeries(wealth_rank, weights=person_weight) # A zero-weight household at the start of the order would get 0. decile = np.clip(weighted_wealth.decile_rank().values, 1, 10) # Set negatives to -1.