diff --git a/changelog.d/2617.fixed.md b/changelog.d/2617.fixed.md new file mode 100644 index 0000000000..88c2111180 --- /dev/null +++ b/changelog.d/2617.fixed.md @@ -0,0 +1 @@ +- Break wealth-decile ties by a fixed hash of `household_id` instead of row order, and give every household with non-negative wealth a decile from 1 to 10, also when every household has zero weight. diff --git a/policyengine_uk/tests/test_household_wealth_decile.py b/policyengine_uk/tests/test_household_wealth_decile.py index 0ef5d5b91e..d50558826c 100644 --- a/policyengine_uk/tests/test_household_wealth_decile.py +++ b/policyengine_uk/tests/test_household_wealth_decile.py @@ -1,19 +1,33 @@ -"""Wealth ranks split ties in original household order (issue #2579).""" +"""Wealth ranks split ties by a fixed hash of household_id (#2579, #2617).""" import numpy as np import pytest from policyengine_uk.variables.household.wealth.household_wealth_decile import ( + _fixed_hash, household_wealth_decile, ) +def ids_in_hash_order(count): + """Household ids whose fixed-hash order is their row order. + + With these ids, a tie splits in row order, so a test can state the + expected deciles row by row. + """ + ids = np.arange(count * 20) + return ids[np.argsort(_fixed_hash(ids))][:count] + + class FakeHousehold: - def __init__(self, wealth, weights, sizes): + def __init__(self, wealth, weights, sizes, ids=None): self.values = { "total_wealth": np.array(wealth, dtype=float), "household_weight": np.array(weights, dtype=float), "household_count_people": np.array(sizes), + "household_id": np.array( + ids_in_hash_order(len(wealth)) if ids is None else ids, dtype=int + ), } def __call__(self, variable, period): @@ -22,7 +36,8 @@ def __call__(self, variable, period): def test_zero_wealth_ties_fill_the_bottom_deciles(): # Fifty equally weighted people: each five successive households fill - # one decile, including the fifteen households with zero wealth. + # one decile, including the fifteen households with zero wealth. The + # default ids put the tied households in hash order = row order. household = FakeHousehold([0] * 15 + list(range(1, 36)), [1] * 50, [1] * 50) result = household_wealth_decile.formula(household, 2026, None) @@ -35,7 +50,7 @@ def test_zero_wealth_ties_fill_the_bottom_deciles(): [ # Cumulative person weights 1, 2, 4, 10 out of 10 determine deciles. ([0, 0, 0, 10], [1, 1, 2, 6], [1, 1, 1, 1], [1, 2, 4, 10]), - # Sorting must restore original rows, retaining order within ties. + # Sorting must restore original rows, retaining hash order within ties. ([10, 0, 0, 0], [6, 1, 1, 2], [1, 1, 1, 1], [10, 1, 2, 4]), # The same effective weights, factored through household sizes. ([0, 0, 0, 10], [1, 1, 1, 1], [1, 1, 2, 6], [1, 2, 4, 10]), @@ -64,8 +79,52 @@ def test_person_weighted_wealth_ranks(wealth, weights, sizes, expected): np.testing.assert_array_equal(household.values["total_wealth"], wealth) -def test_zero_total_weight_still_raises(): - household = FakeHousehold([0, 0], [0, 0], [1, 1]) +def test_zero_total_weight_counts_each_person_equally(): + # With no weight at all, the ranking falls back to one per person, so + # a household of four people takes four tenths of an eight-person frame. + household = FakeHousehold([0, 5, 10], [0, 0, 0], [2, 2, 4]) + + result = household_wealth_decile.formula(household, 2026, None) + + np.testing.assert_array_equal(result, [3, 5, 10]) + + +def test_tied_households_keep_their_decile_when_rows_are_shuffled(): + # Issue #2617: ties split by a fixed hash of household_id, so a + # dataset rebuild that reorders households moves nobody's decile. + ids = np.arange(1000, 1050) * 7 + wealth = [0] * 15 + list(range(1, 36)) + household = FakeHousehold(wealth, [1] * 50, [1] * 50, ids) + by_id = dict(zip(ids, household_wealth_decile.formula(household, 2026, None))) + + order = np.random.default_rng(0).permutation(50) + shuffled = FakeHousehold(np.array(wealth)[order], [1] * 50, [1] * 50, ids[order]) + result = household_wealth_decile.formula(shuffled, 2026, None) + + assert dict(zip(ids[order], result)) == by_id + # The fifteen £0 households still fill deciles 1 to 3, five each. + zero_deciles = [by_id[i] for i in ids[:15]] + assert sorted(zero_deciles) == [1] * 5 + [2] * 5 + [3] * 5 + + +def test_tie_order_does_not_follow_household_id_order(): + # Row order and household_id order agree here, so a rule that + # follows either one would put the lowest ids in decile 1. + ids = np.arange(20) + household = FakeHousehold([0] * 20, [1] * 20, [1] * 20, ids) + + result = household_wealth_decile.formula(household, 2026, None) + + assert not np.array_equal(result, np.sort(result)) + + +def test_zero_weight_household_gets_a_decile_of_at_least_1(): + # Issue #2617: a zero-weight household at the start of the order + # used to get decile 0, outside the 1 to 10 range. + for ids in ([0, 1, 2], [1, 0, 2], [5, 9, 3]): + household = FakeHousehold([0, 0, 5], [0, 1, 1], [1, 1, 1], ids) + + result = household_wealth_decile.formula(household, 2026, None) - with pytest.raises(ZeroDivisionError, match="zero total weight"): - household_wealth_decile.formula(household, 2026, None) + assert result.min() >= 1 + assert result[2] == 10 diff --git a/policyengine_uk/variables/household/wealth/household_wealth_decile.py b/policyengine_uk/variables/household/wealth/household_wealth_decile.py index 35eb39e993..7163af38be 100644 --- a/policyengine_uk/variables/household/wealth/household_wealth_decile.py +++ b/policyengine_uk/variables/household/wealth/household_wealth_decile.py @@ -1,16 +1,28 @@ from policyengine_uk.model_api import * +def _fixed_hash(values): + """A splitmix64 mix of integer ids: the same on every run, unlike hash().""" + x = np.asarray(values).astype(np.int64).astype(np.uint64) + with np.errstate(over="ignore"): + x = x + np.uint64(0x9E3779B97F4A7C15) + x = (x ^ (x >> np.uint64(30))) * np.uint64(0xBF58476D1CE4E5B9) + x = (x ^ (x >> np.uint64(27))) * np.uint64(0x94D049BB133111EB) + return x ^ (x >> np.uint64(31)) + + class household_wealth_decile(Variable): label = "household wealth decile" documentation = ( "Person-weighted decile of gross total_wealth, before debt deductions. " "Each household's weight is household_weight * household_count_people. " - "Equal wealth values are ranked in original household/entity order. " - "Changing this order, including when rebuilding a dataset, can change " - "tied households' decile assignments. Negative total_wealth participates " - "in the ranking before its output is set to -1. Household weights are " - "indivisible, so decile bins can still be empty." + "Equal wealth values are ranked by a fixed hash of household_id, then " + "by row order, so tied households keep their deciles when a dataset " + "is rebuilt in another order. Negative total_wealth participates in " + "the ranking before its output is set to -1. Every other household " + "gets a decile from 1 to 10, also one with zero weight. If every " + "household has zero weight, each person counts equally. Household " + "weights are indivisible, so decile bins can still be empty." ) entity = Household definition_period = YEAR @@ -20,13 +32,20 @@ def formula(household, period, parameters): wealth = household("total_wealth", period) count_people = household("household_count_people", period) household_weight = household("household_weight", period) - # Split equal wealth in original household order without changing wealth. + tie_key = _fixed_hash(household("household_id", period)) + # Split equal wealth by the fixed hash, not by file order, without + # changing wealth. np.lexsort sorts by its last key first. + order = np.lexsort((np.arange(len(wealth)), tie_key, wealth)) wealth_rank = np.empty(len(wealth), dtype=int) - wealth_rank[np.argsort(wealth, kind="stable")] = np.arange(len(wealth)) - weighted_wealth = MicroSeries( - wealth_rank, weights=household_weight * count_people - ) - decile = weighted_wealth.decile_rank().values + wealth_rank[order] = np.arange(len(wealth)) + person_weight = household_weight * count_people + if person_weight.sum() == 0: + # A frame with no weight at all (a test or an unweighted input) + # still gets deciles: each person counts equally. + person_weight = count_people + weighted_wealth = MicroSeries(wealth_rank, weights=person_weight) + # A zero-weight household at the start of the order would get 0. + decile = np.clip(weighted_wealth.decile_rank().values, 1, 10) # Set negatives to -1. # This avoids the bottom decile summing to a negative number, # which would flip the % change in the interface.