Break wealth-decile ties by a fixed hash of household_id

changelog.d/2617.fixed.md

@@ -0,0 +1 @@
+- Break wealth-decile ties by a fixed hash of `household_id` instead of row order, and give every household with non-negative wealth a decile from 1 to 10, also when every household has zero weight.

policyengine_uk/tests/test_household_wealth_decile.py

@@ -1,19 +1,33 @@
-"""Wealth ranks split ties in original household order (issue #2579)."""
+"""Wealth ranks split ties by a fixed hash of household_id (#2579, #2617)."""
 
 import numpy as np
 import pytest
 
 from policyengine_uk.variables.household.wealth.household_wealth_decile import (
+    _fixed_hash,
     household_wealth_decile,
 )
 
 
+def ids_in_hash_order(count):
+    """Household ids whose fixed-hash order is their row order.
+
+    With these ids, a tie splits in row order, so a test can state the
+    expected deciles row by row.
+    """
+    ids = np.arange(count * 20)
+    return ids[np.argsort(_fixed_hash(ids))][:count]
+
+
 class FakeHousehold:
-    def __init__(self, wealth, weights, sizes):
+    def __init__(self, wealth, weights, sizes, ids=None):
         self.values = {
             "total_wealth": np.array(wealth, dtype=float),
             "household_weight": np.array(weights, dtype=float),
             "household_count_people": np.array(sizes),
+            "household_id": np.array(
+                ids_in_hash_order(len(wealth)) if ids is None else ids, dtype=int
+            ),
         }
 
     def __call__(self, variable, period):
@@ -22,7 +36,8 @@ def __call__(self, variable, period):
 
 def test_zero_wealth_ties_fill_the_bottom_deciles():
     # Fifty equally weighted people: each five successive households fill
-    # one decile, including the fifteen households with zero wealth.
+    # one decile, including the fifteen households with zero wealth. The
+    # default ids put the tied households in hash order = row order.
     household = FakeHousehold([0] * 15 + list(range(1, 36)), [1] * 50, [1] * 50)
 
     result = household_wealth_decile.formula(household, 2026, None)
@@ -35,7 +50,7 @@ def test_zero_wealth_ties_fill_the_bottom_deciles():
     [
         # Cumulative person weights 1, 2, 4, 10 out of 10 determine deciles.
         ([0, 0, 0, 10], [1, 1, 2, 6], [1, 1, 1, 1], [1, 2, 4, 10]),
-        # Sorting must restore original rows, retaining order within ties.
+        # Sorting must restore original rows, retaining hash order within ties.
         ([10, 0, 0, 0], [6, 1, 1, 2], [1, 1, 1, 1], [10, 1, 2, 4]),
         # The same effective weights, factored through household sizes.
         ([0, 0, 0, 10], [1, 1, 1, 1], [1, 1, 2, 6], [1, 2, 4, 10]),
@@ -64,8 +79,52 @@ def test_person_weighted_wealth_ranks(wealth, weights, sizes, expected):
         np.testing.assert_array_equal(household.values["total_wealth"], wealth)
 
 
-def test_zero_total_weight_still_raises():
-    household = FakeHousehold([0, 0], [0, 0], [1, 1])
+def test_zero_total_weight_counts_each_person_equally():
+    # With no weight at all, the ranking falls back to one per person, so
+    # a household of four people takes four tenths of an eight-person frame.
+    household = FakeHousehold([0, 5, 10], [0, 0, 0], [2, 2, 4])
+
+    result = household_wealth_decile.formula(household, 2026, None)
+
+    np.testing.assert_array_equal(result, [3, 5, 10])
+
+
+def test_tied_households_keep_their_decile_when_rows_are_shuffled():
+    # Issue #2617: ties split by a fixed hash of household_id, so a
+    # dataset rebuild that reorders households moves nobody's decile.
+    ids = np.arange(1000, 1050) * 7
+    wealth = [0] * 15 + list(range(1, 36))
+    household = FakeHousehold(wealth, [1] * 50, [1] * 50, ids)
+    by_id = dict(zip(ids, household_wealth_decile.formula(household, 2026, None)))
+
+    order = np.random.default_rng(0).permutation(50)
+    shuffled = FakeHousehold(np.array(wealth)[order], [1] * 50, [1] * 50, ids[order])
+    result = household_wealth_decile.formula(shuffled, 2026, None)
+
+    assert dict(zip(ids[order], result)) == by_id
+    # The fifteen £0 households still fill deciles 1 to 3, five each.
+    zero_deciles = [by_id[i] for i in ids[:15]]
+    assert sorted(zero_deciles) == [1] * 5 + [2] * 5 + [3] * 5
+
+
+def test_tie_order_does_not_follow_household_id_order():
+    # Row order and household_id order agree here, so a rule that
+    # follows either one would put the lowest ids in decile 1.
+    ids = np.arange(20)
+    household = FakeHousehold([0] * 20, [1] * 20, [1] * 20, ids)
+
+    result = household_wealth_decile.formula(household, 2026, None)
+
+    assert not np.array_equal(result, np.sort(result))
+
+
+def test_zero_weight_household_gets_a_decile_of_at_least_1():
+    # Issue #2617: a zero-weight household at the start of the order
+    # used to get decile 0, outside the 1 to 10 range.
+    for ids in ([0, 1, 2], [1, 0, 2], [5, 9, 3]):
+        household = FakeHousehold([0, 0, 5], [0, 1, 1], [1, 1, 1], ids)
+
+        result = household_wealth_decile.formula(household, 2026, None)
 
-    with pytest.raises(ZeroDivisionError, match="zero total weight"):
-        household_wealth_decile.formula(household, 2026, None)
+        assert result.min() >= 1
+        assert result[2] == 10

policyengine_uk/variables/household/wealth/household_wealth_decile.py

@@ -1,16 +1,28 @@
 from policyengine_uk.model_api import *
 
 
+def _fixed_hash(values):
+    """A splitmix64 mix of integer ids: the same on every run, unlike hash()."""
+    x = np.asarray(values).astype(np.int64).astype(np.uint64)
+    with np.errstate(over="ignore"):
+        x = x + np.uint64(0x9E3779B97F4A7C15)
+        x = (x ^ (x >> np.uint64(30))) * np.uint64(0xBF58476D1CE4E5B9)
+        x = (x ^ (x >> np.uint64(27))) * np.uint64(0x94D049BB133111EB)
+    return x ^ (x >> np.uint64(31))
+
+
 class household_wealth_decile(Variable):
     label = "household wealth decile"
     documentation = (
         "Person-weighted decile of gross total_wealth, before debt deductions. "
         "Each household's weight is household_weight * household_count_people. "
-        "Equal wealth values are ranked in original household/entity order. "
-        "Changing this order, including when rebuilding a dataset, can change "
-        "tied households' decile assignments. Negative total_wealth participates "
-        "in the ranking before its output is set to -1. Household weights are "
-        "indivisible, so decile bins can still be empty."
+        "Equal wealth values are ranked by a fixed hash of household_id, then "
+        "by row order, so tied households keep their deciles when a dataset "
+        "is rebuilt in another order. Negative total_wealth participates in "
+        "the ranking before its output is set to -1. Every other household "
+        "gets a decile from 1 to 10, also one with zero weight. If every "
+        "household has zero weight, each person counts equally. Household "
+        "weights are indivisible, so decile bins can still be empty."
     )
     entity = Household
     definition_period = YEAR
@@ -20,13 +32,20 @@ def formula(household, period, parameters):
         wealth = household("total_wealth", period)
         count_people = household("household_count_people", period)
         household_weight = household("household_weight", period)
-        # Split equal wealth in original household order without changing wealth.
+        tie_key = _fixed_hash(household("household_id", period))
+        # Split equal wealth by the fixed hash, not by file order, without
+        # changing wealth. np.lexsort sorts by its last key first.
+        order = np.lexsort((np.arange(len(wealth)), tie_key, wealth))
         wealth_rank = np.empty(len(wealth), dtype=int)
-        wealth_rank[np.argsort(wealth, kind="stable")] = np.arange(len(wealth))
-        weighted_wealth = MicroSeries(
-            wealth_rank, weights=household_weight * count_people
-        )
-        decile = weighted_wealth.decile_rank().values
+        wealth_rank[order] = np.arange(len(wealth))
+        person_weight = household_weight * count_people
+        if person_weight.sum() == 0:
+            # A frame with no weight at all (a test or an unweighted input)
+            # still gets deciles: each person counts equally.
+            person_weight = count_people
+        weighted_wealth = MicroSeries(wealth_rank, weights=person_weight)
+        # A zero-weight household at the start of the order would get 0.
+        decile = np.clip(weighted_wealth.decile_rank().values, 1, 10)
         # Set negatives to -1.
         # This avoids the bottom decile summing to a negative number,
         # which would flip the % change in the interface.