Build a reform simulation's baseline from the situation under the baseline system

changelog.d/reform-baseline-situation-defaults.fixed.md

@@ -0,0 +1 @@
+A reform simulation built from a situation now gives its baseline the inputs the baseline system makes of that situation: people the situation gives no value take the baseline's default instead of the reform's, and valid inputs for a variable the reform neutralizes reach the baseline.

policyengine_core/simulations/simulation.py

@@ -918,7 +918,16 @@ def __init__(
             # looked variables up there, and its holders kept the reform's
             # variables, so a variable the reform neutralized read as the
             # default in the baseline too.
-            self.baseline._bind_to_tax_benefit_system()
+            if (
+                situation is not None
+                and populations is None
+                and self.baseline.tax_benefit_system is not None
+            ):
+                # So were its inputs: people the situation gives no value
+                # held the reform's default.
+                self.baseline._build_from_situation(situation)
+            else:
+                self.baseline._bind_to_tax_benefit_system()
         else:
             self.baseline = None
 
@@ -1303,6 +1312,47 @@ def kept(name):
         if getattr(self, "input_variables", None) is not None:
             self.input_variables = [name for name in self.input_variables if kept(name)]
 
+    def _build_from_situation(self, situation: dict) -> None:
+        """Give this simulation the inputs its own system makes of ``situation``.
+
+        For a reform simulation's baseline branch, which starts as a copy of
+        the reform simulation: the situation builder made that simulation's
+        inputs under the reform's system. A stored input is a whole array, so
+        a person the situation gives no value for a variable it gives others
+        held the reform's default for it, and the reform's system ignored
+        inputs for the variables it neutralizes. Here the simulation gets new
+        populations from its own system, and the situation is built into
+        them as the builder builds it under that system, except that inputs
+        the system cannot take are left out (see
+        ``SimulationBuilder.skip_rejected_inputs``), as
+        ``_bind_to_tax_benefit_system`` drops holders built for the reform's
+        definitions. The records of what the old populations held go with
+        them. The inputs are stored under ``"default"``, as in a simulation
+        built from the situation, which a branch reads through.
+        """
+        from policyengine_core.simulations.simulation_builder import (
+            SimulationBuilder,
+        )  # Import here to avoid circular dependency
+
+        self.build_from_populations(self.tax_benefit_system.instantiate_entities())
+        self._user_input_keys = set()
+        self._store_history = StoreHistory()
+        self._requested_variables = set()
+        self.invalidated_caches = set()
+        self._fast_cache = {}
+        builder = SimulationBuilder()
+        builder.default_period = self.default_input_period
+        builder.skip_rejected_inputs = True
+        # A situation without entities is set through ``set_input``, which
+        # stores under the simulation's branch name; one with entities is
+        # stored under ``"default"`` whatever the branch.
+        branch_name, self.branch_name = self.branch_name, "default"
+        try:
+            builder.build_from_dict(self.tax_benefit_system, situation, self)
+        finally:
+            self.branch_name = branch_name
+        self.has_axes = builder.has_axes
+
     def create_shortcuts(self) -> None:
         for _key, population in self.populations.items():
             # create shortcut simulation.person and simulation.household (for instance)

policyengine_core/simulations/simulation_builder.py

@@ -63,6 +63,20 @@ def __init__(self):
         self.axes_memberships: typing.Dict[Entity.plural, typing.List[int]] = {}
         self.axes_roles: typing.Dict[Entity.plural, typing.List[int]] = {}
 
+        # Leave out, rather than raise for, inputs the tax-benefit system
+        # cannot take: a variable it does not have or defines for another
+        # entity, a value it rejects, an axis over a variable it does not
+        # have (the axis still shapes the population), and an array its
+        # ``set_input`` refuses with a ``ValueError`` (for a period the
+        # variable's definition does not take, say, but also for a NaN or
+        # for yearly and monthly inputs that do not add up). Entities,
+        # roles and the syntax of periods are still checked. A reform
+        # simulation builds its baseline's inputs this way, from a situation
+        # the reform's system took (see ``Simulation._build_from_situation``).
+        # That system may define a variable differently, and does not check
+        # inputs for the variables it neutralizes.
+        self.skip_rejected_inputs = False
+
     def build_from_dict(
         self,
         tax_benefit_system: "TaxBenefitSystem",
@@ -219,10 +233,14 @@ def build_from_variables(
                         [variable],
                         "Can't deal with type: expected object. Input variables should be set for specific periods. For instance: {'salary': {'2017-01': 2000, '2017-02': 2500}}, or {'birth_date': {'ETERNITY': '1980-01-01'}}.",
                     )
-                simulation.set_input(variable, self.default_period, value)
-            else:
-                for period_str, dated_value in value.items():
-                    simulation.set_input(variable, period_str, dated_value)
+                value = {self.default_period: value}
+            for period_str, dated_value in value.items():
+                period = periods.period(period_str)
+                try:
+                    simulation.set_input(variable, period, dated_value)
+                except (VariableNotFoundError, ValueError):
+                    if not self.skip_rejected_inputs:
+                        raise
         return simulation
 
     def build_default_simulation(
@@ -508,8 +526,12 @@ def init_variable_values(self, entity, instance_object, instance_id):
             try:
                 entity.check_variable_defined_for_entity(variable_name)
             except ValueError as e:  # The variable is defined for another entity
+                if self.skip_rejected_inputs:
+                    continue
                 raise SituationParsingError(path_in_json, e.args[0])
             except VariableNotFoundError as e:  # The variable doesn't exist
+                if self.skip_rejected_inputs:
+                    continue
                 raise SituationParsingError(path_in_json, str(e), code=404)
 
             instance_index = self.get_ids(entity.plural).index(instance_id)
@@ -554,6 +576,9 @@ def add_variable_value(
         try:
             value = variable.check_set_value(value)
         except ValueError as error:
+            if self.skip_rejected_inputs:
+                # The entry is left as if the situation did not give it.
+                return
             raise SituationParsingError(path_in_json, *error.args)
 
         array[instance_index] = value
@@ -591,7 +616,13 @@ def finalize_variables_init(self, population):
                 # TODO - this duplicates the check in Simulation.set_input, but
                 # fixing that requires improving Simulation's handling of entities
                 if (variable.end is None) or (period_value.start.date <= variable.end):
-                    holder.set_input(period_value, array)
+                    try:
+                        holder.set_input(period_value, array)
+                    except ValueError:
+                        # ``Holder.set_input`` and core's ``set_input``
+                        # handlers raise before storing any of the input.
+                        if not self.skip_rejected_inputs:
+                            raise
 
     def raise_period_mismatch(self, entity, json, e):
         # This error happens when we try to set a variable value for a period that doesn't match its definition period
@@ -688,11 +719,11 @@ def expand_axes(self):
         # Now generate input values along the specified axes
         # TODO - factor out the common logic here
         if len(self.axes) == 1 and len(self.axes[0]):
-            parallel_axes = self.axes[0]
-            first_axis = parallel_axes[0]
-            axis_count: int = first_axis["count"]
-            axis_entity = self.get_variable_entity(first_axis["name"])
-            axis_entity_step_size = self.entity_counts[axis_entity.plural]
+            axis_count: int = self.axes[0][0]["count"]
+            parallel_axes = self._axes_to_set(self.axes[0])
+            if parallel_axes:
+                axis_entity = self.get_variable_entity(parallel_axes[0]["name"])
+                axis_entity_step_size = self.entity_counts[axis_entity.plural]
             # Distribute values along axes
             for axis in parallel_axes:
                 axis_index = axis.get("index", 0)
@@ -721,8 +752,11 @@ def expand_axes(self):
             ]
             axes_meshes = np.meshgrid(*axes_linspaces)
             for parallel_axes, mesh in zip(self.axes, axes_meshes):
+                axis_count = parallel_axes[0]["count"]
+                parallel_axes = self._axes_to_set(parallel_axes)
+                if not parallel_axes:
+                    continue
                 first_axis = parallel_axes[0]
-                axis_count = first_axis["count"]
                 axis_entity = self.get_variable_entity(first_axis["name"])
                 axis_entity_step_size = self.entity_counts[axis_entity.plural]
                 # Distribute values along the grid
@@ -753,6 +787,18 @@ def expand_axes(self):
 
         self.has_axes = True
 
+    def _axes_to_set(self, parallel_axes: list) -> list:
+        """The axes of one dimension whose variable's values are set.
+
+        All of them, except, when skipping rejected inputs, those over a
+        variable the tax-benefit system does not have.
+        """
+        if not self.skip_rejected_inputs:
+            return parallel_axes
+        return [
+            axis for axis in parallel_axes if axis["name"] in self.variable_entities
+        ]
+
     def get_variable_entity(self, variable_name):
         return self.variable_entities[variable_name]