diff --git a/README.md b/README.md index b7da84050..ea84a595c 100644 --- a/README.md +++ b/README.md @@ -48,11 +48,11 @@ from policyengine.core import Simulation from policyengine.outputs.aggregate import Aggregate, AggregateType datasets = pe.uk.ensure_datasets( - datasets=["enhanced_frs_2023_24"], + datasets=["microcosm_uk_2024_25"], years=[2026], data_folder="./data", ) -dataset = datasets["enhanced_frs_2023_24_2026"] +dataset = datasets["microcosm_uk_2024_25_2026"] simulation = Simulation(dataset=dataset, tax_benefit_model_version=pe.uk.model) simulation.run() @@ -71,11 +71,20 @@ For baseline-vs-reform comparisons, see `pe.uk.economic_impact_analysis` and its US counterpart. UK population data is stored in private Hugging Face repositories: the -Microcosm default in a dataset repository, and the enhanced FRS in a model -repository. Set `HUGGING_FACE_TOKEN` to a token from an account with access -before running UK population examples. To download the raw `.h5` file directly, see +Microcosm default (`microcosm_uk_2024_25`) in a dataset repository, and the +enhanced FRS in a model repository. Set `HUGGING_FACE_TOKEN` to a token from an +account with access before running UK population examples. To download the raw +`.h5` file directly, see [Microsimulation](docs/microsim.md#uk-private-data-and-raw-h5-access). +The UK files derive from licensed UK Data Service surveys, so access is granted +per Hugging Face account: email [contact@policyengine.org](mailto:contact@policyengine.org) +with your Hugging Face username and enough information to verify your UK Data +Service access. Without access, UK household calculations +(`pe.uk.calculate_household`) still work, and US population data is public. See +[Microsimulation](docs/microsim.md#working-without-uk-data-access) for what else +works without access. + ## Documentation **Core concepts:** @@ -202,7 +211,9 @@ Use the pinned interpreter and the UK extra to run the checked-in paper repro: uv run --python 3.14 --extra uk python examples/paper_repro_uk.py ``` -On first run this will create `./data/enhanced_frs_2023_24_year_2026.h5`. +On first run this will create `./data/enhanced_frs_2023_24_year_2026.h5`. The +paper repro is pinned to the Enhanced FRS 2023-24 the paper used; new UK analysis +should use the certified default, `microcosm_uk_2024_25`. ## Features diff --git a/changelog.d/473.fixed.md b/changelog.d/473.fixed.md new file mode 100644 index 000000000..79914cdfe --- /dev/null +++ b/changelog.d/473.fixed.md @@ -0,0 +1 @@ +The README quickstart and the UK examples now load the certified `microcosm_uk_2024_25` dataset, and the docs say how to request UK data access and what works without it. diff --git a/docs/microsim.md b/docs/microsim.md index b0aaf8988..8c4d34773 100644 --- a/docs/microsim.md +++ b/docs/microsim.md @@ -210,6 +210,12 @@ token from a Hugging Face account with access: export HUGGING_FACE_TOKEN=hf_... ``` +PolicyEngine grants access per Hugging Face account, not per token. To request +it, email [contact@policyengine.org](mailto:contact@policyengine.org) with your +Hugging Face username and enough information to verify your licensed access to +the UK Data Service source data. Once access is granted, create a read token in +your Hugging Face settings. + For `policyengine.py` analyses, use the logical dataset name from the release bundle. `ensure_datasets` resolves it to the pinned private Hugging Face file, downloads it, caches it locally, and creates year-specific uprated datasets: @@ -252,6 +258,22 @@ not need repository-specific download logic. Authentication or authorization failures are reported directly and do not cause a retry against another repository type. +### Working without UK data access + +Without access to the private repositories: + +- UK household calculations (`pe.uk.calculate_household`) need no data and work + as normal. +- US population data is public, so the US examples run without a token. +- A UK population run fails with a `DatasetMaterializationError` from the + download. No public or synthetic UK population file is published. +- `pe.uk.managed_microsimulation(dataset=path, allow_unmanaged=True)` runs on a + local `.h5` file in policyengine-uk's dataset format, for example one you + built yourself from your own licensed survey data. Such a file is not + certified: its results can differ from published PolicyEngine figures, and + `sim.policyengine_bundle` records the local path as the source with no + certified SHA-256. + ### UK year files and the data year `ensure_datasets` and `create_datasets` cut one file per requested year from diff --git a/examples/income_bands_uk.py b/examples/income_bands_uk.py index f4f43c726..cb9e60fac 100644 --- a/examples/income_bands_uk.py +++ b/examples/income_bands_uk.py @@ -6,11 +6,9 @@ 3. Using Aggregate to calculate statistics within income deciles using quantile filters 4. Visualising results with Plotly -Run: python examples/income_bands.py +Run: python examples/income_bands_uk.py """ -from pathlib import Path - import plotly.graph_objects as go from plotly.subplots import make_subplots @@ -18,26 +16,23 @@ from policyengine.outputs.aggregate import Aggregate, AggregateType from policyengine.tax_benefit_models.uk import ( PolicyEngineUKDataset, + ensure_datasets, uk_latest, ) def load_representative_data(year: int = 2026) -> PolicyEngineUKDataset: - """Load representative household microdata for a given year.""" - dataset_path = Path(f"./data/enhanced_frs_2023_24_year_{year}.h5") - - if not dataset_path.exists(): - raise FileNotFoundError( - f"Dataset not found at {dataset_path}. " - "Run create_datasets() from policyengine.tax_benefit_models.uk first." - ) - - dataset = PolicyEngineUKDataset( - name=f"Enhanced FRS {year}", - description=f"Representative household microdata for {year}", - filepath=str(dataset_path), - year=year, + """Load the certified UK microdata (Microcosm UK 2024-25) for a given year. + + The file is private: set HUGGING_FACE_TOKEN to a token from a Hugging Face + account with access (see docs/microsim.md). + """ + datasets = ensure_datasets( + datasets=["microcosm_uk_2024_25"], + years=[year], + data_folder="./data", ) + dataset = datasets[f"microcosm_uk_2024_25_{year}"] dataset.load() return dataset diff --git a/examples/policy_change_uk.py b/examples/policy_change_uk.py index 00162d497..1d2665f21 100644 --- a/examples/policy_change_uk.py +++ b/examples/policy_change_uk.py @@ -8,11 +8,10 @@ 5. Using quantile filters for decile-based analysis 6. Visualising results with Plotly -Run: python examples/policy_change.py +Run: python examples/policy_change_uk.py """ import datetime -from pathlib import Path import plotly.graph_objects as go from plotly.subplots import make_subplots @@ -24,26 +23,23 @@ ) from policyengine.tax_benefit_models.uk import ( PolicyEngineUKDataset, + ensure_datasets, uk_latest, ) def load_representative_data(year: int = 2026) -> PolicyEngineUKDataset: - """Load representative household microdata for a given year.""" - dataset_path = Path(f"./data/enhanced_frs_2023_24_year_{year}.h5") - - if not dataset_path.exists(): - raise FileNotFoundError( - f"Dataset not found at {dataset_path}. " - "Run create_datasets() from policyengine.tax_benefit_models.uk first." - ) - - dataset = PolicyEngineUKDataset( - name=f"Enhanced FRS {year}", - description=f"Representative household microdata for {year}", - filepath=str(dataset_path), - year=year, + """Load the certified UK microdata (Microcosm UK 2024-25) for a given year. + + The file is private: set HUGGING_FACE_TOKEN to a token from a Hugging Face + account with access (see docs/microsim.md). + """ + datasets = ensure_datasets( + datasets=["microcosm_uk_2024_25"], + years=[year], + data_folder="./data", ) + dataset = datasets[f"microcosm_uk_2024_25_{year}"] dataset.load() return dataset