From dc98d457aaa7509fd607c5ed76d57c4f41552f96 Mon Sep 17 00:00:00 2001 From: Aniket Khanal Date: Thu, 8 Oct 2026 15:50:48 -0400 Subject: [PATCH 01/33] trid readme --- README.md | 106 +++++++++++++++++++++++++++++++++++------------------- 1 file changed, 70 insertions(+), 36 deletions(-) diff --git a/README.md b/README.md index fea9a54..ca1e55e 100644 --- a/README.md +++ b/README.md @@ -72,7 +72,7 @@ The decay chain of the HH → bb̄bb̄ example is ```yaml objects: - jets: {features: [pt, eta, phi, mass, btag], max: 12} # INPUTS/Source (alias of "jets") in the H5 file + jets: {features: [pt, eta, phi, mass], max: 12} # INPUTS/Source (alias of "jets") in the H5 file b: jets btag 4 # the 4 highest-btag jets other: jets rest 8 # up to 8 remaining jets event: [nSelJets, HT] # computed from the jet collection @@ -88,12 +88,13 @@ outputs: combine: [HH] ``` -`jets` is the jet collection, with up to 12 jets per event; the listed `features` are the columns to -read from the input file. `btag` is only used to pick which jets are treated as b-jet candidates -(`b: jets btag 4`, the four jets most likely to be b jets); it is not an input to the network. The -file holds all jets of the event, and the b jets are chosen when it is loaded, so one file serves any -way of choosing them. [docs/EventInfo.md](docs/EventInfo.md) explains every -line of such a file and [docs/Dataset.md](docs/Dataset.md) the input file. +`jets` is the jet collection: `max: 12` is the number of jet slots read per event, and the input file must +have at least that many. The listed `features` are the inputs of the network: these columns of the input +file, and only these, are fed to it. \ +`jets btag 4` selects the four jets most likely to be b jets (alternatively `jets pt 4` would select the four +leading jets). The `btag` selector reads the `btag` column of the file for this selection only; it need not be +a feature, and a feature named `btag` would be one more network input. The input +file holds all jets of the event, and the b jets are chosen when it is loaded. For more details on specifying the event decay, check [docs/EventInfo.md](docs/EventInfo.md), and for the input file itself, check [docs/Dataset.md](docs/Dataset.md). The full-statistics files (454,261 HH → bb̄bb̄ events, 292,285 HH → bb̄WW events) are distributed separately ([DOI / URL to be added]); download them into `examples/data/` as `hh4b_all.h5` and @@ -101,30 +102,43 @@ separately ([DOI / URL to be added]); download them into `examples/data/` as `hh ## Usage -Three steps: describe the final state ([docs/EventInfo.md](docs/EventInfo.md)), prepare the input -file ([docs/Dataset.md](docs/Dataset.md)), and choose the settings ([docs/Options.md](docs/Options.md)). -Then check the two files against each other: +1. Describe the final state in a configuration file ([docs/EventInfo.md](docs/EventInfo.md)). +2. Prepare the input file ([docs/Dataset.md](docs/Dataset.md)). +3. Choose the network and training settings ([docs/Options.md](docs/Options.md)). +4. Before training, check that the configuration and the input file are correct and consistent with each other: -```bash -corenet-validate --config my_config.yaml --input my_data.h5 -``` + ```bash + corenet-validate --config my_config.yaml --input my_data.h5 + ``` + + The report ends with `valid` or `NOT VALID`; `--json report.json` also writes it to a file. -checks the event description against the input file; `--json report.json` also writes the report to a file. +The validator only reads the file, so it can run on a machine without a GPU: pixi checks the environment's +CUDA requirement against the host driver, and `CONDA_OVERRIDE_CUDA=12.0 pixi run corenet-validate ...` +lifts that check. ### Training ```bash corenet-train --config examples/bbww/bbww.yaml --input examples/data/bbww_all.h5 \ - --fold 0 --exclude-folds 1 --output-dir runs/bbww --device cuda + --fold 0 --exclude-folds 1 --output-dir runs/bbww ``` -`--fold k` holds out the events of fold `k` for testing (`--fold -1`, the default, trains one model per -fold in turn); the folds in `--exclude-folds` are kept out of training and used for validation. For each fold -the output directory receives the checkpoint `model_fold{k}.pkl` (self-contained: the evaluation and -the export need no configuration file), the training history `history_fold{k}.json` and the -predictions on the held-out fold `predictions_fold{k}.h5`. Epochs, learning rate, batch size and the -other settings are changed with flags (`--epochs`, `--lr`, ...) or in the configuration file; see -[docs/Options.md](docs/Options.md) and `corenet-train --help`. +* `--fold 0`: hold out fold 0 for testing. With `--fold -1` (the default), train one model per fold in turn. +* `--exclude-folds 1`: keep fold 1 out of the training and use it for validation. +* `--epochs`, `--lr`, `--seed`, ...: override the training settings of the configuration file. + See [docs/Options.md](docs/Options.md) or `corenet-train --help`. + +The number of folds is not a training option: it is set when the input file is written, as the set of +values in `EVENT/fold` ([docs/Dataset.md](docs/Dataset.md)). The example files have 5 folds. + +Per fold `k`, the output directory receives: + +| file | content | +|---|---| +| `model_fold{k}.pkl` | checkpoint; evaluation and export need no configuration file | +| `history_fold{k}.json` | training settings and per-epoch history | +| `predictions_fold{k}.h5` | predictions on the held-out fold | ### Evaluation @@ -133,12 +147,24 @@ corenet-evaluate --checkpoint runs/bbww/model_fold0.pkl --input examples/data/bb --output runs/bbww/eval_fold0.h5 ``` -evaluates the fold that was held out in training (`--fold k` for another fold, `--fold -1` for every -event of the file) and writes the metrics to `eval_fold0.h5.json` and the predictions to `eval_fold0.h5`: -`class_probabilities` (one column per class, column 1 is the signal score), `views/` (the -pairing probabilities, for example the three pairings of the four b jets), `heads/` (the extra -score of every further output), the ROC curve `roc/{fpr,tpr,threshold}`, and the event labels, -weights and folds. `--unweighted` ignores the event weights in the metrics. +* `--fold k`: evaluate fold `k` instead of the fold held out in training. `--fold -1` evaluates every event of the file. +* `--unweighted`: ignore the event weights in the metrics. +* `--device cuda`: evaluate on the GPU (default: CPU). + +Output: + +| file | content | +|---|---| +| `eval_fold0.h5.json` | the metrics | +| `eval_fold0.h5` | the predictions, as below | + +| dataset in `eval_fold0.h5` | content | +|---|---| +| `class_probabilities` | one column per class; column 1 is the signal score | +| `views/HH`, `views/tt` | probability of every assignment of the jets to the particles of that hypothesis: for HH → bb̄bb̄ the three ways of pairing the four b jets into two Higgs bosons, for HH → bb̄WW the two ways of assigning the b jets to the top quarks of tt̄ | +| `heads/ttbar` | the tt̄ score (one dataset per auxiliary output) | +| `roc/{fpr,tpr,threshold}` | the ROC curve of the signal score | +| `EVENT/{index,event,label,weight,fold}` | row index in the input file, event number, label, weight and fold of every evaluated event | ### Exporting @@ -147,10 +173,17 @@ corenet-export --checkpoint runs/bbww/model_fold0.pkl --input examples/data/bbww --output-dir runs/bbww/export ``` -writes a TorchScript file (`model_fold0.pt`), an ONNX file (`model_fold0.onnx`, dynamic batch size) -and a JSON description of the inputs and outputs, then reloads both and checks that they reproduce -the PyTorch outputs on events of `--input` to within `--atol` (1e-4). `--formats torchscript` or -`--formats onnx` exports only one of the two; `--opset` sets the ONNX opset (17). +* `--formats torchscript` or `--formats onnx`: export only one of the two formats. +* `--opset 17`: ONNX opset version. +* `--n-events 1024`, `--atol 1e-4`: number of events of `--input` on which the exported models are checked against PyTorch, and the tolerance. + +Output, in `--output-dir`: + +| file | content | +|---|---| +| `model_fold0.pt` | TorchScript model | +| `model_fold0.onnx` | ONNX model, dynamic batch size | +| `model_fold0.json` | description of the inputs and outputs | Note that the exported model starts after the jets have been assigned: it takes the jets already assigned to the b quarks, the light quarks and so on. An application outside this package has to @@ -160,13 +193,14 @@ assign the jets the same way (for `b: jets btag 4`, the four jets most likely to | input | shape | content | |---|---|---| -| one per set of assigned jets, e.g. `b`, `other` | (N, features + 1, n) | the file's features of the `n` jets, in configuration order, plus a presence flag | +| one per set of assigned jets, e.g. `b`, `other` | (N, features + 1, n) | the `features` of the `n` jets, in configuration order, plus a presence flag | | one per single object, e.g. `lepton`, `met` | (N, features + 1, 1) | the same for a single object | | `event` | (N, n_event) | the event quantities (`nSelJets`, `HT`, ...) | -For HH → bb̄bb̄ the inputs are `b` (N, 6, 4), `other` (N, 6, 8) and `event` (N, 2); for HH → bb̄WW -`b` (N, 6, 2), `q` (N, 6, 2), `lepton` (N, 7, 1), `met` (N, 3, 1) and `event` (N, 2). The -exported model takes the same inputs as the training file, without any preprocessing step. +`features` is the number of features listed for the collection in the configuration; a column read only +by a selector (`btag`) is not an input. For HH → bb̄bb̄ the inputs are `b` (N, 5, 4), `other` (N, 5, 8) +and `event` (N, 2); for HH → bb̄WW `b` (N, 5, 2), `q` (N, 5, 2), `lepton` (N, 7, 1), `met` (N, 3, 1) and +`event` (N, 2). The exported model takes the same inputs as the training file, without any preprocessing step. #### Outputs From 13ffeb25ea7c9199bf1d7c23e5b20f8f2fd775ef Mon Sep 17 00:00:00 2001 From: Aniket Khanal Date: Thu, 8 Oct 2026 15:51:08 -0400 Subject: [PATCH 02/33] clarify features list --- docs/BBWW.md | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/docs/BBWW.md b/docs/BBWW.md index 46b2997..b65b4e7 100644 --- a/docs/BBWW.md +++ b/docs/BBWW.md @@ -28,7 +28,7 @@ The decay-chain part of `examples/bbww/bbww.yaml`: ```yaml objects: - jets: {features: [pt, eta, phi, mass, btag], max: 12} # INPUTS/Jets + jets: {features: [pt, eta, phi, mass], max: 12} # INPUTS/Jets b: jets btag 2 # the 2 highest-btag jets q: jets rest 2 # the 2 leading remaining jets lepton: {features: [pt, eta, phi, mass, isE, isM]} # INPUTS/Lepton, a single object (max 1) @@ -70,8 +70,10 @@ training: ### `objects` -* `jets`: the jet collection, up to 12 jets per event, read from `INPUTS/Jets`. -* `b: jets btag 2`: the two jets most likely to be b jets. +* `jets`: the jet collection, up to 12 jets per event, read from `INPUTS/Jets`; the four kinematic + columns are its network inputs. +* `b: jets btag 2`: the two jets most likely to be b jets; the rule reads `INPUTS/Jets/btag` for this + choice only. * `q: jets rest 2`: the two leading remaining jets, the light-quark jets. * `lepton`: a single object per event; `isE` and `isM` tell the network whether it is an electron or a muon. From 430a0428e34385989843b26ea09530ed1eb8f957 Mon Sep 17 00:00:00 2001 From: Aniket Khanal Date: Thu, 8 Oct 2026 15:51:18 -0400 Subject: [PATCH 03/33] clarify features list --- docs/Dataset.md | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/docs/Dataset.md b/docs/Dataset.md index 94867e1..6e15a5a 100644 --- a/docs/Dataset.md +++ b/docs/Dataset.md @@ -44,9 +44,11 @@ of shape `[NUM_EVENTS, MAX_JETS]`. Every event is stored with the same number of slot with `False`. Store all jets of the event, not only the b-jet candidates: which jets are the b jets is decided by the event description when the file is loaded, so one file serves any choice. -The feature names are the ones used in the event description. `pt`, `eta`, `phi` and `mass` are the -kinematics; `btag` is only used to pick the b-jet candidates; any other feature (`isE`, `isM`, ...) is -an additional input for that object. +The column names are the ones used in the event description. The columns listed as `features` there +are the inputs of the network: `pt`, `eta`, `phi` and `mass` are the kinematics, any other feature +(`isE`, `isM`, ...) is an additional input for that object. A column named in a candidate rule (`btag` +for `b: jets btag 4`) is read for that choice only and need not be a feature; listing it under `features` +would make it an input as well. ## Single objects @@ -88,3 +90,6 @@ description with ```bash corenet-validate --config examples/bbww/bbww.yaml --input examples/data/bbww_small.h5 ``` + +On a machine without a GPU (no CUDA driver), prefix the command with `CONDA_OVERRIDE_CUDA=12.0`; the +validator only reads the file. From b984343e1dd1e41d615f8dbe8a0ad71dac8e73da Mon Sep 17 00:00:00 2001 From: Aniket Khanal Date: Thu, 8 Oct 2026 15:51:25 -0400 Subject: [PATCH 04/33] clarify features list --- docs/EventInfo.md | 18 +++++++++++------- 1 file changed, 11 insertions(+), 7 deletions(-) diff --git a/docs/EventInfo.md b/docs/EventInfo.md index 436cf9e..2f3b4f9 100644 --- a/docs/EventInfo.md +++ b/docs/EventInfo.md @@ -28,17 +28,19 @@ members are the candidate jets of the decay chain. ```yaml objects: - jets: {features: [pt, eta, phi, mass, btag], max: 12} + jets: {features: [pt, eta, phi, mass], max: 12} b: jets btag 4 other: jets rest 8 lepton: {features: [pt, eta, phi, mass, isE, isM]} met: {features: [pt, phi]} ``` -* `jets`: the jet collection, with up to 12 jets per event. The listed features are the columns to - read from the input file ([Dataset.md](./Dataset.md)). `btag` is only used to pick which jets are - treated as b-jet candidates; it is not an input to the network. -* `b: jets btag 4`: the four jets most likely to be b jets, taken from the jet collection. +* `jets`: the jet collection, with up to 12 jets per event. The listed features are the inputs of the + network: exactly these columns of the input file ([Dataset.md](./Dataset.md)) are fed to it, `pt`, + `eta`, `phi` and `mass` as the kinematics and any other feature as an additional input. +* `b: jets btag 4`: the four jets most likely to be b jets, taken from the jet collection. The `btag` + rule reads the `btag` column of the file for this choice only; it need not be a feature. (Listing + `btag` under `features` would make it one more input of the network.) * `other: jets rest 8`: up to eight of the remaining jets. * `lepton`: a single object per event (no `max`), with two extra inputs `isE` and `isM`. * `met`: the missing transverse momentum, a single object given by `pt` and `phi`. @@ -49,10 +51,12 @@ through candidate lines of the form `: `: | rule | the `n` jets taken | |---|---| -| `btag` | the `n` jets most likely to be b jets (needs a `btag` feature) | -| `pt` | the `n` leading jets | +| `btag` | the `n` jets most likely to be b jets; reads the `btag` (and `pt`) column of the file, which need not be a feature | +| `pt` | the `n` leading jets; reads the `pt` column | | `rest` | up to `n` of the jets not taken by an earlier line | +A rule reads its column from the input file for the selection only; the network sees the `features`. + Lines are filled in order and a jet is used by at most one line. An event with fewer jets than needed is kept; the missing ones are simply absent. From 025a585179b28f13162c8c091c044d6a98001385 Mon Sep 17 00:00:00 2001 From: Aniket Khanal Date: Thu, 8 Oct 2026 15:51:31 -0400 Subject: [PATCH 05/33] clarify features list --- docs/HH4b.md | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/docs/HH4b.md b/docs/HH4b.md index 9654dde..5656da9 100644 --- a/docs/HH4b.md +++ b/docs/HH4b.md @@ -35,7 +35,7 @@ The decay-chain part of `examples/hh4b/hh4b.yaml`: ```yaml objects: - jets: {features: [pt, eta, phi, mass, btag], max: 12} # INPUTS/Source (alias of "jets") in the H5 file + jets: {features: [pt, eta, phi, mass], max: 12} # INPUTS/Source (alias of "jets") in the H5 file b: jets btag 4 # the 4 highest-btag jets other: jets rest 8 # up to 8 remaining jets event: [nSelJets, HT] # computed from the jet collection @@ -61,9 +61,9 @@ training: ### `objects` * `jets`: the jet collection, with up to 12 jets per event, read from `INPUTS/Source`. The listed - features are the columns to read from the input file. `btag` is only used to pick which jets are - treated as b-jet candidates; it is not an input to the network. -* `b: jets btag 4`: the four jets most likely to be b jets, taken from the jet collection. When fewer + features are the inputs of the network: the four kinematic columns and nothing else. +* `b: jets btag 4`: the four jets most likely to be b jets, taken from the jet collection. The rule + reads `INPUTS/Source/btag` for this choice only; the b tag is not an input of the network. When fewer than four jets are tagged, the remaining places go to untagged jets. * `other: jets rest 8`: up to eight of the remaining jets. Events with fewer jets are kept. * `event: [nSelJets, HT]`: the number of jets and their scalar momentum sum, given to the network as From 76bf2fe88d0d65030dc87da5ad0d9b804e87ee76 Mon Sep 17 00:00:00 2001 From: Aniket Khanal Date: Thu, 8 Oct 2026 15:51:43 -0400 Subject: [PATCH 06/33] update features list --- examples/bbww/bbww.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/examples/bbww/bbww.yaml b/examples/bbww/bbww.yaml index 2c4ab1c..afee0e8 100644 --- a/examples/bbww/bbww.yaml +++ b/examples/bbww/bbww.yaml @@ -1,8 +1,8 @@ # HH -> bbWW (semileptonic), with a competing ttbar hypothesis on the same objects. # See docs/BBWW.md. objects: - jets: {features: [pt, eta, phi, mass, btag], max: 12} # INPUTS/Jets - b: jets btag 2 # the 2 highest-btag jets + jets: {features: [pt, eta, phi, mass], max: 12} # INPUTS/Jets; features = network inputs + b: jets btag 2 # the 2 highest-btag jets (reads INPUTS/Jets/btag for the selection only) q: jets rest 2 # the 2 leading remaining jets lepton: {features: [pt, eta, phi, mass, isE, isM]} # INPUTS/Lepton, a single object (max 1) met: {features: [pt, phi]} # INPUTS/Met, a single object (max 1) From 6170be1c968460c257a8ef91cd77311f3f9f664a Mon Sep 17 00:00:00 2001 From: Aniket Khanal Date: Thu, 8 Oct 2026 15:51:51 -0400 Subject: [PATCH 07/33] update features list --- examples/bbww/bbww_hyperparameters.yaml | 2 +- examples/hh4b/hh4b.yaml | 4 ++-- examples/hh4b/hh4b_hyperparameters.yaml | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/examples/bbww/bbww_hyperparameters.yaml b/examples/bbww/bbww_hyperparameters.yaml index c21472b..ca8ebd1 100644 --- a/examples/bbww/bbww_hyperparameters.yaml +++ b/examples/bbww/bbww_hyperparameters.yaml @@ -2,7 +2,7 @@ # (except reflect_eta, which this example switches off). Training with this file gives the same # result as with bbww.yaml. Copy it and edit the values. Reference: docs/Options.md. objects: - jets: {features: [pt, eta, phi, mass, btag], max: 12} + jets: {features: [pt, eta, phi, mass], max: 12} b: jets btag 2 q: jets rest 2 lepton: {features: [pt, eta, phi, mass, isE, isM]} diff --git a/examples/hh4b/hh4b.yaml b/examples/hh4b/hh4b.yaml index 23784f6..a1eb07a 100644 --- a/examples/hh4b/hh4b.yaml +++ b/examples/hh4b/hh4b.yaml @@ -1,8 +1,8 @@ # HH -> 4b: two Higgs bosons, each decaying to a pair of b jets. # See docs/HH4b.md. objects: - jets: {features: [pt, eta, phi, mass, btag], max: 12} # INPUTS/Source (alias of "jets") in the H5 file - b: jets btag 4 # the 4 highest-btag jets + jets: {features: [pt, eta, phi, mass], max: 12} # INPUTS/Source (alias of "jets") in the H5 file; features = network inputs + b: jets btag 4 # the 4 highest-btag jets (reads INPUTS/Source/btag for the selection only) other: jets rest 8 # up to 8 remaining jets event: [nSelJets, HT] # computed from the jet collection diff --git a/examples/hh4b/hh4b_hyperparameters.yaml b/examples/hh4b/hh4b_hyperparameters.yaml index efcbad8..318e958 100644 --- a/examples/hh4b/hh4b_hyperparameters.yaml +++ b/examples/hh4b/hh4b_hyperparameters.yaml @@ -2,7 +2,7 @@ # Training with this file gives the same result as with hh4b.yaml. Copy it and edit the values. # Reference: docs/Options.md. objects: - jets: {features: [pt, eta, phi, mass, btag], max: 12} + jets: {features: [pt, eta, phi, mass], max: 12} b: jets btag 4 other: jets rest 8 event: [nSelJets, HT] From e8da7f174be77a128e0a4c476801b65108c36fa8 Mon Sep 17 00:00:00 2001 From: Aniket Khanal Date: Thu, 8 Oct 2026 15:52:18 -0400 Subject: [PATCH 08/33] define skipped features --- pixi.lock | 1 + src/corenet/checkpoint.py | 45 ++++++++++++++++++++++++++++++++++++--- 2 files changed, 43 insertions(+), 3 deletions(-) diff --git a/pixi.lock b/pixi.lock index e770cad..b4bde6c 100644 --- a/pixi.lock +++ b/pixi.lock @@ -2548,4 +2548,5 @@ packages: - onnx>=1.14 ; extra == 'export' - onnxruntime>=1.15 ; extra == 'export' - pytest>=7 ; extra == 'test' + - ruff ; extra == 'lint' requires_python: '>=3.9' diff --git a/src/corenet/checkpoint.py b/src/corenet/checkpoint.py index 235ed71..a30f68b 100644 --- a/src/corenet/checkpoint.py +++ b/src/corenet/checkpoint.py @@ -4,11 +4,19 @@ "config": the configuration mapping the network was built from (so a checkpoint is self-contained), "config_path": path of the configuration file at training time, "classes": ["rest", ...], - "metadata": {"fold", "epoch", "exclude_folds", "input", }} + "metadata": {"fold", "epoch", "exclude_folds", "input", }, + "format": CHECKPOINT_FORMAT} + +``format`` 2: every feature of a collection is a network input. Checkpoints without ``format`` were +written when a ``btag`` feature was skipped by the network (used by the ``btag`` selector only); their +stored configuration is translated on loading by dropping ``btag`` from every ``features`` list, which +describes the trained network exactly. """ from __future__ import annotations +import copy +import logging from pathlib import Path import torch @@ -16,6 +24,9 @@ from .models.config import NetworkSpec, parse_config, read_config_dict from .models.network import CoReNet +CHECKPOINT_FORMAT = 2 +LEGACY_SKIPPED = ("btag",) # features the network ignored before format 2 + def save_checkpoint( path, model: CoReNet, config: dict, config_path, metadata: dict @@ -28,12 +39,27 @@ def save_checkpoint( "config_path": str(Path(config_path).resolve()), "classes": model.spec.classes, "metadata": metadata, + "format": CHECKPOINT_FORMAT, }, path, ) return path +def legacy_config(cfg: dict) -> tuple[dict, list[str]]: + """The configuration of a pre-format-2 checkpoint as the current code reads it: ``btag`` removed from + every ``features`` list (the network never used it). Returns (config, names of the edited collections).""" + cfg = copy.deepcopy(cfg) + edited = [] + for name, body in (cfg.get("objects") or {}).items(): + if isinstance(body, dict) and isinstance(body.get("features"), list): + kept = [f for f in body["features"] if f not in LEGACY_SKIPPED] + if kept != body["features"]: + body["features"] = kept + edited.append(name) + return cfg, edited + + def load_checkpoint( path, config=None, device="cpu" ) -> tuple[CoReNet, NetworkSpec, dict]: @@ -47,16 +73,29 @@ def load_checkpoint( cfg = read_config_dict(config) elif isinstance(ckpt.get("config"), dict): cfg = ckpt["config"] + if "format" not in ckpt: + cfg, edited = legacy_config(cfg) + if edited: + logging.info(f"{path}: checkpoint written before format {CHECKPOINT_FORMAT}; " + f"{list(LEGACY_SKIPPED)} dropped from the features of {edited} (the network never used them)") else: raise ValueError( f"{path}: the checkpoint stores no configuration; pass the configuration file explicitly" ) spec = parse_config(cfg) model = CoReNet(spec, device=str(device)).to(device) - model.load_state_dict(ckpt["model"], strict=True) + try: + model.load_state_dict(ckpt["model"], strict=True) + except RuntimeError as e: + hint = "" + if "format" not in ckpt and config is not None: + hint = (f"; the checkpoint predates format {CHECKPOINT_FORMAT}, in which every feature is a network input: " + f"if the network was trained with {list(LEGACY_SKIPPED)} listed under features, remove them from " + f"the features of {config} (or load without --config, which does so automatically)") + raise RuntimeError(f"{path}: the state dict does not fit the network built from the configuration{hint}\n{e}") from e model.eval() model.setGhostBatches(0) return model, spec, ckpt -__all__ = ["load_checkpoint", "save_checkpoint"] +__all__ = ["save_checkpoint", "load_checkpoint", "legacy_config", "CHECKPOINT_FORMAT"] From 0e781c11dd7b7c5670499b3d78f7a87dce77749e Mon Sep 17 00:00:00 2001 From: Aniket Khanal Date: Thu, 8 Oct 2026 15:52:39 -0400 Subject: [PATCH 09/33] add option to read columns for just selection and not training --- src/corenet/data/h5.py | 59 +++++++++++++++++------------------------- 1 file changed, 24 insertions(+), 35 deletions(-) diff --git a/src/corenet/data/h5.py b/src/corenet/data/h5.py index 29e0696..cb61919 100644 --- a/src/corenet/data/h5.py +++ b/src/corenet/data/h5.py @@ -14,6 +14,11 @@ (``EVENT/weight``), ``folds`` (``EVENT/fold``) and ``events`` (``EVENT/event``). No generator-level information is read: the roles are chosen from reconstructed quantities only. +A collection's ``features`` are exactly the channels of its tensors. A selector reads the column it +needs (``btag``: the ``btag`` and ``pt`` columns) from the file on its own, whether or not it is a +feature; such a column is used for the selection only (``columns_to_read``). A column listed under +``features`` is always a network input, ``btag`` included. + ds = H5Dataset("examples/data/bbww_small.h5", load_config("examples/bbww/bbww.yaml")) out = model(ds.inputs) @@ -71,6 +76,11 @@ def used_collections(spec: NetworkSpec) -> list[Collection]: ] +def columns_to_read(spec: NetworkSpec, coll: Collection) -> list[str]: + """Datasets read from a collection's group: its features, then the selector columns that are not features.""" + return spec.columns_to_read(coll.name) + + def event_source(spec: NetworkSpec, name: str) -> tuple[str, str]: """Where an event scalar comes from: ("computed", collection) for nSelJets / HT when a read collection has max > 1 (the first one), else ("dataset", "EVENT/").""" @@ -92,20 +102,16 @@ def resolve_group(f: h5py.File, name: str) -> str: ) -def _read_collection( - f: h5py.File, coll: Collection -) -> tuple[dict[str, np.ndarray], np.ndarray]: - """(features, present) of one collection, both shaped (N, slots in the file).""" +def _read_collection(f: h5py.File, coll: Collection, columns: list[str]) -> tuple[dict[str, np.ndarray], np.ndarray]: + """(columns, present) of one collection, both shaped (N, slots in the file); ``columns`` are the + features plus the selector columns (``columns_to_read``).""" group = f["INPUTS"][resolve_group(f, coll.name)] - missing = [k for k in coll.features if k not in group] + missing = [k for k in columns if k not in group] if missing: - raise KeyError( - f"collection {coll.name!r} (INPUTS/{group.name.split('/')[-1]}): missing features {missing}" - ) - feats = { - k: np.atleast_2d(group[k][...].astype(np.float32).T).T for k in coll.features - } - n, slots = feats[coll.features[0]].shape + raise KeyError(f"collection {coll.name!r} (INPUTS/{group.name.split('/')[-1]}): missing columns {missing} " + f"(features {coll.features}, selector columns {[c for c in columns if c not in coll.features]})") + feats = {k: np.atleast_2d(group[k][...].astype(np.float32).T).T for k in columns} + n, slots = feats[columns[0]].shape if coll.max > 1 and slots < coll.max: raise ValueError( f"collection {coll.name!r} has {slots} slots in the file, configuration asks for max {coll.max}" @@ -150,9 +156,7 @@ def __init__(self, path: str | Path, spec: NetworkSpec, rest_order: str = "pt"): inputs: dict[str, torch.Tensor] = {} with h5py.File(path, "r") as f: - collections = { - c.name: _read_collection(f, c) for c in used_collections(spec) - } + collections = {c.name: _read_collection(f, c, columns_to_read(spec, c)) for c in used_collections(spec)} for role in spec.roles.values(): feats, present = collections[role.collection] inputs[role.name] = self._role_tensor( @@ -211,13 +215,9 @@ def _role_tensor( return self._stack(coll, feats, idx, valid) @staticmethod - def _stack( - coll: Collection, - feats: dict[str, np.ndarray], - idx: np.ndarray, - valid: np.ndarray, - ) -> torch.Tensor: - """(N, len(features) + 1, n): features in configuration order, presence flag last, padding -1.""" + def _stack(coll: Collection, feats: dict[str, np.ndarray], idx: np.ndarray, valid: np.ndarray) -> torch.Tensor: + """(N, len(features) + 1, n): features in configuration order, presence flag last, padding -1. + Columns read only for a selector are not stacked.""" n, slots = idx.shape rows = np.arange(n)[:, None] x = np.full((n, len(coll.features) + 1, slots), -1.0, dtype=np.float32) @@ -285,16 +285,5 @@ def fold_indices(self, fold: int) -> tuple[np.ndarray, np.ndarray]: return np.flatnonzero(~val), np.flatnonzero(val) -__all__ = [ - "COMPUTED_EVENT", - "EVENT_KEY", - "FOLD_KEY", - "GROUP_ALIASES", - "LABEL_KEYS", - "WEIGHT_KEY", - "H5Dataset", - "event_source", - "group_candidates", - "resolve_group", - "used_collections", -] +__all__ = ["H5Dataset", "GROUP_ALIASES", "COMPUTED_EVENT", "LABEL_KEYS", "WEIGHT_KEY", "FOLD_KEY", "EVENT_KEY", + "group_candidates", "used_collections", "columns_to_read", "event_source", "resolve_group"] From 014d3d67a749f9070cd772a6657dff32a40d6a6b Mon Sep 17 00:00:00 2001 From: Aniket Khanal Date: Wed, 21 Oct 2026 12:29:39 -0400 Subject: [PATCH 10/33] add early stop --- README.md | 8 +++++--- src/corenet/models/config.py | 28 ++++++++++++++++++++++------ 2 files changed, 27 insertions(+), 9 deletions(-) diff --git a/README.md b/README.md index ca1e55e..78731c5 100644 --- a/README.md +++ b/README.md @@ -91,9 +91,10 @@ combine: [HH] `jets` is the jet collection: `max: 12` is the number of jet slots read per event, and the input file must have at least that many. The listed `features` are the inputs of the network: these columns of the input file, and only these, are fed to it. \ -`jets btag 4` selects the four jets most likely to be b jets (alternatively `jets pt 4` would select the four -leading jets). The `btag` selector reads the `btag` column of the file for this selection only; it need not be -a feature, and a feature named `btag` would be one more network input. The input +`jets btag 4` selects the four jets with the highest `btag` (alternatively `jets pt 4` would select the four +leading jets): any column of the jets in the file can be used this way, and `jets mass 2 lowest` would take the +two jets with the lowest value instead. The selector reads its column from the file for this selection only; it +need not be a feature, and a feature named `btag` would be one more network input. The input file holds all jets of the event, and the b jets are chosen when it is loaded. For more details on specifying the event decay, check [docs/EventInfo.md](docs/EventInfo.md), and for the input file itself, check [docs/Dataset.md](docs/Dataset.md). The full-statistics files (454,261 HH → bb̄bb̄ events, 292,285 HH → bb̄WW events) are distributed @@ -126,6 +127,7 @@ corenet-train --config examples/bbww/bbww.yaml --input examples/data/bbww_all.h5 * `--fold 0`: hold out fold 0 for testing. With `--fold -1` (the default), train one model per fold in turn. * `--exclude-folds 1`: keep fold 1 out of the training and use it for validation. +* `--early-stop-patience 5`: stop when the validation AUC has not improved for 5 epochs and keep the best epoch. * `--epochs`, `--lr`, `--seed`, ...: override the training settings of the configuration file. See [docs/Options.md](docs/Options.md) or `corenet-train --help`. diff --git a/src/corenet/models/config.py b/src/corenet/models/config.py index 471e907..f23aa29 100644 --- a/src/corenet/models/config.py +++ b/src/corenet/models/config.py @@ -6,7 +6,7 @@ The file grammar:: - objects: jets: {features: [pt, eta, phi, mass, btag], max: 12} # collection + objects: jets: {features: [pt, eta, phi, mass], max: 12} # collection: features = network inputs b: jets btag 4 # role: event: [nSelJets, HT] hypotheses: HH: {H1: [b, b], H2: [b, b], context: other} @@ -33,6 +33,9 @@ KINEMATIC = ("pt", "eta", "phi", "mass") SELECTORS = ("btag", "pt", "rest") +# columns a selector reads from the input file, for the selection only (``rest`` orders by pT, or by the +# previous selector's key); they need not be features +SELECTOR_COLUMNS = {"btag": ("btag", "pt"), "pt": ("pt",), "rest": ("pt",)} SECTIONS = ( "objects", "event", @@ -53,7 +56,7 @@ class ConfigError(ValueError): @dataclass class Collection: name: str - features: list[str] + features: list[str] # exactly the columns fed to the network (kinematics + extras), in this order max: int = 1 @@ -145,6 +148,20 @@ def collection_of(self, leaf: str) -> Collection: else self.collections[leaf] ) + def selector_columns(self, collection: str) -> list[str]: + """Columns the selectors of the roles on ``collection`` read from the file (selection only).""" + out: list[str] = [] + for r in self.roles.values(): + if r.collection == collection: + out.extend(c for c in SELECTOR_COLUMNS[r.selector] if c not in out) + return out + + def columns_to_read(self, collection: str) -> list[str]: + """Every column the loader reads for ``collection``: its features (the network inputs), then the + selector columns that are not features.""" + feats = list(self.collections[collection].features) + return feats + [c for c in self.selector_columns(collection) if c not in feats] + def nodes_in(self, hypothesis: str) -> set[str]: """Composite nodes reachable from a hypothesis' top node.""" seen, stack = set(), [hypothesis] @@ -229,10 +246,6 @@ def _parse_role(name: str, text: str, collections: dict[str, Collection]) -> Rol raise ConfigError( f"objects.{name}: selector must be one of {SELECTORS}, got {sel!r}" ) - if sel == "btag" and "btag" not in collections[coll].features: - raise ConfigError( - f"objects.{name}: selector 'btag' needs a 'btag' feature on {coll}" - ) if n < 1 or n > collections[coll].max: raise ConfigError( f"objects.{name}: n={n} must be within 1..max={collections[coll].max} of {coll}" @@ -484,7 +497,10 @@ def load_config(path: str | Path) -> NetworkSpec: __all__ = [ + "KINEMATIC", "SECTIONS", + "SELECTOR_COLUMNS", + "SELECTORS", "AttentionSpec", "Collection", "ConfigError", From 69bb2cf990b606d47064e30e234e6d5027beee3d Mon Sep 17 00:00:00 2001 From: Aniket Khanal Date: Wed, 21 Oct 2026 12:30:05 -0400 Subject: [PATCH 11/33] update documentation with njets change --- docs/Dataset.md | 6 +++--- docs/EventInfo.md | 23 +++++++++++++++++------ docs/Options.md | 7 ++++--- 3 files changed, 24 insertions(+), 12 deletions(-) diff --git a/docs/Dataset.md b/docs/Dataset.md index 6e15a5a..ee78c2f 100644 --- a/docs/Dataset.md +++ b/docs/Dataset.md @@ -46,9 +46,9 @@ jets is decided by the event description when the file is loaded, so one file se The column names are the ones used in the event description. The columns listed as `features` there are the inputs of the network: `pt`, `eta`, `phi` and `mass` are the kinematics, any other feature -(`isE`, `isM`, ...) is an additional input for that object. A column named in a candidate rule (`btag` -for `b: jets btag 4`) is read for that choice only and need not be a feature; listing it under `features` -would make it an input as well. +(`isE`, `isM`, ...) is an additional input for that object. Any column of a collection can be named in a +candidate line (`btag` for `b: jets btag 4`, [EventInfo.md](./EventInfo.md)); it is read for that choice +only and need not be a feature; listing it under `features` would make it an input as well. ## Single objects diff --git a/docs/EventInfo.md b/docs/EventInfo.md index 2f3b4f9..9487f3b 100644 --- a/docs/EventInfo.md +++ b/docs/EventInfo.md @@ -47,15 +47,18 @@ objects: A collection has `features` (required) and `max` (the number of jets per event, default 1). A single object enters the decay chains by its own name; a collection with several jets is used -through candidate lines of the form `: `: +through candidate lines of the form `: `, where `` is any +per-jet column of the collection's group in the input file (`btag`, `pt`, `mass`, ...): -| rule | the `n` jets taken | +| line | the `n` jets taken | |---|---| -| `btag` | the `n` jets most likely to be b jets; reads the `btag` (and `pt`) column of the file, which need not be a feature | -| `pt` | the `n` leading jets; reads the `pt` column | -| `rest` | up to `n` of the jets not taken by an earlier line | +| ` ` | the `n` jets with the highest value of the column (`jets btag 4`: the four jets most likely to be b jets; `jets pt 4`: the four leading jets); ties go to the higher `pt` | +| ` lowest` | the `n` jets with the lowest value of the column (`jets mass 2 lowest`), ties again to the higher `pt` | +| `rest ` | up to `n` of the jets not taken by an earlier line, leading `pt` first ([Options.md](./Options.md): `rest_order`) | -A rule reads its column from the input file for the selection only; the network sees the `features`. +`rest` is the only reserved word; every other word in that position names a column. The column (and +`pt`) is read from the input file for the selection only and need not be a feature; the network sees +the `features`. Nothing else may follow `` but `lowest`. Lines are filled in order and a jet is used by at most one line. An event with fewer jets than needed is kept; the missing ones are simply absent. @@ -69,6 +72,14 @@ event: [nSelJets, HT] Optional per-event quantities. `nSelJets` (the number of jets) and `HT` (their scalar momentum sum) are computed from the jet collection; any other name is read from the input file. +```yaml +min_jets: 4 +``` + +Optional, default 4: the minimum number of jets per event in your selection. The jet multiplicity enters +the network as `log(nSelJets - (min_jets - 1))`, so that the minimum multiplicity maps to zero; an event +with fewer jets than `min_jets` would give a non-finite input. + ## `hypotheses` The second **required** section: the decay chain of each process, as a tree over the candidate diff --git a/docs/Options.md b/docs/Options.md index b5561bf..4f1e0fc 100644 --- a/docs/Options.md +++ b/docs/Options.md @@ -26,7 +26,7 @@ Changing one of these gives a different network: validate the file with `corenet | key | flag | default | meaning | |---|---|---|---| -| `epochs` | `--epochs` | 20 | number of epochs; the model after the last epoch is kept | +| `epochs` | `--epochs` | 20 | number of epochs; the model after the last epoch is kept unless `early_stop_patience` is set | | `lr` | `--lr` | 0.01 | learning rate | | `lr_milestones` | `--lr-milestones` | 15, 16, ..., 24 | epochs after which the learning rate is multiplied by `lr_gamma` | | `lr_gamma` | `--lr-gamma` | 0.25 | learning-rate decay factor | @@ -34,7 +34,8 @@ Changing one of these gives a different network: validate the file with `corenet | `freeze_epoch` | `--freeze-epoch` | 15 | from this epoch on only the output layers are trained (a value ≥ `epochs` disables this) | | `class_norm` | `--class-norm` / `--no-class-norm` | true | give every class the same total weight in training | | `seed` | `--seed` | 42 | random seed; fold `k` uses `seed + k` | -| `rest_order` | `--rest-order` | `pt` | order of the jets taken by a `rest` line: `pt`, or `selector` to keep the order of the previous `btag`/`pt` line | +| `early_stop_patience` | `--early-stop-patience` | 0 | stop when the validation AUC (on the `--exclude-folds` fold) has not improved for this many epochs, and keep the weights of the best epoch; 0 trains every epoch | +| `rest_order` | `--rest-order` | `pt` | order of the jets taken by a `rest` line: `pt`, or `selector` to keep the order of the previous candidate line on the same collection (e.g. `b: jets btag 4`) | Write floating-point numbers with a decimal point (`0.005` or `5.0e-3`): YAML reads `5e-3` as a string, which the validation rejects. @@ -51,7 +52,7 @@ These choices are part of the training procedure and the network and have no set * the batch size doubles at epochs 1, 3, 6 and 10, and the batches are split into 64, 16, 4, 1 and 0 ghost batches over those stages; * from `freeze_epoch` on, only the output layers keep training; * the input file needs no standardization of its values; -* the event-level quantities `nSelJets` and `HT` are computed from the jets in the file. +* the event-level quantities `nSelJets` and `HT` are computed from the jets in the file; `nSelJets` enters as `log(nSelJets - (min_jets - 1))` ([EventInfo.md](./EventInfo.md)). ## Choosing the jets From 9c1bba5af8368fa0d468170d3618e62134f650f7 Mon Sep 17 00:00:00 2001 From: Aniket Khanal Date: Wed, 21 Oct 2026 12:30:25 -0400 Subject: [PATCH 12/33] update input config file structure --- src/corenet/data/h5.py | 70 +++++++++++++++++------------------------- 1 file changed, 29 insertions(+), 41 deletions(-) diff --git a/src/corenet/data/h5.py b/src/corenet/data/h5.py index cb61919..6997c92 100644 --- a/src/corenet/data/h5.py +++ b/src/corenet/data/h5.py @@ -15,20 +15,21 @@ information is read: the roles are chosen from reconstructed quantities only. A collection's ``features`` are exactly the channels of its tensors. A selector reads the column it -needs (``btag``: the ``btag`` and ``pt`` columns) from the file on its own, whether or not it is a -feature; such a column is used for the selection only (``columns_to_read``). A column listed under -``features`` is always a network input, ``btag`` included. +needs (its column and ``pt``) from the file on its own, whether or not it is a feature; such a column +is used for the selection only (``columns_to_read``). A column listed under ``features`` is always a +network input, ``btag`` included. ds = H5Dataset("examples/data/bbww_small.h5", load_config("examples/bbww/bbww.yaml")) out = model(ds.inputs) Selectors, applied to the collection's present members in configuration order: - btag n the n highest-btag members, ties broken by pT - pt n the n highest-pT members - rest n up to n of the members no earlier role consumed; ``rest_order`` decides their order: - "pt" (default) or "selector", which continues the ordering of the preceding role on the - same collection. + n the n members with the highest value of (any dataset of the collection's + group: btag, pt, mass, ...), ties broken by the highest pT, then file order + n lowest the n members with the lowest value of , ties again by the highest pT + rest n up to n of the members no earlier role consumed; ``rest_order`` decides their order: + "pt" (default) or "selector", which continues the ordering of the preceding role on + the same collection. ``nSelJets`` and ``HT`` are always computed from the first collection with ``max > 1`` (number of present members and scalar sum of their pT); every other event scalar is read from ``EVENT/``. @@ -43,7 +44,7 @@ import torch from torch.utils.data import Dataset -from ..models.config import Collection, NetworkSpec, RoleSpec +from ..models.config import REST, Collection, NetworkSpec, RoleSpec # collection name (lower case) -> group names accepted in INPUTS, in order of preference GROUP_ALIASES = { @@ -53,11 +54,7 @@ "met": ("Met", "MET"), } COMPUTED_EVENT = ("nSelJets", "HT") -BTAG_PRIORITY = 1e6 # btag dominates the sort key, pT breaks ties -LABEL_KEYS = ( - "EVENT/label", - "CLASSIFICATIONS/EVENT/signal", -) # first one present is used +LABEL_KEYS = ("EVENT/label", "CLASSIFICATIONS/EVENT/signal") # first one present is used WEIGHT_KEY, FOLD_KEY, EVENT_KEY = "EVENT/weight", "EVENT/fold", "EVENT/event" @@ -124,19 +121,18 @@ def _read_collection(f: h5py.File, coll: Collection, columns: list[str]) -> tupl return feats, present -def _selector_key(selector: str, feats: dict[str, np.ndarray]) -> np.ndarray: - """Sort key (ascending = preferred first) of a selector.""" - if selector == "pt": - return -feats["pt"] - if selector == "btag": - return -(feats["btag"] * BTAG_PRIORITY + feats["pt"]) - raise ValueError(f"unknown selector {selector!r}") +def _selector_keys(role: RoleSpec, feats: dict[str, np.ndarray]) -> tuple[np.ndarray, ...]: + """Sort keys (smaller = preferred) of an explicit selector: the column, negated unless ``lowest``, then + -pt as the tie breaker. A lexicographic sort, so the column's scale does not matter.""" + col = feats[role.selector] + return (col if role.ascending else -col, -feats["pt"]) -def _take(key: np.ndarray, avail: np.ndarray, n: int) -> tuple[np.ndarray, np.ndarray]: - """(indices, valid) of the n preferred available members: (N, n) each, stable ties.""" - order = np.argsort(np.where(avail, key, np.inf), axis=1, kind="stable")[:, :n] - rows = np.arange(len(key))[:, None] +def _take(keys: tuple[np.ndarray, ...], avail: np.ndarray, n: int) -> tuple[np.ndarray, np.ndarray]: + """(indices, valid) of the n preferred available members: (N, n) each. Available members first, then + by ``keys`` in order (the first key dominates), remaining ties in file order (stable).""" + order = np.lexsort((*reversed(keys), ~avail), axis=1)[:, :n] # lexsort: the last key is the primary one + rows = np.arange(len(avail))[:, None] return order, avail[rows, order] @@ -147,12 +143,8 @@ def __init__(self, path: str | Path, spec: NetworkSpec, rest_order: str = "pt"): f"rest_order must be 'pt' or 'selector', got {rest_order!r}" ) self.spec, self.path, self.rest_order = spec, Path(path), rest_order - self._taken: dict[ - str, np.ndarray - ] = {} # per collection: members already consumed by a role - self._last_key: dict[ - str, np.ndarray - ] = {} # per collection: sort key of the last explicit selector + self._taken: dict[str, np.ndarray] = {} # per collection: members already consumed by a role + self._last_keys: dict[str, tuple[np.ndarray, ...]] = {} # per collection: sort keys of the last explicit selector inputs: dict[str, torch.Tensor] = {} with h5py.File(path, "r") as f: @@ -198,17 +190,13 @@ def _role_tensor( present: np.ndarray, ) -> torch.Tensor: taken = self._taken.setdefault(role.collection, np.zeros_like(present)) - if role.selector == "rest": - key = ( - self._last_key.get(role.collection) - if self.rest_order == "selector" - else None - ) - key = _selector_key("pt", feats) if key is None else key + if role.selector == REST: + keys = self._last_keys.get(role.collection) if self.rest_order == "selector" else None + keys = (-feats["pt"],) if keys is None else keys else: - key = _selector_key(role.selector, feats) - self._last_key[role.collection] = key - idx, valid = _take(key, present & ~taken, role.n) + keys = _selector_keys(role, feats) + self._last_keys[role.collection] = keys + idx, valid = _take(keys, present & ~taken, role.n) rows = np.arange(len(present))[:, None] for j in range(role.n): # mark the chosen members as consumed taken[rows[:, 0], idx[:, j]] |= valid[:, j] From ebae23fbee744bbf55b49edb055ff2769efd97b9 Mon Sep 17 00:00:00 2001 From: Aniket Khanal Date: Wed, 21 Oct 2026 12:30:37 -0400 Subject: [PATCH 13/33] update input config file structure --- src/corenet/models/config.py | 97 ++++++++++++------------------------ 1 file changed, 32 insertions(+), 65 deletions(-) diff --git a/src/corenet/models/config.py b/src/corenet/models/config.py index f23aa29..b4fa68a 100644 --- a/src/corenet/models/config.py +++ b/src/corenet/models/config.py @@ -7,7 +7,7 @@ The file grammar:: objects: jets: {features: [pt, eta, phi, mass], max: 12} # collection: features = network inputs - b: jets btag 4 # role: + b: jets btag 4 # role: [lowest] event: [nSelJets, HT] hypotheses: HH: {H1: [b, b], H2: [b, b], context: other} tt: {t_had: [b, W_had], t_lep: [b, W_lep]} # W_had / W_lep reused by name @@ -32,21 +32,9 @@ from .combinatorics import Node, Role KINEMATIC = ("pt", "eta", "phi", "mass") -SELECTORS = ("btag", "pt", "rest") -# columns a selector reads from the input file, for the selection only (``rest`` orders by pT, or by the -# previous selector's key); they need not be features -SELECTOR_COLUMNS = {"btag": ("btag", "pt"), "pt": ("pt",), "rest": ("pt",)} -SECTIONS = ( - "objects", - "event", - "hypotheses", - "attention", - "relations", - "outputs", - "combine", - "training", - "trainer", -) +REST = "rest" # the only reserved selector word: the members no earlier role took +LOWEST = "lowest" # trailing word of a role line: lowest value of the column first +SECTIONS = ("objects", "event", "min_jets", "hypotheses", "attention", "relations", "outputs", "combine", "training", "trainer") class ConfigError(ValueError): @@ -64,8 +52,15 @@ class Collection: class RoleSpec: name: str collection: str - selector: str # btag | pt | rest + selector: str # a column of the collection's group in the file (btag, pt, mass, ...) or ``rest`` n: int + ascending: bool = False # ``lowest``: the n members with the lowest value first (default: highest first) + + @property + def columns(self) -> tuple[str, ...]: + """Columns the selector reads from the file, for the selection only: the column and pt (the tie + breaker; ``rest`` orders by pt or continues the previous role's order).""" + return ("pt",) if self.selector in (REST, "pt") else (self.selector, "pt") @dataclass @@ -119,9 +114,8 @@ class NetworkSpec: combine: list[str] training: TrainingSpec relations: list[RelationSpec] = field(default_factory=list) - owner: dict[str, str] = field( - default_factory=dict - ) # node -> hypothesis in which it is declared + owner: dict[str, str] = field(default_factory=dict) # node -> hypothesis in which it is declared + min_jets: int = 4 # minimum jet multiplicity of the selection: nSelJets enters as log(nSelJets - (min_jets - 1)) # ---- derived @property @@ -153,7 +147,7 @@ def selector_columns(self, collection: str) -> list[str]: out: list[str] = [] for r in self.roles.values(): if r.collection == collection: - out.extend(c for c in SELECTOR_COLUMNS[r.selector] if c not in out) + out.extend(c for c in r.columns if c not in out) return out def columns_to_read(self, collection: str) -> list[str]: @@ -234,23 +228,20 @@ def _check_training(tr: dict) -> None: def _parse_role(name: str, text: str, collections: dict[str, Collection]) -> RoleSpec: - m = re.fullmatch(r"\s*(\w+)\s+(\w+)\s+(\d+)\s*", text) + """`` ``: the n members with the highest value of ``column`` (ties: highest pt); + `` lowest``: the lowest first; `` rest ``: the members no earlier + role took. ``column`` is any per-member dataset of the collection's group in the file.""" + m = re.fullmatch(rf"\s*(\w+)\s+(\w+)\s+(\d+)(?:\s+({LOWEST}))?\s*", text) if not m: - raise ConfigError( - f"objects.{name}: expected ' ', got {text!r}" - ) - coll, sel, n = m.group(1), m.group(2), int(m.group(3)) + raise ConfigError(f"objects.{name}: expected ' [{LOWEST}]', got {text!r}") + coll, sel, n, tail = m.group(1), m.group(2), int(m.group(3)), m.group(4) if coll not in collections: raise ConfigError(f"objects.{name}: unknown collection {coll!r}") - if sel not in SELECTORS: - raise ConfigError( - f"objects.{name}: selector must be one of {SELECTORS}, got {sel!r}" - ) + if sel == REST and tail is not None: + raise ConfigError(f"objects.{name}: '{REST}' takes no {LOWEST!r} (its order is the rest_order option)") if n < 1 or n > collections[coll].max: - raise ConfigError( - f"objects.{name}: n={n} must be within 1..max={collections[coll].max} of {coll}" - ) - return RoleSpec(name, coll, sel, n) + raise ConfigError(f"objects.{name}: n={n} must be within 1..max={collections[coll].max} of {coll}") + return RoleSpec(name, coll, sel, n, ascending=tail == LOWEST) def _parse_tree( @@ -343,6 +334,9 @@ def parse_config(cfg: dict) -> NetworkSpec: ) event = _str_list(cfg.get("event", []) or [], "event") + min_jets = cfg.get("min_jets", 4) + if isinstance(min_jets, bool) or not isinstance(min_jets, int) or min_jets < 2: + raise ConfigError(f"min_jets must be an integer >= 2 (the minimum jet multiplicity of the selection), got {min_jets!r}") # hypotheses --------------------------------------------------------------------------- leaves = set(roles) | {c.name for c in collections.values() if c.max == 1} @@ -466,19 +460,7 @@ def parse_config(cfg: dict) -> NetworkSpec: if h not in [o.name for o in outputs[1:]]: raise ConfigError(f"training.head_weights: {h!r} is not a secondary output") - return NetworkSpec( - collections, - roles, - event, - nodes, - hypotheses, - attention, - outputs, - combine, - training, - relations, - owner, - ) + return NetworkSpec(collections, roles, event, nodes, hypotheses, attention, outputs, combine, training, relations, owner, min_jets) def read_config_dict(path: str | Path) -> dict: @@ -496,21 +478,6 @@ def load_config(path: str | Path) -> NetworkSpec: return parse_config(read_config_dict(path)) -__all__ = [ - "KINEMATIC", - "SECTIONS", - "SELECTOR_COLUMNS", - "SELECTORS", - "AttentionSpec", - "Collection", - "ConfigError", - "HypothesisSpec", - "NetworkSpec", - "OutputSpec", - "RelationSpec", - "RoleSpec", - "TrainingSpec", - "load_config", - "parse_config", - "read_config_dict", -] +__all__ = ["ConfigError", "Collection", "RoleSpec", "HypothesisSpec", "AttentionSpec", "RelationSpec", "OutputSpec", + "TrainingSpec", "NetworkSpec", "parse_config", "read_config_dict", "load_config", "SECTIONS", "REST", + "LOWEST", "KINEMATIC"] From 03659085dd5b35f96c89a8a16e5ce57b340111b3 Mon Sep 17 00:00:00 2001 From: Aniket Khanal Date: Wed, 21 Oct 2026 12:30:49 -0400 Subject: [PATCH 14/33] add dynamic njets option --- src/corenet/models/network.py | 29 +++++++++-------------------- 1 file changed, 9 insertions(+), 20 deletions(-) diff --git a/src/corenet/models/network.py b/src/corenet/models/network.py index c5996df..8c5d254 100644 --- a/src/corenet/models/network.py +++ b/src/corenet/models/network.py @@ -9,7 +9,7 @@ children; a massless MET leaf enters with eta = m = 0) 2. features log(1+pt), eta, dphi-to-parent, log(1+m) for every child slot; the top node drops phi; event scalars whose name contains ``nSelJets`` enter as - log(nSelJets - 3) + log(nSelJets - (min_jets - 1)) 3. embedding GBN embed (+ event scalars for leaves) -> conv, per role / node 4. reinforce bottom-up ResNetBlock per composite node (a node reused by several hypotheses is computed once); ``context`` attention after the first level; @@ -36,12 +36,7 @@ from .config import KINEMATIC, ConfigError, NetworkSpec from .fourvectors import NonLU, PxPyPzE, addFourVectors, diObjectMass -TAG_FEATURES = ( - "btag", -) # selection tags: used by the role selectors, never fed to the network -MASKED_LOGIT = ( - -1e4 -) # selector logit of an absent view (finite, so an all-absent event stays finite) +MASKED_LOGIT = -1e4 # selector logit of an absent view (finite, so an all-absent event stays finite) # ------------------------------------------------------------------------------- small helpers @@ -405,7 +400,7 @@ def dep(n): def _extras(self, role): feats = self.spec.collection_of(role).features - return [f for f in feats if f not in KINEMATIC and f not in TAG_FEATURES] + return [f for f in feats if f not in KINEMATIC] # every non-kinematic feature is an input def _kinematic_channels(self, role): feats = self.spec.collection_of(role).features @@ -437,8 +432,8 @@ def _partner(self, key): def _prepare(self, inputs): """Kinematics, features and masks for every hypothesis. Returns a dict of intermediate tensors.""" n = next(iter(inputs.values())).shape[0] - # per leaf (role or single object): four-vectors (n, 4, slots), its non-kinematic features (isE / isM; btag - # is never used), and which slots hold a real object + # per leaf (role or single object): four-vectors (n, 4, slots), its non-kinematic features (isE / isM, ...; + # every feature of the collection is an input), and which slots hold a real object leaf_kin, leaf_extra, leaf_present, etaSign = {}, {}, {}, None for r in self.leaf_order: if r not in inputs: @@ -460,21 +455,15 @@ def _prepare(self, inputs): kin[:, 1, :] = etaSign * kin[:, 1, :] leaf_kin[r] = kin - # event scalars; a jet multiplicity (name containing ``nSelJets``) enters as log(nSelJets - 3) + # event scalars; a jet multiplicity (name containing ``nSelJets``) enters as log(nSelJets - (min_jets - 1)), + # so that the minimum multiplicity of the selection maps to log(1) = 0 a = None if self.dA: a = inputs["event"].view(n, self.dA, 1) for idx, name in enumerate(self.spec.event): if "nSelJets" in name: - a = torch.cat( - [ - a[:, :idx], - torch.log(a[:, idx : idx + 1] - 3), - a[:, idx + 1 :], - ], - dim=1, - ) - a = finite(a) # nSelJets <= 3 would give -inf / NaN + a = torch.cat([a[:, :idx], torch.log(a[:, idx:idx + 1] - (self.spec.min_jets - 1)), a[:, idx + 1:]], dim=1) + a = finite(a) # nSelJets < min_jets would give -inf / NaN out = { "n": n, From 6dfd2cd357630e61b0e73c45d14a9d55308bcfee Mon Sep 17 00:00:00 2001 From: Aniket Khanal Date: Wed, 21 Oct 2026 12:31:00 -0400 Subject: [PATCH 15/33] add early stopping function --- src/corenet/train.py | 165 ++++++++++++++----------------------------- 1 file changed, 54 insertions(+), 111 deletions(-) diff --git a/src/corenet/train.py b/src/corenet/train.py index e896975..97f158f 100644 --- a/src/corenet/train.py +++ b/src/corenet/train.py @@ -62,6 +62,7 @@ "class_norm": True, "seed": 42, "rest_order": "pt", + "early_stop_patience": 0, } @@ -225,6 +226,7 @@ def train_fold( out_dir.mkdir(parents=True, exist_ok=True) recipe = {k: getattr(args, k) for k in RECIPE_DEFAULTS} history, history_path = [], out_dir / f"history_fold{fold}.json" + best_auc, best_epoch, best_state, bad, stopped = None, None, None, 0, False # early stopping state for epoch in range(args.epochs): bs, n_gb = batch_schedule(epoch, args.batch_size_init) model.setGhostBatches(n_gb) @@ -272,44 +274,37 @@ def train_fold( entry.update(validation_metrics(model, data, val_idx, labels, val_weights)) model.setGhostBatches(n_gb) # predict() set them to 0 history.append(entry) - history_path.write_text( - json.dumps( - { - "fold": fold, - "exclude_folds": args.exclude_folds, - "config": str(args.config), - "classes": spec.classes, - "recipe": recipe, - "n_train": len(train_idx), - "n_val": len(val_idx), - "history": history, - }, - indent=1, - ) - ) - logging.info( - f"fold {fold} | epoch {epoch:02d} | bs {bs:6d} | gbn {n_gb:2d} | lr {optimizer.param_groups[0]['lr']:.2e} | loss {entry['train_loss']:.5f}" - + ( - f" | val loss {entry['val_loss']:.5f} | val auc {entry.get('val_auc', float('nan')):.5f}" - if len(val_idx) - else "" - ) - ) - + history_path.write_text(json.dumps({"fold": fold, "exclude_folds": args.exclude_folds, "config": str(args.config), + "classes": spec.classes, "recipe": recipe, + "n_train": int(len(train_idx)), "n_val": int(len(val_idx)), + "history": history}, indent=1)) + logging.info(f"fold {fold} | epoch {epoch:02d} | bs {bs:6d} | gbn {n_gb:2d} | lr {optimizer.param_groups[0]['lr']:.2e} | loss {entry['train_loss']:.5f}" + + (f" | val loss {entry['val_loss']:.5f} | val auc {entry.get('val_auc', float('nan')):.5f}" if len(val_idx) else "")) + if args.early_stop_patience and len(val_idx): + auc = entry.get("val_auc") + if auc is not None and (best_auc is None or auc > best_auc): + best_auc, best_epoch, bad = auc, epoch, 0 + best_state = {k: v.detach().clone() for k, v in model.state_dict().items()} + else: + bad += 1 + if bad >= args.early_stop_patience: + stopped = True + logging.info(f"fold {fold}: early stopping after epoch {epoch}: the validation AUC did not improve for " + f"{bad} epochs (best {best_auc:.5f} at epoch {best_epoch})") + break + + final_epoch = args.epochs - 1 if not stopped else epoch + if args.early_stop_patience and best_state is not None: + model.load_state_dict(best_state) # keep the weights of the best validation epoch + final_epoch = best_epoch + doc = json.loads(history_path.read_text()) + doc.update({"best_epoch": best_epoch, "stopped_after_epoch": epoch if stopped else None, + "early_stop_patience": args.early_stop_patience}) + history_path.write_text(json.dumps(doc, indent=1)) + logging.info(f"fold {fold}: keeping the weights of epoch {best_epoch} (validation AUC {best_auc:.5f})") ckpt = out_dir / f"model_fold{fold}.pkl" - save_checkpoint( - ckpt, - model, - config, - args.config, - { - "fold": fold, - "epoch": args.epochs - 1, - "exclude_folds": args.exclude_folds, - "input": str(args.input), - **recipe, - }, - ) + save_checkpoint(ckpt, model, config, args.config, + {"fold": fold, "epoch": final_epoch, "exclude_folds": args.exclude_folds, "input": str(args.input), **recipe}) probs, views, heads = predict(model, data, test_idx) write_predictions( out_dir / f"predictions_fold{fold}.h5", @@ -345,75 +340,24 @@ def build_parser() -> argparse.ArgumentParser: "--input", required=True, help="HDF5 file with all folds (docs/Dataset.md)" ) p.add_argument("--output-dir", default="runs/corenet") - p.add_argument( - "--fold", - type=int, - default=-1, - help="held-out (test) fold, or -1 for every fold in turn", - ) - p.add_argument( - "--exclude-folds", - type=int, - nargs="*", - default=[], - help="folds left out of training and used for per-epoch validation", - ) - g = p.add_argument_group( - "training recipe (default: the `trainer:` section of --config, else the built-in value)" - ) - g.add_argument( - "--epochs", type=int, help=f"number of epochs [{RECIPE_DEFAULTS['epochs']}]" - ) - g.add_argument( - "--freeze-epoch", - type=int, - help=f"epoch at which all but the output heads are frozen [{RECIPE_DEFAULTS['freeze_epoch']}]", - ) - g.add_argument( - "--lr", type=float, help=f"Adam learning rate [{RECIPE_DEFAULTS['lr']}]" - ) - g.add_argument( - "--lr-milestones", - type=int, - nargs="*", - help="epochs after which the learning rate is multiplied by --lr-gamma [15 .. 24]", - ) - g.add_argument( - "--lr-gamma", - type=float, - help=f"learning-rate decay factor [{RECIPE_DEFAULTS['lr_gamma']}]", - ) - g.add_argument( - "--batch-size-init", - type=int, - help=f"epoch-0 batch size; doubles at epochs 1, 3, 6, 10 [{RECIPE_DEFAULTS['batch_size_init']}]", - ) - g.add_argument( - "--class-norm", - action=argparse.BooleanOptionalAction, - default=None, - help="normalise the weights of every class to 1 over the training set [on]", - ) - g.add_argument( - "--seed", - type=int, - help=f"random seed (the seed of fold k is seed + k) [{RECIPE_DEFAULTS['seed']}]", - ) - g.add_argument( - "--rest-order", - choices=["pt", "selector"], - help="ordering of the 'rest' selector [pt]", - ) - p.add_argument( - "--save-every-epoch", - action="store_true", - help="also save model_fold{k}_epoch{e}.pkl after every epoch", - ) - p.add_argument( - "--device", - default="cuda", - help="torch device; falls back to cpu when CUDA is unavailable", - ) + p.add_argument("--fold", type=int, default=-1, help="held-out (test) fold, or -1 for every fold in turn") + p.add_argument("--exclude-folds", type=int, nargs="*", default=[], + help="folds left out of training and used for per-epoch validation") + g = p.add_argument_group("training recipe (default: the `trainer:` section of --config, else the built-in value)") + g.add_argument("--epochs", type=int, help=f"number of epochs [{RECIPE_DEFAULTS['epochs']}]") + g.add_argument("--freeze-epoch", type=int, help=f"epoch at which all but the output heads are frozen [{RECIPE_DEFAULTS['freeze_epoch']}]") + g.add_argument("--lr", type=float, help=f"Adam learning rate [{RECIPE_DEFAULTS['lr']}]") + g.add_argument("--lr-milestones", type=int, nargs="*", help="epochs after which the learning rate is multiplied by --lr-gamma [15 .. 24]") + g.add_argument("--lr-gamma", type=float, help=f"learning-rate decay factor [{RECIPE_DEFAULTS['lr_gamma']}]") + g.add_argument("--batch-size-init", type=int, help=f"epoch-0 batch size; doubles at epochs 1, 3, 6, 10 [{RECIPE_DEFAULTS['batch_size_init']}]") + g.add_argument("--class-norm", action=argparse.BooleanOptionalAction, default=None, + help="normalise the weights of every class to 1 over the training set [on]") + g.add_argument("--seed", type=int, help=f"random seed (the seed of fold k is seed + k) [{RECIPE_DEFAULTS['seed']}]") + g.add_argument("--early-stop-patience", type=int, help="stop when the validation AUC has not improved for this many epochs " + f"and keep the weights of the best epoch; 0 = train every epoch; needs --exclude-folds [{RECIPE_DEFAULTS['early_stop_patience']}]") + g.add_argument("--rest-order", choices=["pt", "selector"], help="ordering of the 'rest' selector [pt]") + p.add_argument("--save-every-epoch", action="store_true", help="also save model_fold{k}_epoch{e}.pkl after every epoch") + p.add_argument("--device", default="cuda", help="torch device; falls back to cpu when CUDA is unavailable") return p @@ -439,6 +383,7 @@ def is_num(x): "class_norm": (lambda x: isinstance(x, bool), "true or false"), "seed": (lambda x: is_int(x), "an integer"), "rest_order": (lambda x: x in ("pt", "selector"), "'pt' or 'selector'"), + "early_stop_patience": (lambda x: is_int(x) and x >= 0, "an integer >= 0"), } for key, (ok, what) in rules.items(): if key in r and not ok(r[key]): @@ -470,11 +415,9 @@ def main(argv=None): config = read_config_dict(args.config) spec = parse_config(config) resolve_recipe(args, config) - device = torch.device( - args.device - if torch.cuda.is_available() and args.device.startswith("cuda") - else "cpu" - ) + if args.early_stop_patience and not args.exclude_folds: + build_parser().error("--early-stop-patience needs a validation fold: pass --exclude-folds") + device = torch.device(args.device if torch.cuda.is_available() and args.device.startswith("cuda") else "cpu") ds = H5Dataset(args.input, spec, rest_order=args.rest_order) logging.info( f"{args.input}: {len(ds):,} events, device={device}, recipe " From d6f1a4faa9ee5e3ab36e48254a0fafa0f619f822 Mon Sep 17 00:00:00 2001 From: Aniket Khanal Date: Wed, 21 Oct 2026 12:31:36 -0400 Subject: [PATCH 16/33] update validation function with new changes --- src/corenet/validate.py | 36 +++++++++++++++++------------------- 1 file changed, 17 insertions(+), 19 deletions(-) diff --git a/src/corenet/validate.py b/src/corenet/validate.py index 8a8b812..71f07c1 100644 --- a/src/corenet/validate.py +++ b/src/corenet/validate.py @@ -30,7 +30,7 @@ import torch from .data import h5 as loader -from .models.config import ConfigError, NetworkSpec, parse_config, read_config_dict +from .models.config import REST, ConfigError, NetworkSpec, parse_config, read_config_dict from .models.network import CoReNet @@ -116,11 +116,11 @@ def check_config(path, report: Report) -> NetworkSpec | None: return None for c in spec.collections.values(): - report.ok(sec, f"collection `{c.name}`: max {c.max}, features {c.features}") + report.ok(sec, f"collection `{c.name}`: max {c.max}, features {c.features} (the network inputs)") for r in spec.roles.values(): - report.ok( - sec, f"role `{r.name}`: {r.n} x `{r.collection}` selected by `{r.selector}`" - ) + how = "" if r.selector == REST else (" lowest first" if r.ascending else " highest first") + report.ok(sec, f"role `{r.name}`: {r.n} x `{r.collection}` selected by `{r.selector}`{how} " + f"(reads {', '.join(r.columns)} for the selection)") for h, T in model.tables.items(): nodes = ", ".join(f"{n} ({T.nodes[n].n})" for n in model.levels[h]) ctx = spec.hypotheses[h].context @@ -208,13 +208,15 @@ def length(name, ds): continue group = f["INPUTS"][gname] slots = None - for feat in coll.features: + columns = loader.columns_to_read(spec, coll) + selection = [c for c in columns if c not in coll.features] # read for a selector only + for feat in columns: name = f"INPUTS/{gname}/{feat}" + readers = [f"`{r.name}`" for r in spec.roles.values() + if r.collection == coll.name and feat in r.columns] + why = (f" (read for the selection by {', '.join(readers)}, not a feature)" if feat in selection else "") if feat not in group or not isinstance(group[feat], h5py.Dataset): - report.fail( - sec, - f"{who} expects dataset {name}: not found (INPUTS/{gname} has {sorted(group)})", - ) + report.fail(sec, f"{who} expects dataset {name}{why}: not found (INPUTS/{gname} has {sorted(group)})") continue ds = group[feat] if not _numeric(ds): @@ -271,20 +273,16 @@ def length(name, ds): k for k in ("pt", "eta", "phi", "mass") if k not in coll.features ] note = f"; {', '.join(missing_kin)} taken as 0" if missing_kin else "" - report.ok( - sec, - f"{who}: INPUTS/{gname} with {coll.features}" - + (f", {slots} slots" if slots else "") - + note, - ) + report.ok(sec, f"{who}: INPUTS/{gname} with features {coll.features}" + (f", {slots} slots" if slots else "") + note) + if selection: + report.ok(sec, f"{who}: {selection} read for the selection only (not network inputs)") # ---- event scalars for name in spec.event: kind, where = loader.event_source(spec, name) if kind == "computed": - report.ok( - sec, f"event scalar `{name}` computed from collection `{where}`" - ) + note = f" and enters as log({name} - {spec.min_jets - 1}) (min_jets {spec.min_jets})" if "nSelJets" in name else "" + report.ok(sec, f"event scalar `{name}` computed from collection `{where}`{note}") elif where not in f: report.fail( sec, f"event scalar `{name}` expects dataset {where}: not found" From cd797433e4492e902c4decde16cf520861b4f7dc Mon Sep 17 00:00:00 2001 From: Aniket Khanal Date: Wed, 21 Oct 2026 12:31:47 -0400 Subject: [PATCH 17/33] update tests for new config structure --- tests/test_checkpoint.py | 101 +++++++++++++++++++++ tests/test_config.py | 156 ++++++++++++++++---------------- tests/test_h5.py | 177 ++++++++++++++++++++++++++++--------- tests/test_network_bbww.py | 37 ++------ tests/test_train.py | 30 +++++++ tests/test_validate.py | 170 +++++++++++++---------------------- 6 files changed, 417 insertions(+), 254 deletions(-) create mode 100644 tests/test_checkpoint.py diff --git a/tests/test_checkpoint.py b/tests/test_checkpoint.py new file mode 100644 index 0000000..6050769 --- /dev/null +++ b/tests/test_checkpoint.py @@ -0,0 +1,101 @@ +"""Checkpoint format: new checkpoints carry ``format``; checkpoints written before every feature became a +network input (btag listed under features but skipped) still load, from their stored configuration.""" +from __future__ import annotations + +import copy +from pathlib import Path + +import pytest +import torch + +from corenet import evaluate +from corenet.checkpoint import CHECKPOINT_FORMAT, legacy_config, load_checkpoint, save_checkpoint +from corenet.data import H5Dataset +from corenet.models.config import parse_config, read_config_dict +from corenet.models.network import CoReNet +from synthetic import write_synthetic + +repo = Path(__file__).resolve().parent.parent + + +def _legacy_checkpoint(tmp_path): + """A checkpoint as corenet-train wrote it before ``format``: configuration with btag under features, state + dict of the network that ignored it (= today's network without the btag feature).""" + config = repo / "examples/hh4b/hh4b.yaml" + cfg = read_config_dict(config) + old_cfg = copy.deepcopy(cfg) + old_cfg["objects"]["jets"]["features"] = ["pt", "eta", "phi", "mass", "btag"] + torch.manual_seed(0) + model = CoReNet(parse_config(cfg), device="cpu") + path = tmp_path / "legacy.pkl" + torch.save({"model": model.state_dict(), "config": old_cfg, "config_path": str(config), "classes": model.spec.classes, + "metadata": {"fold": 0, "rest_order": "pt"}}, path) + return path, old_cfg, model + + +def test_legacy_config_translation(): + cfg = {"objects": {"jets": {"features": ["pt", "eta", "phi", "mass", "btag"], "max": 12}, "b": "jets btag 4", + "lepton": {"features": ["pt", "eta", "phi", "mass", "isE"]}}} + new, edited = legacy_config(cfg) + assert edited == ["jets"] and new["objects"]["jets"]["features"] == ["pt", "eta", "phi", "mass"] + assert new["objects"]["lepton"]["features"] == ["pt", "eta", "phi", "mass", "isE"] + assert cfg["objects"]["jets"]["features"][-1] == "btag" # the input is not modified + assert legacy_config({"objects": {"jets": {"features": ["pt", "eta", "phi", "mass"], "max": 12}}})[1] == [] + + +def _same_state(a, b) -> bool: + """Same keys, every tensor equal.""" + sa, sb = a.state_dict(), b.state_dict() + return list(sa) == list(sb) and all(torch.equal(sa[k], sb[k]) for k in sa) + + +def test_new_checkpoint_has_format(tmp_path): + cfg = read_config_dict(repo / "examples/hh4b/hh4b.yaml") + model = CoReNet(parse_config(cfg), device="cpu") + path = save_checkpoint(tmp_path / "m.pkl", model, cfg, repo / "examples/hh4b/hh4b.yaml", {"fold": 0}) + ck = torch.load(path, map_location="cpu", weights_only=False) + assert ck["format"] == CHECKPOINT_FORMAT == 2 + reloaded, spec, _ = load_checkpoint(path) + assert spec.collections["jets"].features == ["pt", "eta", "phi", "mass"] + model.setGhostBatches(0) # load_checkpoint sets the nGhostBatches buffers to 0 (eval mode) + assert _same_state(model, reloaded) + + +def test_legacy_checkpoint_loads(tmp_path): + path, old_cfg, model = _legacy_checkpoint(tmp_path) + reloaded, spec, ck = load_checkpoint(path) + assert "format" not in ck + assert spec.collections["jets"].features == ["pt", "eta", "phi", "mass"] # btag dropped: the trained network + model.setGhostBatches(0) + assert _same_state(model, reloaded) + # corenet-evaluate on it: the loader builds (N, 5, n) tensors from the translated configuration + data = write_synthetic(tmp_path / "d.h5", "hh4b", n=300) + out = tmp_path / "eval.h5" + metrics = evaluate.main(["--checkpoint", str(path), "--input", str(data), "--output", str(out), "--fold", "-1"]) + assert metrics["n_events"] == 300 and out.exists() + # the same legacy checkpoint with an explicit, untranslated configuration: a clear error + import yaml + old = tmp_path / "old.yaml" + old.write_text(yaml.safe_dump(old_cfg, sort_keys=False)) + with pytest.raises(RuntimeError, match="predates format 2"): + load_checkpoint(path, config=old) + # with btag as a feature the network is a different (larger) one: loading its own checkpoint works + big = CoReNet(parse_config(old_cfg), device="cpu") + big_ck = save_checkpoint(tmp_path / "big.pkl", big, old_cfg, old, {"fold": 0}) + _, spec_big, ck_big = load_checkpoint(big_ck) + assert ck_big["format"] == 2 and spec_big.collections["jets"].features[-1] == "btag" + assert H5Dataset(data, spec_big).inputs["b"].shape == (300, 6, 4) + + +@pytest.mark.parametrize("run", ["runs/repro_quickstart", "runs/repro_hh4b_full"]) +def test_shipped_legacy_runs(run): + """Checkpoints trained in this repository before the format marker: they load and evaluate.""" + path, data = repo / run / "model_fold0.pkl", repo / "examples/data/hh4b_small.h5" + if not (path.exists() and data.exists()): + pytest.skip(f"{path} or {data} not found") + model, spec, ck = load_checkpoint(path) + assert "format" not in ck and spec.collections["jets"].features == ["pt", "eta", "phi", "mass"] + ds = H5Dataset(data, spec, rest_order=ck.get("metadata", {}).get("rest_order", "pt")) + with torch.no_grad(): + out = model({k: v[:256] for k, v in ds.inputs.items()}) + assert out["logits"].shape == (256, 2) and torch.isfinite(out["logits"]).all() diff --git a/tests/test_config.py b/tests/test_config.py index 155c164..ae4e457 100644 --- a/tests/test_config.py +++ b/tests/test_config.py @@ -4,27 +4,20 @@ import copy +import pytest + from corenet.models.config import ConfigError, parse_config HH4B = { - "objects": { - "jets": {"features": ["pt", "eta", "phi", "mass", "btag"], "max": 12}, - "b": "jets btag 4", - "other": "jets rest 8", - }, + "objects": {"jets": {"features": ["pt", "eta", "phi", "mass"], "max": 12}, "b": "jets btag 4", "other": "jets rest 8"}, "event": ["nSelJets", "HT"], "hypotheses": {"HH": {"H1": ["b", "b"], "H2": ["b", "b"], "context": "other"}}, "outputs": {"signal": "HH"}, "combine": ["HH"], } BBWW = { - "objects": { - "jets": {"features": ["pt", "eta", "phi", "mass", "btag"], "max": 12}, - "b": "jets btag 2", - "q": "jets rest 2", - "lepton": {"features": ["pt", "eta", "phi", "mass", "isE", "isM"]}, - "met": {"features": ["pt", "phi"]}, - }, + "objects": {"jets": {"features": ["pt", "eta", "phi", "mass"], "max": 12}, "b": "jets btag 2", "q": "jets rest 2", + "lepton": {"features": ["pt", "eta", "phi", "mass", "isE", "isM"]}, "met": {"features": ["pt", "phi"]}}, "event": ["nSelJets", "HT"], "hypotheses": { "HH": { @@ -66,6 +59,22 @@ def test_hh4b_parses(): assert s.leaves()["b"].n == 4 and s.nodes_in("HH") == {"H1", "H2", "HH"} +def test_selector_columns_are_not_features(): + """A btag selector needs no btag feature: the selector reads its column from the file on its own.""" + s = parse_config(HH4B) + assert s.collections["jets"].features == ["pt", "eta", "phi", "mass"] + assert s.selector_columns("jets") == ["btag", "pt"] + assert s.columns_to_read("jets") == ["pt", "eta", "phi", "mass", "btag"] + # btag listed as a feature is accepted and stays a feature (an ordinary extra input) + c = copy.deepcopy(HH4B); c["objects"]["jets"]["features"] = ["pt", "eta", "phi", "mass", "btag"] + s = parse_config(c) + assert s.collections["jets"].features == ["pt", "eta", "phi", "mass", "btag"] + assert s.columns_to_read("jets") == ["pt", "eta", "phi", "mass", "btag"] + # a pt selector reads pt, which a max > 1 collection always has as a feature + c = copy.deepcopy(HH4B); c["objects"]["b"] = "jets pt 4" + assert parse_config(c).columns_to_read("jets") == ["pt", "eta", "phi", "mass"] + + def test_bbww_parses(): s = parse_config(BBWW) assert s.nodes["W_lep"].children == [ @@ -89,64 +98,52 @@ def test_bbww_parses(): assert s.nodes_in("tt") == {"t_had", "t_lep", "W_had", "W_lep", "tt"} +def test_role_grammar(): + """ [lowest]: any column of the file, highest first unless `lowest`; `rest` reserved.""" + c = copy.deepcopy(HH4B); c["objects"]["b"] = "jets mass 4" + s = parse_config(c) + r = s.roles["b"] + assert (r.selector, r.n, r.ascending, r.columns) == ("mass", 4, False, ("mass", "pt")) + assert s.columns_to_read("jets") == ["pt", "eta", "phi", "mass"] # mass is a feature already + c = copy.deepcopy(HH4B); c["objects"]["b"] = "jets mass 4 lowest" + r = parse_config(c).roles["b"] + assert (r.selector, r.ascending, r.columns) == ("mass", True, ("mass", "pt")) + c = copy.deepcopy(HH4B); c["objects"]["b"] = "jets dnn_score 4 lowest " + r = parse_config(c).roles["b"] + assert (r.selector, r.ascending) == ("dnn_score", True) and parse_config(c).columns_to_read("jets")[-1] == "dnn_score" + c = copy.deepcopy(HH4B); c["objects"]["b"] = "jets pt 4" + r = parse_config(c).roles["b"] + assert (r.selector, r.ascending, r.columns) == ("pt", False, ("pt",)) + assert parse_config(HH4B).roles["other"].columns == ("pt",) and parse_config(HH4B).roles["b"].columns == ("btag", "pt") + for bad, msg in (("jets btag 4 highest", "expected ' [lowest]'"), + ("jets btag 4 Lowest", "expected ' [lowest]'"), + ("jets btag 4 lowest extra", "expected ' [lowest]'"), + ("jets rest 8 lowest", "'rest' takes no 'lowest'"), + ("jets btag", "expected ' [lowest]'"), + ("jets btag four", "expected ' [lowest]'")): + c = copy.deepcopy(HH4B); c["objects"]["b"] = bad; _fails(c, msg) + + def test_validation_rules(): - c = copy.deepcopy(HH4B) - c["combine"] = [] - _fails(c, "combine must list the main hypothesis") - c = copy.deepcopy(HH4B) - c["objects"]["b"] = "jets btag 13" - _fails(c, "within 1..max") - c = copy.deepcopy(HH4B) - c["objects"]["b"] = "jets weird 4" - _fails(c, "selector must be one of") - c = copy.deepcopy(HH4B) - c["objects"]["jets"]["features"] = ["pt", "eta", "phi", "mass"] - _fails(c, "needs a 'btag' feature") - c = copy.deepcopy(HH4B) - c["hypotheses"]["HH"]["context"] = "nothing" - _fails(c, "is not a declared role") - c = copy.deepcopy(HH4B) - c["hypotheses"]["HH"]["H1"] = ["b", "c"] - _fails(c, "neither a role") - c = copy.deepcopy(HH4B) - c["hypotheses"]["ZZ"] = {"Z1": ["b", "b"], "Z2": ["b", "b"]} - _fails(c, "used by no output") - c = copy.deepcopy(HH4B) - c["outputs"] = {"signal": "XX"} - _fails(c, "is not a declared hypothesis") - c = copy.deepcopy(HH4B) - c["outputs"] = {"signal": None} - _fails(c, "main score) needs a hypothesis") - c = copy.deepcopy(HH4B) - c["training"] = {"features": 8, "attention_heads": 3} - _fails(c, "divisible") - c = copy.deepcopy(HH4B) - c["training"] = {"foo": 1} - _fails(c, "unknown keys") - c = copy.deepcopy(BBWW) - c["hypotheses"]["HH"]["H_WW"]["W_lep"] = "lepton + met" - _fails(c, "the '+' form was removed; write [lepton, met]") - c = copy.deepcopy(BBWW) - c["hypotheses"]["HH"]["H_WW"]["W_lep"] = "lepton met" - _fails(c, "cannot interpret") - c = copy.deepcopy(BBWW) - c["hypotheses"]["tt"]["W_had"] = ["q", "b"] - _fails(c, "defined twice with different children") - c = copy.deepcopy(BBWW) - c["attention"] = [{"W_lep": "nowhere"}] - _fails(c, "is not a node") - c = copy.deepcopy(BBWW) - c["objects"]["q"] = "jets rest 11" - _fails(c, "slots > max") - c = copy.deepcopy(HH4B) - del c["combine"] - _fails(c, "missing required section") - c = copy.deepcopy(HH4B) - c["relation"] = [] - _fails(c, "unknown top-level sections") - c = copy.deepcopy(HH4B) - c["trainer"] = {"epochs": 3} - parse_config(c) # read by corenet-train only + c = copy.deepcopy(HH4B); c["combine"] = []; _fails(c, "combine must list the main hypothesis") + c = copy.deepcopy(HH4B); c["objects"]["b"] = "jets btag 13"; _fails(c, "within 1..max") + c = copy.deepcopy(HH4B); c["objects"]["b"] = "jets btag 4 highest"; _fails(c, "expected ' [lowest]'") + c = copy.deepcopy(HH4B); c["objects"]["jets"]["features"] = ["eta", "phi", "mass"]; _fails(c, "need pt, eta, phi features") + c = copy.deepcopy(HH4B); c["hypotheses"]["HH"]["context"] = "nothing"; _fails(c, "is not a declared role") + c = copy.deepcopy(HH4B); c["hypotheses"]["HH"]["H1"] = ["b", "c"]; _fails(c, "neither a role") + c = copy.deepcopy(HH4B); c["hypotheses"]["ZZ"] = {"Z1": ["b", "b"], "Z2": ["b", "b"]}; _fails(c, "used by no output") + c = copy.deepcopy(HH4B); c["outputs"] = {"signal": "XX"}; _fails(c, "is not a declared hypothesis") + c = copy.deepcopy(HH4B); c["outputs"] = {"signal": None}; _fails(c, "main score) needs a hypothesis") + c = copy.deepcopy(HH4B); c["training"] = {"features": 8, "attention_heads": 3}; _fails(c, "divisible") + c = copy.deepcopy(HH4B); c["training"] = {"foo": 1}; _fails(c, "unknown keys") + c = copy.deepcopy(BBWW); c["hypotheses"]["HH"]["H_WW"]["W_lep"] = "lepton + met"; _fails(c, "the '+' form was removed; write [lepton, met]") + c = copy.deepcopy(BBWW); c["hypotheses"]["HH"]["H_WW"]["W_lep"] = "lepton met"; _fails(c, "cannot interpret") + c = copy.deepcopy(BBWW); c["hypotheses"]["tt"]["W_had"] = ["q", "b"]; _fails(c, "defined twice with different children") + c = copy.deepcopy(BBWW); c["attention"] = [{"W_lep": "nowhere"}]; _fails(c, "is not a node") + c = copy.deepcopy(BBWW); c["objects"]["q"] = "jets rest 11"; _fails(c, "slots > max") + c = copy.deepcopy(HH4B); del c["combine"]; _fails(c, "missing required section") + c = copy.deepcopy(HH4B); c["relation"] = []; _fails(c, "unknown top-level sections") + c = copy.deepcopy(HH4B); c["trainer"] = {"epochs": 3}; parse_config(c) # read by corenet-train only def test_relations_and_single_object_children(): @@ -171,11 +168,16 @@ def test_relations_and_single_object_children(): c["hypotheses"]["HH"]["H_WW"]["W_lep"] = ["l", "met"] c["relations"] = [["l", "met"]] s = parse_config(c) - assert ( - s.nodes["W_lep"].children == ["l", "met"] - and s.leaves()["l"].n == 1 - and s.leaves()["met"].n == 1 - ) - c = copy.deepcopy(BBWW) - c["hypotheses"]["HH"]["H_WW"]["W_lep"] = "l + met" - _fails(c, "the '+' form was removed; write [l, met]") + assert s.nodes["W_lep"].children == ["l", "met"] and s.leaves()["l"].n == 1 and s.leaves()["met"].n == 1 + c = copy.deepcopy(BBWW); c["hypotheses"]["HH"]["H_WW"]["W_lep"] = "l + met"; _fails(c, "the '+' form was removed; write [l, met]") + + +def test_min_jets(): + """min_jets (optional, default 4) sets the nSelJets offset; rejected unless an integer >= 2.""" + assert parse_config(copy.deepcopy(HH4B)).min_jets == 4 + c = copy.deepcopy(HH4B); c["min_jets"] = 2 + assert parse_config(c).min_jets == 2 + for bad in (1, 0, "4", True, 4.0): + c = copy.deepcopy(HH4B); c["min_jets"] = bad + with pytest.raises(ConfigError): + parse_config(c) diff --git a/tests/test_h5.py b/tests/test_h5.py index 3850b32..e36f8ad 100644 --- a/tests/test_h5.py +++ b/tests/test_h5.py @@ -43,12 +43,7 @@ def test_bbww_small(): ds = H5Dataset(path, spec) x, n = ds.inputs, len(ds) assert {k: tuple(v.shape) for k, v in x.items()} == { - "b": (n, 6, 2), - "q": (n, 6, 2), - "lepton": (n, 7, 1), - "met": (n, 3, 1), - "event": (n, 2), - } + "b": (n, 5, 2), "q": (n, 5, 2), "lepton": (n, 7, 1), "met": (n, 3, 1), "event": (n, 2)} assert all(v.dtype == torch.float32 for v in x.values()) _check_flags(x) assert (x["lepton"][:, -1] == 1).all() and (x["met"][:, -1] == 1).all(), ( @@ -82,13 +77,9 @@ def test_bbww_small(): :2 ] rest = sorted(set(present) - set(b_rank), key=lambda j: -jet["pt"][i, j])[:2] - assert np.array_equal( - x["b"][i, 0, : len(b_rank)].numpy(), jet["pt"][i, b_rank] - ), i - assert np.array_equal( - x["b"][i, 4, : len(b_rank)].numpy(), jet["btag"][i, b_rank] - ), i - assert np.array_equal(x["q"][i, 0, : len(rest)].numpy(), jet["pt"][i, rest]), i + assert np.array_equal(x["b"][i, 0, :len(b_rank)].numpy(), jet["pt"][i, b_rank]), i # btag-ranked, btag not a channel + assert np.array_equal(x["b"][i, 3, :len(b_rank)].numpy(), jet["mass"][i, b_rank]), i + assert np.array_equal(x["q"][i, 0, :len(rest)].numpy(), jet["pt"][i, rest]), i assert (ds.labels.numpy() == label).all() and set(np.unique(label)) == {1, 2} assert np.array_equal(ds.weights.numpy(), weight) and np.array_equal( ds.folds.numpy(), fold @@ -110,11 +101,7 @@ def test_hh4b_small(): for rest_order in ("pt", "selector"): ds = H5Dataset(path, spec, rest_order=rest_order) x, n = ds.inputs, len(ds) - assert ( - x["b"].shape == (n, 6, 4) - and x["other"].shape == (n, 6, 8) - and x["event"].shape == (n, 2) - ) + assert x["b"].shape == (n, 5, 4) and x["other"].shape == (n, 5, 8) and x["event"].shape == (n, 2) _check_flags(x) for i in range(N_CHECK): present = np.flatnonzero(mask[i]) @@ -137,30 +124,140 @@ def test_hh4b_small(): ) # no EVENT/label: CLASSIFICATIONS/EVENT/signal +def _hh4b_cfg(features): + return {"objects": {"jets": {"features": features, "max": 12}, "b": "jets btag 4", "other": "jets rest 8"}, + "event": ["nSelJets", "HT"], "hypotheses": {"HH": {"H1": ["b", "b"], "H2": ["b", "b"], "context": "other"}}, + "outputs": {"signal": "HH"}, "combine": ["HH"]} + + +def test_selector_column_read_without_feature(): + """(a) `b: jets btag 4` with btag absent from features: the loader reads INPUTS/Source/btag on its own, + the selection is the btag ranking, and the tensor holds the features only.""" + path = _need(DATA / "hh4b_small.h5") + spec = parse_config(_hh4b_cfg(["pt", "eta", "phi", "mass"])) + ds = H5Dataset(path, spec) + x, n = ds.inputs, len(ds) + assert x["b"].shape == (n, 5, 4) and x["other"].shape == (n, 5, 8) + with h5py.File(path, "r") as f: + mask = f["INPUTS/Source/MASK"][...] + jet = {k: f[f"INPUTS/Source/{k}"][...] for k in ("pt", "btag")} + for i in range(N_CHECK): + present = np.flatnonzero(mask[i]) + b = sorted(present, key=lambda j: (-jet["btag"][i, j], -jet["pt"][i, j]))[:4] + assert np.array_equal(x["b"][i, 0, :len(b)].numpy(), jet["pt"][i, b]), i + # the same selection with btag as a pt-only selector would differ: the file is really read for btag + cfg_pt = _hh4b_cfg(["pt", "eta", "phi", "mass"]); cfg_pt["objects"]["b"] = "jets pt 4" + assert not torch.equal(H5Dataset(path, parse_config(cfg_pt)).inputs["b"], x["b"]) + + +def test_btag_feature_is_an_extra_channel(): + """(c) btag listed under features enters the tensor as an ordinary extra channel (and the network as an input).""" + from corenet.models.network import CoReNet + path = _need(DATA / "hh4b_small.h5") + spec4, spec5 = parse_config(_hh4b_cfg(["pt", "eta", "phi", "mass"])), parse_config(_hh4b_cfg(["pt", "eta", "phi", "mass", "btag"])) + x4, x5 = H5Dataset(path, spec4).inputs, H5Dataset(path, spec5).inputs + n = len(x4["event"]) + assert x5["b"].shape == (n, 6, 4) and x5["other"].shape == (n, 6, 8) + assert torch.equal(x5["b"][:, :4], x4["b"][:, :4]) and torch.equal(x5["b"][:, -1], x4["b"][:, -1]) # same jets + assert set(np.unique(x5["b"][:, 4][x5["b"][:, -1] > 0].numpy())) <= {0.0, 1.0} # the btag column + with h5py.File(path, "r") as f: + mask, pt, btag = (f[f"INPUTS/Source/{k}"][...] for k in ("MASK", "pt", "btag")) + for i in range(N_CHECK): + present = np.flatnonzero(mask[i]) + b = sorted(present, key=lambda j: (-btag[i, j], -pt[i, j]))[:4] + assert np.array_equal(x5["b"][i, 4, :len(b)].numpy(), btag[i, b]), i + m4, m5 = CoReNet(spec4, device="cpu"), CoReNet(spec5, device="cpu") + assert m4._extras("b") == [] and m5._extras("b") == ["btag"] and m5._extras("other") == ["btag"] + n4, n5 = (sum(p.numel() for p in m.parameters()) for m in (m4, m5)) + assert n4 == 1508 and n5 > n4, (n4, n5) + batch = {k: v[:256] for k, v in x5.items()} + m5.train(); m5.updateMeanStd(batch); m5.initMeanStd(); m5.eval(); m5.setGhostBatches(0) + with torch.no_grad(): + out = m5(batch) + assert out["logits"].shape == (256, 2) and torch.isfinite(out["logits"]).all() + assert torch.isfinite(out["views"]["HH"]).all() + + +def _brute_force(mask, cols, pt, column, n, ascending, taken=None): + """Per-event python sort: available members by the column (lowest first when ascending), ties by highest pt.""" + out = [] + for i in range(len(mask)): + avail = [j for j in np.flatnonzero(mask[i]) if taken is None or j not in taken[i]] + key = (lambda j: (cols[i, j], -pt[i, j])) if ascending else (lambda j: (-cols[i, j], -pt[i, j])) + out.append(sorted(avail, key=key)[:n]) + return out + + +def test_generic_selectors_synthetic(tmp_path): + """Selection by any column, highest or lowest first, against a brute-force per-event sort.""" + from synthetic import write_synthetic + path = write_synthetic(tmp_path / "s.h5", "hh4b", n=600) + with h5py.File(path, "r") as f: + mask = f["INPUTS/Source/MASK"][...] + jet = {k: f[f"INPUTS/Source/{k}"][...] for k in ("pt", "eta", "phi", "mass", "btag")} + for line, column, n, asc in (("jets mass 3", "mass", 3, False), ("jets mass 2 lowest", "mass", 2, True), + ("jets btag 4", "btag", 4, False), ("jets pt 4 lowest", "pt", 4, True)): + cfg = _hh4b_cfg(["pt", "eta", "phi", "mass"]) + cfg["objects"] = {"jets": {"features": ["pt", "eta", "phi", "mass"], "max": 12}, "b": line, "other": "jets rest 8"} + ds = H5Dataset(path, parse_config(cfg)) + x = ds.inputs + assert x["b"].shape == (600, 5, n) and x["other"].shape == (600, 5, 8) + want = _brute_force(mask, jet[column], jet["pt"], column, n, asc) + for i in range(600): + b = want[i] + assert np.array_equal(x["b"][i, 0, :len(b)].numpy(), jet["pt"][i, b]), (line, i) + assert np.array_equal(x["b"][i, 1, :len(b)].numpy(), jet["eta"][i, b]), (line, i) + assert (x["b"][i, -1, len(b):] == -1).all() + rest = sorted(set(np.flatnonzero(mask[i])) - set(b), key=lambda j: -jet["pt"][i, j])[:8] + assert np.array_equal(x["other"][i, 0, :len(rest)].numpy(), jet["pt"][i, rest]), (line, i) + # rest_order="selector" continues the previous line's order, lowest first included + cfg = _hh4b_cfg(["pt", "eta", "phi", "mass"]) + cfg["objects"] = {"jets": {"features": ["pt", "eta", "phi", "mass"], "max": 12}, "b": "jets mass 2 lowest", "other": "jets rest 8"} + x = H5Dataset(path, parse_config(cfg), rest_order="selector").inputs + for i in range(600): + order = _brute_force(mask, jet["mass"], jet["pt"], "mass", 12, True)[i] + rest = order[2:10] + assert np.array_equal(x["other"][i, 0, :len(rest)].numpy(), jet["pt"][i, rest]), i + + +@pytest.mark.parametrize("name,group", [("hh4b_small.h5", "Source"), ("bbww_small.h5", "Jets")]) +def test_lexicographic_order_matches_old_key(name, group): + """The lexicographic (column, pt) sort gives the same btag selection as the former weighted key + -(btag * 1e6 + pt) on every event of the example files.""" + path = _need(DATA / name) + with h5py.File(path, "r") as f: + mask = f[f"INPUTS/{group}/MASK"][...] + pt, btag = (f[f"INPUTS/{group}/{k}"][...].astype(np.float32) for k in ("pt", "btag")) + old = np.argsort(np.where(mask, -(btag * 1e6 + pt), np.inf), axis=1, kind="stable") + from corenet.data.h5 import _take + new, _ = _take((-btag, -pt), mask, mask.shape[1]) + n_avail = mask.sum(1) + for i in range(len(mask)): + assert np.array_equal(old[i, :n_avail[i]], new[i, :n_avail[i]]), i + + +def test_missing_selector_column(tmp_path): + """The selector column must exist in the file even when it is not a feature.""" + src = _need(DATA / "hh4b_small.h5") + dst = tmp_path / "nobtag.h5" + with h5py.File(src, "r") as f, h5py.File(dst, "w") as g: + for key in ("INPUTS/Source/MASK", "INPUTS/Source/pt", "INPUTS/Source/eta", "INPUTS/Source/phi", "INPUTS/Source/mass", + "CLASSIFICATIONS/EVENT/signal", "EVENT/weight", "EVENT/fold"): + g.create_dataset(key, data=f[key][:500]) + with pytest.raises(KeyError, match=r"missing columns \['btag'\].*selector columns \['btag'\]"): + H5Dataset(dst, parse_config(_hh4b_cfg(["pt", "eta", "phi", "mass"]))) + cfg = _hh4b_cfg(["pt", "eta", "phi", "mass"]); cfg["objects"]["b"] = "jets pt 4" + assert H5Dataset(dst, parse_config(cfg)).inputs["b"].shape == (500, 5, 4) # pt selector: no btag needed + + def test_errors(tmp_path): path = _need(DATA / "bbww_small.h5") - cfg = { - "objects": { - "jets": {"features": ["pt", "eta", "phi", "mass", "btag"], "max": 12}, - "b": "jets btag 2", - "photon": {"features": ["pt", "eta", "phi"]}, - }, - "hypotheses": {"HH": {"H_bb": ["b", "b"], "X": ["b", "photon"]}}, - "outputs": {"signal": "HH"}, - "combine": ["HH"], - } + cfg = {"objects": {"jets": {"features": ["pt", "eta", "phi", "mass"], "max": 12}, "b": "jets btag 2", + "photon": {"features": ["pt", "eta", "phi"]}}, + "hypotheses": {"HH": {"H_bb": ["b", "b"], "X": ["b", "photon"]}}, "outputs": {"signal": "HH"}, "combine": ["HH"]} with pytest.raises(KeyError, match="photon"): H5Dataset(path, parse_config(cfg)) - cfg = { - "objects": { - "jets": {"features": ["pt", "eta", "phi", "mass", "btag"], "max": 12}, - "b": "jets btag 4", - }, - "hypotheses": {"HH": {"H1": ["b", "b"], "H2": ["b", "b"]}}, - "outputs": {"signal": "HH"}, - "combine": ["HH"], - } - with pytest.raises( - ValueError, match="labels" - ): # bbWW labels 1, 2 with a single output + cfg = {"objects": {"jets": {"features": ["pt", "eta", "phi", "mass"], "max": 12}, "b": "jets btag 4"}, + "hypotheses": {"HH": {"H1": ["b", "b"], "H2": ["b", "b"]}}, "outputs": {"signal": "HH"}, "combine": ["HH"]} + with pytest.raises(ValueError, match="labels"): # bbWW labels 1, 2 with a single output H5Dataset(path, parse_config(cfg)) diff --git a/tests/test_network_bbww.py b/tests/test_network_bbww.py index 74d1628..ed1e60e 100644 --- a/tests/test_network_bbww.py +++ b/tests/test_network_bbww.py @@ -30,9 +30,8 @@ def jets(k): eta = 4.8 * torch.rand(n, k, generator=g) - 2.4 phi = math.tau * torch.rand(n, k, generator=g) - math.pi m = 5 + 20 * torch.rand(n, k, generator=g) - btag = torch.rand(n, k, generator=g) flag = torch.ones(n, k) - return torch.stack([pt, eta, phi, m, btag, flag], dim=1) # (n, 5+1, k) + return torch.stack([pt, eta, phi, m, flag], dim=1) # (n, 4+1, k): the features + flag b, q = jets(2), jets(2) if pad_q: # second q jet missing @@ -200,20 +199,10 @@ def test_padded_leaves_stay_finite(): def test_relation_needs_unique_candidates(): cfg = { - "objects": { - "jets": {"features": ["pt", "eta", "phi", "mass", "btag"], "max": 12}, - "b": "jets btag 2", - "q": "jets rest 2", - "lepton": {"features": ["pt", "eta", "phi", "mass"]}, - "met": {"features": ["pt", "phi"]}, - }, - "hypotheses": { - "HH": { - "H_bb": ["b", "b"], - "H_WW": {"W_had": ["q", "q"], "W_lep": ["lepton", "met"]}, - }, - "tt": {"t_had": ["b", "W_had"], "t_lep": ["b", "W_lep"]}, - }, + "objects": {"jets": {"features": ["pt", "eta", "phi", "mass"], "max": 12}, "b": "jets btag 2", "q": "jets rest 2", + "lepton": {"features": ["pt", "eta", "phi", "mass"]}, "met": {"features": ["pt", "phi"]}}, + "hypotheses": {"HH": {"H_bb": ["b", "b"], "H_WW": {"W_had": ["q", "q"], "W_lep": ["lepton", "met"]}}, + "tt": {"t_had": ["b", "W_had"], "t_lep": ["b", "W_lep"]}}, "relations": [["t_had", "met"]], "outputs": {"signal": "HH", "ttbar": "tt"}, "combine": ["HH", "tt"], @@ -300,18 +289,10 @@ def test_self_attention_masks_diagonal_only(): def test_edge_with_many_keys_is_attention(): """An edge whose queries have several admissible keys stays real attention: HH -> 4b dijets attending to the other dijets (6 candidates, diagonal masked -> 5 keys each).""" - cfg = { - "objects": { - "jets": {"features": ["pt", "eta", "phi", "mass", "btag"], "max": 12}, - "b": "jets btag 4", - "other": "jets rest 8", - }, - "event": ["nSelJets", "HT"], - "hypotheses": {"HH": {"H1": ["b", "b"], "H2": ["b", "b"], "context": "other"}}, - "attention": [{"H1": "H1"}], - "outputs": {"signal": "HH"}, - "combine": ["HH"], - } + # btag listed as a feature here: it is one more input channel (the 5th of the 6 stacked below) + cfg = {"objects": {"jets": {"features": ["pt", "eta", "phi", "mass", "btag"], "max": 12}, "b": "jets btag 4", "other": "jets rest 8"}, + "event": ["nSelJets", "HT"], "hypotheses": {"HH": {"H1": ["b", "b"], "H2": ["b", "b"], "context": "other"}}, + "attention": [{"H1": "H1"}], "outputs": {"signal": "HH"}, "combine": ["HH"]} m = CoReNet(parse_config(cfg), "cpu") assert "H1>H1" in m.attention and not m.merge same = m._same("H1>H1")[0] diff --git a/tests/test_train.py b/tests/test_train.py index e7bec49..b9a25f1 100644 --- a/tests/test_train.py +++ b/tests/test_train.py @@ -248,3 +248,33 @@ def test_train_and_evaluate() -> None: with tempfile.TemporaryDirectory(prefix="corenet_train_") as tmp: run_one(Path(tmp), "hh4b", HH4B, 1508, ["rest", "signal"]) run_one(Path(tmp), "bbww", BBWW, 3832, ["rest", "signal", "ttbar"]) + + +def test_early_stopping() -> None: + """--early-stop-patience: stop after N epochs without a better validation AUC and keep the best epoch.""" + import torch + with tempfile.TemporaryDirectory(prefix="corenet_es_") as tmp: + tmp = Path(tmp) + data = write_synthetic(tmp / "hh4b_all.h5", "hh4b") + out_dir = tmp / "m" + train.main(["--config", str(HH4B), "--input", str(data), "--output-dir", str(out_dir), "--fold", "0", + "--exclude-folds", "1", "--epochs", "12", "--batch-size-init", "128", "--early-stop-patience", "1", + "--device", "cpu"]) + h = json.loads((out_dir / "history_fold0.json").read_text()) + aucs = [e["val_auc"] for e in h["history"]] + best = max(range(len(aucs)), key=lambda i: aucs[i]) + assert h["best_epoch"] == best and h["early_stop_patience"] == 1 + if h["stopped_after_epoch"] is not None: # stopped: one non-improving epoch after the best + assert h["stopped_after_epoch"] == best + 1 and len(aucs) == best + 2 + else: + assert len(aucs) == 12 + ck = torch.load(out_dir / "model_fold0.pkl", map_location="cpu", weights_only=False) + assert ck["metadata"]["epoch"] == best and ck["metadata"]["early_stop_patience"] == 1 + # without a validation fold the option is refused + try: + train.main(["--config", str(HH4B), "--input", str(data), "--output-dir", str(tmp / "n"), "--fold", "0", + "--epochs", "2", "--early-stop-patience", "1", "--device", "cpu"]) + except SystemExit: + pass + else: + raise AssertionError("expected an error without --exclude-folds") diff --git a/tests/test_validate.py b/tests/test_validate.py index 0672838..851f049 100644 --- a/tests/test_validate.py +++ b/tests/test_validate.py @@ -39,19 +39,12 @@ def test_valid_synthetic(tmp_path, topology): in msgs ) assert "loader + forward pass" in msgs - assert ( - validate.main( - [ - "--config", - str(cfg), - "--input", - str(data), - "--json", - str(tmp_path / "r.json"), - ] - ) - == 0 - ) + assert "selected by `btag` highest first (reads btag, pt for the selection)" in msgs + assert "selected by `rest` (reads pt for the selection)" in msgs + assert "['btag'] read for the selection only (not network inputs)" in msgs + assert "features ['pt', 'eta', 'phi', 'mass'] (the network inputs)" in msgs + assert ("leaf `b`: tensor (400, 5, 4)" if topology == "hh4b" else "leaf `b`: tensor (400, 5, 2)") in msgs + assert validate.main(["--config", str(cfg), "--input", str(data), "--json", str(tmp_path / "r.json")]) == 0 @pytest.mark.parametrize( @@ -88,46 +81,18 @@ def _bbww(): ), (lambda c: c["objects"]["jets"].__setitem__("max", "12"), "max must be an integer"), (lambda c: c["objects"].__setitem__("b", "jet btag 2"), "unknown collection 'jet'"), - ( - lambda c: c["hypotheses"]["tt"].__setitem__("t_had", ["bb", "W_had"]), - "child 'bb' is neither", - ), - ( - lambda c: c["hypotheses"]["HH"]["H_WW"].__setitem__("W_lep", "lepton + met"), - "the '+' form was removed; write [lepton, met]", - ), - ( - lambda c: c.__setitem__("attention", [{"W_lep": "W_hadronic"}]), - "'W_hadronic' is not a node", - ), - ( - lambda c: c.__setitem__("relations", [["lepton", "neutrino"]]), - "'neutrino' is neither a node nor a leaf", - ), - ( - lambda c: c.__setitem__("relations", [["t_had", "met"]]), - "t_had has 2 candidates", - ), - ( - lambda c: c.__setitem__("combine", ["tt"]), - "combine must list the main hypothesis 'HH'", - ), - ( - lambda c: c.__setitem__("combine", ["HH", "HH"]), - "combine: duplicate entries ['HH']", - ), - ( - lambda c: c["outputs"].__setitem__("ttbar", "ttbar"), - "outputs.ttbar: 'ttbar' is not a declared hypothesis", - ), - ( - lambda c: c["training"].__setitem__("features", "8"), - "training.features must be an integer", - ), - ( - lambda c: c["training"].__setitem__("features", 7), - "must be divisible by training.attention_heads", - ), + (lambda c: c["objects"].__setitem__("b", "jets btag 2 top"), "expected ' [lowest]', got 'jets btag 2 top'"), + (lambda c: c["objects"].__setitem__("q", "jets rest 2 lowest"), "'rest' takes no 'lowest'"), + (lambda c: c["hypotheses"]["tt"].__setitem__("t_had", ["bb", "W_had"]), "child 'bb' is neither"), + (lambda c: c["hypotheses"]["HH"]["H_WW"].__setitem__("W_lep", "lepton + met"), "the '+' form was removed; write [lepton, met]"), + (lambda c: c.__setitem__("attention", [{"W_lep": "W_hadronic"}]), "'W_hadronic' is not a node"), + (lambda c: c.__setitem__("relations", [["lepton", "neutrino"]]), "'neutrino' is neither a node nor a leaf"), + (lambda c: c.__setitem__("relations", [["t_had", "met"]]), "t_had has 2 candidates"), + (lambda c: c.__setitem__("combine", ["tt"]), "combine must list the main hypothesis 'HH'"), + (lambda c: c.__setitem__("combine", ["HH", "HH"]), "combine: duplicate entries ['HH']"), + (lambda c: c["outputs"].__setitem__("ttbar", "ttbar"), "outputs.ttbar: 'ttbar' is not a declared hypothesis"), + (lambda c: c["training"].__setitem__("features", "8"), "training.features must be an integer"), + (lambda c: c["training"].__setitem__("features", 7), "must be divisible by training.attention_heads"), (lambda c: c["training"].update(features=9, attention_heads=3), "must be even"), ( lambda c: c["training"].__setitem__("phase_symetric", True), @@ -163,6 +128,34 @@ def test_broken_config(tmp_path, edit, message): assert r.errors and message in _errors(r), _errors(r) +def test_missing_selector_column(tmp_path): + """A column named in a candidate line must exist in the file (FAIL), whether or not it is a feature.""" + data = write_synthetic(tmp_path / "d.h5", "bbww", n=400) + cfg = _bbww() + cfg["objects"]["b"] = "jets dnn_score 2 lowest" + r = validate.validate(_write(tmp_path, cfg), data) + msgs = "\n".join(m for _, _, m in r.items) + assert "selected by `dnn_score` lowest first (reads dnn_score, pt for the selection)" in msgs + assert "expects dataset INPUTS/Jets/dnn_score (read for the selection by `b`, not a feature): not found" in _errors(r) + cfg["objects"]["b"] = "jets mass 2 lowest" # a feature column: no selection-only line + r = validate.validate(_write(tmp_path, cfg), data) + assert not r.errors, r.errors + assert "read for the selection only" not in "\n".join(m for _, _, m in r.items) + + +def test_btag_feature_is_an_input(tmp_path): + """btag listed under features: accepted, reported as a network input, one more channel.""" + cfg = _bbww() + cfg["objects"]["jets"]["features"] = ["pt", "eta", "phi", "mass", "btag"] + data = write_synthetic(tmp_path / "d.h5", "bbww", n=400) + r = validate.validate(_write(tmp_path, cfg), data) + assert not r.errors, r.errors + msgs = "\n".join(m for _, _, m in r.items) + assert "features ['pt', 'eta', 'phi', 'mass', 'btag'] (the network inputs)" in msgs + assert "read for the selection only" not in msgs + assert "leaf `b`: tensor (400, 6, 2)" in msgs and "network built: 3,832 parameters" not in msgs + + def test_yaml_syntax_error(tmp_path): p = tmp_path / "bad.yaml" p.write_text("objects: {jets: [\n") @@ -184,62 +177,21 @@ def _replace(f, name, data): BROKEN_FILES = [ - ( - lambda f: f.move("INPUTS/Lepton", "INPUTS/Electron"), - "collection `lepton` (used by `lepton`) expects a group INPUTS/lepton or INPUTS/Lepton or INPUTS/Leptons: not found", - ), - ( - lambda f: f.__delitem__("INPUTS/Lepton/isE"), - "expects dataset INPUTS/Lepton/isE: not found", - ), - ( - lambda f: _replace(f, "INPUTS/Jets/pt", f["INPUTS/Jets/pt"][:, :10]), - "INPUTS/Jets/pt: 10 slots, the configuration asks for max 12", - ), - ( - lambda f: _replace(f, "INPUTS/Met/pt", f["INPUTS/Met/pt"][:-5]), - "INPUTS/Met/pt has 395 entries, expected 400", - ), - ( - lambda f: _replace( - f, "INPUTS/Lepton/pt", np.stack([f["INPUTS/Lepton/pt"][...]] * 2, 1) - ), - "expected (events,) for a single-object collection", - ), - ( - lambda f: _replace(f, "INPUTS/Jets/btag", np.array(["x"] * 400, dtype="S1")), - "INPUTS/Jets/btag: dtype |S1 is not numeric", - ), - ( - lambda f: _replace(f, "INPUTS/Jets/MASK", f["INPUTS/Jets/MASK"][:, :11]), - "INPUTS/Jets/MASK: shape (400, 11), expected (400, 12)", - ), - ( - lambda f: _replace(f, "EVENT/label", np.full(400, 3)), - "EVENT/label: labels [3] outside 0..2", - ), - ( - lambda f: _replace(f, "EVENT/label", np.full(400, 1.0)), - "EVENT/label: shape (400,) dtype float64, expected integer", - ), - ( - lambda f: f.__delitem__("EVENT/label"), - "no labels: expected EVENT/label or CLASSIFICATIONS/EVENT/signal", - ), - ( - lambda f: f.__delitem__("EVENT/fold"), - "EVENT/fold not found: corenet-train needs a fold index", - ), - ( - lambda f: _replace(f, "EVENT/weight", np.full(400, np.nan, dtype=np.float32)), - "EVENT/weight: 400 non-finite weights", - ), - ( - lambda f: _replace( - f, "INPUTS/Lepton/pt", np.full(400, np.inf, dtype=np.float32) - ), - "leaf `lepton`: non-finite features in 400 present objects", - ), + (lambda f: f.move("INPUTS/Lepton", "INPUTS/Electron"), "collection `lepton` (used by `lepton`) expects a group INPUTS/lepton or INPUTS/Lepton or INPUTS/Leptons: not found"), + (lambda f: f.__delitem__("INPUTS/Lepton/isE"), "expects dataset INPUTS/Lepton/isE: not found"), + (lambda f: _replace(f, "INPUTS/Jets/pt", f["INPUTS/Jets/pt"][:, :10]), "INPUTS/Jets/pt: 10 slots, the configuration asks for max 12"), + (lambda f: _replace(f, "INPUTS/Met/pt", f["INPUTS/Met/pt"][:-5]), "INPUTS/Met/pt has 395 entries, expected 400"), + (lambda f: _replace(f, "INPUTS/Lepton/pt", np.stack([f["INPUTS/Lepton/pt"][...]] * 2, 1)), "expected (events,) for a single-object collection"), + (lambda f: _replace(f, "INPUTS/Jets/btag", np.array(["x"] * 400, dtype="S1")), "INPUTS/Jets/btag: dtype |S1 is not numeric"), + (lambda f: f.__delitem__("INPUTS/Jets/btag"), "expects dataset INPUTS/Jets/btag (read for the selection by `b`, not a feature): not found"), + (lambda f: _replace(f, "INPUTS/Jets/btag", f["INPUTS/Jets/btag"][:, :10]), "INPUTS/Jets/btag: 10 slots, the configuration asks for max 12"), + (lambda f: _replace(f, "INPUTS/Jets/MASK", f["INPUTS/Jets/MASK"][:, :11]), "INPUTS/Jets/MASK: shape (400, 11), expected (400, 12)"), + (lambda f: _replace(f, "EVENT/label", np.full(400, 3)), "EVENT/label: labels [3] outside 0..2"), + (lambda f: _replace(f, "EVENT/label", np.full(400, 1.0)), "EVENT/label: shape (400,) dtype float64, expected integer"), + (lambda f: f.__delitem__("EVENT/label"), "no labels: expected EVENT/label or CLASSIFICATIONS/EVENT/signal"), + (lambda f: f.__delitem__("EVENT/fold"), "EVENT/fold not found: corenet-train needs a fold index"), + (lambda f: _replace(f, "EVENT/weight", np.full(400, np.nan, dtype=np.float32)), "EVENT/weight: 400 non-finite weights"), + (lambda f: _replace(f, "INPUTS/Lepton/pt", np.full(400, np.inf, dtype=np.float32)), "leaf `lepton`: non-finite features in 400 present objects"), ] From fcfcd93ebac86e017615344219f2f87bbef7c007 Mon Sep 17 00:00:00 2001 From: Aniket Khanal Date: Wed, 21 Oct 2026 17:47:02 -0400 Subject: [PATCH 18/33] add fold as configurable parameter --- README.md | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 78731c5..b70be15 100644 --- a/README.md +++ b/README.md @@ -126,13 +126,21 @@ corenet-train --config examples/bbww/bbww.yaml --input examples/data/bbww_all.h5 ``` * `--fold 0`: hold out fold 0 for testing. With `--fold -1` (the default), train one model per fold in turn. -* `--exclude-folds 1`: keep fold 1 out of the training and use it for validation. +* `--exclude-folds 1`: keep fold 1 out of the training and use it for validation. `--exclude-folds next` + uses the fold `(k + 1) % n_folds` of the held-out fold `k`. +* `--n-folds 5`: take the fold of an event as `event % 5` from `EVENT/event` instead of the `EVENT/fold` + column ([docs/Dataset.md](docs/Dataset.md)); the rule is stored in the checkpoint and reused by + `corenet-evaluate`. * `--early-stop-patience 5`: stop when the validation AUC has not improved for 5 epochs and keep the best epoch. +* `--freeze-epoch 15 --finetune-layers head_main. head_reduce. aux_heads. --finetune-lr 1.0e-3 --finetune-batch-size 4096`: + the fine-tuning stage; from epoch 15 on only the parameters whose names start with the given prefixes + (default: the output layers; `none` freezes nothing) are trained, at this learning rate and batch size + (default: the schedule continues). * `--epochs`, `--lr`, `--seed`, ...: override the training settings of the configuration file. See [docs/Options.md](docs/Options.md) or `corenet-train --help`. -The number of folds is not a training option: it is set when the input file is written, as the set of -values in `EVENT/fold` ([docs/Dataset.md](docs/Dataset.md)). The example files have 5 folds. +The folds come from the input file: the `EVENT/fold` column, or `event % K` with `--n-folds K` +([docs/Dataset.md](docs/Dataset.md)). The example files have 5 folds in `EVENT/fold`. Per fold `k`, the output directory receives: From c260de025fabc02e355a91ae00bc591f7aba0fef Mon Sep 17 00:00:00 2001 From: Aniket Khanal Date: Wed, 21 Oct 2026 17:47:13 -0400 Subject: [PATCH 19/33] update dataset documentation --- docs/Dataset.md | 24 +++++++++++++++++------- docs/HH4b.md | 6 +++--- docs/Options.md | 13 +++++++++---- 3 files changed, 29 insertions(+), 14 deletions(-) diff --git a/docs/Dataset.md b/docs/Dataset.md index ee78c2f..400a9ef 100644 --- a/docs/Dataset.md +++ b/docs/Dataset.md @@ -29,8 +29,8 @@ event description. Next to each key is the expected shape and data type. - EVENT ---- label : [NUM_EVENTS, ] : int (or CLASSIFICATIONS/EVENT/signal) ---- weight : [NUM_EVENTS, ] : float (optional) ----- fold : [NUM_EVENTS, ] : int (needed for training) ----- event : [NUM_EVENTS, ] : int (optional event identifier) +---- fold : [NUM_EVENTS, ] : int (optional: cross-validation fold; else --n-folds) +---- event : [NUM_EVENTS, ] : int (event number; needed with --n-folds) ---- other_scalar : [NUM_EVENTS, ] : float (any quantity named under `event:`) ``` @@ -67,9 +67,16 @@ A collection named `x` is read from `INPUTS/x`, matched case-insensitively, or f of the event description in order (`1` = signal in both examples). * **Weights.** `EVENT/weight` is the event weight (1 when absent). Each class is normalised during training, so only the relative weights within a class matter. Negative weights count as 0. -* **Folds.** `EVENT/fold` is the cross-validation fold of the event. `corenet-train --fold k` holds - out fold `k` for testing and `--exclude-folds j` uses fold `j` for validation. -* **Event identifier.** `EVENT/event` is copied to the prediction files. +* **Folds.** The cross-validation fold of an event is `EVENT/fold` when the file has that column. + Without it, pass `--n-folds K` to `corenet-train` (and to `corenet-validate`): the fold is then + `event % K`, computed from the event numbers in `EVENT/event` (integers ≥ 0), and no random split + is ever made. With `--n-folds K` an existing `EVENT/fold` column is ignored. The rule used + (`EVENT/fold` or `event % n_folds`) and the number of folds are stored in the checkpoint, so + `corenet-evaluate` applies the same rule to any file. `corenet-train --fold k` holds out fold `k` + for testing and `--exclude-folds j` uses fold `j` for validation (`--exclude-folds next`: fold + `(k + 1) % n_folds`). +* **Event identifier.** `EVENT/event` is copied to the prediction files and is the input of the + `event % n_folds` rule. ## Example @@ -84,8 +91,11 @@ follow this layout: `EVENT/event`. They are random subsamples of the full files `hh4b_all.h5` and `bbww_all.h5`, written with -`python scripts/make_small_h5.py --events 20000`. Check any file against an event -description with +`python scripts/make_small_h5.py --events 20000`. The `EVENT/fold` column of the +example files was assigned from a hash of the event number when the files were produced and is not +`event % 5`: to reproduce the folds of the paper, train and evaluate on these files without +`--n-folds`. `corenet-validate --n-folds 5` reports how many events the two rules assign differently. +Check any file against an event description with ```bash corenet-validate --config examples/bbww/bbww.yaml --input examples/data/bbww_small.h5 diff --git a/docs/HH4b.md b/docs/HH4b.md index 5656da9..01d5b84 100644 --- a/docs/HH4b.md +++ b/docs/HH4b.md @@ -122,12 +122,12 @@ In `--output-dir`: | `predictions_fold0.h5` | predictions on the held-out fold 0, in the layout of the evaluation below (without the ROC curve) | **All five folds.** `--fold -1` (the default) trains one model per fold in turn. To validate each -fold on the next one, (k + 1) mod 5, run the folds one by one: +fold on the next one, (k + 1) mod 5, run the folds one by one with `--exclude-folds next`: ```bash for k in 0 1 2 3 4; do corenet-train --config examples/hh4b/hh4b.yaml --input examples/data/hh4b_all.h5 \ - --fold $k --exclude-folds $(( (k + 1) % 5 )) --device cuda --output-dir runs/hh4b_full + --fold $k --exclude-folds next --device cuda --output-dir runs/hh4b_full done ``` @@ -184,7 +184,7 @@ of the b jets has to be reproduced wherever the model is deployed; see Exporting | epochs, learning rate, decay | `--epochs`, `--lr`, `--lr-milestones`, `--lr-gamma`, `--freeze-epoch`, or the `trainer:` section | `--epochs 30 --lr-milestones 25 26 27 28 29 --freeze-epoch 25` | | batch size | `--batch-size-init` or `trainer: batch_size_init` | | | seed | `--seed` or `trainer: seed` | fold k uses seed + k | -| validation / test folds | `--fold`, `--exclude-folds` | Training above | +| validation / test folds | `--fold`, `--exclude-folds` (`next` = the fold after the held-out one), `--n-folds` (fold = `event % K` instead of `EVENT/fold`) | Training above | A change in `objects` or `training:` gives a different network: validate the new file with `corenet-validate` and retrain ([Options.md](./Options.md)). diff --git a/docs/Options.md b/docs/Options.md index 4f1e0fc..051f243 100644 --- a/docs/Options.md +++ b/docs/Options.md @@ -31,7 +31,10 @@ Changing one of these gives a different network: validate the file with `corenet | `lr_milestones` | `--lr-milestones` | 15, 16, ..., 24 | epochs after which the learning rate is multiplied by `lr_gamma` | | `lr_gamma` | `--lr-gamma` | 0.25 | learning-rate decay factor | | `batch_size_init` | `--batch-size-init` | 1024 | batch size of the first epoch; it is doubled at epochs 1, 3, 6 and 10 | -| `freeze_epoch` | `--freeze-epoch` | 15 | from this epoch on only the output layers are trained (a value ≥ `epochs` disables this) | +| `freeze_epoch` | `--freeze-epoch` | 15 | start of the fine-tuning stage: from this epoch on only the layers named by `finetune_layers` are trained (a value ≥ `epochs` disables this) | +| `finetune_layers` | `--finetune-layers PREFIX [PREFIX ...]` | `head_main.`, `head_reduce.`, `aux_heads.` (the output layers) | parameter-name prefixes that stay trainable from `freeze_epoch` on; `none` (the word, alone; `[]` in the file) freezes nothing | +| `finetune_lr` | `--finetune-lr` | unset | learning rate set at `freeze_epoch`; the `lr_milestones` after it multiply this value by `lr_gamma`; unset keeps the schedule | +| `finetune_batch_size` | `--finetune-batch-size` | unset | batch size from `freeze_epoch` on (the ghost batches of the stage are kept, capped so that none has fewer than 8 events); unset keeps the schedule | | `class_norm` | `--class-norm` / `--no-class-norm` | true | give every class the same total weight in training | | `seed` | `--seed` | 42 | random seed; fold `k` uses `seed + k` | | `early_stop_patience` | `--early-stop-patience` | 0 | stop when the validation AUC (on the `--exclude-folds` fold) has not improved for this many epochs, and keep the weights of the best epoch; 0 trains every epoch | @@ -41,8 +44,10 @@ Write floating-point numbers with a decimal point (`0.005` or `5.0e-3`): YAML re which the validation rejects. Other flags of `corenet-train`: `--config`, `--input`, `--output-dir`, `--fold` (held-out fold, `-1` -for every fold in turn), `--exclude-folds` (validation folds), `--save-every-epoch` (also save -`model_fold{k}_epoch{e}.pkl` after every epoch) and `--device`. +for every fold in turn), `--exclude-folds` (validation folds; `next` = the fold `(k + 1) % n_folds` of +the held-out fold `k`), `--n-folds K` (fold = `event % K` from `EVENT/event` instead of the `EVENT/fold` +column, [Dataset.md](./Dataset.md)), `--save-every-epoch` (also save `model_fold{k}_epoch{e}.pkl` after +every epoch) and `--device`. ## Fixed in code @@ -50,7 +55,7 @@ These choices are part of the training procedure and the network and have no set * the optimizer is Adam, and the learning rate is multiplied by `lr_gamma` after each epoch listed in `lr_milestones`; * the batch size doubles at epochs 1, 3, 6 and 10, and the batches are split into 64, 16, 4, 1 and 0 ghost batches over those stages; -* from `freeze_epoch` on, only the output layers keep training; +* from `freeze_epoch` on, only the layers named by `finetune_layers` (by default the output layers) keep training; * the input file needs no standardization of its values; * the event-level quantities `nSelJets` and `HT` are computed from the jets in the file; `nSelJets` enters as `log(nSelJets - (min_jets - 1))` ([EventInfo.md](./EventInfo.md)). From 5c327ed538492f4c78acc918ccb306809aea9737 Mon Sep 17 00:00:00 2001 From: Aniket Khanal Date: Wed, 21 Oct 2026 17:47:40 -0400 Subject: [PATCH 20/33] make training recipe configurable --- src/corenet/checkpoint.py | 40 ++++++++++++++++++++++++++++++--------- 1 file changed, 31 insertions(+), 9 deletions(-) diff --git a/src/corenet/checkpoint.py b/src/corenet/checkpoint.py index a30f68b..a0df2a0 100644 --- a/src/corenet/checkpoint.py +++ b/src/corenet/checkpoint.py @@ -4,7 +4,8 @@ "config": the configuration mapping the network was built from (so a checkpoint is self-contained), "config_path": path of the configuration file at training time, "classes": ["rest", ...], - "metadata": {"fold", "epoch", "exclude_folds", "input", }, + "metadata": {"fold", "epoch", "exclude_folds", "input", "n_folds", "fold_rule" ("EVENT/fold" or + "event % n_folds"), , "finetune_trainable_parameters"}, "format": CHECKPOINT_FORMAT} ``format`` 2: every feature of a collection is a network input. Checkpoints without ``format`` were @@ -21,11 +22,20 @@ import torch +from .data.h5 import FOLD_RULE_EVENT from .models.config import NetworkSpec, parse_config, read_config_dict from .models.network import CoReNet CHECKPOINT_FORMAT = 2 -LEGACY_SKIPPED = ("btag",) # features the network ignored before format 2 +LEGACY_SKIPPED = ("btag",) # features the network ignored before format 2 + + +def n_folds_of(metadata: dict) -> int | None: + """The ``n_folds`` to pass to ``H5Dataset`` for a checkpoint: its ``n_folds`` when it was trained with + fold = event % n_folds, else None (the ``EVENT/fold`` column).""" + if metadata.get("fold_rule") == FOLD_RULE_EVENT: + return metadata.get("n_folds") + return None def save_checkpoint( @@ -76,8 +86,10 @@ def load_checkpoint( if "format" not in ckpt: cfg, edited = legacy_config(cfg) if edited: - logging.info(f"{path}: checkpoint written before format {CHECKPOINT_FORMAT}; " - f"{list(LEGACY_SKIPPED)} dropped from the features of {edited} (the network never used them)") + logging.info( + f"{path}: checkpoint written before format {CHECKPOINT_FORMAT}; " + f"{list(LEGACY_SKIPPED)} dropped from the features of {edited} (the network never used them)" + ) else: raise ValueError( f"{path}: the checkpoint stores no configuration; pass the configuration file explicitly" @@ -89,13 +101,23 @@ def load_checkpoint( except RuntimeError as e: hint = "" if "format" not in ckpt and config is not None: - hint = (f"; the checkpoint predates format {CHECKPOINT_FORMAT}, in which every feature is a network input: " - f"if the network was trained with {list(LEGACY_SKIPPED)} listed under features, remove them from " - f"the features of {config} (or load without --config, which does so automatically)") - raise RuntimeError(f"{path}: the state dict does not fit the network built from the configuration{hint}\n{e}") from e + hint = ( + f"; the checkpoint predates format {CHECKPOINT_FORMAT}, in which every feature is a network input: " + f"if the network was trained with {list(LEGACY_SKIPPED)} listed under features, remove them from " + f"the features of {config} (or load without --config, which does so automatically)" + ) + raise RuntimeError( + f"{path}: the state dict does not fit the network built from the configuration{hint}\n{e}" + ) from e model.eval() model.setGhostBatches(0) return model, spec, ckpt -__all__ = ["save_checkpoint", "load_checkpoint", "legacy_config", "CHECKPOINT_FORMAT"] +__all__ = [ + "CHECKPOINT_FORMAT", + "legacy_config", + "load_checkpoint", + "n_folds_of", + "save_checkpoint", +] From e12d9b3436eb5ac3ee6bd91477f9da3fd8d98c40 Mon Sep 17 00:00:00 2001 From: Aniket Khanal Date: Wed, 21 Oct 2026 17:48:17 -0400 Subject: [PATCH 21/33] update training recipe rules --- src/corenet/data/h5.py | 138 ++++++++++++++++++++++++++++++++++------- 1 file changed, 117 insertions(+), 21 deletions(-) diff --git a/src/corenet/data/h5.py b/src/corenet/data/h5.py index 6997c92..4b2845e 100644 --- a/src/corenet/data/h5.py +++ b/src/corenet/data/h5.py @@ -11,7 +11,9 @@ inputs["event"] (N, len(spec.event)) event scalars in the configured order plus ``labels`` (``EVENT/label``, or ``CLASSIFICATIONS/EVENT/signal`` when absent), ``weights`` -(``EVENT/weight``), ``folds`` (``EVENT/fold``) and ``events`` (``EVENT/event``). No generator-level +(``EVENT/weight``), ``folds`` and ``events`` (``EVENT/event``). The cross-validation fold of an event +is ``EVENT/fold`` by default; with ``n_folds`` it is ``fold_of(event, n_folds)`` = ``event % n_folds`` +computed from the event numbers, and the file needs no ``EVENT/fold`` column. No generator-level information is read: the roles are chosen from reconstructed quantities only. A collection's ``features`` are exactly the channels of its tensors. A selector reads the column it @@ -37,6 +39,7 @@ from __future__ import annotations +import logging from pathlib import Path import h5py @@ -54,8 +57,33 @@ "met": ("Met", "MET"), } COMPUTED_EVENT = ("nSelJets", "HT") -LABEL_KEYS = ("EVENT/label", "CLASSIFICATIONS/EVENT/signal") # first one present is used +LABEL_KEYS = ( + "EVENT/label", + "CLASSIFICATIONS/EVENT/signal", +) # first one present is used WEIGHT_KEY, FOLD_KEY, EVENT_KEY = "EVENT/weight", "EVENT/fold", "EVENT/event" +FOLD_RULE_EVENT, FOLD_RULE_COLUMN = ( + "event % n_folds", + "EVENT/fold", +) # H5Dataset.fold_rule + + +def fold_of(event: np.ndarray, n_folds: int) -> np.ndarray: + """The fold rule: fold = event % n_folds on the (non-negative, int64) event numbers.""" + if not isinstance(n_folds, int) or isinstance(n_folds, bool) or n_folds < 1: + raise ValueError(f"n_folds must be an integer >= 1, got {n_folds!r}") + event = np.asarray(event) + if event.dtype.kind not in "iu": + raise ValueError( + f"event numbers must be integers to compute fold = event % n_folds, got dtype {event.dtype}" + ) + event = event.astype(np.int64) + if (event < 0).any(): + raise ValueError( + f"{int((event < 0).sum()):,} event numbers are negative (or exceed int64): " + "fold = event % n_folds needs event numbers >= 0" + ) + return event % n_folds # ------------------------------------------------------------------ what the loader reads (shared with corenet-validate) @@ -99,14 +127,18 @@ def resolve_group(f: h5py.File, name: str) -> str: ) -def _read_collection(f: h5py.File, coll: Collection, columns: list[str]) -> tuple[dict[str, np.ndarray], np.ndarray]: +def _read_collection( + f: h5py.File, coll: Collection, columns: list[str] +) -> tuple[dict[str, np.ndarray], np.ndarray]: """(columns, present) of one collection, both shaped (N, slots in the file); ``columns`` are the features plus the selector columns (``columns_to_read``).""" group = f["INPUTS"][resolve_group(f, coll.name)] missing = [k for k in columns if k not in group] if missing: - raise KeyError(f"collection {coll.name!r} (INPUTS/{group.name.split('/')[-1]}): missing columns {missing} " - f"(features {coll.features}, selector columns {[c for c in columns if c not in coll.features]})") + raise KeyError( + f"collection {coll.name!r} (INPUTS/{group.name.split('/')[-1]}): missing columns {missing} " + f"(features {coll.features}, selector columns {[c for c in columns if c not in coll.features]})" + ) feats = {k: np.atleast_2d(group[k][...].astype(np.float32).T).T for k in columns} n, slots = feats[columns[0]].shape if coll.max > 1 and slots < coll.max: @@ -121,34 +153,54 @@ def _read_collection(f: h5py.File, coll: Collection, columns: list[str]) -> tupl return feats, present -def _selector_keys(role: RoleSpec, feats: dict[str, np.ndarray]) -> tuple[np.ndarray, ...]: +def _selector_keys( + role: RoleSpec, feats: dict[str, np.ndarray] +) -> tuple[np.ndarray, ...]: """Sort keys (smaller = preferred) of an explicit selector: the column, negated unless ``lowest``, then -pt as the tie breaker. A lexicographic sort, so the column's scale does not matter.""" col = feats[role.selector] return (col if role.ascending else -col, -feats["pt"]) -def _take(keys: tuple[np.ndarray, ...], avail: np.ndarray, n: int) -> tuple[np.ndarray, np.ndarray]: +def _take( + keys: tuple[np.ndarray, ...], avail: np.ndarray, n: int +) -> tuple[np.ndarray, np.ndarray]: """(indices, valid) of the n preferred available members: (N, n) each. Available members first, then by ``keys`` in order (the first key dominates), remaining ties in file order (stable).""" - order = np.lexsort((*reversed(keys), ~avail), axis=1)[:, :n] # lexsort: the last key is the primary one + order = np.lexsort((*reversed(keys), ~avail), axis=1)[ + :, :n + ] # lexsort: the last key is the primary one rows = np.arange(len(avail))[:, None] return order, avail[rows, order] class H5Dataset(Dataset): - def __init__(self, path: str | Path, spec: NetworkSpec, rest_order: str = "pt"): + def __init__( + self, + path: str | Path, + spec: NetworkSpec, + rest_order: str = "pt", + n_folds: int | None = None, + ): + """``n_folds``: use fold = event % n_folds (``fold_of``) instead of the ``EVENT/fold`` column.""" if rest_order not in ("pt", "selector"): raise ValueError( f"rest_order must be 'pt' or 'selector', got {rest_order!r}" ) self.spec, self.path, self.rest_order = spec, Path(path), rest_order - self._taken: dict[str, np.ndarray] = {} # per collection: members already consumed by a role - self._last_keys: dict[str, tuple[np.ndarray, ...]] = {} # per collection: sort keys of the last explicit selector + self._taken: dict[ + str, np.ndarray + ] = {} # per collection: members already consumed by a role + self._last_keys: dict[ + str, tuple[np.ndarray, ...] + ] = {} # per collection: sort keys of the last explicit selector inputs: dict[str, torch.Tensor] = {} with h5py.File(path, "r") as f: - collections = {c.name: _read_collection(f, c, columns_to_read(spec, c)) for c in used_collections(spec)} + collections = { + c.name: _read_collection(f, c, columns_to_read(spec, c)) + for c in used_collections(spec) + } for role in spec.roles.values(): feats, present = collections[role.collection] inputs[role.name] = self._role_tensor( @@ -169,18 +221,38 @@ def __init__(self, path: str | Path, spec: NetworkSpec, rest_order: str = "pt"): if WEIGHT_KEY in f else torch.ones(n, dtype=torch.float32) ) - self.folds = ( - torch.from_numpy(f[FOLD_KEY][...].astype(np.int64)) - if FOLD_KEY in f - else torch.zeros(n, dtype=torch.int64) - ) self.events = ( torch.from_numpy(f[EVENT_KEY][...].astype(np.int64)) if EVENT_KEY in f else torch.arange(n, dtype=torch.int64) ) + self.folds, self.n_folds, self.fold_rule = self._folds(f, n_folds) self.inputs = inputs + def _folds( + self, f: h5py.File, n_folds: int | None + ) -> tuple[torch.Tensor, int, str]: + """(fold of every event, number of folds, rule): ``EVENT/fold`` (n_folds = the number of distinct + values) or, with ``n_folds``, ``fold_of(EVENT/event, n_folds)``.""" + if n_folds is not None: + if EVENT_KEY not in f: + raise KeyError( + f"{self.path.name}: no {EVENT_KEY} dataset; fold = event % {n_folds} needs the event numbers" + ) + if FOLD_KEY in f: + logging.info( + f"{self.path.name}: {FOLD_KEY} ignored, folds = event % {n_folds} from {EVENT_KEY}" + ) + folds = fold_of(f[EVENT_KEY][...], n_folds) + return torch.from_numpy(folds), n_folds, FOLD_RULE_EVENT + if FOLD_KEY not in f: + raise KeyError( + f"{self.path.name}: no {FOLD_KEY} dataset; pass --n-folds K (H5Dataset(n_folds=K)) " + f"to use fold = event % K from {EVENT_KEY}" + ) + folds = f[FOLD_KEY][...].astype(np.int64) + return torch.from_numpy(folds), len(np.unique(folds)), FOLD_RULE_COLUMN + # ---------------------------------------------------------------- construction def _role_tensor( self, @@ -191,7 +263,11 @@ def _role_tensor( ) -> torch.Tensor: taken = self._taken.setdefault(role.collection, np.zeros_like(present)) if role.selector == REST: - keys = self._last_keys.get(role.collection) if self.rest_order == "selector" else None + keys = ( + self._last_keys.get(role.collection) + if self.rest_order == "selector" + else None + ) keys = (-feats["pt"],) if keys is None else keys else: keys = _selector_keys(role, feats) @@ -203,7 +279,12 @@ def _role_tensor( return self._stack(coll, feats, idx, valid) @staticmethod - def _stack(coll: Collection, feats: dict[str, np.ndarray], idx: np.ndarray, valid: np.ndarray) -> torch.Tensor: + def _stack( + coll: Collection, + feats: dict[str, np.ndarray], + idx: np.ndarray, + valid: np.ndarray, + ) -> torch.Tensor: """(N, len(features) + 1, n): features in configuration order, presence flag last, padding -1. Columns read only for a selector are not stacked.""" n, slots = idx.shape @@ -273,5 +354,20 @@ def fold_indices(self, fold: int) -> tuple[np.ndarray, np.ndarray]: return np.flatnonzero(~val), np.flatnonzero(val) -__all__ = ["H5Dataset", "GROUP_ALIASES", "COMPUTED_EVENT", "LABEL_KEYS", "WEIGHT_KEY", "FOLD_KEY", "EVENT_KEY", - "group_candidates", "used_collections", "columns_to_read", "event_source", "resolve_group"] +__all__ = [ + "COMPUTED_EVENT", + "EVENT_KEY", + "FOLD_KEY", + "FOLD_RULE_COLUMN", + "FOLD_RULE_EVENT", + "GROUP_ALIASES", + "LABEL_KEYS", + "WEIGHT_KEY", + "H5Dataset", + "columns_to_read", + "event_source", + "fold_of", + "group_candidates", + "resolve_group", + "used_collections", +] From dc1b339c08e61149fc09d13a4863856f053bf2f4 Mon Sep 17 00:00:00 2001 From: Aniket Khanal Date: Wed, 21 Oct 2026 17:48:46 -0400 Subject: [PATCH 22/33] make evaluation consistent with user specified n_folds --- src/corenet/evaluate.py | 25 ++++++++++++++++++------- 1 file changed, 18 insertions(+), 7 deletions(-) diff --git a/src/corenet/evaluate.py b/src/corenet/evaluate.py index f08e8e6..6e9cd5e 100644 --- a/src/corenet/evaluate.py +++ b/src/corenet/evaluate.py @@ -5,10 +5,11 @@ --output runs/hh4b/eval_fold0.h5 Events: by default the fold held out when the checkpoint was trained (``--fold -1`` = every event of -the file). The output file (layout in docs/Dataset.md) holds the class probabilities, the view -(candidate) probabilities of every multi-candidate hypothesis, the auxiliary head probabilities, -the event labels / weights and, for the main output, the ROC curve; the AUCs are printed and -written as JSON next to it (``.json``). +the file), with the checkpoint's fold rule: ``EVENT/fold`` of the file, or fold = event % n_folds when +the model was trained with ``--n-folds`` (``--n-folds K`` overrides). The output file (layout in +docs/Dataset.md) holds the class probabilities, the view (candidate) probabilities of every +multi-candidate hypothesis, the auxiliary head probabilities, the event labels / weights and, for the +main output, the ROC curve; the AUCs are printed and written as JSON next to it (``.json``). Metrics use the event weights of the file, clipped at >= 0 (``--unweighted`` ignores them): * main AUC: class 1 (the first output) against every other class, scored by P(class 1); @@ -191,6 +192,13 @@ def build_parser() -> argparse.ArgumentParser: default=None, help="evaluate the events of this fold; -1 = every event (default: the checkpoint's held-out fold)", ) + p.add_argument( + "--n-folds", + type=int, + default=None, + help="use fold = event %% K computed from EVENT/event instead of the EVENT/fold column " + "(default: the checkpoint's rule)", + ) p.add_argument( "--rest-order", choices=["pt", "selector"], @@ -208,7 +216,7 @@ def build_parser() -> argparse.ArgumentParser: def main(argv=None): - from .checkpoint import load_checkpoint + from .checkpoint import load_checkpoint, n_folds_of args = build_parser().parse_args(argv) logging.basicConfig( @@ -222,7 +230,8 @@ def main(argv=None): model, spec, ckpt = load_checkpoint(args.checkpoint, args.config, device) meta = ckpt.get("metadata", {}) rest_order = args.rest_order or meta.get("rest_order", "pt") - ds = H5Dataset(args.input, spec, rest_order=rest_order) + n_folds = args.n_folds if args.n_folds is not None else n_folds_of(meta) + ds = H5Dataset(args.input, spec, rest_order=rest_order, n_folds=n_folds) fold = args.fold if args.fold is not None else meta.get("fold", -1) idx = ( np.arange(len(ds)) @@ -233,7 +242,7 @@ def main(argv=None): raise SystemExit(f"{args.input}: no events in fold {fold}") logging.info( f"{args.checkpoint}: {count_parameters(model):,} parameters; evaluating {len(idx):,} events " - f"({'all folds' if fold is None or fold < 0 else f'fold {fold}'}) of {args.input}" + f"({'all folds' if fold is None or fold < 0 else f'fold {fold}'}; folds: {ds.fold_rule}) of {args.input}" ) probs, views, heads = predict(model, Batches(ds, device), idx, args.batch_size) @@ -260,6 +269,8 @@ def main(argv=None): attrs={ "checkpoint": str(args.checkpoint), "fold": -1 if fold is None else int(fold), + "n_folds": ds.n_folds, + "fold_rule": ds.fold_rule, "n_parameters": count_parameters(model), "auc_main": metrics["auc"]["main"] or float("nan"), }, From 6c26ea30e72e6c066ecdec57cc133aa2d487e80d Mon Sep 17 00:00:00 2001 From: Aniket Khanal Date: Wed, 21 Oct 2026 17:49:05 -0400 Subject: [PATCH 23/33] add support for dynamic number of folds --- src/corenet/export.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/src/corenet/export.py b/src/corenet/export.py index d98190f..8f5cc77 100644 --- a/src/corenet/export.py +++ b/src/corenet/export.py @@ -32,7 +32,7 @@ import torch.nn.functional as F from torch import nn -from .checkpoint import load_checkpoint +from .checkpoint import load_checkpoint, n_folds_of from .data.h5 import H5Dataset @@ -158,8 +158,12 @@ def main(argv=None): ) model, spec, ckpt = load_checkpoint(args.checkpoint, args.config, "cpu") wrapper = ExportWrapper(model).eval() + meta = ckpt.get("metadata", {}) ds = H5Dataset( - args.input, spec, rest_order=ckpt.get("metadata", {}).get("rest_order", "pt") + args.input, + spec, + rest_order=meta.get("rest_order", "pt"), + n_folds=n_folds_of(meta), ) out_dir = Path(args.output_dir) out_dir.mkdir(parents=True, exist_ok=True) From 421eaeb412deca1057090f927a9a25238bb625cc Mon Sep 17 00:00:00 2001 From: Aniket Khanal Date: Wed, 21 Oct 2026 17:49:27 -0400 Subject: [PATCH 24/33] update features list --- src/corenet/models/config.py | 83 +++++++++++++++++++++++++++++------- 1 file changed, 67 insertions(+), 16 deletions(-) diff --git a/src/corenet/models/config.py b/src/corenet/models/config.py index b4fa68a..f3a0473 100644 --- a/src/corenet/models/config.py +++ b/src/corenet/models/config.py @@ -32,9 +32,20 @@ from .combinatorics import Node, Role KINEMATIC = ("pt", "eta", "phi", "mass") -REST = "rest" # the only reserved selector word: the members no earlier role took -LOWEST = "lowest" # trailing word of a role line: lowest value of the column first -SECTIONS = ("objects", "event", "min_jets", "hypotheses", "attention", "relations", "outputs", "combine", "training", "trainer") +REST = "rest" # the only reserved selector word: the members no earlier role took +LOWEST = "lowest" # trailing word of a role line: lowest value of the column first +SECTIONS = ( + "objects", + "event", + "min_jets", + "hypotheses", + "attention", + "relations", + "outputs", + "combine", + "training", + "trainer", +) class ConfigError(ValueError): @@ -44,7 +55,9 @@ class ConfigError(ValueError): @dataclass class Collection: name: str - features: list[str] # exactly the columns fed to the network (kinematics + extras), in this order + features: list[ + str + ] # exactly the columns fed to the network (kinematics + extras), in this order max: int = 1 @@ -52,9 +65,9 @@ class Collection: class RoleSpec: name: str collection: str - selector: str # a column of the collection's group in the file (btag, pt, mass, ...) or ``rest`` + selector: str # a column of the collection's group in the file (btag, pt, mass, ...) or ``rest`` n: int - ascending: bool = False # ``lowest``: the n members with the lowest value first (default: highest first) + ascending: bool = False # ``lowest``: the n members with the lowest value first (default: highest first) @property def columns(self) -> tuple[str, ...]: @@ -114,8 +127,10 @@ class NetworkSpec: combine: list[str] training: TrainingSpec relations: list[RelationSpec] = field(default_factory=list) - owner: dict[str, str] = field(default_factory=dict) # node -> hypothesis in which it is declared - min_jets: int = 4 # minimum jet multiplicity of the selection: nSelJets enters as log(nSelJets - (min_jets - 1)) + owner: dict[str, str] = field( + default_factory=dict + ) # node -> hypothesis in which it is declared + min_jets: int = 4 # minimum jet multiplicity of the selection: nSelJets enters as log(nSelJets - (min_jets - 1)) # ---- derived @property @@ -233,14 +248,20 @@ def _parse_role(name: str, text: str, collections: dict[str, Collection]) -> Rol role took. ``column`` is any per-member dataset of the collection's group in the file.""" m = re.fullmatch(rf"\s*(\w+)\s+(\w+)\s+(\d+)(?:\s+({LOWEST}))?\s*", text) if not m: - raise ConfigError(f"objects.{name}: expected ' [{LOWEST}]', got {text!r}") + raise ConfigError( + f"objects.{name}: expected ' [{LOWEST}]', got {text!r}" + ) coll, sel, n, tail = m.group(1), m.group(2), int(m.group(3)), m.group(4) if coll not in collections: raise ConfigError(f"objects.{name}: unknown collection {coll!r}") if sel == REST and tail is not None: - raise ConfigError(f"objects.{name}: '{REST}' takes no {LOWEST!r} (its order is the rest_order option)") + raise ConfigError( + f"objects.{name}: '{REST}' takes no {LOWEST!r} (its order is the rest_order option)" + ) if n < 1 or n > collections[coll].max: - raise ConfigError(f"objects.{name}: n={n} must be within 1..max={collections[coll].max} of {coll}") + raise ConfigError( + f"objects.{name}: n={n} must be within 1..max={collections[coll].max} of {coll}" + ) return RoleSpec(name, coll, sel, n, ascending=tail == LOWEST) @@ -336,7 +357,9 @@ def parse_config(cfg: dict) -> NetworkSpec: event = _str_list(cfg.get("event", []) or [], "event") min_jets = cfg.get("min_jets", 4) if isinstance(min_jets, bool) or not isinstance(min_jets, int) or min_jets < 2: - raise ConfigError(f"min_jets must be an integer >= 2 (the minimum jet multiplicity of the selection), got {min_jets!r}") + raise ConfigError( + f"min_jets must be an integer >= 2 (the minimum jet multiplicity of the selection), got {min_jets!r}" + ) # hypotheses --------------------------------------------------------------------------- leaves = set(roles) | {c.name for c in collections.values() if c.max == 1} @@ -460,7 +483,20 @@ def parse_config(cfg: dict) -> NetworkSpec: if h not in [o.name for o in outputs[1:]]: raise ConfigError(f"training.head_weights: {h!r} is not a secondary output") - return NetworkSpec(collections, roles, event, nodes, hypotheses, attention, outputs, combine, training, relations, owner, min_jets) + return NetworkSpec( + collections, + roles, + event, + nodes, + hypotheses, + attention, + outputs, + combine, + training, + relations, + owner, + min_jets, + ) def read_config_dict(path: str | Path) -> dict: @@ -478,6 +514,21 @@ def load_config(path: str | Path) -> NetworkSpec: return parse_config(read_config_dict(path)) -__all__ = ["ConfigError", "Collection", "RoleSpec", "HypothesisSpec", "AttentionSpec", "RelationSpec", "OutputSpec", - "TrainingSpec", "NetworkSpec", "parse_config", "read_config_dict", "load_config", "SECTIONS", "REST", - "LOWEST", "KINEMATIC"] +__all__ = [ + "KINEMATIC", + "LOWEST", + "REST", + "SECTIONS", + "AttentionSpec", + "Collection", + "ConfigError", + "HypothesisSpec", + "NetworkSpec", + "OutputSpec", + "RelationSpec", + "RoleSpec", + "TrainingSpec", + "load_config", + "parse_config", + "read_config_dict", +] From f175488a99cad264d439a68196778cb6dc06267b Mon Sep 17 00:00:00 2001 From: Aniket Khanal Date: Wed, 21 Oct 2026 17:49:46 -0400 Subject: [PATCH 25/33] make n_jets dynamic input --- src/corenet/models/network.py | 19 +++++++++++++++---- 1 file changed, 15 insertions(+), 4 deletions(-) diff --git a/src/corenet/models/network.py b/src/corenet/models/network.py index 8c5d254..32b17ba 100644 --- a/src/corenet/models/network.py +++ b/src/corenet/models/network.py @@ -36,7 +36,9 @@ from .config import KINEMATIC, ConfigError, NetworkSpec from .fourvectors import NonLU, PxPyPzE, addFourVectors, diObjectMass -MASKED_LOGIT = -1e4 # selector logit of an absent view (finite, so an all-absent event stays finite) +MASKED_LOGIT = ( + -1e4 +) # selector logit of an absent view (finite, so an all-absent event stays finite) # ------------------------------------------------------------------------------- small helpers @@ -400,7 +402,9 @@ def dep(n): def _extras(self, role): feats = self.spec.collection_of(role).features - return [f for f in feats if f not in KINEMATIC] # every non-kinematic feature is an input + return [ + f for f in feats if f not in KINEMATIC + ] # every non-kinematic feature is an input def _kinematic_channels(self, role): feats = self.spec.collection_of(role).features @@ -462,8 +466,15 @@ def _prepare(self, inputs): a = inputs["event"].view(n, self.dA, 1) for idx, name in enumerate(self.spec.event): if "nSelJets" in name: - a = torch.cat([a[:, :idx], torch.log(a[:, idx:idx + 1] - (self.spec.min_jets - 1)), a[:, idx + 1:]], dim=1) - a = finite(a) # nSelJets < min_jets would give -inf / NaN + a = torch.cat( + [ + a[:, :idx], + torch.log(a[:, idx : idx + 1] - (self.spec.min_jets - 1)), + a[:, idx + 1 :], + ], + dim=1, + ) + a = finite(a) # nSelJets < min_jets would give -inf / NaN out = { "n": n, From 8ba6b856e87b299ae72504f6558fce1fc57c2a33 Mon Sep 17 00:00:00 2001 From: Aniket Khanal Date: Wed, 21 Oct 2026 17:50:03 -0400 Subject: [PATCH 26/33] make training recipe configurable --- src/corenet/train.py | 343 +++++++++++++++++++++++++++++++++++-------- 1 file changed, 285 insertions(+), 58 deletions(-) diff --git a/src/corenet/train.py b/src/corenet/train.py index 97f158f..3d22ec6 100644 --- a/src/corenet/train.py +++ b/src/corenet/train.py @@ -10,7 +10,10 @@ * ``epochs`` (20) epochs; batch size ``batch_size_init`` (1024) at epoch 0, doubled at epochs 1, 3, 6 and 10, with (64, 16, 4, 1, 0) ghost batches; * the input normalisation of every embedder is frozen from the training set before epoch 0; - * at epoch ``freeze_epoch`` (15) every parameter except the output heads is frozen; + * at epoch ``freeze_epoch`` (15) the fine-tuning stage starts: every parameter whose name does not + start with one of the ``finetune_layers`` prefixes (the output heads) is frozen; ``finetune_lr`` + and ``finetune_batch_size`` (unset: the schedule continues) set the learning rate, from which the + milestones continue, and the batch size of that stage; * per-class weight normalisation (``class_norm``): the event weights of every class sum to 1 over the training set; weights are clamped at >= 0 inside the loss. @@ -19,6 +22,9 @@ binary cross entropy of its head against ``label == class index of o``. The view (candidate selection) scores are not supervised. +Folds: ``EVENT/fold`` of the file, or ``--n-folds K`` for fold = event % K from the event numbers +(``corenet.data.h5.fold_of``); ``--exclude-folds next`` is the fold (k + 1) % n_folds of held-out fold k. + For each held-out fold k this writes, in ``--output-dir``: model_fold{k}.pkl checkpoint: state dict, configuration, classes and metadata history_fold{k}.json per-epoch training loss, and validation loss/AUC on --exclude-folds @@ -47,7 +53,8 @@ "head_main.", "head_reduce.", "aux_heads.", -) # trainable after the freeze epoch +) # default finetune_layers: trainable after the freeze epoch +NO_FINETUNE_LAYERS = "none" # --finetune-layers none: freeze nothing MIN_GHOST_BATCH_SIZE = 8 # single-pixel GBNs go NaN with one-event ghost batches # training-recipe options: name -> default. Settable in the `trainer:` section of the configuration; @@ -63,6 +70,9 @@ "seed": 42, "rest_order": "pt", "early_stop_patience": 0, + "finetune_layers": list(HEAD_PREFIXES), + "finetune_lr": None, + "finetune_batch_size": None, } @@ -84,13 +94,46 @@ def batch_schedule(epoch: int, bs_init: int) -> tuple[int, int]: return bs, min(n_gb, max(1, bs // MIN_GHOST_BATCH_SIZE)) # no-op for bs_init >= 512 -def freeze_heads(model: CoReNet) -> int: - """Freeze everything but the output heads (head_main, head_reduce, aux_heads); returns #trainable.""" - n = 0 +def finetune_batch(epoch: int, bs_init: int, finetune_batch_size: int | None): + """(batch size, ghost batches) of the fine-tuning stage: ``finetune_batch_size`` when set (ghost batches + capped so that none has fewer than MIN_GHOST_BATCH_SIZE events, 0 stays 0), else the schedule.""" + bs, n_gb = batch_schedule(epoch, bs_init) + if finetune_batch_size is None: + return bs, n_gb + bs = finetune_batch_size + return bs, min(n_gb, max(1, bs // MIN_GHOST_BATCH_SIZE)) + + +def freeze_heads(model: CoReNet, prefixes=HEAD_PREFIXES) -> int: + """Freeze every parameter whose name does not start with one of ``prefixes`` (default: the output heads + head_main, head_reduce, aux_heads); parameters that are non-trainable by construction stay so. + Returns the number of trainable parameters.""" + prefixes = tuple(prefixes) for name, param in model.named_parameters(): - param.requires_grad = name.startswith(HEAD_PREFIXES) - n += int(param.requires_grad) * param.numel() - return n + if not name.startswith(prefixes): + param.requires_grad = False + return count_trainable(model) + + +def count_trainable(model: torch.nn.Module) -> int: + return int(sum(p.numel() for p in model.parameters() if p.requires_grad)) + + +def parse_finetune_layers(value) -> list[str]: + """``finetune_layers`` as a list of prefixes: a list of strings, or ``none`` (alone) for an empty list.""" + if isinstance(value, str): + value = [value] + if not isinstance(value, list) or not all(isinstance(e, str) and e for e in value): + raise ConfigError( + f"trainer.finetune_layers must be a list of parameter-name prefixes or 'none', got {value!r}" + ) + if NO_FINETUNE_LAYERS in value: + if len(value) > 1: + raise ConfigError( + f"trainer.finetune_layers: 'none' cannot be combined with prefixes, got {value!r}" + ) + return [] + return list(value) def normalize_class_weights( @@ -184,8 +227,9 @@ def train_fold( labels = ds.labels.numpy() train_idx, test_idx = ds.fold_indices(fold) val_idx = train_idx[:0] - if args.exclude_folds: - keep = ~np.isin(ds.folds.numpy()[train_idx], args.exclude_folds) + exclude_folds = resolve_exclude_folds(args.exclude_folds, fold, ds.n_folds) + if exclude_folds: + keep = ~np.isin(ds.folds.numpy()[train_idx], exclude_folds) val_idx, train_idx = train_idx[~keep], train_idx[keep] weights = ( normalize_class_weights(labels, ds.weights, train_idx, n_classes) @@ -225,15 +269,53 @@ def train_fold( out_dir = Path(args.output_dir) out_dir.mkdir(parents=True, exist_ok=True) recipe = {k: getattr(args, k) for k in RECIPE_DEFAULTS} + # trainable parameters of the fine-tuning stage, set at the freeze epoch + n_trainable = None + + def metadata(epoch: int) -> dict: + return { + "fold": fold, + "epoch": epoch, + "exclude_folds": exclude_folds, + "input": str(args.input), + "n_folds": ds.n_folds, + "fold_rule": ds.fold_rule, + **recipe, + "finetune_trainable_parameters": n_trainable, + } + history, history_path = [], out_dir / f"history_fold{fold}.json" - best_auc, best_epoch, best_state, bad, stopped = None, None, None, 0, False # early stopping state + best_auc, best_epoch, best_state, bad, stopped = ( + None, + None, + None, + 0, + False, + ) # early stopping state for epoch in range(args.epochs): - bs, n_gb = batch_schedule(epoch, args.batch_size_init) + bs, n_gb = ( + finetune_batch(epoch, args.batch_size_init, args.finetune_batch_size) + if epoch >= args.freeze_epoch + else batch_schedule(epoch, args.batch_size_init) + ) model.setGhostBatches(n_gb) if epoch == args.freeze_epoch: - n_trainable = freeze_heads(model) + if args.finetune_layers: + n_trainable = freeze_heads(model, args.finetune_layers) + else: + n_trainable = count_trainable(model) + if args.finetune_lr is not None: + for group in optimizer.param_groups: + group["lr"] = args.finetune_lr logging.info( - f"fold {fold}: frozen at epoch {epoch}, {n_trainable:,} trainable head parameters" + f"fold {fold}: fine-tuning from epoch {epoch}: trainable layers " + + ( + f"{args.finetune_layers}" + if args.finetune_layers + else "all (nothing frozen)" + ) + + f", {n_trainable:,} trainable parameters, lr {optimizer.param_groups[0]['lr']:.2e}, " + f"batch size {bs}, {n_gb} ghost batches" ) model.train() loss_sum, w_sum = 0.0, 0.0 @@ -255,13 +337,7 @@ def train_fold( model, config, args.config, - { - "fold": fold, - "epoch": epoch, - "exclude_folds": args.exclude_folds, - "input": str(args.input), - **recipe, - }, + metadata(epoch), ) entry = { "epoch": epoch, @@ -274,37 +350,69 @@ def train_fold( entry.update(validation_metrics(model, data, val_idx, labels, val_weights)) model.setGhostBatches(n_gb) # predict() set them to 0 history.append(entry) - history_path.write_text(json.dumps({"fold": fold, "exclude_folds": args.exclude_folds, "config": str(args.config), - "classes": spec.classes, "recipe": recipe, - "n_train": int(len(train_idx)), "n_val": int(len(val_idx)), - "history": history}, indent=1)) - logging.info(f"fold {fold} | epoch {epoch:02d} | bs {bs:6d} | gbn {n_gb:2d} | lr {optimizer.param_groups[0]['lr']:.2e} | loss {entry['train_loss']:.5f}" - + (f" | val loss {entry['val_loss']:.5f} | val auc {entry.get('val_auc', float('nan')):.5f}" if len(val_idx) else "")) + history_path.write_text( + json.dumps( + { + "fold": fold, + "exclude_folds": exclude_folds, + "n_folds": ds.n_folds, + "fold_rule": ds.fold_rule, + "config": str(args.config), + "classes": spec.classes, + "recipe": recipe, + "finetune_trainable_parameters": n_trainable, + "n_train": len(train_idx), + "n_val": len(val_idx), + "history": history, + }, + indent=1, + ) + ) + logging.info( + f"fold {fold} | epoch {epoch:02d} | bs {bs:6d} | gbn {n_gb:2d} | lr {optimizer.param_groups[0]['lr']:.2e} | loss {entry['train_loss']:.5f}" + + ( + f" | val loss {entry['val_loss']:.5f} | val auc {entry.get('val_auc', float('nan')):.5f}" + if len(val_idx) + else "" + ) + ) if args.early_stop_patience and len(val_idx): auc = entry.get("val_auc") if auc is not None and (best_auc is None or auc > best_auc): best_auc, best_epoch, bad = auc, epoch, 0 - best_state = {k: v.detach().clone() for k, v in model.state_dict().items()} + best_state = { + k: v.detach().clone() for k, v in model.state_dict().items() + } else: bad += 1 if bad >= args.early_stop_patience: stopped = True - logging.info(f"fold {fold}: early stopping after epoch {epoch}: the validation AUC did not improve for " - f"{bad} epochs (best {best_auc:.5f} at epoch {best_epoch})") + logging.info( + f"fold {fold}: early stopping after epoch {epoch}: the validation AUC did not improve for " + f"{bad} epochs (best {best_auc:.5f} at epoch {best_epoch})" + ) break final_epoch = args.epochs - 1 if not stopped else epoch if args.early_stop_patience and best_state is not None: - model.load_state_dict(best_state) # keep the weights of the best validation epoch + model.load_state_dict( + best_state + ) # keep the weights of the best validation epoch final_epoch = best_epoch doc = json.loads(history_path.read_text()) - doc.update({"best_epoch": best_epoch, "stopped_after_epoch": epoch if stopped else None, - "early_stop_patience": args.early_stop_patience}) + doc.update( + { + "best_epoch": best_epoch, + "stopped_after_epoch": epoch if stopped else None, + "early_stop_patience": args.early_stop_patience, + } + ) history_path.write_text(json.dumps(doc, indent=1)) - logging.info(f"fold {fold}: keeping the weights of epoch {best_epoch} (validation AUC {best_auc:.5f})") + logging.info( + f"fold {fold}: keeping the weights of epoch {best_epoch} (validation AUC {best_auc:.5f})" + ) ckpt = out_dir / f"model_fold{fold}.pkl" - save_checkpoint(ckpt, model, config, args.config, - {"fold": fold, "epoch": final_epoch, "exclude_folds": args.exclude_folds, "input": str(args.input), **recipe}) + save_checkpoint(ckpt, model, config, args.config, metadata(final_epoch)) probs, views, heads = predict(model, data, test_idx) write_predictions( out_dir / f"predictions_fold{fold}.h5", @@ -319,6 +427,8 @@ def train_fold( "config": str(Path(args.config).resolve()), "n_parameters": count_parameters(model), "fold": fold, + "n_folds": ds.n_folds, + "fold_rule": ds.fold_rule, }, ) logging.info( @@ -340,27 +450,126 @@ def build_parser() -> argparse.ArgumentParser: "--input", required=True, help="HDF5 file with all folds (docs/Dataset.md)" ) p.add_argument("--output-dir", default="runs/corenet") - p.add_argument("--fold", type=int, default=-1, help="held-out (test) fold, or -1 for every fold in turn") - p.add_argument("--exclude-folds", type=int, nargs="*", default=[], - help="folds left out of training and used for per-epoch validation") - g = p.add_argument_group("training recipe (default: the `trainer:` section of --config, else the built-in value)") - g.add_argument("--epochs", type=int, help=f"number of epochs [{RECIPE_DEFAULTS['epochs']}]") - g.add_argument("--freeze-epoch", type=int, help=f"epoch at which all but the output heads are frozen [{RECIPE_DEFAULTS['freeze_epoch']}]") - g.add_argument("--lr", type=float, help=f"Adam learning rate [{RECIPE_DEFAULTS['lr']}]") - g.add_argument("--lr-milestones", type=int, nargs="*", help="epochs after which the learning rate is multiplied by --lr-gamma [15 .. 24]") - g.add_argument("--lr-gamma", type=float, help=f"learning-rate decay factor [{RECIPE_DEFAULTS['lr_gamma']}]") - g.add_argument("--batch-size-init", type=int, help=f"epoch-0 batch size; doubles at epochs 1, 3, 6, 10 [{RECIPE_DEFAULTS['batch_size_init']}]") - g.add_argument("--class-norm", action=argparse.BooleanOptionalAction, default=None, - help="normalise the weights of every class to 1 over the training set [on]") - g.add_argument("--seed", type=int, help=f"random seed (the seed of fold k is seed + k) [{RECIPE_DEFAULTS['seed']}]") - g.add_argument("--early-stop-patience", type=int, help="stop when the validation AUC has not improved for this many epochs " - f"and keep the weights of the best epoch; 0 = train every epoch; needs --exclude-folds [{RECIPE_DEFAULTS['early_stop_patience']}]") - g.add_argument("--rest-order", choices=["pt", "selector"], help="ordering of the 'rest' selector [pt]") - p.add_argument("--save-every-epoch", action="store_true", help="also save model_fold{k}_epoch{e}.pkl after every epoch") - p.add_argument("--device", default="cuda", help="torch device; falls back to cpu when CUDA is unavailable") + p.add_argument( + "--fold", + type=int, + default=-1, + help="held-out (test) fold, or -1 for every fold in turn", + ) + p.add_argument( + "--n-folds", + type=int, + default=None, + help="use fold = event %% K computed from EVENT/event instead of the EVENT/fold column", + ) + p.add_argument( + "--exclude-folds", + type=exclude_fold, + nargs="*", + default=[], + help="folds left out of training and used for per-epoch validation; " + "`next` = the fold (k + 1) %% n_folds of the held-out fold k", + ) + g = p.add_argument_group( + "training recipe (default: the `trainer:` section of --config, else the built-in value)" + ) + g.add_argument( + "--epochs", type=int, help=f"number of epochs [{RECIPE_DEFAULTS['epochs']}]" + ) + g.add_argument( + "--freeze-epoch", + type=int, + help="epoch at which the fine-tuning stage starts: all but --finetune-layers are frozen " + f"[{RECIPE_DEFAULTS['freeze_epoch']}]", + ) + g.add_argument( + "--finetune-layers", + nargs="+", + metavar="PREFIX", + help="parameter-name prefixes that stay trainable from --freeze-epoch on, or `none` to freeze " + f"nothing [{' '.join(HEAD_PREFIXES)}]", + ) + g.add_argument( + "--finetune-lr", + type=float, + help="learning rate set at --freeze-epoch; the milestones continue from it [keep the schedule]", + ) + g.add_argument( + "--finetune-batch-size", + type=int, + help="batch size from --freeze-epoch on [keep the schedule]", + ) + g.add_argument( + "--lr", type=float, help=f"Adam learning rate [{RECIPE_DEFAULTS['lr']}]" + ) + g.add_argument( + "--lr-milestones", + type=int, + nargs="*", + help="epochs after which the learning rate is multiplied by --lr-gamma [15 .. 24]", + ) + g.add_argument( + "--lr-gamma", + type=float, + help=f"learning-rate decay factor [{RECIPE_DEFAULTS['lr_gamma']}]", + ) + g.add_argument( + "--batch-size-init", + type=int, + help=f"epoch-0 batch size; doubles at epochs 1, 3, 6, 10 [{RECIPE_DEFAULTS['batch_size_init']}]", + ) + g.add_argument( + "--class-norm", + action=argparse.BooleanOptionalAction, + default=None, + help="normalise the weights of every class to 1 over the training set [on]", + ) + g.add_argument( + "--seed", + type=int, + help=f"random seed (the seed of fold k is seed + k) [{RECIPE_DEFAULTS['seed']}]", + ) + g.add_argument( + "--early-stop-patience", + type=int, + help="stop when the validation AUC has not improved for this many epochs " + f"and keep the weights of the best epoch; 0 = train every epoch; needs --exclude-folds [{RECIPE_DEFAULTS['early_stop_patience']}]", + ) + g.add_argument( + "--rest-order", + choices=["pt", "selector"], + help="ordering of the 'rest' selector [pt]", + ) + p.add_argument( + "--save-every-epoch", + action="store_true", + help="also save model_fold{k}_epoch{e}.pkl after every epoch", + ) + p.add_argument( + "--device", + default="cuda", + help="torch device; falls back to cpu when CUDA is unavailable", + ) return p +def exclude_fold(text: str): + """An --exclude-folds entry: a fold number or the word `next`.""" + if text == "next": + return text + try: + return int(text) + except ValueError: + raise argparse.ArgumentTypeError( + f"expected a fold number or `next`, got {text!r}" + ) from None + + +def resolve_exclude_folds(entries: list, fold: int, n_folds: int) -> list[int]: + """The validation folds of held-out fold ``fold``: `next` becomes (fold + 1) % n_folds.""" + return [(fold + 1) % n_folds if e == "next" else int(e) for e in entries] + + def _check_recipe(r: dict) -> None: """Types and ranges of the training-recipe options.""" @@ -384,6 +593,15 @@ def is_num(x): "seed": (lambda x: is_int(x), "an integer"), "rest_order": (lambda x: x in ("pt", "selector"), "'pt' or 'selector'"), "early_stop_patience": (lambda x: is_int(x) and x >= 0, "an integer >= 0"), + "finetune_layers": ( + lambda x: isinstance(x, list) and all(isinstance(e, str) and e for e in x), + "a list of parameter-name prefixes (or 'none')", + ), + "finetune_lr": (lambda x: x is None or (is_num(x) and x > 0), "a number > 0"), + "finetune_batch_size": ( + lambda x: x is None or (is_int(x) and x >= 1), + "an integer >= 1", + ), } for key, (ok, what) in rules.items(): if key in r and not ok(r[key]): @@ -403,6 +621,7 @@ def resolve_recipe(args, config: dict): for key, default in RECIPE_DEFAULTS.items(): if getattr(args, key) is None: setattr(args, key, section.get(key, default)) + args.finetune_layers = parse_finetune_layers(args.finetune_layers) _check_recipe({k: getattr(args, k) for k in RECIPE_DEFAULTS}) return args @@ -416,11 +635,19 @@ def main(argv=None): spec = parse_config(config) resolve_recipe(args, config) if args.early_stop_patience and not args.exclude_folds: - build_parser().error("--early-stop-patience needs a validation fold: pass --exclude-folds") - device = torch.device(args.device if torch.cuda.is_available() and args.device.startswith("cuda") else "cpu") - ds = H5Dataset(args.input, spec, rest_order=args.rest_order) + build_parser().error( + "--early-stop-patience needs a validation fold: pass --exclude-folds" + ) + if args.n_folds is not None and args.n_folds < 1: + build_parser().error(f"--n-folds must be >= 1, got {args.n_folds}") + device = torch.device( + args.device + if torch.cuda.is_available() and args.device.startswith("cuda") + else "cpu" + ) + ds = H5Dataset(args.input, spec, rest_order=args.rest_order, n_folds=args.n_folds) logging.info( - f"{args.input}: {len(ds):,} events, device={device}, recipe " + f"{args.input}: {len(ds):,} events, {ds.n_folds} folds ({ds.fold_rule}), device={device}, recipe " + ", ".join(f"{k}={getattr(args, k)}" for k in RECIPE_DEFAULTS) ) folds = sorted(set(ds.folds.tolist())) if args.fold < 0 else [args.fold] From 05ccf86f25b95e8f51ee7536862dbde141bbdee9 Mon Sep 17 00:00:00 2001 From: Aniket Khanal Date: Wed, 21 Oct 2026 17:50:25 -0400 Subject: [PATCH 27/33] update validation for dynamic training recipe --- src/corenet/validate.py | 148 +++++++++++++++++++++++++++++++--------- 1 file changed, 115 insertions(+), 33 deletions(-) diff --git a/src/corenet/validate.py b/src/corenet/validate.py index 71f07c1..6c4077d 100644 --- a/src/corenet/validate.py +++ b/src/corenet/validate.py @@ -10,9 +10,10 @@ 2. Data (with ``--input``): every dataset the loader reads is looked up with the loader's own rules (``corenet.data.h5``: group aliases, used collections, computed event scalars, label keys) and its dtype and shape are checked; the labels must lie in 0..len(outputs), the weights must be finite, - the folds must exist for training; the present members must have finite features. Finally the - actual loader (``H5Dataset``) reads the file and the network runs a forward pass on the first - events, so a file that passes here also loads in corenet-train. + the folds must exist for training (``EVENT/fold``, or ``--n-folds K`` for fold = event % K from + the event numbers; the report gives the fold source and the events per fold); the present members + must have finite features. Finally the actual loader (``H5Dataset``) reads the file and the network + runs a forward pass on the first events, so a file that passes here also loads in corenet-train. Every finding is printed as OK / WARN / FAIL with the dataset or key it concerns; the exit status is 1 when anything FAILed. ``--json`` also writes the report as JSON. @@ -30,7 +31,13 @@ import torch from .data import h5 as loader -from .models.config import REST, ConfigError, NetworkSpec, parse_config, read_config_dict +from .models.config import ( + REST, + ConfigError, + NetworkSpec, + parse_config, + read_config_dict, +) from .models.network import CoReNet @@ -116,11 +123,21 @@ def check_config(path, report: Report) -> NetworkSpec | None: return None for c in spec.collections.values(): - report.ok(sec, f"collection `{c.name}`: max {c.max}, features {c.features} (the network inputs)") + report.ok( + sec, + f"collection `{c.name}`: max {c.max}, features {c.features} (the network inputs)", + ) for r in spec.roles.values(): - how = "" if r.selector == REST else (" lowest first" if r.ascending else " highest first") - report.ok(sec, f"role `{r.name}`: {r.n} x `{r.collection}` selected by `{r.selector}`{how} " - f"(reads {', '.join(r.columns)} for the selection)") + how = ( + "" + if r.selector == REST + else (" lowest first" if r.ascending else " highest first") + ) + report.ok( + sec, + f"role `{r.name}`: {r.n} x `{r.collection}` selected by `{r.selector}`{how} " + f"(reads {', '.join(r.columns)} for the selection)", + ) for h, T in model.tables.items(): nodes = ", ".join(f"{n} ({T.nodes[n].n})" for n in model.levels[h]) ctx = spec.hypotheses[h].context @@ -160,6 +177,7 @@ def check_data( report: Report, rest_order: str = "pt", n_forward: int = 512, + n_folds: int | None = None, ) -> None: sec = f"data {path}" try: @@ -209,14 +227,26 @@ def length(name, ds): group = f["INPUTS"][gname] slots = None columns = loader.columns_to_read(spec, coll) - selection = [c for c in columns if c not in coll.features] # read for a selector only + selection = [ + c for c in columns if c not in coll.features + ] # read for a selector only for feat in columns: name = f"INPUTS/{gname}/{feat}" - readers = [f"`{r.name}`" for r in spec.roles.values() - if r.collection == coll.name and feat in r.columns] - why = (f" (read for the selection by {', '.join(readers)}, not a feature)" if feat in selection else "") + readers = [ + f"`{r.name}`" + for r in spec.roles.values() + if r.collection == coll.name and feat in r.columns + ] + why = ( + f" (read for the selection by {', '.join(readers)}, not a feature)" + if feat in selection + else "" + ) if feat not in group or not isinstance(group[feat], h5py.Dataset): - report.fail(sec, f"{who} expects dataset {name}{why}: not found (INPUTS/{gname} has {sorted(group)})") + report.fail( + sec, + f"{who} expects dataset {name}{why}: not found (INPUTS/{gname} has {sorted(group)})", + ) continue ds = group[feat] if not _numeric(ds): @@ -273,16 +303,31 @@ def length(name, ds): k for k in ("pt", "eta", "phi", "mass") if k not in coll.features ] note = f"; {', '.join(missing_kin)} taken as 0" if missing_kin else "" - report.ok(sec, f"{who}: INPUTS/{gname} with features {coll.features}" + (f", {slots} slots" if slots else "") + note) + report.ok( + sec, + f"{who}: INPUTS/{gname} with features {coll.features}" + + (f", {slots} slots" if slots else "") + + note, + ) if selection: - report.ok(sec, f"{who}: {selection} read for the selection only (not network inputs)") + report.ok( + sec, + f"{who}: {selection} read for the selection only (not network inputs)", + ) # ---- event scalars for name in spec.event: kind, where = loader.event_source(spec, name) if kind == "computed": - note = f" and enters as log({name} - {spec.min_jets - 1}) (min_jets {spec.min_jets})" if "nSelJets" in name else "" - report.ok(sec, f"event scalar `{name}` computed from collection `{where}`{note}") + note = ( + f" and enters as log({name} - {spec.min_jets - 1}) (min_jets {spec.min_jets})" + if "nSelJets" in name + else "" + ) + report.ok( + sec, + f"event scalar `{name}` computed from collection `{where}`{note}", + ) elif where not in f: report.fail( sec, f"event scalar `{name}` expects dataset {where}: not found" @@ -353,13 +398,8 @@ def length(name, ds): f"{loader.WEIGHT_KEY}: {n_neg:,} negative weights ({n_neg / len(wv):.3g} of the " "events); they count as 0 in the loss and in the metrics", ) - if loader.FOLD_KEY not in f: - report.fail( - sec, - f"{loader.FOLD_KEY} not found: corenet-train needs a fold index per event " - "(the loader would put every event in fold 0)", - ) - else: + column = None # EVENT/fold, when present and well-formed + if loader.FOLD_KEY in f: fo = f[loader.FOLD_KEY] if fo.dtype.kind not in "iu" or fo.ndim != 1: report.fail( @@ -367,16 +407,49 @@ def length(name, ds): f"{loader.FOLD_KEY}: shape {fo.shape} dtype {fo.dtype}, expected integer (events,)", ) elif length(loader.FOLD_KEY, fo): - vals, cnt = np.unique(fo[...], return_counts=True) - report.ok( + column = fo[...].astype(np.int64) + elif n_folds is None: + report.fail( + sec, + f"{loader.FOLD_KEY} not found: corenet-train needs a fold index per event; " + "pass --n-folds K to use fold = event % K from EVENT/event", + ) + folds, source = None, None + if n_folds is not None: + source = f"event % {n_folds} (--n-folds)" + if loader.EVENT_KEY not in f: + report.fail( + sec, + f"folds = {source}: {loader.EVENT_KEY} not found", + ) + else: + try: + folds = loader.fold_of(f[loader.EVENT_KEY][...], n_folds) + except ValueError as e: + report.fail(sec, f"folds = {source}: {e}") + elif column is not None: + folds, source = column, loader.FOLD_KEY + if folds is not None: + vals, cnt = np.unique(folds, return_counts=True) + report.ok( + sec, + f"folds from {source}: events per fold {dict(zip(vals.tolist(), cnt.tolist()))}", + ) + if len(vals) < 2: + report.warn( sec, - f"{loader.FOLD_KEY}: folds {dict(zip(vals.tolist(), cnt.tolist()))}", + f"folds from {source}: a single fold; training on fold k uses the other folds", ) - if len(vals) < 2: + if n_folds is not None and column is not None: + n_diff = int((column != folds).sum()) + if n_diff: report.warn( sec, - f"{loader.FOLD_KEY}: a single fold; training on fold k uses the other folds", + f"{loader.FOLD_KEY} disagrees with event % {n_folds} for {n_diff:,} of " + f"{len(folds):,} events; corenet-train --n-folds {n_folds} ignores the column", ) + else: + report.ok(sec, f"{loader.FOLD_KEY} equals event % {n_folds}") if loader.EVENT_KEY not in f: report.warn( sec, @@ -387,7 +460,7 @@ def length(name, ds): return # ---- the actual loader and a forward pass try: - ds = loader.H5Dataset(path, spec, rest_order=rest_order) + ds = loader.H5Dataset(path, spec, rest_order=rest_order, n_folds=n_folds) except Exception as e: report.fail(sec, f"the loader (H5Dataset) fails: {type(e).__name__}: {e}") return @@ -456,13 +529,20 @@ def build_parser() -> argparse.ArgumentParser: default=None, help="ordering of the 'rest' selector (default: the trainer section, else pt)", ) + p.add_argument( + "--n-folds", + type=int, + default=None, + help="report the fold = event %% K split of EVENT/event (as corenet-train --n-folds K uses) " + "instead of the EVENT/fold column", + ) p.add_argument( "--json", default=None, help="also write the report to this JSON file" ) return p -def validate(config, data=None, rest_order=None) -> Report: +def validate(config, data=None, rest_order=None, n_folds=None) -> Report: report = Report() spec = check_config(config, report) if spec is not None and data is not None: @@ -473,7 +553,7 @@ def validate(config, data=None, rest_order=None) -> Report: ) except Exception: rest_order = "pt" - check_data(data, spec, report, rest_order=rest_order) + check_data(data, spec, report, rest_order=rest_order, n_folds=n_folds) elif data is not None: report.fail(f"data {data}", "not checked: the configuration is invalid") return report @@ -481,7 +561,9 @@ def validate(config, data=None, rest_order=None) -> Report: def main(argv=None) -> int: args = build_parser().parse_args(argv) - report = validate(args.config, args.input, args.rest_order) + if args.n_folds is not None and args.n_folds < 1: + build_parser().error(f"--n-folds must be >= 1, got {args.n_folds}") + report = validate(args.config, args.input, args.rest_order, args.n_folds) report.print() if args.json: Path(args.json).write_text(json.dumps(report.to_dict(), indent=1)) From cea1b7d80dc87ecd432df4d1c95a5a140b4c13eb Mon Sep 17 00:00:00 2001 From: Aniket Khanal Date: Wed, 21 Oct 2026 17:50:36 -0400 Subject: [PATCH 28/33] update tests with checkpoint support --- tests/test_checkpoint.py | 87 ++++++++-- tests/test_config.py | 165 +++++++++++++----- tests/test_h5.py | 260 +++++++++++++++++++++++----- tests/test_network_bbww.py | 37 +++- tests/test_train.py | 336 ++++++++++++++++++++++++++++++++++++- tests/test_validate.py | 227 +++++++++++++++++++++---- 6 files changed, 961 insertions(+), 151 deletions(-) diff --git a/tests/test_checkpoint.py b/tests/test_checkpoint.py index 6050769..be5e834 100644 --- a/tests/test_checkpoint.py +++ b/tests/test_checkpoint.py @@ -1,5 +1,6 @@ """Checkpoint format: new checkpoints carry ``format``; checkpoints written before every feature became a network input (btag listed under features but skipped) still load, from their stored configuration.""" + from __future__ import annotations import copy @@ -7,13 +8,18 @@ import pytest import torch +from synthetic import write_synthetic from corenet import evaluate -from corenet.checkpoint import CHECKPOINT_FORMAT, legacy_config, load_checkpoint, save_checkpoint +from corenet.checkpoint import ( + CHECKPOINT_FORMAT, + legacy_config, + load_checkpoint, + save_checkpoint, +) from corenet.data import H5Dataset from corenet.models.config import parse_config, read_config_dict from corenet.models.network import CoReNet -from synthetic import write_synthetic repo = Path(__file__).resolve().parent.parent @@ -28,19 +34,42 @@ def _legacy_checkpoint(tmp_path): torch.manual_seed(0) model = CoReNet(parse_config(cfg), device="cpu") path = tmp_path / "legacy.pkl" - torch.save({"model": model.state_dict(), "config": old_cfg, "config_path": str(config), "classes": model.spec.classes, - "metadata": {"fold": 0, "rest_order": "pt"}}, path) + torch.save( + { + "model": model.state_dict(), + "config": old_cfg, + "config_path": str(config), + "classes": model.spec.classes, + "metadata": {"fold": 0, "rest_order": "pt"}, + }, + path, + ) return path, old_cfg, model def test_legacy_config_translation(): - cfg = {"objects": {"jets": {"features": ["pt", "eta", "phi", "mass", "btag"], "max": 12}, "b": "jets btag 4", - "lepton": {"features": ["pt", "eta", "phi", "mass", "isE"]}}} + cfg = { + "objects": { + "jets": {"features": ["pt", "eta", "phi", "mass", "btag"], "max": 12}, + "b": "jets btag 4", + "lepton": {"features": ["pt", "eta", "phi", "mass", "isE"]}, + } + } new, edited = legacy_config(cfg) - assert edited == ["jets"] and new["objects"]["jets"]["features"] == ["pt", "eta", "phi", "mass"] + assert edited == ["jets"] and new["objects"]["jets"]["features"] == [ + "pt", + "eta", + "phi", + "mass", + ] assert new["objects"]["lepton"]["features"] == ["pt", "eta", "phi", "mass", "isE"] - assert cfg["objects"]["jets"]["features"][-1] == "btag" # the input is not modified - assert legacy_config({"objects": {"jets": {"features": ["pt", "eta", "phi", "mass"], "max": 12}}})[1] == [] + assert cfg["objects"]["jets"]["features"][-1] == "btag" # the input is not modified + assert ( + legacy_config( + {"objects": {"jets": {"features": ["pt", "eta", "phi", "mass"], "max": 12}}} + )[1] + == [] + ) def _same_state(a, b) -> bool: @@ -52,12 +81,16 @@ def _same_state(a, b) -> bool: def test_new_checkpoint_has_format(tmp_path): cfg = read_config_dict(repo / "examples/hh4b/hh4b.yaml") model = CoReNet(parse_config(cfg), device="cpu") - path = save_checkpoint(tmp_path / "m.pkl", model, cfg, repo / "examples/hh4b/hh4b.yaml", {"fold": 0}) + path = save_checkpoint( + tmp_path / "m.pkl", model, cfg, repo / "examples/hh4b/hh4b.yaml", {"fold": 0} + ) ck = torch.load(path, map_location="cpu", weights_only=False) assert ck["format"] == CHECKPOINT_FORMAT == 2 reloaded, spec, _ = load_checkpoint(path) assert spec.collections["jets"].features == ["pt", "eta", "phi", "mass"] - model.setGhostBatches(0) # load_checkpoint sets the nGhostBatches buffers to 0 (eval mode) + model.setGhostBatches( + 0 + ) # load_checkpoint sets the nGhostBatches buffers to 0 (eval mode) assert _same_state(model, reloaded) @@ -65,16 +98,33 @@ def test_legacy_checkpoint_loads(tmp_path): path, old_cfg, model = _legacy_checkpoint(tmp_path) reloaded, spec, ck = load_checkpoint(path) assert "format" not in ck - assert spec.collections["jets"].features == ["pt", "eta", "phi", "mass"] # btag dropped: the trained network + assert spec.collections["jets"].features == [ + "pt", + "eta", + "phi", + "mass", + ] # btag dropped: the trained network model.setGhostBatches(0) assert _same_state(model, reloaded) # corenet-evaluate on it: the loader builds (N, 5, n) tensors from the translated configuration data = write_synthetic(tmp_path / "d.h5", "hh4b", n=300) out = tmp_path / "eval.h5" - metrics = evaluate.main(["--checkpoint", str(path), "--input", str(data), "--output", str(out), "--fold", "-1"]) + metrics = evaluate.main( + [ + "--checkpoint", + str(path), + "--input", + str(data), + "--output", + str(out), + "--fold", + "-1", + ] + ) assert metrics["n_events"] == 300 and out.exists() # the same legacy checkpoint with an explicit, untranslated configuration: a clear error import yaml + old = tmp_path / "old.yaml" old.write_text(yaml.safe_dump(old_cfg, sort_keys=False)) with pytest.raises(RuntimeError, match="predates format 2"): @@ -94,8 +144,15 @@ def test_shipped_legacy_runs(run): if not (path.exists() and data.exists()): pytest.skip(f"{path} or {data} not found") model, spec, ck = load_checkpoint(path) - assert "format" not in ck and spec.collections["jets"].features == ["pt", "eta", "phi", "mass"] - ds = H5Dataset(data, spec, rest_order=ck.get("metadata", {}).get("rest_order", "pt")) + assert "format" not in ck and spec.collections["jets"].features == [ + "pt", + "eta", + "phi", + "mass", + ] + ds = H5Dataset( + data, spec, rest_order=ck.get("metadata", {}).get("rest_order", "pt") + ) with torch.no_grad(): out = model({k: v[:256] for k, v in ds.inputs.items()}) assert out["logits"].shape == (256, 2) and torch.isfinite(out["logits"]).all() diff --git a/tests/test_config.py b/tests/test_config.py index ae4e457..5c0e20b 100644 --- a/tests/test_config.py +++ b/tests/test_config.py @@ -9,15 +9,24 @@ from corenet.models.config import ConfigError, parse_config HH4B = { - "objects": {"jets": {"features": ["pt", "eta", "phi", "mass"], "max": 12}, "b": "jets btag 4", "other": "jets rest 8"}, + "objects": { + "jets": {"features": ["pt", "eta", "phi", "mass"], "max": 12}, + "b": "jets btag 4", + "other": "jets rest 8", + }, "event": ["nSelJets", "HT"], "hypotheses": {"HH": {"H1": ["b", "b"], "H2": ["b", "b"], "context": "other"}}, "outputs": {"signal": "HH"}, "combine": ["HH"], } BBWW = { - "objects": {"jets": {"features": ["pt", "eta", "phi", "mass"], "max": 12}, "b": "jets btag 2", "q": "jets rest 2", - "lepton": {"features": ["pt", "eta", "phi", "mass", "isE", "isM"]}, "met": {"features": ["pt", "phi"]}}, + "objects": { + "jets": {"features": ["pt", "eta", "phi", "mass"], "max": 12}, + "b": "jets btag 2", + "q": "jets rest 2", + "lepton": {"features": ["pt", "eta", "phi", "mass", "isE", "isM"]}, + "met": {"features": ["pt", "phi"]}, + }, "event": ["nSelJets", "HT"], "hypotheses": { "HH": { @@ -66,12 +75,14 @@ def test_selector_columns_are_not_features(): assert s.selector_columns("jets") == ["btag", "pt"] assert s.columns_to_read("jets") == ["pt", "eta", "phi", "mass", "btag"] # btag listed as a feature is accepted and stays a feature (an ordinary extra input) - c = copy.deepcopy(HH4B); c["objects"]["jets"]["features"] = ["pt", "eta", "phi", "mass", "btag"] + c = copy.deepcopy(HH4B) + c["objects"]["jets"]["features"] = ["pt", "eta", "phi", "mass", "btag"] s = parse_config(c) assert s.collections["jets"].features == ["pt", "eta", "phi", "mass", "btag"] assert s.columns_to_read("jets") == ["pt", "eta", "phi", "mass", "btag"] # a pt selector reads pt, which a max > 1 collection always has as a feature - c = copy.deepcopy(HH4B); c["objects"]["b"] = "jets pt 4" + c = copy.deepcopy(HH4B) + c["objects"]["b"] = "jets pt 4" assert parse_config(c).columns_to_read("jets") == ["pt", "eta", "phi", "mass"] @@ -100,50 +111,110 @@ def test_bbww_parses(): def test_role_grammar(): """ [lowest]: any column of the file, highest first unless `lowest`; `rest` reserved.""" - c = copy.deepcopy(HH4B); c["objects"]["b"] = "jets mass 4" + c = copy.deepcopy(HH4B) + c["objects"]["b"] = "jets mass 4" s = parse_config(c) r = s.roles["b"] - assert (r.selector, r.n, r.ascending, r.columns) == ("mass", 4, False, ("mass", "pt")) - assert s.columns_to_read("jets") == ["pt", "eta", "phi", "mass"] # mass is a feature already - c = copy.deepcopy(HH4B); c["objects"]["b"] = "jets mass 4 lowest" + assert (r.selector, r.n, r.ascending, r.columns) == ( + "mass", + 4, + False, + ("mass", "pt"), + ) + assert s.columns_to_read("jets") == [ + "pt", + "eta", + "phi", + "mass", + ] # mass is a feature already + c = copy.deepcopy(HH4B) + c["objects"]["b"] = "jets mass 4 lowest" r = parse_config(c).roles["b"] assert (r.selector, r.ascending, r.columns) == ("mass", True, ("mass", "pt")) - c = copy.deepcopy(HH4B); c["objects"]["b"] = "jets dnn_score 4 lowest " + c = copy.deepcopy(HH4B) + c["objects"]["b"] = "jets dnn_score 4 lowest " r = parse_config(c).roles["b"] - assert (r.selector, r.ascending) == ("dnn_score", True) and parse_config(c).columns_to_read("jets")[-1] == "dnn_score" - c = copy.deepcopy(HH4B); c["objects"]["b"] = "jets pt 4" + assert (r.selector, r.ascending) == ("dnn_score", True) and parse_config( + c + ).columns_to_read("jets")[-1] == "dnn_score" + c = copy.deepcopy(HH4B) + c["objects"]["b"] = "jets pt 4" r = parse_config(c).roles["b"] assert (r.selector, r.ascending, r.columns) == ("pt", False, ("pt",)) - assert parse_config(HH4B).roles["other"].columns == ("pt",) and parse_config(HH4B).roles["b"].columns == ("btag", "pt") - for bad, msg in (("jets btag 4 highest", "expected ' [lowest]'"), - ("jets btag 4 Lowest", "expected ' [lowest]'"), - ("jets btag 4 lowest extra", "expected ' [lowest]'"), - ("jets rest 8 lowest", "'rest' takes no 'lowest'"), - ("jets btag", "expected ' [lowest]'"), - ("jets btag four", "expected ' [lowest]'")): - c = copy.deepcopy(HH4B); c["objects"]["b"] = bad; _fails(c, msg) + assert parse_config(HH4B).roles["other"].columns == ("pt",) and parse_config( + HH4B + ).roles["b"].columns == ("btag", "pt") + for bad, msg in ( + ("jets btag 4 highest", "expected ' [lowest]'"), + ("jets btag 4 Lowest", "expected ' [lowest]'"), + ("jets btag 4 lowest extra", "expected ' [lowest]'"), + ("jets rest 8 lowest", "'rest' takes no 'lowest'"), + ("jets btag", "expected ' [lowest]'"), + ("jets btag four", "expected ' [lowest]'"), + ): + c = copy.deepcopy(HH4B) + c["objects"]["b"] = bad + _fails(c, msg) def test_validation_rules(): - c = copy.deepcopy(HH4B); c["combine"] = []; _fails(c, "combine must list the main hypothesis") - c = copy.deepcopy(HH4B); c["objects"]["b"] = "jets btag 13"; _fails(c, "within 1..max") - c = copy.deepcopy(HH4B); c["objects"]["b"] = "jets btag 4 highest"; _fails(c, "expected ' [lowest]'") - c = copy.deepcopy(HH4B); c["objects"]["jets"]["features"] = ["eta", "phi", "mass"]; _fails(c, "need pt, eta, phi features") - c = copy.deepcopy(HH4B); c["hypotheses"]["HH"]["context"] = "nothing"; _fails(c, "is not a declared role") - c = copy.deepcopy(HH4B); c["hypotheses"]["HH"]["H1"] = ["b", "c"]; _fails(c, "neither a role") - c = copy.deepcopy(HH4B); c["hypotheses"]["ZZ"] = {"Z1": ["b", "b"], "Z2": ["b", "b"]}; _fails(c, "used by no output") - c = copy.deepcopy(HH4B); c["outputs"] = {"signal": "XX"}; _fails(c, "is not a declared hypothesis") - c = copy.deepcopy(HH4B); c["outputs"] = {"signal": None}; _fails(c, "main score) needs a hypothesis") - c = copy.deepcopy(HH4B); c["training"] = {"features": 8, "attention_heads": 3}; _fails(c, "divisible") - c = copy.deepcopy(HH4B); c["training"] = {"foo": 1}; _fails(c, "unknown keys") - c = copy.deepcopy(BBWW); c["hypotheses"]["HH"]["H_WW"]["W_lep"] = "lepton + met"; _fails(c, "the '+' form was removed; write [lepton, met]") - c = copy.deepcopy(BBWW); c["hypotheses"]["HH"]["H_WW"]["W_lep"] = "lepton met"; _fails(c, "cannot interpret") - c = copy.deepcopy(BBWW); c["hypotheses"]["tt"]["W_had"] = ["q", "b"]; _fails(c, "defined twice with different children") - c = copy.deepcopy(BBWW); c["attention"] = [{"W_lep": "nowhere"}]; _fails(c, "is not a node") - c = copy.deepcopy(BBWW); c["objects"]["q"] = "jets rest 11"; _fails(c, "slots > max") - c = copy.deepcopy(HH4B); del c["combine"]; _fails(c, "missing required section") - c = copy.deepcopy(HH4B); c["relation"] = []; _fails(c, "unknown top-level sections") - c = copy.deepcopy(HH4B); c["trainer"] = {"epochs": 3}; parse_config(c) # read by corenet-train only + c = copy.deepcopy(HH4B) + c["combine"] = [] + _fails(c, "combine must list the main hypothesis") + c = copy.deepcopy(HH4B) + c["objects"]["b"] = "jets btag 13" + _fails(c, "within 1..max") + c = copy.deepcopy(HH4B) + c["objects"]["b"] = "jets btag 4 highest" + _fails(c, "expected ' [lowest]'") + c = copy.deepcopy(HH4B) + c["objects"]["jets"]["features"] = ["eta", "phi", "mass"] + _fails(c, "need pt, eta, phi features") + c = copy.deepcopy(HH4B) + c["hypotheses"]["HH"]["context"] = "nothing" + _fails(c, "is not a declared role") + c = copy.deepcopy(HH4B) + c["hypotheses"]["HH"]["H1"] = ["b", "c"] + _fails(c, "neither a role") + c = copy.deepcopy(HH4B) + c["hypotheses"]["ZZ"] = {"Z1": ["b", "b"], "Z2": ["b", "b"]} + _fails(c, "used by no output") + c = copy.deepcopy(HH4B) + c["outputs"] = {"signal": "XX"} + _fails(c, "is not a declared hypothesis") + c = copy.deepcopy(HH4B) + c["outputs"] = {"signal": None} + _fails(c, "main score) needs a hypothesis") + c = copy.deepcopy(HH4B) + c["training"] = {"features": 8, "attention_heads": 3} + _fails(c, "divisible") + c = copy.deepcopy(HH4B) + c["training"] = {"foo": 1} + _fails(c, "unknown keys") + c = copy.deepcopy(BBWW) + c["hypotheses"]["HH"]["H_WW"]["W_lep"] = "lepton + met" + _fails(c, "the '+' form was removed; write [lepton, met]") + c = copy.deepcopy(BBWW) + c["hypotheses"]["HH"]["H_WW"]["W_lep"] = "lepton met" + _fails(c, "cannot interpret") + c = copy.deepcopy(BBWW) + c["hypotheses"]["tt"]["W_had"] = ["q", "b"] + _fails(c, "defined twice with different children") + c = copy.deepcopy(BBWW) + c["attention"] = [{"W_lep": "nowhere"}] + _fails(c, "is not a node") + c = copy.deepcopy(BBWW) + c["objects"]["q"] = "jets rest 11" + _fails(c, "slots > max") + c = copy.deepcopy(HH4B) + del c["combine"] + _fails(c, "missing required section") + c = copy.deepcopy(HH4B) + c["relation"] = [] + _fails(c, "unknown top-level sections") + c = copy.deepcopy(HH4B) + c["trainer"] = {"epochs": 3} + parse_config(c) # read by corenet-train only def test_relations_and_single_object_children(): @@ -168,16 +239,24 @@ def test_relations_and_single_object_children(): c["hypotheses"]["HH"]["H_WW"]["W_lep"] = ["l", "met"] c["relations"] = [["l", "met"]] s = parse_config(c) - assert s.nodes["W_lep"].children == ["l", "met"] and s.leaves()["l"].n == 1 and s.leaves()["met"].n == 1 - c = copy.deepcopy(BBWW); c["hypotheses"]["HH"]["H_WW"]["W_lep"] = "l + met"; _fails(c, "the '+' form was removed; write [l, met]") + assert ( + s.nodes["W_lep"].children == ["l", "met"] + and s.leaves()["l"].n == 1 + and s.leaves()["met"].n == 1 + ) + c = copy.deepcopy(BBWW) + c["hypotheses"]["HH"]["H_WW"]["W_lep"] = "l + met" + _fails(c, "the '+' form was removed; write [l, met]") def test_min_jets(): """min_jets (optional, default 4) sets the nSelJets offset; rejected unless an integer >= 2.""" assert parse_config(copy.deepcopy(HH4B)).min_jets == 4 - c = copy.deepcopy(HH4B); c["min_jets"] = 2 + c = copy.deepcopy(HH4B) + c["min_jets"] = 2 assert parse_config(c).min_jets == 2 for bad in (1, 0, "4", True, 4.0): - c = copy.deepcopy(HH4B); c["min_jets"] = bad + c = copy.deepcopy(HH4B) + c["min_jets"] = bad with pytest.raises(ConfigError): parse_config(c) diff --git a/tests/test_h5.py b/tests/test_h5.py index e36f8ad..2fff14b 100644 --- a/tests/test_h5.py +++ b/tests/test_h5.py @@ -43,7 +43,12 @@ def test_bbww_small(): ds = H5Dataset(path, spec) x, n = ds.inputs, len(ds) assert {k: tuple(v.shape) for k, v in x.items()} == { - "b": (n, 5, 2), "q": (n, 5, 2), "lepton": (n, 7, 1), "met": (n, 3, 1), "event": (n, 2)} + "b": (n, 5, 2), + "q": (n, 5, 2), + "lepton": (n, 7, 1), + "met": (n, 3, 1), + "event": (n, 2), + } assert all(v.dtype == torch.float32 for v in x.values()) _check_flags(x) assert (x["lepton"][:, -1] == 1).all() and (x["met"][:, -1] == 1).all(), ( @@ -77,9 +82,13 @@ def test_bbww_small(): :2 ] rest = sorted(set(present) - set(b_rank), key=lambda j: -jet["pt"][i, j])[:2] - assert np.array_equal(x["b"][i, 0, :len(b_rank)].numpy(), jet["pt"][i, b_rank]), i # btag-ranked, btag not a channel - assert np.array_equal(x["b"][i, 3, :len(b_rank)].numpy(), jet["mass"][i, b_rank]), i - assert np.array_equal(x["q"][i, 0, :len(rest)].numpy(), jet["pt"][i, rest]), i + assert np.array_equal( + x["b"][i, 0, : len(b_rank)].numpy(), jet["pt"][i, b_rank] + ), i # btag-ranked, btag not a channel + assert np.array_equal( + x["b"][i, 3, : len(b_rank)].numpy(), jet["mass"][i, b_rank] + ), i + assert np.array_equal(x["q"][i, 0, : len(rest)].numpy(), jet["pt"][i, rest]), i assert (ds.labels.numpy() == label).all() and set(np.unique(label)) == {1, 2} assert np.array_equal(ds.weights.numpy(), weight) and np.array_equal( ds.folds.numpy(), fold @@ -101,7 +110,11 @@ def test_hh4b_small(): for rest_order in ("pt", "selector"): ds = H5Dataset(path, spec, rest_order=rest_order) x, n = ds.inputs, len(ds) - assert x["b"].shape == (n, 5, 4) and x["other"].shape == (n, 5, 8) and x["event"].shape == (n, 2) + assert ( + x["b"].shape == (n, 5, 4) + and x["other"].shape == (n, 5, 8) + and x["event"].shape == (n, 2) + ) _check_flags(x) for i in range(N_CHECK): present = np.flatnonzero(mask[i]) @@ -125,9 +138,17 @@ def test_hh4b_small(): def _hh4b_cfg(features): - return {"objects": {"jets": {"features": features, "max": 12}, "b": "jets btag 4", "other": "jets rest 8"}, - "event": ["nSelJets", "HT"], "hypotheses": {"HH": {"H1": ["b", "b"], "H2": ["b", "b"], "context": "other"}}, - "outputs": {"signal": "HH"}, "combine": ["HH"]} + return { + "objects": { + "jets": {"features": features, "max": 12}, + "b": "jets btag 4", + "other": "jets rest 8", + }, + "event": ["nSelJets", "HT"], + "hypotheses": {"HH": {"H1": ["b", "b"], "H2": ["b", "b"], "context": "other"}}, + "outputs": {"signal": "HH"}, + "combine": ["HH"], + } def test_selector_column_read_without_feature(): @@ -144,34 +165,52 @@ def test_selector_column_read_without_feature(): for i in range(N_CHECK): present = np.flatnonzero(mask[i]) b = sorted(present, key=lambda j: (-jet["btag"][i, j], -jet["pt"][i, j]))[:4] - assert np.array_equal(x["b"][i, 0, :len(b)].numpy(), jet["pt"][i, b]), i + assert np.array_equal(x["b"][i, 0, : len(b)].numpy(), jet["pt"][i, b]), i # the same selection with btag as a pt-only selector would differ: the file is really read for btag - cfg_pt = _hh4b_cfg(["pt", "eta", "phi", "mass"]); cfg_pt["objects"]["b"] = "jets pt 4" + cfg_pt = _hh4b_cfg(["pt", "eta", "phi", "mass"]) + cfg_pt["objects"]["b"] = "jets pt 4" assert not torch.equal(H5Dataset(path, parse_config(cfg_pt)).inputs["b"], x["b"]) def test_btag_feature_is_an_extra_channel(): """(c) btag listed under features enters the tensor as an ordinary extra channel (and the network as an input).""" from corenet.models.network import CoReNet + path = _need(DATA / "hh4b_small.h5") - spec4, spec5 = parse_config(_hh4b_cfg(["pt", "eta", "phi", "mass"])), parse_config(_hh4b_cfg(["pt", "eta", "phi", "mass", "btag"])) + spec4, spec5 = ( + parse_config(_hh4b_cfg(["pt", "eta", "phi", "mass"])), + parse_config(_hh4b_cfg(["pt", "eta", "phi", "mass", "btag"])), + ) x4, x5 = H5Dataset(path, spec4).inputs, H5Dataset(path, spec5).inputs n = len(x4["event"]) assert x5["b"].shape == (n, 6, 4) and x5["other"].shape == (n, 6, 8) - assert torch.equal(x5["b"][:, :4], x4["b"][:, :4]) and torch.equal(x5["b"][:, -1], x4["b"][:, -1]) # same jets - assert set(np.unique(x5["b"][:, 4][x5["b"][:, -1] > 0].numpy())) <= {0.0, 1.0} # the btag column + assert torch.equal(x5["b"][:, :4], x4["b"][:, :4]) and torch.equal( + x5["b"][:, -1], x4["b"][:, -1] + ) # same jets + assert set(np.unique(x5["b"][:, 4][x5["b"][:, -1] > 0].numpy())) <= { + 0.0, + 1.0, + } # the btag column with h5py.File(path, "r") as f: mask, pt, btag = (f[f"INPUTS/Source/{k}"][...] for k in ("MASK", "pt", "btag")) for i in range(N_CHECK): present = np.flatnonzero(mask[i]) b = sorted(present, key=lambda j: (-btag[i, j], -pt[i, j]))[:4] - assert np.array_equal(x5["b"][i, 4, :len(b)].numpy(), btag[i, b]), i + assert np.array_equal(x5["b"][i, 4, : len(b)].numpy(), btag[i, b]), i m4, m5 = CoReNet(spec4, device="cpu"), CoReNet(spec5, device="cpu") - assert m4._extras("b") == [] and m5._extras("b") == ["btag"] and m5._extras("other") == ["btag"] + assert ( + m4._extras("b") == [] + and m5._extras("b") == ["btag"] + and m5._extras("other") == ["btag"] + ) n4, n5 = (sum(p.numel() for p in m.parameters()) for m in (m4, m5)) assert n4 == 1508 and n5 > n4, (n4, n5) batch = {k: v[:256] for k, v in x5.items()} - m5.train(); m5.updateMeanStd(batch); m5.initMeanStd(); m5.eval(); m5.setGhostBatches(0) + m5.train() + m5.updateMeanStd(batch) + m5.initMeanStd() + m5.eval() + m5.setGhostBatches(0) with torch.no_grad(): out = m5(batch) assert out["logits"].shape == (256, 2) and torch.isfinite(out["logits"]).all() @@ -182,8 +221,14 @@ def _brute_force(mask, cols, pt, column, n, ascending, taken=None): """Per-event python sort: available members by the column (lowest first when ascending), ties by highest pt.""" out = [] for i in range(len(mask)): - avail = [j for j in np.flatnonzero(mask[i]) if taken is None or j not in taken[i]] - key = (lambda j: (cols[i, j], -pt[i, j])) if ascending else (lambda j: (-cols[i, j], -pt[i, j])) + avail = [ + j for j in np.flatnonzero(mask[i]) if taken is None or j not in taken[i] + ] + key = ( + (lambda j: (cols[i, j], -pt[i, j])) + if ascending + else (lambda j: (-cols[i, j], -pt[i, j])) + ) out.append(sorted(avail, key=key)[:n]) return out @@ -191,49 +236,82 @@ def _brute_force(mask, cols, pt, column, n, ascending, taken=None): def test_generic_selectors_synthetic(tmp_path): """Selection by any column, highest or lowest first, against a brute-force per-event sort.""" from synthetic import write_synthetic + path = write_synthetic(tmp_path / "s.h5", "hh4b", n=600) with h5py.File(path, "r") as f: mask = f["INPUTS/Source/MASK"][...] - jet = {k: f[f"INPUTS/Source/{k}"][...] for k in ("pt", "eta", "phi", "mass", "btag")} - for line, column, n, asc in (("jets mass 3", "mass", 3, False), ("jets mass 2 lowest", "mass", 2, True), - ("jets btag 4", "btag", 4, False), ("jets pt 4 lowest", "pt", 4, True)): + jet = { + k: f[f"INPUTS/Source/{k}"][...] + for k in ("pt", "eta", "phi", "mass", "btag") + } + for line, column, n, asc in ( + ("jets mass 3", "mass", 3, False), + ("jets mass 2 lowest", "mass", 2, True), + ("jets btag 4", "btag", 4, False), + ("jets pt 4 lowest", "pt", 4, True), + ): cfg = _hh4b_cfg(["pt", "eta", "phi", "mass"]) - cfg["objects"] = {"jets": {"features": ["pt", "eta", "phi", "mass"], "max": 12}, "b": line, "other": "jets rest 8"} + cfg["objects"] = { + "jets": {"features": ["pt", "eta", "phi", "mass"], "max": 12}, + "b": line, + "other": "jets rest 8", + } ds = H5Dataset(path, parse_config(cfg)) x = ds.inputs assert x["b"].shape == (600, 5, n) and x["other"].shape == (600, 5, 8) want = _brute_force(mask, jet[column], jet["pt"], column, n, asc) for i in range(600): b = want[i] - assert np.array_equal(x["b"][i, 0, :len(b)].numpy(), jet["pt"][i, b]), (line, i) - assert np.array_equal(x["b"][i, 1, :len(b)].numpy(), jet["eta"][i, b]), (line, i) - assert (x["b"][i, -1, len(b):] == -1).all() - rest = sorted(set(np.flatnonzero(mask[i])) - set(b), key=lambda j: -jet["pt"][i, j])[:8] - assert np.array_equal(x["other"][i, 0, :len(rest)].numpy(), jet["pt"][i, rest]), (line, i) + assert np.array_equal(x["b"][i, 0, : len(b)].numpy(), jet["pt"][i, b]), ( + line, + i, + ) + assert np.array_equal(x["b"][i, 1, : len(b)].numpy(), jet["eta"][i, b]), ( + line, + i, + ) + assert (x["b"][i, -1, len(b) :] == -1).all() + rest = sorted( + set(np.flatnonzero(mask[i])) - set(b), key=lambda j: -jet["pt"][i, j] + )[:8] + assert np.array_equal( + x["other"][i, 0, : len(rest)].numpy(), jet["pt"][i, rest] + ), (line, i) # rest_order="selector" continues the previous line's order, lowest first included cfg = _hh4b_cfg(["pt", "eta", "phi", "mass"]) - cfg["objects"] = {"jets": {"features": ["pt", "eta", "phi", "mass"], "max": 12}, "b": "jets mass 2 lowest", "other": "jets rest 8"} + cfg["objects"] = { + "jets": {"features": ["pt", "eta", "phi", "mass"], "max": 12}, + "b": "jets mass 2 lowest", + "other": "jets rest 8", + } x = H5Dataset(path, parse_config(cfg), rest_order="selector").inputs for i in range(600): order = _brute_force(mask, jet["mass"], jet["pt"], "mass", 12, True)[i] rest = order[2:10] - assert np.array_equal(x["other"][i, 0, :len(rest)].numpy(), jet["pt"][i, rest]), i + assert np.array_equal( + x["other"][i, 0, : len(rest)].numpy(), jet["pt"][i, rest] + ), i -@pytest.mark.parametrize("name,group", [("hh4b_small.h5", "Source"), ("bbww_small.h5", "Jets")]) +@pytest.mark.parametrize( + "name,group", [("hh4b_small.h5", "Source"), ("bbww_small.h5", "Jets")] +) def test_lexicographic_order_matches_old_key(name, group): """The lexicographic (column, pt) sort gives the same btag selection as the former weighted key -(btag * 1e6 + pt) on every event of the example files.""" path = _need(DATA / name) with h5py.File(path, "r") as f: mask = f[f"INPUTS/{group}/MASK"][...] - pt, btag = (f[f"INPUTS/{group}/{k}"][...].astype(np.float32) for k in ("pt", "btag")) + pt, btag = ( + f[f"INPUTS/{group}/{k}"][...].astype(np.float32) for k in ("pt", "btag") + ) old = np.argsort(np.where(mask, -(btag * 1e6 + pt), np.inf), axis=1, kind="stable") from corenet.data.h5 import _take + new, _ = _take((-btag, -pt), mask, mask.shape[1]) n_avail = mask.sum(1) for i in range(len(mask)): - assert np.array_equal(old[i, :n_avail[i]], new[i, :n_avail[i]]), i + assert np.array_equal(old[i, : n_avail[i]], new[i, : n_avail[i]]), i def test_missing_selector_column(tmp_path): @@ -241,23 +319,119 @@ def test_missing_selector_column(tmp_path): src = _need(DATA / "hh4b_small.h5") dst = tmp_path / "nobtag.h5" with h5py.File(src, "r") as f, h5py.File(dst, "w") as g: - for key in ("INPUTS/Source/MASK", "INPUTS/Source/pt", "INPUTS/Source/eta", "INPUTS/Source/phi", "INPUTS/Source/mass", - "CLASSIFICATIONS/EVENT/signal", "EVENT/weight", "EVENT/fold"): + for key in ( + "INPUTS/Source/MASK", + "INPUTS/Source/pt", + "INPUTS/Source/eta", + "INPUTS/Source/phi", + "INPUTS/Source/mass", + "CLASSIFICATIONS/EVENT/signal", + "EVENT/weight", + "EVENT/fold", + ): g.create_dataset(key, data=f[key][:500]) - with pytest.raises(KeyError, match=r"missing columns \['btag'\].*selector columns \['btag'\]"): + with pytest.raises( + KeyError, match=r"missing columns \['btag'\].*selector columns \['btag'\]" + ): H5Dataset(dst, parse_config(_hh4b_cfg(["pt", "eta", "phi", "mass"]))) - cfg = _hh4b_cfg(["pt", "eta", "phi", "mass"]); cfg["objects"]["b"] = "jets pt 4" - assert H5Dataset(dst, parse_config(cfg)).inputs["b"].shape == (500, 5, 4) # pt selector: no btag needed + cfg = _hh4b_cfg(["pt", "eta", "phi", "mass"]) + cfg["objects"]["b"] = "jets pt 4" + assert H5Dataset(dst, parse_config(cfg)).inputs["b"].shape == ( + 500, + 5, + 4, + ) # pt selector: no btag needed def test_errors(tmp_path): path = _need(DATA / "bbww_small.h5") - cfg = {"objects": {"jets": {"features": ["pt", "eta", "phi", "mass"], "max": 12}, "b": "jets btag 2", - "photon": {"features": ["pt", "eta", "phi"]}}, - "hypotheses": {"HH": {"H_bb": ["b", "b"], "X": ["b", "photon"]}}, "outputs": {"signal": "HH"}, "combine": ["HH"]} + cfg = { + "objects": { + "jets": {"features": ["pt", "eta", "phi", "mass"], "max": 12}, + "b": "jets btag 2", + "photon": {"features": ["pt", "eta", "phi"]}, + }, + "hypotheses": {"HH": {"H_bb": ["b", "b"], "X": ["b", "photon"]}}, + "outputs": {"signal": "HH"}, + "combine": ["HH"], + } with pytest.raises(KeyError, match="photon"): H5Dataset(path, parse_config(cfg)) - cfg = {"objects": {"jets": {"features": ["pt", "eta", "phi", "mass"], "max": 12}, "b": "jets btag 4"}, - "hypotheses": {"HH": {"H1": ["b", "b"], "H2": ["b", "b"]}}, "outputs": {"signal": "HH"}, "combine": ["HH"]} - with pytest.raises(ValueError, match="labels"): # bbWW labels 1, 2 with a single output + cfg = { + "objects": { + "jets": {"features": ["pt", "eta", "phi", "mass"], "max": 12}, + "b": "jets btag 4", + }, + "hypotheses": {"HH": {"H1": ["b", "b"], "H2": ["b", "b"]}}, + "outputs": {"signal": "HH"}, + "combine": ["HH"], + } + with pytest.raises( + ValueError, match="labels" + ): # bbWW labels 1, 2 with a single output H5Dataset(path, parse_config(cfg)) + + +def test_fold_rule(): + """fold_of = event % n_folds; the default loader reads EVENT/fold; the two differ on the shipped files.""" + from corenet.data.h5 import FOLD_RULE_COLUMN, FOLD_RULE_EVENT, fold_of + + path = _need(DATA / "hh4b_small.h5") + spec = load_config(repo / "examples/hh4b/hh4b.yaml") + with h5py.File(path, "r") as f: + event, column = f["EVENT/event"][...], f["EVENT/fold"][...] + default = H5Dataset(path, spec) + assert default.fold_rule == FOLD_RULE_COLUMN and default.n_folds == 5 + assert (default.folds.numpy() == column).all() + ds5 = H5Dataset(path, spec, n_folds=5) + assert ds5.fold_rule == FOLD_RULE_EVENT and ds5.n_folds == 5 + assert (ds5.folds.numpy() == event % 5).all() + assert (ds5.folds.numpy() == fold_of(event, 5)).all() + assert set(ds5.folds.tolist()) == set(range(5)) + # the EVENT/fold column of the example files was assigned from a hash of the event number when + # the files were produced (docs/Dataset.md): it is not event % 5, so the default (column) and the + # rule give different folds on these files + assert not (ds5.folds.numpy() == column).all() + ds3 = H5Dataset(path, spec, n_folds=3) + assert ds3.n_folds == 3 and set(ds3.folds.tolist()) == set(range(3)) + assert (ds3.folds.numpy() == event % 3).all() + tr, te = ds3.fold_indices(0) + assert len(tr) + len(te) == len(ds3) and (event[te] % 3 == 0).all() + assert torch.equal( + ds3.inputs["b"], default.inputs["b"] + ) # the rule changes folds only + with pytest.raises(ValueError, match="negative"): + fold_of(np.array([3, -1]), 5) + with pytest.raises(ValueError, match="n_folds"): + fold_of(event[:10], 0) + with pytest.raises(ValueError, match="integers"): + fold_of(event[:10].astype(np.float64), 5) + + +def test_missing_fold_column(tmp_path): + """Without EVENT/fold the loader asks for --n-folds; with it the folds come from EVENT/event.""" + src = _need(DATA / "hh4b_small.h5") + spec = load_config(repo / "examples/hh4b/hh4b.yaml") + dst = tmp_path / "nofold.h5" + with h5py.File(src, "r") as f, h5py.File(dst, "w") as g: + for key in ( + "INPUTS/Source/MASK", + "INPUTS/Source/pt", + "INPUTS/Source/eta", + "INPUTS/Source/phi", + "INPUTS/Source/mass", + "INPUTS/Source/btag", + "CLASSIFICATIONS/EVENT/signal", + "EVENT/weight", + "EVENT/event", + ): + g.create_dataset(key, data=f[key][:500]) + event = f["EVENT/event"][:500] + with pytest.raises(KeyError, match="no EVENT/fold dataset; pass --n-folds"): + H5Dataset(dst, spec) + ds = H5Dataset(dst, spec, n_folds=5) + assert ds.n_folds == 5 and (ds.folds.numpy() == event % 5).all() + with h5py.File(dst, "a") as g: + del g["EVENT/event"] + with pytest.raises(KeyError, match="no EVENT/event dataset"): + H5Dataset(dst, spec, n_folds=5) diff --git a/tests/test_network_bbww.py b/tests/test_network_bbww.py index ed1e60e..35d4f33 100644 --- a/tests/test_network_bbww.py +++ b/tests/test_network_bbww.py @@ -31,7 +31,9 @@ def jets(k): phi = math.tau * torch.rand(n, k, generator=g) - math.pi m = 5 + 20 * torch.rand(n, k, generator=g) flag = torch.ones(n, k) - return torch.stack([pt, eta, phi, m, flag], dim=1) # (n, 4+1, k): the features + flag + return torch.stack( + [pt, eta, phi, m, flag], dim=1 + ) # (n, 4+1, k): the features + flag b, q = jets(2), jets(2) if pad_q: # second q jet missing @@ -199,10 +201,20 @@ def test_padded_leaves_stay_finite(): def test_relation_needs_unique_candidates(): cfg = { - "objects": {"jets": {"features": ["pt", "eta", "phi", "mass"], "max": 12}, "b": "jets btag 2", "q": "jets rest 2", - "lepton": {"features": ["pt", "eta", "phi", "mass"]}, "met": {"features": ["pt", "phi"]}}, - "hypotheses": {"HH": {"H_bb": ["b", "b"], "H_WW": {"W_had": ["q", "q"], "W_lep": ["lepton", "met"]}}, - "tt": {"t_had": ["b", "W_had"], "t_lep": ["b", "W_lep"]}}, + "objects": { + "jets": {"features": ["pt", "eta", "phi", "mass"], "max": 12}, + "b": "jets btag 2", + "q": "jets rest 2", + "lepton": {"features": ["pt", "eta", "phi", "mass"]}, + "met": {"features": ["pt", "phi"]}, + }, + "hypotheses": { + "HH": { + "H_bb": ["b", "b"], + "H_WW": {"W_had": ["q", "q"], "W_lep": ["lepton", "met"]}, + }, + "tt": {"t_had": ["b", "W_had"], "t_lep": ["b", "W_lep"]}, + }, "relations": [["t_had", "met"]], "outputs": {"signal": "HH", "ttbar": "tt"}, "combine": ["HH", "tt"], @@ -290,9 +302,18 @@ def test_edge_with_many_keys_is_attention(): """An edge whose queries have several admissible keys stays real attention: HH -> 4b dijets attending to the other dijets (6 candidates, diagonal masked -> 5 keys each).""" # btag listed as a feature here: it is one more input channel (the 5th of the 6 stacked below) - cfg = {"objects": {"jets": {"features": ["pt", "eta", "phi", "mass", "btag"], "max": 12}, "b": "jets btag 4", "other": "jets rest 8"}, - "event": ["nSelJets", "HT"], "hypotheses": {"HH": {"H1": ["b", "b"], "H2": ["b", "b"], "context": "other"}}, - "attention": [{"H1": "H1"}], "outputs": {"signal": "HH"}, "combine": ["HH"]} + cfg = { + "objects": { + "jets": {"features": ["pt", "eta", "phi", "mass", "btag"], "max": 12}, + "b": "jets btag 4", + "other": "jets rest 8", + }, + "event": ["nSelJets", "HT"], + "hypotheses": {"HH": {"H1": ["b", "b"], "H2": ["b", "b"], "context": "other"}}, + "attention": [{"H1": "H1"}], + "outputs": {"signal": "HH"}, + "combine": ["HH"], + } m = CoReNet(parse_config(cfg), "cpu") assert "H1>H1" in m.attention and not m.merge same = m._same("H1>H1")[0] diff --git a/tests/test_train.py b/tests/test_train.py index b9a25f1..244df2c 100644 --- a/tests/test_train.py +++ b/tests/test_train.py @@ -5,7 +5,9 @@ 2. the prediction file has the documented layout, normalised probabilities, the held-out events, 3. the freeze leaves exactly the output-head parameters trainable, parameter counts 1,508 / 3,832, 4. the checkpoint is self-contained and corenet-evaluate reproduces the training-time predictions, - 5. the `trainer:` section of the configuration sets the recipe and the command line overrides it. + 5. the `trainer:` section of the configuration sets the recipe and the command line overrides it, + 6. the fine-tuning stage (finetune_layers / finetune_lr / finetune_batch_size) and its defaults, + 7. the fold rule (--n-folds, --exclude-folds next) and its record in checkpoint and predictions. """ from __future__ import annotations @@ -16,6 +18,7 @@ import h5py import numpy as np +import pytest import yaml from synthetic import write_synthetic @@ -253,28 +256,343 @@ def test_train_and_evaluate() -> None: def test_early_stopping() -> None: """--early-stop-patience: stop after N epochs without a better validation AUC and keep the best epoch.""" import torch + with tempfile.TemporaryDirectory(prefix="corenet_es_") as tmp: tmp = Path(tmp) data = write_synthetic(tmp / "hh4b_all.h5", "hh4b") out_dir = tmp / "m" - train.main(["--config", str(HH4B), "--input", str(data), "--output-dir", str(out_dir), "--fold", "0", - "--exclude-folds", "1", "--epochs", "12", "--batch-size-init", "128", "--early-stop-patience", "1", - "--device", "cpu"]) + train.main( + [ + "--config", + str(HH4B), + "--input", + str(data), + "--output-dir", + str(out_dir), + "--fold", + "0", + "--exclude-folds", + "1", + "--epochs", + "12", + "--batch-size-init", + "128", + "--early-stop-patience", + "1", + "--device", + "cpu", + ] + ) h = json.loads((out_dir / "history_fold0.json").read_text()) aucs = [e["val_auc"] for e in h["history"]] best = max(range(len(aucs)), key=lambda i: aucs[i]) assert h["best_epoch"] == best and h["early_stop_patience"] == 1 - if h["stopped_after_epoch"] is not None: # stopped: one non-improving epoch after the best + if ( + h["stopped_after_epoch"] is not None + ): # stopped: one non-improving epoch after the best assert h["stopped_after_epoch"] == best + 1 and len(aucs) == best + 2 else: assert len(aucs) == 12 - ck = torch.load(out_dir / "model_fold0.pkl", map_location="cpu", weights_only=False) - assert ck["metadata"]["epoch"] == best and ck["metadata"]["early_stop_patience"] == 1 + ck = torch.load( + out_dir / "model_fold0.pkl", map_location="cpu", weights_only=False + ) + assert ( + ck["metadata"]["epoch"] == best + and ck["metadata"]["early_stop_patience"] == 1 + ) # without a validation fold the option is refused try: - train.main(["--config", str(HH4B), "--input", str(data), "--output-dir", str(tmp / "n"), "--fold", "0", - "--epochs", "2", "--early-stop-patience", "1", "--device", "cpu"]) + train.main( + [ + "--config", + str(HH4B), + "--input", + str(data), + "--output-dir", + str(tmp / "n"), + "--fold", + "0", + "--epochs", + "2", + "--early-stop-patience", + "1", + "--device", + "cpu", + ] + ) except SystemExit: pass else: raise AssertionError("expected an error without --exclude-folds") + + +def _train(data: Path, out_dir: Path, *extra: str, epochs: int = 2) -> dict: + """corenet-train on the synthetic hh4b file (fold 0 held out, fold 1 validation, freeze at epoch 1).""" + train.main( + [ + "--config", + str(HH4B), + "--input", + str(data), + "--output-dir", + str(out_dir), + "--fold", + "0", + "--exclude-folds", + "1", + "--epochs", + str(epochs), + "--batch-size-init", + "128", + "--freeze-epoch", + "1", + "--device", + "cpu", + *extra, + ] + ) + return json.loads((out_dir / "history_fold0.json").read_text()) + + +def _metadata(out_dir: Path, fold: int = 0) -> dict: + import torch + + return torch.load( + out_dir / f"model_fold{fold}.pkl", map_location="cpu", weights_only=False + )["metadata"] + + +def test_finetune_layers() -> None: + """The default fine-tuning stage equals the explicit head prefixes bit for bit; `none` freezes nothing.""" + with tempfile.TemporaryDirectory(prefix="corenet_ft_") as tmp: + tmp = Path(tmp) + data = write_synthetic(tmp / "hh4b_all.h5", "hh4b") + default = _train(data, tmp / "default") + explicit = _train( + data, tmp / "explicit", "--finetune-layers", *train.HEAD_PREFIXES + ) + assert default["history"] == explicit["history"] + assert default["recipe"] == explicit["recipe"] + assert default["recipe"]["finetune_layers"] == list(train.HEAD_PREFIXES) + assert ( + default["recipe"]["finetune_lr"] is None + and default["recipe"]["finetune_batch_size"] is None + ) + fresh = CoReNet(load_config(HH4B), device="cpu") + n_all = train.count_trainable( + fresh + ) # some parameters are non-trainable by construction + n_heads = train.freeze_heads(fresh) + assert 0 < n_heads < n_all < evaluate.count_parameters(fresh) + assert default["finetune_trainable_parameters"] == n_heads + meta = _metadata(tmp / "default") + assert meta["finetune_trainable_parameters"] == n_heads + assert meta["finetune_layers"] == list(train.HEAD_PREFIXES) + assert meta["finetune_lr"] is None and meta["finetune_batch_size"] is None + # `none`: nothing is frozen, every trainable parameter of a fresh model stays trainable + none = _train(data, tmp / "none", "--finetune-layers", "none") + assert none["recipe"]["finetune_layers"] == [] + assert none["finetune_trainable_parameters"] == n_all + assert _metadata(tmp / "none")["finetune_trainable_parameters"] == n_all + assert ( + none["history"][0] == default["history"][0] + ) # identical before the freeze + assert none["history"][1]["train_loss"] != default["history"][1]["train_loss"] + with pytest.raises(ConfigError, match="'none' cannot be combined"): + _train(data, tmp / "bad", "--finetune-layers", "none", "head_main.") + + +def test_finetune_lr_and_batch_size() -> None: + """--finetune-lr / --finetune-batch-size apply from the freeze epoch; the milestones continue from the lr.""" + with tempfile.TemporaryDirectory(prefix="corenet_ft_") as tmp: + tmp = Path(tmp) + data = write_synthetic(tmp / "hh4b_all.h5", "hh4b") + h = _train( + data, + tmp / "m", + "--finetune-lr", + "1e-4", + "--finetune-batch-size", + "256", + "--lr-milestones", + "3", + epochs=3, + ) + e0, e1, e2 = h["history"] + assert ( + e0["batch_size"] == 128 and e0["ghost_batches"] == 16 + ) # schedule: 128 // 8 + assert e0["lr"] == 1e-2 + assert ( + e1["batch_size"] == 256 and e1["ghost_batches"] == 16 + ) # schedule: min(16, 256 // 8) + assert e1["lr"] == 1e-4 + assert e2["batch_size"] == 256 and e2["ghost_batches"] == 16 + assert e2["lr"] == pytest.approx( + 1e-4 * 0.25 + ) # milestone 3 multiplies the fine-tuning lr + assert h["recipe"]["finetune_lr"] == 1e-4 + assert h["recipe"]["finetune_batch_size"] == 256 + meta = _metadata(tmp / "m") + assert meta["finetune_lr"] == 1e-4 and meta["finetune_batch_size"] == 256 + assert ( + meta["finetune_trainable_parameters"] == h["finetune_trainable_parameters"] + ) + assert train.finetune_batch(10, 1024, None) == (16384, 0) + assert train.finetune_batch(10, 1024, 256) == (256, 0) # 0 ghost batches stay 0 + assert train.finetune_batch(1, 1024, 256) == (256, 16) + assert train.finetune_batch(1, 1024, 32) == (32, 4) + assert train.finetune_batch(0, 1024, 4) == (4, 1) + + +def test_finetune_recipe_from_config() -> None: + """The three fine-tuning keys in the trainer: section, and their validation.""" + cfg = yaml.safe_load(HH4B.read_text()) + cfg["trainer"] = { + "finetune_layers": ["head_main.", "resnet."], + "finetune_lr": 0.0001, + "finetune_batch_size": 256, + } + args = train.build_parser().parse_args(["--config", "x", "--input", "y"]) + train.resolve_recipe(args, cfg) + assert args.finetune_layers == ["head_main.", "resnet."] + assert args.finetune_lr == 0.0001 and args.finetune_batch_size == 256 + args = train.build_parser().parse_args( + ["--config", "x", "--input", "y", "--finetune-layers", "none"] + ) + train.resolve_recipe(args, cfg) # the command line wins + assert args.finetune_layers == [] + cfg["trainer"] = {"finetune_layers": "none"} + args = train.build_parser().parse_args(["--config", "x", "--input", "y"]) + assert train.resolve_recipe(args, cfg).finetune_layers == [] + for bad, message in [ + ({"finetune_layers": ["none", "head_main."]}, "cannot be combined"), + ({"finetune_layers": [1]}, "finetune_layers must be a list"), + ({"finetune_lr": 0.0}, "finetune_lr must be a number > 0"), + ({"finetune_lr": "1e-4"}, "finetune_lr must be a number > 0"), + ({"finetune_batch_size": 0}, "finetune_batch_size must be an integer >= 1"), + ]: + cfg["trainer"] = bad + with pytest.raises(ConfigError, match=message): + train.resolve_recipe( + train.build_parser().parse_args(["--config", "x", "--input", "y"]), cfg + ) + + +def test_n_folds_rule() -> None: + """--n-folds K: folds = event % K; --exclude-folds next; the rule is stored and reused by corenet-evaluate.""" + from corenet.data.h5 import FOLD_RULE_COLUMN, FOLD_RULE_EVENT + + assert train.resolve_exclude_folds(["next"], 4, 5) == [0] + assert train.resolve_exclude_folds([2, "next"], 0, 3) == [2, 1] + with tempfile.TemporaryDirectory(prefix="corenet_nf_") as tmp: + tmp = Path(tmp) + data = write_synthetic( + tmp / "hh4b_all.h5", "hh4b" + ) # events 0..N-1, 5 random folds + out_dir = tmp / "m" + train.main( + [ + "--config", + str(HH4B), + "--input", + str(data), + "--output-dir", + str(out_dir), + "--n-folds", + "3", + "--fold", + "0", + "--exclude-folds", + "next", + "--epochs", + "1", + "--batch-size-init", + "128", + "--device", + "cpu", + ] + ) + h = json.loads((out_dir / "history_fold0.json").read_text()) + assert h["exclude_folds"] == [1] and h["n_folds"] == 3 + assert h["fold_rule"] == FOLD_RULE_EVENT == "event % n_folds" + meta = _metadata(out_dir) + assert meta["n_folds"] == 3 and meta["fold_rule"] == FOLD_RULE_EVENT + assert meta["exclude_folds"] == [1] + with h5py.File(data, "r") as f: + event = f["EVENT/event"][...] + n_test, n_val = int((event % 3 == 0).sum()), int((event % 3 == 1).sum()) + assert h["n_val"] == n_val and h["n_train"] == len(event) - n_test - n_val + with h5py.File(out_dir / "predictions_fold0.h5", "r") as f: + assert f.attrs["n_folds"] == 3 and f.attrs["fold_rule"] == FOLD_RULE_EVENT + assert (f["EVENT/event"][...] % 3 == 0).all() and len( + f["EVENT/event"] + ) == n_test + assert (f["EVENT/fold"][...] == 0).all() + # corenet-evaluate applies the checkpoint's rule, also to a file without the column + with h5py.File(data, "a") as f: + del f["EVENT/fold"] + metrics = evaluate.main( + [ + "--checkpoint", + str(out_dir / "model_fold0.pkl"), + "--input", + str(data), + "--output", + str(tmp / "eval.h5"), + ] + ) + assert metrics["n_events"] == n_test + with h5py.File(tmp / "eval.h5", "r") as f: + assert f.attrs["n_folds"] == 3 and f.attrs["fold_rule"] == FOLD_RULE_EVENT + assert evaluate.main( + [ + "--checkpoint", + str(out_dir / "model_fold0.pkl"), + "--input", + str(data), + "--output", + str(tmp / "eval5.h5"), + "--n-folds", + "5", + ] + )["n_events"] == int((event % 5 == 0).sum()) + # a checkpoint trained on the column keeps the column rule and needs it in the file + with_column = write_synthetic(tmp / "col.h5", "hh4b") + train.main( + [ + "--config", + str(HH4B), + "--input", + str(with_column), + "--output-dir", + str(tmp / "c"), + "--fold", + "4", + "--exclude-folds", + "next", + "--epochs", + "1", + "--batch-size-init", + "128", + "--device", + "cpu", + ] + ) + meta = _metadata(tmp / "c", fold=4) + assert ( + meta["n_folds"] == 5 + and meta["fold_rule"] == FOLD_RULE_COLUMN == "EVENT/fold" + ) + assert meta["exclude_folds"] == [0] + with pytest.raises(KeyError, match="no EVENT/fold dataset"): + evaluate.main( + [ + "--checkpoint", + str(tmp / "c/model_fold4.pkl"), + "--input", + str(data), + "--output", + str(tmp / "eval_c.h5"), + ] + ) diff --git a/tests/test_validate.py b/tests/test_validate.py index 851f049..6479fe8 100644 --- a/tests/test_validate.py +++ b/tests/test_validate.py @@ -43,8 +43,24 @@ def test_valid_synthetic(tmp_path, topology): assert "selected by `rest` (reads pt for the selection)" in msgs assert "['btag'] read for the selection only (not network inputs)" in msgs assert "features ['pt', 'eta', 'phi', 'mass'] (the network inputs)" in msgs - assert ("leaf `b`: tensor (400, 5, 4)" if topology == "hh4b" else "leaf `b`: tensor (400, 5, 2)") in msgs - assert validate.main(["--config", str(cfg), "--input", str(data), "--json", str(tmp_path / "r.json")]) == 0 + assert ( + "leaf `b`: tensor (400, 5, 4)" + if topology == "hh4b" + else "leaf `b`: tensor (400, 5, 2)" + ) in msgs + assert ( + validate.main( + [ + "--config", + str(cfg), + "--input", + str(data), + "--json", + str(tmp_path / "r.json"), + ] + ) + == 0 + ) @pytest.mark.parametrize( @@ -81,18 +97,54 @@ def _bbww(): ), (lambda c: c["objects"]["jets"].__setitem__("max", "12"), "max must be an integer"), (lambda c: c["objects"].__setitem__("b", "jet btag 2"), "unknown collection 'jet'"), - (lambda c: c["objects"].__setitem__("b", "jets btag 2 top"), "expected ' [lowest]', got 'jets btag 2 top'"), - (lambda c: c["objects"].__setitem__("q", "jets rest 2 lowest"), "'rest' takes no 'lowest'"), - (lambda c: c["hypotheses"]["tt"].__setitem__("t_had", ["bb", "W_had"]), "child 'bb' is neither"), - (lambda c: c["hypotheses"]["HH"]["H_WW"].__setitem__("W_lep", "lepton + met"), "the '+' form was removed; write [lepton, met]"), - (lambda c: c.__setitem__("attention", [{"W_lep": "W_hadronic"}]), "'W_hadronic' is not a node"), - (lambda c: c.__setitem__("relations", [["lepton", "neutrino"]]), "'neutrino' is neither a node nor a leaf"), - (lambda c: c.__setitem__("relations", [["t_had", "met"]]), "t_had has 2 candidates"), - (lambda c: c.__setitem__("combine", ["tt"]), "combine must list the main hypothesis 'HH'"), - (lambda c: c.__setitem__("combine", ["HH", "HH"]), "combine: duplicate entries ['HH']"), - (lambda c: c["outputs"].__setitem__("ttbar", "ttbar"), "outputs.ttbar: 'ttbar' is not a declared hypothesis"), - (lambda c: c["training"].__setitem__("features", "8"), "training.features must be an integer"), - (lambda c: c["training"].__setitem__("features", 7), "must be divisible by training.attention_heads"), + ( + lambda c: c["objects"].__setitem__("b", "jets btag 2 top"), + "expected ' [lowest]', got 'jets btag 2 top'", + ), + ( + lambda c: c["objects"].__setitem__("q", "jets rest 2 lowest"), + "'rest' takes no 'lowest'", + ), + ( + lambda c: c["hypotheses"]["tt"].__setitem__("t_had", ["bb", "W_had"]), + "child 'bb' is neither", + ), + ( + lambda c: c["hypotheses"]["HH"]["H_WW"].__setitem__("W_lep", "lepton + met"), + "the '+' form was removed; write [lepton, met]", + ), + ( + lambda c: c.__setitem__("attention", [{"W_lep": "W_hadronic"}]), + "'W_hadronic' is not a node", + ), + ( + lambda c: c.__setitem__("relations", [["lepton", "neutrino"]]), + "'neutrino' is neither a node nor a leaf", + ), + ( + lambda c: c.__setitem__("relations", [["t_had", "met"]]), + "t_had has 2 candidates", + ), + ( + lambda c: c.__setitem__("combine", ["tt"]), + "combine must list the main hypothesis 'HH'", + ), + ( + lambda c: c.__setitem__("combine", ["HH", "HH"]), + "combine: duplicate entries ['HH']", + ), + ( + lambda c: c["outputs"].__setitem__("ttbar", "ttbar"), + "outputs.ttbar: 'ttbar' is not a declared hypothesis", + ), + ( + lambda c: c["training"].__setitem__("features", "8"), + "training.features must be an integer", + ), + ( + lambda c: c["training"].__setitem__("features", 7), + "must be divisible by training.attention_heads", + ), (lambda c: c["training"].update(features=9, attention_heads=3), "must be even"), ( lambda c: c["training"].__setitem__("phase_symetric", True), @@ -135,9 +187,17 @@ def test_missing_selector_column(tmp_path): cfg["objects"]["b"] = "jets dnn_score 2 lowest" r = validate.validate(_write(tmp_path, cfg), data) msgs = "\n".join(m for _, _, m in r.items) - assert "selected by `dnn_score` lowest first (reads dnn_score, pt for the selection)" in msgs - assert "expects dataset INPUTS/Jets/dnn_score (read for the selection by `b`, not a feature): not found" in _errors(r) - cfg["objects"]["b"] = "jets mass 2 lowest" # a feature column: no selection-only line + assert ( + "selected by `dnn_score` lowest first (reads dnn_score, pt for the selection)" + in msgs + ) + assert ( + "expects dataset INPUTS/Jets/dnn_score (read for the selection by `b`, not a feature): not found" + in _errors(r) + ) + cfg["objects"]["b"] = ( + "jets mass 2 lowest" # a feature column: no selection-only line + ) r = validate.validate(_write(tmp_path, cfg), data) assert not r.errors, r.errors assert "read for the selection only" not in "\n".join(m for _, _, m in r.items) @@ -153,7 +213,10 @@ def test_btag_feature_is_an_input(tmp_path): msgs = "\n".join(m for _, _, m in r.items) assert "features ['pt', 'eta', 'phi', 'mass', 'btag'] (the network inputs)" in msgs assert "read for the selection only" not in msgs - assert "leaf `b`: tensor (400, 6, 2)" in msgs and "network built: 3,832 parameters" not in msgs + assert ( + "leaf `b`: tensor (400, 6, 2)" in msgs + and "network built: 3,832 parameters" not in msgs + ) def test_yaml_syntax_error(tmp_path): @@ -177,21 +240,70 @@ def _replace(f, name, data): BROKEN_FILES = [ - (lambda f: f.move("INPUTS/Lepton", "INPUTS/Electron"), "collection `lepton` (used by `lepton`) expects a group INPUTS/lepton or INPUTS/Lepton or INPUTS/Leptons: not found"), - (lambda f: f.__delitem__("INPUTS/Lepton/isE"), "expects dataset INPUTS/Lepton/isE: not found"), - (lambda f: _replace(f, "INPUTS/Jets/pt", f["INPUTS/Jets/pt"][:, :10]), "INPUTS/Jets/pt: 10 slots, the configuration asks for max 12"), - (lambda f: _replace(f, "INPUTS/Met/pt", f["INPUTS/Met/pt"][:-5]), "INPUTS/Met/pt has 395 entries, expected 400"), - (lambda f: _replace(f, "INPUTS/Lepton/pt", np.stack([f["INPUTS/Lepton/pt"][...]] * 2, 1)), "expected (events,) for a single-object collection"), - (lambda f: _replace(f, "INPUTS/Jets/btag", np.array(["x"] * 400, dtype="S1")), "INPUTS/Jets/btag: dtype |S1 is not numeric"), - (lambda f: f.__delitem__("INPUTS/Jets/btag"), "expects dataset INPUTS/Jets/btag (read for the selection by `b`, not a feature): not found"), - (lambda f: _replace(f, "INPUTS/Jets/btag", f["INPUTS/Jets/btag"][:, :10]), "INPUTS/Jets/btag: 10 slots, the configuration asks for max 12"), - (lambda f: _replace(f, "INPUTS/Jets/MASK", f["INPUTS/Jets/MASK"][:, :11]), "INPUTS/Jets/MASK: shape (400, 11), expected (400, 12)"), - (lambda f: _replace(f, "EVENT/label", np.full(400, 3)), "EVENT/label: labels [3] outside 0..2"), - (lambda f: _replace(f, "EVENT/label", np.full(400, 1.0)), "EVENT/label: shape (400,) dtype float64, expected integer"), - (lambda f: f.__delitem__("EVENT/label"), "no labels: expected EVENT/label or CLASSIFICATIONS/EVENT/signal"), - (lambda f: f.__delitem__("EVENT/fold"), "EVENT/fold not found: corenet-train needs a fold index"), - (lambda f: _replace(f, "EVENT/weight", np.full(400, np.nan, dtype=np.float32)), "EVENT/weight: 400 non-finite weights"), - (lambda f: _replace(f, "INPUTS/Lepton/pt", np.full(400, np.inf, dtype=np.float32)), "leaf `lepton`: non-finite features in 400 present objects"), + ( + lambda f: f.move("INPUTS/Lepton", "INPUTS/Electron"), + "collection `lepton` (used by `lepton`) expects a group INPUTS/lepton or INPUTS/Lepton or INPUTS/Leptons: not found", + ), + ( + lambda f: f.__delitem__("INPUTS/Lepton/isE"), + "expects dataset INPUTS/Lepton/isE: not found", + ), + ( + lambda f: _replace(f, "INPUTS/Jets/pt", f["INPUTS/Jets/pt"][:, :10]), + "INPUTS/Jets/pt: 10 slots, the configuration asks for max 12", + ), + ( + lambda f: _replace(f, "INPUTS/Met/pt", f["INPUTS/Met/pt"][:-5]), + "INPUTS/Met/pt has 395 entries, expected 400", + ), + ( + lambda f: _replace( + f, "INPUTS/Lepton/pt", np.stack([f["INPUTS/Lepton/pt"][...]] * 2, 1) + ), + "expected (events,) for a single-object collection", + ), + ( + lambda f: _replace(f, "INPUTS/Jets/btag", np.array(["x"] * 400, dtype="S1")), + "INPUTS/Jets/btag: dtype |S1 is not numeric", + ), + ( + lambda f: f.__delitem__("INPUTS/Jets/btag"), + "expects dataset INPUTS/Jets/btag (read for the selection by `b`, not a feature): not found", + ), + ( + lambda f: _replace(f, "INPUTS/Jets/btag", f["INPUTS/Jets/btag"][:, :10]), + "INPUTS/Jets/btag: 10 slots, the configuration asks for max 12", + ), + ( + lambda f: _replace(f, "INPUTS/Jets/MASK", f["INPUTS/Jets/MASK"][:, :11]), + "INPUTS/Jets/MASK: shape (400, 11), expected (400, 12)", + ), + ( + lambda f: _replace(f, "EVENT/label", np.full(400, 3)), + "EVENT/label: labels [3] outside 0..2", + ), + ( + lambda f: _replace(f, "EVENT/label", np.full(400, 1.0)), + "EVENT/label: shape (400,) dtype float64, expected integer", + ), + ( + lambda f: f.__delitem__("EVENT/label"), + "no labels: expected EVENT/label or CLASSIFICATIONS/EVENT/signal", + ), + ( + lambda f: f.__delitem__("EVENT/fold"), + "EVENT/fold not found: corenet-train needs a fold index", + ), + ( + lambda f: _replace(f, "EVENT/weight", np.full(400, np.nan, dtype=np.float32)), + "EVENT/weight: 400 non-finite weights", + ), + ( + lambda f: _replace( + f, "INPUTS/Lepton/pt", np.full(400, np.inf, dtype=np.float32) + ), + "leaf `lepton`: non-finite features in 400 present objects", + ), ] @@ -219,3 +331,52 @@ def test_missing_event_scalar_and_warnings(tmp_path): ) w = "\n".join(r.warnings) assert "EVENT/weight not found" in w and "INPUTS/Jets/MASK not found" in w + + +def test_fold_report(tmp_path): + """The fold source and the events per fold; --n-folds reports the event % K split instead.""" + data = write_synthetic(tmp_path / "d.h5", "bbww", n=400) + r = validate.validate(BBWW, data) + msgs = "\n".join(m for _, _, m in r.items) + assert not r.errors and "folds from EVENT/fold: events per fold {0: " in msgs + assert "--n-folds" not in msgs + # the synthetic folds are random: event % 5 disagrees with the column for most events + r = validate.validate(BBWW, data, n_folds=5) + msgs = "\n".join(m for _, _, m in r.items) + assert not r.errors, r.errors + assert "folds from event % 5 (--n-folds): events per fold {0: " in msgs + assert "EVENT/fold disagrees with event % 5 for " in "\n".join(r.warnings) + assert "loader + forward pass" in msgs + # a column equal to the rule + with h5py.File(data, "a") as f: + _replace(f, "EVENT/fold", f["EVENT/event"][...] % 5) + r = validate.validate(BBWW, data, n_folds=5) + assert not r.errors and "EVENT/fold equals event % 5" in "\n".join( + m for _, _, m in r.items + ) + assert not r.warnings, r.warnings + # no column: FAIL by default (see BROKEN_FILES), valid with --n-folds + with h5py.File(data, "a") as f: + del f["EVENT/fold"] + r = validate.validate(BBWW, data) + assert "EVENT/fold not found" in _errors(r) and "pass --n-folds K" in _errors(r) + assert ( + validate.main(["--config", str(BBWW), "--input", str(data), "--n-folds", "3"]) + == 0 + ) + r = validate.validate(BBWW, data, n_folds=3) + msgs = "\n".join(m for _, _, m in r.items) + assert ( + not r.errors + and "folds from event % 3 (--n-folds): events per fold {0: 134, 1: 133, 2: 133}" + in msgs + ) + # negative event numbers cannot be folded + with h5py.File(data, "a") as f: + _replace(f, "EVENT/event", f["EVENT/event"][...] - 10) + r = validate.validate(BBWW, data, n_folds=3) + assert "folds = event % 3 (--n-folds): 10 event numbers are negative" in _errors(r) + with h5py.File(data, "a") as f: + del f["EVENT/event"] + r = validate.validate(BBWW, data, n_folds=3) + assert "folds = event % 3 (--n-folds): EVENT/event not found" in _errors(r) From 27d3204756b66ee747b6ab393e2019875ee94b30 Mon Sep 17 00:00:00 2001 From: Aniket Khanal Date: Wed, 21 Oct 2026 20:19:58 -0400 Subject: [PATCH 29/33] update readme --- README.md | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/README.md b/README.md index b70be15..082ef44 100644 --- a/README.md +++ b/README.md @@ -128,9 +128,11 @@ corenet-train --config examples/bbww/bbww.yaml --input examples/data/bbww_all.h5 * `--fold 0`: hold out fold 0 for testing. With `--fold -1` (the default), train one model per fold in turn. * `--exclude-folds 1`: keep fold 1 out of the training and use it for validation. `--exclude-folds next` uses the fold `(k + 1) % n_folds` of the held-out fold `k`. -* `--n-folds 5`: take the fold of an event as `event % 5` from `EVENT/event` instead of the `EVENT/fold` - column ([docs/Dataset.md](docs/Dataset.md)); the rule is stored in the checkpoint and reused by - `corenet-evaluate`. +* `--n-folds 5`: take the fold of an event as `squares(event) % 5`, a hash of the event number in + `EVENT/event`, instead of the `EVENT/fold` column ([docs/Dataset.md](docs/Dataset.md)); `--fold-seed` + (default 0) selects the hash key. The rule, the number of folds and the seed are stored in the + checkpoint and reused by `corenet-evaluate`. With 5 folds and seed 0 the rule equals the + `EVENT/fold` column of the example files. * `--early-stop-patience 5`: stop when the validation AUC has not improved for 5 epochs and keep the best epoch. * `--freeze-epoch 15 --finetune-layers head_main. head_reduce. aux_heads. --finetune-lr 1.0e-3 --finetune-batch-size 4096`: the fine-tuning stage; from epoch 15 on only the parameters whose names start with the given prefixes @@ -139,8 +141,8 @@ corenet-train --config examples/bbww/bbww.yaml --input examples/data/bbww_all.h5 * `--epochs`, `--lr`, `--seed`, ...: override the training settings of the configuration file. See [docs/Options.md](docs/Options.md) or `corenet-train --help`. -The folds come from the input file: the `EVENT/fold` column, or `event % K` with `--n-folds K` -([docs/Dataset.md](docs/Dataset.md)). The example files have 5 folds in `EVENT/fold`. +The folds come from the input file: the `EVENT/fold` column, or `squares(event) % K` with +`--n-folds K` ([docs/Dataset.md](docs/Dataset.md)). The example files have 5 folds in `EVENT/fold`. Per fold `k`, the output directory receives: From 68b1529d1ec0556f70b1fe5fc58d39e4fc00bf87 Mon Sep 17 00:00:00 2001 From: Aniket Khanal Date: Wed, 21 Oct 2026 20:20:11 -0400 Subject: [PATCH 30/33] update new fold setup --- docs/Dataset.md | 19 ++++++++++--------- docs/HH4b.md | 2 +- docs/Options.md | 7 ++++--- 3 files changed, 15 insertions(+), 13 deletions(-) diff --git a/docs/Dataset.md b/docs/Dataset.md index 400a9ef..86aa270 100644 --- a/docs/Dataset.md +++ b/docs/Dataset.md @@ -69,14 +69,16 @@ A collection named `x` is read from `INPUTS/x`, matched case-insensitively, or f training, so only the relative weights within a class matter. Negative weights count as 0. * **Folds.** The cross-validation fold of an event is `EVENT/fold` when the file has that column. Without it, pass `--n-folds K` to `corenet-train` (and to `corenet-validate`): the fold is then - `event % K`, computed from the event numbers in `EVENT/event` (integers ≥ 0), and no random split - is ever made. With `--n-folds K` an existing `EVENT/fold` column is ignored. The rule used - (`EVENT/fold` or `event % n_folds`) and the number of folds are stored in the checkpoint, so + `squares(event) % K`, a hash of the event number in `EVENT/event` (integers ≥ 0) with the Squares + counter-based generator (`corenet.data.folds.fold_of`; `--fold-seed`, default 0, selects the hash + key), so any subset of a sample gets the same folds and no random split is ever made. With + `--n-folds K` an existing `EVENT/fold` column is ignored. The rule used (`EVENT/fold` or + `squares(event) % n_folds`), the number of folds and the seed are stored in the checkpoint, so `corenet-evaluate` applies the same rule to any file. `corenet-train --fold k` holds out fold `k` for testing and `--exclude-folds j` uses fold `j` for validation (`--exclude-folds next`: fold `(k + 1) % n_folds`). * **Event identifier.** `EVENT/event` is copied to the prediction files and is the input of the - `event % n_folds` rule. + `squares(event) % n_folds` rule. ## Example @@ -91,11 +93,10 @@ follow this layout: `EVENT/event`. They are random subsamples of the full files `hh4b_all.h5` and `bbww_all.h5`, written with -`python scripts/make_small_h5.py --events 20000`. The `EVENT/fold` column of the -example files was assigned from a hash of the event number when the files were produced and is not -`event % 5`: to reproduce the folds of the paper, train and evaluate on these files without -`--n-folds`. `corenet-validate --n-folds 5` reports how many events the two rules assign differently. -Check any file against an event description with +`python scripts/make_small_h5.py --events 20000`. Their `EVENT/fold` column equals the +fold rule with 5 folds and seed 0 (`--n-folds 5 --fold-seed 0`), so the column and `--n-folds 5` +give the same folds on these files; `corenet-validate --n-folds 5` reports that agreement. Check any +file against an event description with ```bash corenet-validate --config examples/bbww/bbww.yaml --input examples/data/bbww_small.h5 diff --git a/docs/HH4b.md b/docs/HH4b.md index 01d5b84..c9b233e 100644 --- a/docs/HH4b.md +++ b/docs/HH4b.md @@ -184,7 +184,7 @@ of the b jets has to be reproduced wherever the model is deployed; see Exporting | epochs, learning rate, decay | `--epochs`, `--lr`, `--lr-milestones`, `--lr-gamma`, `--freeze-epoch`, or the `trainer:` section | `--epochs 30 --lr-milestones 25 26 27 28 29 --freeze-epoch 25` | | batch size | `--batch-size-init` or `trainer: batch_size_init` | | | seed | `--seed` or `trainer: seed` | fold k uses seed + k | -| validation / test folds | `--fold`, `--exclude-folds` (`next` = the fold after the held-out one), `--n-folds` (fold = `event % K` instead of `EVENT/fold`) | Training above | +| validation / test folds | `--fold`, `--exclude-folds` (`next` = the fold after the held-out one), `--n-folds` (fold = `squares(event) % K` instead of `EVENT/fold`), `--fold-seed` | Training above | A change in `objects` or `training:` gives a different network: validate the new file with `corenet-validate` and retrain ([Options.md](./Options.md)). diff --git a/docs/Options.md b/docs/Options.md index 051f243..87e4423 100644 --- a/docs/Options.md +++ b/docs/Options.md @@ -45,9 +45,10 @@ which the validation rejects. Other flags of `corenet-train`: `--config`, `--input`, `--output-dir`, `--fold` (held-out fold, `-1` for every fold in turn), `--exclude-folds` (validation folds; `next` = the fold `(k + 1) % n_folds` of -the held-out fold `k`), `--n-folds K` (fold = `event % K` from `EVENT/event` instead of the `EVENT/fold` -column, [Dataset.md](./Dataset.md)), `--save-every-epoch` (also save `model_fold{k}_epoch{e}.pkl` after -every epoch) and `--device`. +the held-out fold `k`), `--n-folds K` (fold = `squares(event) % K` from `EVENT/event` instead of the +`EVENT/fold` column, [Dataset.md](./Dataset.md)), `--fold-seed` (seed of that rule, default 0, the seed +of the example files), `--save-every-epoch` (also save `model_fold{k}_epoch{e}.pkl` after every epoch) +and `--device`. ## Fixed in code From a917c245914ce45e1c03edf5d77d9880768a85e5 Mon Sep 17 00:00:00 2001 From: Aniket Khanal Date: Wed, 21 Oct 2026 20:20:24 -0400 Subject: [PATCH 31/33] create fold with hash rule --- src/corenet/checkpoint.py | 17 ++++++++++------- 1 file changed, 10 insertions(+), 7 deletions(-) diff --git a/src/corenet/checkpoint.py b/src/corenet/checkpoint.py index a0df2a0..1cbe27b 100644 --- a/src/corenet/checkpoint.py +++ b/src/corenet/checkpoint.py @@ -5,7 +5,8 @@ "config_path": path of the configuration file at training time, "classes": ["rest", ...], "metadata": {"fold", "epoch", "exclude_folds", "input", "n_folds", "fold_rule" ("EVENT/fold" or - "event % n_folds"), , "finetune_trainable_parameters"}, + "squares(event) % n_folds"), "fold_seed", , + "finetune_trainable_parameters"}, "format": CHECKPOINT_FORMAT} ``format`` 2: every feature of a collection is a network input. Checkpoints without ``format`` were @@ -22,6 +23,7 @@ import torch +from .data.folds import FOLD_SEED from .data.h5 import FOLD_RULE_EVENT from .models.config import NetworkSpec, parse_config, read_config_dict from .models.network import CoReNet @@ -30,12 +32,13 @@ LEGACY_SKIPPED = ("btag",) # features the network ignored before format 2 -def n_folds_of(metadata: dict) -> int | None: - """The ``n_folds`` to pass to ``H5Dataset`` for a checkpoint: its ``n_folds`` when it was trained with - fold = event % n_folds, else None (the ``EVENT/fold`` column).""" +def fold_rule_of(metadata: dict) -> tuple[int | None, int]: + """(``n_folds``, ``fold_seed``) to pass to ``H5Dataset`` for a checkpoint: its ``n_folds`` and seed when + it was trained with fold = squares(event) % n_folds, else (None, seed) for the ``EVENT/fold`` column.""" + seed = int(metadata.get("fold_seed", FOLD_SEED)) if metadata.get("fold_rule") == FOLD_RULE_EVENT: - return metadata.get("n_folds") - return None + return metadata.get("n_folds"), seed + return None, seed def save_checkpoint( @@ -116,8 +119,8 @@ def load_checkpoint( __all__ = [ "CHECKPOINT_FORMAT", + "fold_rule_of", "legacy_config", "load_checkpoint", - "n_folds_of", "save_checkpoint", ] From b5dad139562e92a48ab5a10e7ebf69b6a313e13a Mon Sep 17 00:00:00 2001 From: Aniket Khanal Date: Wed, 21 Oct 2026 20:20:45 -0400 Subject: [PATCH 32/33] update new fold creation rule --- src/corenet/data/folds.py | 164 ++++++++++++++++++++++++++++++++++++++ src/corenet/data/h5.py | 42 ++++------ src/corenet/evaluate.py | 26 ++++-- src/corenet/export.py | 6 +- src/corenet/train.py | 25 +++++- src/corenet/validate.py | 52 ++++++++---- 6 files changed, 262 insertions(+), 53 deletions(-) create mode 100644 src/corenet/data/folds.py diff --git a/src/corenet/data/folds.py b/src/corenet/data/folds.py new file mode 100644 index 0000000..885862a --- /dev/null +++ b/src/corenet/data/folds.py @@ -0,0 +1,164 @@ +""" +The fold rule: a deterministic cross-validation fold from the event number. + + fold = squares(event) % n_folds (``fold_of(event, n_folds, seed)``) + +``squares`` is the Squares counter-based random number generator (Widynski, arXiv:2004.06278) keyed by +the strings ``"delphes_h5"``, ``"fold"`` and the integer ``seed``; the event number is the counter. +The fold of an event therefore depends only on its event number, the number of folds and the seed: +any subset of a sample, in any order, gets the same folds, and no random split is ever made. + +This is a copy of the rule of the research converter that wrote the example files +(``hcr.data.delphes_h5.assign_folds`` with ``hcr.math_tools.random.Squares``), kept bit for bit so +that the package reproduces their ``EVENT/fold`` column: the shipped files +``examples/data/{hh4b,bbww}_{small,all}.h5`` were written with ``n_folds=5`` and ``seed=0``, so +``--n-folds 5`` (``--fold-seed 0``) gives exactly the folds stored in them. +""" + +from __future__ import annotations + +import hashlib +from collections.abc import Iterable + +import numpy as np + +_UINT64_32 = np.uint64(32) + +SeedLike = int | str | Iterable[int | str] + + +def _str_to_entropy(text: str) -> list[np.uint64]: + return np.frombuffer(hashlib.md5(text.encode()).digest(), dtype=np.uint64).tolist() + + +def _seed(*entropy: SeedLike) -> tuple[int, ...]: + seeds = [] + for e in entropy: + if isinstance(e, str): + seeds.extend(_str_to_entropy(e)) + elif isinstance(e, Iterable): + seeds.extend(_seed(*e)) + else: + seeds.append(e) + return (*seeds,) + + +class Squares: + """Squares: a counter-based random number generator (CBRNG), https://arxiv.org/abs/2004.06278. + + The key is drawn with ``numpy.random.PCG64`` seeded by the entropy of ``seed`` (strings enter + through their MD5 digest, integers as they are). + """ + + def __init__(self, *seed: SeedLike): + self._seed = _seed(seed) + self._key_value: np.uint64 | None = None + + @property + def _key(self) -> np.uint64: + if self._key_value is None: + self._key_value = self.key(np.random.Generator(np.random.PCG64(self._seed))) + return self._key_value + + @staticmethod + def key(gen: np.random.Generator) -> np.uint64: + bits = np.arange(1, 16, dtype=np.uint64) + offsets = np.arange(0, 29, 4, dtype=np.uint64) + lower8 = gen.choice(bits, 8, replace=False) + for i in range(16): + if lower8[i] % 2 == 1: + lower8 = np.roll(lower8, -i) + break + higher8 = np.zeros(8, dtype=np.uint64) + higher8[0:1] = gen.choice(np.delete(bits, int(lower8[-1]) - 1), 1) + higher8[1:] = gen.choice(np.delete(bits, int(higher8[0]) - 1), 7, replace=False) + return np.sum(lower8 << offsets) + (np.sum(higher8 << offsets) << _UINT64_32) + + @staticmethod + def _round(lr: np.ndarray, shift: np.ndarray, last: bool = False): + lr *= lr + lr += shift + if last: + yield lr.copy() + low = lr >> _UINT64_32 + lr <<= _UINT64_32 + lr |= low + yield lr + + def bit32(self, ctrs: np.ndarray) -> np.ndarray: + x = ctrs * self._key + y = x.copy() + z = y + self._key + # round 1-3 + for i in [y, z, y]: + (_,) = self._round(x, i) + # round 4 + x *= x + x += z + x >>= _UINT64_32 + return x.astype(np.uint32) + + def bit64(self, ctrs: np.ndarray) -> np.ndarray: + x = ctrs * self._key + y = x.copy() + z = y + self._key + # round 1-3 + for i in [y, z, y]: + (_,) = self._round(x, i) + # round 4 + (t, _) = self._round(x, z, last=True) + # round 5 + x *= x + x += y + x >>= _UINT64_32 + x ^= t + return x + + def uint64(self, counters) -> np.ndarray: + """One uint64 per counter; a trailing dimension of several counters is reduced.""" + counters = np.asarray(counters, dtype=np.uint64) + if counters.ndim == 1: + return self.bit64(counters) + while True: + shape = counters.shape[-1] + if shape == 1: + return self.bit64(counters).reshape(counters.shape[:-1]) + elif shape % 2 == 0: + counters = self.bit32(counters).view(np.uint64) + else: + counters = np.concatenate( + [ + counters[..., -1:], + self.bit32(counters[..., :-1]).view(np.uint64), + ], + axis=-1, + ) + + +FOLD_SEED = 0 # the seed the example files were written with + + +def fold_rule(n_folds: int, seed: int = FOLD_SEED) -> str: + """The rule in words, as the validator and the loader report it.""" + return f"squares(event) % {n_folds}, seed {seed}" + + +def fold_of(event, n_folds: int, seed: int = FOLD_SEED) -> np.ndarray: + """fold = squares(event) % n_folds on the (non-negative, integer) event numbers; int64 (N,).""" + if not isinstance(n_folds, int) or isinstance(n_folds, bool) or n_folds < 1: + raise ValueError(f"n_folds must be an integer >= 1, got {n_folds!r}") + event = np.asarray(event) + if event.dtype.kind not in "iu": + raise ValueError( + f"event numbers must be integers to compute fold = squares(event) % n_folds, got dtype {event.dtype}" + ) + if (event.astype(np.int64) < 0).any(): + raise ValueError( + f"{int((event.astype(np.int64) < 0).sum()):,} event numbers are negative (or exceed int64): " + "fold = squares(event) % n_folds needs event numbers >= 0" + ) + rng = Squares("delphes_h5", "fold", int(seed)) + return (rng.uint64(event.astype(np.uint64)) % np.uint64(n_folds)).astype(np.int64) + + +__all__ = ["FOLD_SEED", "Squares", "fold_of", "fold_rule"] diff --git a/src/corenet/data/h5.py b/src/corenet/data/h5.py index 4b2845e..56b5d46 100644 --- a/src/corenet/data/h5.py +++ b/src/corenet/data/h5.py @@ -12,8 +12,9 @@ plus ``labels`` (``EVENT/label``, or ``CLASSIFICATIONS/EVENT/signal`` when absent), ``weights`` (``EVENT/weight``), ``folds`` and ``events`` (``EVENT/event``). The cross-validation fold of an event -is ``EVENT/fold`` by default; with ``n_folds`` it is ``fold_of(event, n_folds)`` = ``event % n_folds`` -computed from the event numbers, and the file needs no ``EVENT/fold`` column. No generator-level +is ``EVENT/fold`` by default; with ``n_folds`` it is ``fold_of(event, n_folds, fold_seed)`` = +``squares(event) % n_folds`` (``corenet.data.folds``) computed from the event numbers, and the file +needs no ``EVENT/fold`` column. No generator-level information is read: the roles are chosen from reconstructed quantities only. A collection's ``features`` are exactly the channels of its tensors. A selector reads the column it @@ -48,6 +49,7 @@ from torch.utils.data import Dataset from ..models.config import REST, Collection, NetworkSpec, RoleSpec +from .folds import FOLD_SEED, fold_of, fold_rule # collection name (lower case) -> group names accepted in INPUTS, in order of preference GROUP_ALIASES = { @@ -63,29 +65,11 @@ ) # first one present is used WEIGHT_KEY, FOLD_KEY, EVENT_KEY = "EVENT/weight", "EVENT/fold", "EVENT/event" FOLD_RULE_EVENT, FOLD_RULE_COLUMN = ( - "event % n_folds", + "squares(event) % n_folds", "EVENT/fold", ) # H5Dataset.fold_rule -def fold_of(event: np.ndarray, n_folds: int) -> np.ndarray: - """The fold rule: fold = event % n_folds on the (non-negative, int64) event numbers.""" - if not isinstance(n_folds, int) or isinstance(n_folds, bool) or n_folds < 1: - raise ValueError(f"n_folds must be an integer >= 1, got {n_folds!r}") - event = np.asarray(event) - if event.dtype.kind not in "iu": - raise ValueError( - f"event numbers must be integers to compute fold = event % n_folds, got dtype {event.dtype}" - ) - event = event.astype(np.int64) - if (event < 0).any(): - raise ValueError( - f"{int((event < 0).sum()):,} event numbers are negative (or exceed int64): " - "fold = event % n_folds needs event numbers >= 0" - ) - return event % n_folds - - # ------------------------------------------------------------------ what the loader reads (shared with corenet-validate) def group_candidates(name: str) -> tuple[str, ...]: """INPUTS group names accepted for a collection, in order of preference (case-insensitive).""" @@ -181,8 +165,10 @@ def __init__( spec: NetworkSpec, rest_order: str = "pt", n_folds: int | None = None, + fold_seed: int = FOLD_SEED, ): - """``n_folds``: use fold = event % n_folds (``fold_of``) instead of the ``EVENT/fold`` column.""" + """``n_folds``: use fold = squares(event) % n_folds (``fold_of``, seed ``fold_seed``) instead of the + ``EVENT/fold`` column.""" if rest_order not in ("pt", "selector"): raise ValueError( f"rest_order must be 'pt' or 'selector', got {rest_order!r}" @@ -226,6 +212,7 @@ def __init__( if EVENT_KEY in f else torch.arange(n, dtype=torch.int64) ) + self.fold_seed = int(fold_seed) self.folds, self.n_folds, self.fold_rule = self._folds(f, n_folds) self.inputs = inputs @@ -233,22 +220,23 @@ def _folds( self, f: h5py.File, n_folds: int | None ) -> tuple[torch.Tensor, int, str]: """(fold of every event, number of folds, rule): ``EVENT/fold`` (n_folds = the number of distinct - values) or, with ``n_folds``, ``fold_of(EVENT/event, n_folds)``.""" + values) or, with ``n_folds``, ``fold_of(EVENT/event, n_folds, self.fold_seed)``.""" if n_folds is not None: + rule = fold_rule(n_folds, self.fold_seed) if EVENT_KEY not in f: raise KeyError( - f"{self.path.name}: no {EVENT_KEY} dataset; fold = event % {n_folds} needs the event numbers" + f"{self.path.name}: no {EVENT_KEY} dataset; fold = {rule} needs the event numbers" ) if FOLD_KEY in f: logging.info( - f"{self.path.name}: {FOLD_KEY} ignored, folds = event % {n_folds} from {EVENT_KEY}" + f"{self.path.name}: {FOLD_KEY} ignored, folds = {rule} from {EVENT_KEY}" ) - folds = fold_of(f[EVENT_KEY][...], n_folds) + folds = fold_of(f[EVENT_KEY][...], n_folds, self.fold_seed) return torch.from_numpy(folds), n_folds, FOLD_RULE_EVENT if FOLD_KEY not in f: raise KeyError( f"{self.path.name}: no {FOLD_KEY} dataset; pass --n-folds K (H5Dataset(n_folds=K)) " - f"to use fold = event % K from {EVENT_KEY}" + f"to use fold = squares(event) % K from {EVENT_KEY}" ) folds = f[FOLD_KEY][...].astype(np.int64) return torch.from_numpy(folds), len(np.unique(folds)), FOLD_RULE_COLUMN diff --git a/src/corenet/evaluate.py b/src/corenet/evaluate.py index 6e9cd5e..d327fc9 100644 --- a/src/corenet/evaluate.py +++ b/src/corenet/evaluate.py @@ -5,8 +5,9 @@ --output runs/hh4b/eval_fold0.h5 Events: by default the fold held out when the checkpoint was trained (``--fold -1`` = every event of -the file), with the checkpoint's fold rule: ``EVENT/fold`` of the file, or fold = event % n_folds when -the model was trained with ``--n-folds`` (``--n-folds K`` overrides). The output file (layout in +the file), with the checkpoint's fold rule: ``EVENT/fold`` of the file, or fold = squares(event) % +n_folds with the checkpoint's seed when the model was trained with ``--n-folds`` (``--n-folds K`` and +``--fold-seed`` override). The output file (layout in docs/Dataset.md) holds the class probabilities, the view (candidate) probabilities of every multi-candidate hypothesis, the auxiliary head probabilities, the event labels / weights and, for the main output, the ROC curve; the AUCs are printed and written as JSON next to it (``.json``). @@ -196,9 +197,15 @@ def build_parser() -> argparse.ArgumentParser: "--n-folds", type=int, default=None, - help="use fold = event %% K computed from EVENT/event instead of the EVENT/fold column " + help="use fold = squares(event) %% K computed from EVENT/event instead of the EVENT/fold column " "(default: the checkpoint's rule)", ) + p.add_argument( + "--fold-seed", + type=int, + default=None, + help="seed of the squares(event) %% K rule (default: the checkpoint's, else 0)", + ) p.add_argument( "--rest-order", choices=["pt", "selector"], @@ -216,7 +223,7 @@ def build_parser() -> argparse.ArgumentParser: def main(argv=None): - from .checkpoint import load_checkpoint, n_folds_of + from .checkpoint import fold_rule_of, load_checkpoint args = build_parser().parse_args(argv) logging.basicConfig( @@ -230,8 +237,14 @@ def main(argv=None): model, spec, ckpt = load_checkpoint(args.checkpoint, args.config, device) meta = ckpt.get("metadata", {}) rest_order = args.rest_order or meta.get("rest_order", "pt") - n_folds = args.n_folds if args.n_folds is not None else n_folds_of(meta) - ds = H5Dataset(args.input, spec, rest_order=rest_order, n_folds=n_folds) + n_folds, fold_seed = fold_rule_of(meta) + if args.n_folds is not None: + n_folds = args.n_folds + if args.fold_seed is not None: + fold_seed = args.fold_seed + ds = H5Dataset( + args.input, spec, rest_order=rest_order, n_folds=n_folds, fold_seed=fold_seed + ) fold = args.fold if args.fold is not None else meta.get("fold", -1) idx = ( np.arange(len(ds)) @@ -271,6 +284,7 @@ def main(argv=None): "fold": -1 if fold is None else int(fold), "n_folds": ds.n_folds, "fold_rule": ds.fold_rule, + "fold_seed": ds.fold_seed, "n_parameters": count_parameters(model), "auc_main": metrics["auc"]["main"] or float("nan"), }, diff --git a/src/corenet/export.py b/src/corenet/export.py index 8f5cc77..a7fd052 100644 --- a/src/corenet/export.py +++ b/src/corenet/export.py @@ -32,7 +32,7 @@ import torch.nn.functional as F from torch import nn -from .checkpoint import load_checkpoint, n_folds_of +from .checkpoint import fold_rule_of, load_checkpoint from .data.h5 import H5Dataset @@ -159,11 +159,13 @@ def main(argv=None): model, spec, ckpt = load_checkpoint(args.checkpoint, args.config, "cpu") wrapper = ExportWrapper(model).eval() meta = ckpt.get("metadata", {}) + n_folds, fold_seed = fold_rule_of(meta) ds = H5Dataset( args.input, spec, rest_order=meta.get("rest_order", "pt"), - n_folds=n_folds_of(meta), + n_folds=n_folds, + fold_seed=fold_seed, ) out_dir = Path(args.output_dir) out_dir.mkdir(parents=True, exist_ok=True) diff --git a/src/corenet/train.py b/src/corenet/train.py index 3d22ec6..0405754 100644 --- a/src/corenet/train.py +++ b/src/corenet/train.py @@ -22,8 +22,9 @@ binary cross entropy of its head against ``label == class index of o``. The view (candidate selection) scores are not supervised. -Folds: ``EVENT/fold`` of the file, or ``--n-folds K`` for fold = event % K from the event numbers -(``corenet.data.h5.fold_of``); ``--exclude-folds next`` is the fold (k + 1) % n_folds of held-out fold k. +Folds: ``EVENT/fold`` of the file, or ``--n-folds K`` for fold = squares(event) % K from the event +numbers (``corenet.data.folds.fold_of``, seed ``--fold-seed``, 0 = the rule of the example files); +``--exclude-folds next`` is the fold (k + 1) % n_folds of held-out fold k. For each held-out fold k this writes, in ``--output-dir``: model_fold{k}.pkl checkpoint: state dict, configuration, classes and metadata @@ -44,6 +45,7 @@ import torch.nn.functional as F from .checkpoint import save_checkpoint +from .data.folds import FOLD_SEED from .data.h5 import H5Dataset from .evaluate import Batches, count_parameters, predict, write_predictions from .models.config import ConfigError, NetworkSpec, parse_config, read_config_dict @@ -280,6 +282,7 @@ def metadata(epoch: int) -> dict: "input": str(args.input), "n_folds": ds.n_folds, "fold_rule": ds.fold_rule, + "fold_seed": ds.fold_seed, **recipe, "finetune_trainable_parameters": n_trainable, } @@ -357,6 +360,7 @@ def metadata(epoch: int) -> dict: "exclude_folds": exclude_folds, "n_folds": ds.n_folds, "fold_rule": ds.fold_rule, + "fold_seed": ds.fold_seed, "config": str(args.config), "classes": spec.classes, "recipe": recipe, @@ -429,6 +433,7 @@ def metadata(epoch: int) -> dict: "fold": fold, "n_folds": ds.n_folds, "fold_rule": ds.fold_rule, + "fold_seed": ds.fold_seed, }, ) logging.info( @@ -460,7 +465,13 @@ def build_parser() -> argparse.ArgumentParser: "--n-folds", type=int, default=None, - help="use fold = event %% K computed from EVENT/event instead of the EVENT/fold column", + help="use fold = squares(event) %% K computed from EVENT/event instead of the EVENT/fold column", + ) + p.add_argument( + "--fold-seed", + type=int, + default=FOLD_SEED, + help=f"seed of the squares(event) %% K rule [{FOLD_SEED}, the seed of the example files]", ) p.add_argument( "--exclude-folds", @@ -645,7 +656,13 @@ def main(argv=None): if torch.cuda.is_available() and args.device.startswith("cuda") else "cpu" ) - ds = H5Dataset(args.input, spec, rest_order=args.rest_order, n_folds=args.n_folds) + ds = H5Dataset( + args.input, + spec, + rest_order=args.rest_order, + n_folds=args.n_folds, + fold_seed=args.fold_seed, + ) logging.info( f"{args.input}: {len(ds):,} events, {ds.n_folds} folds ({ds.fold_rule}), device={device}, recipe " + ", ".join(f"{k}={getattr(args, k)}" for k in RECIPE_DEFAULTS) diff --git a/src/corenet/validate.py b/src/corenet/validate.py index 6c4077d..d5f3a2c 100644 --- a/src/corenet/validate.py +++ b/src/corenet/validate.py @@ -10,8 +10,9 @@ 2. Data (with ``--input``): every dataset the loader reads is looked up with the loader's own rules (``corenet.data.h5``: group aliases, used collections, computed event scalars, label keys) and its dtype and shape are checked; the labels must lie in 0..len(outputs), the weights must be finite, - the folds must exist for training (``EVENT/fold``, or ``--n-folds K`` for fold = event % K from - the event numbers; the report gives the fold source and the events per fold); the present members + the folds must exist for training (``EVENT/fold``, or ``--n-folds K`` for fold = squares(event) % K + from the event numbers, seed ``--fold-seed``; the report gives the fold source and the events per + fold); the present members must have finite features. Finally the actual loader (``H5Dataset``) reads the file and the network runs a forward pass on the first events, so a file that passes here also loads in corenet-train. @@ -31,6 +32,7 @@ import torch from .data import h5 as loader +from .data.folds import FOLD_SEED, fold_of, fold_rule from .models.config import ( REST, ConfigError, @@ -178,6 +180,7 @@ def check_data( rest_order: str = "pt", n_forward: int = 512, n_folds: int | None = None, + fold_seed: int = FOLD_SEED, ) -> None: sec = f"data {path}" try: @@ -412,11 +415,11 @@ def length(name, ds): report.fail( sec, f"{loader.FOLD_KEY} not found: corenet-train needs a fold index per event; " - "pass --n-folds K to use fold = event % K from EVENT/event", + "pass --n-folds K to use fold = squares(event) % K from EVENT/event", ) folds, source = None, None if n_folds is not None: - source = f"event % {n_folds} (--n-folds)" + source = f"{fold_rule(n_folds, fold_seed)} (--n-folds)" if loader.EVENT_KEY not in f: report.fail( sec, @@ -424,7 +427,7 @@ def length(name, ds): ) else: try: - folds = loader.fold_of(f[loader.EVENT_KEY][...], n_folds) + folds = fold_of(f[loader.EVENT_KEY][...], n_folds, fold_seed) except ValueError as e: report.fail(sec, f"folds = {source}: {e}") elif column is not None: @@ -445,11 +448,13 @@ def length(name, ds): if n_diff: report.warn( sec, - f"{loader.FOLD_KEY} disagrees with event % {n_folds} for {n_diff:,} of " - f"{len(folds):,} events; corenet-train --n-folds {n_folds} ignores the column", + f"{loader.FOLD_KEY} disagrees with {fold_rule(n_folds, fold_seed)} for " + f"{n_diff:,} of {len(folds):,} events; corenet-train --n-folds {n_folds} ignores the column", ) else: - report.ok(sec, f"{loader.FOLD_KEY} equals event % {n_folds}") + report.ok( + sec, f"{loader.FOLD_KEY} equals {fold_rule(n_folds, fold_seed)}" + ) if loader.EVENT_KEY not in f: report.warn( sec, @@ -460,7 +465,9 @@ def length(name, ds): return # ---- the actual loader and a forward pass try: - ds = loader.H5Dataset(path, spec, rest_order=rest_order, n_folds=n_folds) + ds = loader.H5Dataset( + path, spec, rest_order=rest_order, n_folds=n_folds, fold_seed=fold_seed + ) except Exception as e: report.fail(sec, f"the loader (H5Dataset) fails: {type(e).__name__}: {e}") return @@ -533,8 +540,14 @@ def build_parser() -> argparse.ArgumentParser: "--n-folds", type=int, default=None, - help="report the fold = event %% K split of EVENT/event (as corenet-train --n-folds K uses) " - "instead of the EVENT/fold column", + help="report the fold = squares(event) %% K split of EVENT/event (as corenet-train --n-folds K " + "uses) instead of the EVENT/fold column", + ) + p.add_argument( + "--fold-seed", + type=int, + default=FOLD_SEED, + help=f"seed of the squares(event) %% K rule [{FOLD_SEED}, the seed of the example files]", ) p.add_argument( "--json", default=None, help="also write the report to this JSON file" @@ -542,7 +555,9 @@ def build_parser() -> argparse.ArgumentParser: return p -def validate(config, data=None, rest_order=None, n_folds=None) -> Report: +def validate( + config, data=None, rest_order=None, n_folds=None, fold_seed=FOLD_SEED +) -> Report: report = Report() spec = check_config(config, report) if spec is not None and data is not None: @@ -553,7 +568,14 @@ def validate(config, data=None, rest_order=None, n_folds=None) -> Report: ) except Exception: rest_order = "pt" - check_data(data, spec, report, rest_order=rest_order, n_folds=n_folds) + check_data( + data, + spec, + report, + rest_order=rest_order, + n_folds=n_folds, + fold_seed=fold_seed, + ) elif data is not None: report.fail(f"data {data}", "not checked: the configuration is invalid") return report @@ -563,7 +585,9 @@ def main(argv=None) -> int: args = build_parser().parse_args(argv) if args.n_folds is not None and args.n_folds < 1: build_parser().error(f"--n-folds must be >= 1, got {args.n_folds}") - report = validate(args.config, args.input, args.rest_order, args.n_folds) + report = validate( + args.config, args.input, args.rest_order, args.n_folds, args.fold_seed + ) report.print() if args.json: Path(args.json).write_text(json.dumps(report.to_dict(), indent=1)) From 6744a679950fba0ac38e8918fedb2a77485c1ae3 Mon Sep 17 00:00:00 2001 From: Aniket Khanal Date: Wed, 21 Oct 2026 20:20:57 -0400 Subject: [PATCH 33/33] update fold rule --- tests/test_h5.py | 66 ++++++++++++++++++++++++------------------ tests/test_train.py | 18 ++++++++---- tests/test_validate.py | 50 +++++++++++++++++++++++--------- 3 files changed, 87 insertions(+), 47 deletions(-) diff --git a/tests/test_h5.py b/tests/test_h5.py index 2fff14b..d2c8aec 100644 --- a/tests/test_h5.py +++ b/tests/test_h5.py @@ -373,33 +373,41 @@ def test_errors(tmp_path): def test_fold_rule(): - """fold_of = event % n_folds; the default loader reads EVENT/fold; the two differ on the shipped files.""" - from corenet.data.h5 import FOLD_RULE_COLUMN, FOLD_RULE_EVENT, fold_of - - path = _need(DATA / "hh4b_small.h5") - spec = load_config(repo / "examples/hh4b/hh4b.yaml") - with h5py.File(path, "r") as f: - event, column = f["EVENT/event"][...], f["EVENT/fold"][...] - default = H5Dataset(path, spec) - assert default.fold_rule == FOLD_RULE_COLUMN and default.n_folds == 5 - assert (default.folds.numpy() == column).all() - ds5 = H5Dataset(path, spec, n_folds=5) - assert ds5.fold_rule == FOLD_RULE_EVENT and ds5.n_folds == 5 - assert (ds5.folds.numpy() == event % 5).all() - assert (ds5.folds.numpy() == fold_of(event, 5)).all() - assert set(ds5.folds.tolist()) == set(range(5)) - # the EVENT/fold column of the example files was assigned from a hash of the event number when - # the files were produced (docs/Dataset.md): it is not event % 5, so the default (column) and the - # rule give different folds on these files - assert not (ds5.folds.numpy() == column).all() - ds3 = H5Dataset(path, spec, n_folds=3) - assert ds3.n_folds == 3 and set(ds3.folds.tolist()) == set(range(3)) - assert (ds3.folds.numpy() == event % 3).all() - tr, te = ds3.fold_indices(0) - assert len(tr) + len(te) == len(ds3) and (event[te] % 3 == 0).all() - assert torch.equal( - ds3.inputs["b"], default.inputs["b"] - ) # the rule changes folds only + """fold_of = squares(event) % n_folds reproduces the EVENT/fold column of the example files + (5 folds, seed 0); the default loader reads the column; another seed or n_folds gives other folds.""" + from corenet.data.folds import FOLD_SEED, fold_of, fold_rule + from corenet.data.h5 import FOLD_RULE_COLUMN, FOLD_RULE_EVENT + + assert FOLD_SEED == 0 and fold_rule(5, 0) == "squares(event) % 5, seed 0" + for name, config in [ + ("hh4b_small.h5", "examples/hh4b/hh4b.yaml"), + ("bbww_small.h5", "examples/bbww/bbww.yaml"), + ]: + path = _need(DATA / name) + spec = load_config(repo / config) + with h5py.File(path, "r") as f: + event, column = f["EVENT/event"][...], f["EVENT/fold"][...] + assert (fold_of(event, 5, 0) == column).all(), name # all events + assert (fold_of(event, 5) == column).all() + default = H5Dataset(path, spec) + assert default.fold_rule == FOLD_RULE_COLUMN and default.n_folds == 5 + assert default.fold_seed == 0 and (default.folds.numpy() == column).all() + ds5 = H5Dataset(path, spec, n_folds=5) + assert ds5.fold_rule == FOLD_RULE_EVENT == "squares(event) % n_folds" + assert ds5.n_folds == 5 and ds5.fold_seed == 0 + assert torch.equal(ds5.folds, default.folds) + seeded = H5Dataset(path, spec, n_folds=5, fold_seed=1) + assert (seeded.folds.numpy() != column).any() and seeded.fold_seed == 1 + assert (fold_of(event, 5, 1) != column).any() + ds3 = H5Dataset(path, spec, n_folds=3) + counts = np.bincount(ds3.folds.numpy(), minlength=3) + assert ds3.n_folds == 3 and len(counts) == 3 and counts.min() > 0.3 * len(ds3) + assert (ds3.folds.numpy() == fold_of(event, 3)).all() + tr, te = ds3.fold_indices(0) + assert len(tr) + len(te) == len(ds3) and (fold_of(event[te], 3) == 0).all() + assert torch.equal( + ds3.inputs["b"], default.inputs["b"] + ) # only the folds change with pytest.raises(ValueError, match="negative"): fold_of(np.array([3, -1]), 5) with pytest.raises(ValueError, match="n_folds"): @@ -410,6 +418,8 @@ def test_fold_rule(): def test_missing_fold_column(tmp_path): """Without EVENT/fold the loader asks for --n-folds; with it the folds come from EVENT/event.""" + from corenet.data.folds import fold_of + src = _need(DATA / "hh4b_small.h5") spec = load_config(repo / "examples/hh4b/hh4b.yaml") dst = tmp_path / "nofold.h5" @@ -430,7 +440,7 @@ def test_missing_fold_column(tmp_path): with pytest.raises(KeyError, match="no EVENT/fold dataset; pass --n-folds"): H5Dataset(dst, spec) ds = H5Dataset(dst, spec, n_folds=5) - assert ds.n_folds == 5 and (ds.folds.numpy() == event % 5).all() + assert ds.n_folds == 5 and (ds.folds.numpy() == fold_of(event, 5)).all() with h5py.File(dst, "a") as g: del g["EVENT/event"] with pytest.raises(KeyError, match="no EVENT/event dataset"): diff --git a/tests/test_train.py b/tests/test_train.py index 244df2c..8d02438 100644 --- a/tests/test_train.py +++ b/tests/test_train.py @@ -480,7 +480,9 @@ def test_finetune_recipe_from_config() -> None: def test_n_folds_rule() -> None: - """--n-folds K: folds = event % K; --exclude-folds next; the rule is stored and reused by corenet-evaluate.""" + """--n-folds K: folds = squares(event) % K; --exclude-folds next; the rule and its seed are stored and + reused by corenet-evaluate.""" + from corenet.data.folds import fold_of from corenet.data.h5 import FOLD_RULE_COLUMN, FOLD_RULE_EVENT assert train.resolve_exclude_folds(["next"], 4, 5) == [0] @@ -515,17 +517,20 @@ def test_n_folds_rule() -> None: ) h = json.loads((out_dir / "history_fold0.json").read_text()) assert h["exclude_folds"] == [1] and h["n_folds"] == 3 - assert h["fold_rule"] == FOLD_RULE_EVENT == "event % n_folds" + assert h["fold_rule"] == FOLD_RULE_EVENT == "squares(event) % n_folds" + assert h["fold_seed"] == 0 meta = _metadata(out_dir) assert meta["n_folds"] == 3 and meta["fold_rule"] == FOLD_RULE_EVENT - assert meta["exclude_folds"] == [1] + assert meta["fold_seed"] == 0 and meta["exclude_folds"] == [1] with h5py.File(data, "r") as f: event = f["EVENT/event"][...] - n_test, n_val = int((event % 3 == 0).sum()), int((event % 3 == 1).sum()) + folds3 = fold_of(event, 3) + n_test, n_val = int((folds3 == 0).sum()), int((folds3 == 1).sum()) assert h["n_val"] == n_val and h["n_train"] == len(event) - n_test - n_val with h5py.File(out_dir / "predictions_fold0.h5", "r") as f: assert f.attrs["n_folds"] == 3 and f.attrs["fold_rule"] == FOLD_RULE_EVENT - assert (f["EVENT/event"][...] % 3 == 0).all() and len( + assert f.attrs["fold_seed"] == 0 + assert (fold_of(f["EVENT/event"][...], 3) == 0).all() and len( f["EVENT/event"] ) == n_test assert (f["EVENT/fold"][...] == 0).all() @@ -545,6 +550,7 @@ def test_n_folds_rule() -> None: assert metrics["n_events"] == n_test with h5py.File(tmp / "eval.h5", "r") as f: assert f.attrs["n_folds"] == 3 and f.attrs["fold_rule"] == FOLD_RULE_EVENT + assert f.attrs["fold_seed"] == 0 assert evaluate.main( [ "--checkpoint", @@ -556,7 +562,7 @@ def test_n_folds_rule() -> None: "--n-folds", "5", ] - )["n_events"] == int((event % 5 == 0).sum()) + )["n_events"] == int((fold_of(event, 5) == 0).sum()) # a checkpoint trained on the column keeps the column rule and needs it in the file with_column = write_synthetic(tmp / "col.h5", "hh4b") train.main( diff --git a/tests/test_validate.py b/tests/test_validate.py index 6479fe8..cf26007 100644 --- a/tests/test_validate.py +++ b/tests/test_validate.py @@ -334,27 +334,43 @@ def test_missing_event_scalar_and_warnings(tmp_path): def test_fold_report(tmp_path): - """The fold source and the events per fold; --n-folds reports the event % K split instead.""" + """The fold source and the events per fold; --n-folds reports the squares(event) % K split instead.""" + from corenet.data.folds import fold_of + data = write_synthetic(tmp_path / "d.h5", "bbww", n=400) r = validate.validate(BBWW, data) msgs = "\n".join(m for _, _, m in r.items) assert not r.errors and "folds from EVENT/fold: events per fold {0: " in msgs assert "--n-folds" not in msgs - # the synthetic folds are random: event % 5 disagrees with the column for most events + # the synthetic folds are random: the rule disagrees with the column for most events r = validate.validate(BBWW, data, n_folds=5) msgs = "\n".join(m for _, _, m in r.items) assert not r.errors, r.errors - assert "folds from event % 5 (--n-folds): events per fold {0: " in msgs - assert "EVENT/fold disagrees with event % 5 for " in "\n".join(r.warnings) + assert ( + "folds from squares(event) % 5, seed 0 (--n-folds): events per fold {0: " + in msgs + ) + assert "EVENT/fold disagrees with squares(event) % 5, seed 0 for " in "\n".join( + r.warnings + ) assert "loader + forward pass" in msgs - # a column equal to the rule + # a column equal to the rule (seed 3) with h5py.File(data, "a") as f: - _replace(f, "EVENT/fold", f["EVENT/event"][...] % 5) - r = validate.validate(BBWW, data, n_folds=5) - assert not r.errors and "EVENT/fold equals event % 5" in "\n".join( + _replace(f, "EVENT/fold", fold_of(f["EVENT/event"][...], 5, 3)) + r = validate.validate(BBWW, data, n_folds=5, fold_seed=3) + assert not r.errors and "EVENT/fold equals squares(event) % 5, seed 3" in "\n".join( m for _, _, m in r.items ) assert not r.warnings, r.warnings + assert "disagrees" in "\n".join(validate.validate(BBWW, data, n_folds=5).warnings) + # the shipped file: its column is the rule with 5 folds and seed 0 + shipped = repo / "examples/data/hh4b_small.h5" + if shipped.exists(): + r = validate.validate(HH4B, shipped, n_folds=5) + assert not r.errors and not r.warnings, (r.errors, r.warnings) + assert "EVENT/fold equals squares(event) % 5, seed 0" in "\n".join( + m for _, _, m in r.items + ) # no column: FAIL by default (see BROKEN_FILES), valid with --n-folds with h5py.File(data, "a") as f: del f["EVENT/fold"] @@ -366,17 +382,25 @@ def test_fold_report(tmp_path): ) r = validate.validate(BBWW, data, n_folds=3) msgs = "\n".join(m for _, _, m in r.items) - assert ( - not r.errors - and "folds from event % 3 (--n-folds): events per fold {0: 134, 1: 133, 2: 133}" + with h5py.File(data, "r") as f: + counts = np.bincount(fold_of(f["EVENT/event"][...], 3), minlength=3).tolist() + expected = {i: c for i, c in enumerate(counts)} + assert not r.errors and ( + f"folds from squares(event) % 3, seed 0 (--n-folds): events per fold {expected}" in msgs ) # negative event numbers cannot be folded with h5py.File(data, "a") as f: _replace(f, "EVENT/event", f["EVENT/event"][...] - 10) r = validate.validate(BBWW, data, n_folds=3) - assert "folds = event % 3 (--n-folds): 10 event numbers are negative" in _errors(r) + assert ( + "folds = squares(event) % 3, seed 0 (--n-folds): 10 event numbers are negative" + in _errors(r) + ) with h5py.File(data, "a") as f: del f["EVENT/event"] r = validate.validate(BBWW, data, n_folds=3) - assert "folds = event % 3 (--n-folds): EVENT/event not found" in _errors(r) + assert ( + "folds = squares(event) % 3, seed 0 (--n-folds): EVENT/event not found" + in _errors(r) + )