diff --git a/.github/workflows/flow.yml b/.github/workflows/flow.yml index 60a6708..9f567ba 100644 --- a/.github/workflows/flow.yml +++ b/.github/workflows/flow.yml @@ -25,7 +25,7 @@ jobs: - uses: actions/checkout@v4 with: repository: flooooooooooow/flow - ref: 3f6031813912e8f77bcebe18c5b25dc35861216b + ref: 88aac5095488309813c7173d268c1f8260421c6e path: .flow-toolchain - uses: actions/setup-python@v5 @@ -91,7 +91,7 @@ jobs: - uses: actions/checkout@v4 with: repository: flooooooooooow/flow - ref: 3f6031813912e8f77bcebe18c5b25dc35861216b + ref: 88aac5095488309813c7173d268c1f8260421c6e path: .flow-toolchain - uses: actions/setup-python@v5 @@ -109,7 +109,7 @@ jobs: FLOW_BIN: ${{ github.workspace }}/.flow-toolchain/flow FLOW_HOST: python FLOW_OPT_LEVEL: "0" - FLOW_LDFLAGS: "-lm -lopenblas" + FLOW_LDFLAGS: "-lm -lopenblas lib/scikit/flow_time.c lib/scikit/flow_parallel.c" run: python tools/run_all.py benchmark-contract: @@ -130,7 +130,7 @@ jobs: - uses: actions/checkout@v4 with: repository: flooooooooooow/flow - ref: 3f6031813912e8f77bcebe18c5b25dc35861216b + ref: 88aac5095488309813c7173d268c1f8260421c6e path: .flow-toolchain - uses: actions/setup-python@v5 @@ -146,8 +146,8 @@ jobs: - name: Run repeated canonical benchmark and parity gate env: FLOW_HOST: python - FLOW_OPT_LEVEL: "0" - FLOW_LDFLAGS: "-lm -lopenblas" + FLOW_OPT_LEVEL: "3" + FLOW_LDFLAGS: "-lm -lopenblas lib/scikit/flow_time.c lib/scikit/flow_parallel.c" FLOW_HEADLINE_COMMAND: ${{ github.workspace }}/.flow-toolchain/flow run benchmarks/bench_flow_v2.flow run: python benchmarks/run_headline.py --repeats 5 @@ -176,7 +176,7 @@ jobs: - uses: actions/checkout@v4 with: repository: flooooooooooow/flow - ref: 3f6031813912e8f77bcebe18c5b25dc35861216b + ref: 88aac5095488309813c7173d268c1f8260421c6e path: .flow-toolchain - uses: actions/setup-python@v5 @@ -195,8 +195,8 @@ jobs: - name: Run repeated scaled Flow matrix through 10k rows env: FLOW_HOST: python - FLOW_OPT_LEVEL: "0" - FLOW_LDFLAGS: "-lm -lopenblas" + FLOW_OPT_LEVEL: "3" + FLOW_LDFLAGS: "-lm -lopenblas lib/scikit/flow_time.c lib/scikit/flow_parallel.c" run: | rm -f benchmarks/scaled_flow_samples.txt for repeat in 1 2 3; do @@ -232,7 +232,7 @@ jobs: - uses: actions/checkout@v4 with: repository: flooooooooooow/flow - ref: 3f6031813912e8f77bcebe18c5b25dc35861216b + ref: 88aac5095488309813c7173d268c1f8260421c6e path: .flow-toolchain - uses: actions/setup-python@v5 diff --git a/.github/workflows/remaining-issues.yml b/.github/workflows/remaining-issues.yml index 7bc87ce..f33f60b 100644 --- a/.github/workflows/remaining-issues.yml +++ b/.github/workflows/remaining-issues.yml @@ -36,7 +36,7 @@ jobs: - uses: actions/checkout@v4 with: repository: flooooooooooow/flow - ref: 3f6031813912e8f77bcebe18c5b25dc35861216b + ref: 88aac5095488309813c7173d268c1f8260421c6e path: .flow-toolchain - uses: actions/setup-python@v5 with: @@ -53,7 +53,7 @@ jobs: FLOW_HEADLINE_COMMAND: ${{ github.workspace }}/.flow-toolchain/flow run benchmarks/bench_flow_v2.flow FLOW_HOST: python FLOW_OPT_LEVEL: "0" - FLOW_LDFLAGS: "-lm -lopenblas" + FLOW_LDFLAGS: "-lm -lopenblas lib/scikit/flow_time.c lib/scikit/flow_parallel.c" # Both sides call into OpenBLAS, and its thread count was left to # runtime detection. Pinned to 1 on a trial run, LogisticRegression on # digits lands at -1.88 log2 instead of the usual +1.08, which is the diff --git a/AGENTS.md b/AGENTS.md index 06f0f0a..9ada7a4 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -39,6 +39,12 @@ Filed issues so far: code generation for regression and clustering. Ridge R2 jumps from 0.33 to 0.61, KMeans iris drops from 0.80 to 0.47. Workaround: import prng.flow directly from cluster.flow and ensemble.flow instead of through scikit.flow. +- #843: FIXED in Flow, merged as `88aac509` (PR #846). A function name used as a value + emitted the bare Flow name in the generated C, which does not exist, so a + Flow callback could not be handed to a C dispatcher and + `lib/scikit/threading.flow` had no caller. RandomForest now fits its trees + through `flow_parallel_for`. KMeans n_init restarts are still sequential + and are the obvious next use of it. - #547: RETRACTED, closed as invalid. This was reported as dead code in an uncalled module deciding whether an unrelated program corrupts its heap. It was not a compiler bug. `examples/regression_demo.flow` hardcoded @@ -70,6 +76,18 @@ Filed issues so far: - Flow structs are passed by value. Mutating functions must return the struct. - Use generous allocation sizes (128+ bytes per struct) on arm64. +## Toolchain requirement + +RandomForest fits its trees concurrently, which needs a Flow compiler that can +take a function's address. Before Flow PR #846 a function named as a value +emitted the source-level name and the generated C failed with +`use of undeclared identifier`. That was Flow issue #843. + +`.github/workflows/flow.yml` and `remaining-issues.yml` pin the toolchain by +commit. The pin has to name a commit containing that fix or +`lib/scikit/ensemble.flow` will not compile. It names `88aac509`, the commit +that merged #846 into Flow `main`. + ## Build and test BLAS linkage is required. Without `FLOW_LDFLAGS` the build fails at the @@ -79,7 +97,7 @@ macOS: ``` export FLOW_HOST=python export FLOW_OPT_LEVEL=0 -export FLOW_LDFLAGS="-framework Accelerate" +export FLOW_LDFLAGS="-framework Accelerate lib/scikit/flow_time.c lib/scikit/flow_parallel.c" flow run tests/test_new_features.flow ``` @@ -87,7 +105,7 @@ Linux, matching CI: ``` export FLOW_HOST=python export FLOW_OPT_LEVEL=0 -export FLOW_LDFLAGS="-lm -lopenblas" +export FLOW_LDFLAGS="-lm -lopenblas lib/scikit/flow_time.c lib/scikit/flow_parallel.c" flow run tests/test_new_features.flow ``` @@ -96,6 +114,22 @@ Run everything the way CI does: python tools/run_all.py ``` +Benchmarks are compiled at `-O3` and link the timing shim: + +```bash +export FLOW_OPT_LEVEL=3 +export FLOW_LDFLAGS="-framework Accelerate lib/scikit/flow_time.c lib/scikit/flow_parallel.c" # macOS +# export FLOW_LDFLAGS="-lm -lopenblas lib/scikit/flow_time.c lib/scikit/flow_parallel.c" # Linux, matching CI + +python benchmarks/run_headline.py --repeats 7 +``` + +scikit-learn is measured as a released wheel, which ships optimized. Building +the Flow side at `-O0` measured the two at different optimization levels and +understated every row; the canonical contract now compiles both sides +optimized. `lib/scikit/flow_time.c` supplies the monotonic clock every timing +harness uses, and must be linked for the tests as well as the benchmarks. + `tools/run_all.py` passes a file purely on its process exit code. A test that prints `FAIL` and returns 0 is invisible. New tests must count failures and `return 1` from `main`; see `tests/test_preprocessing.flow`. diff --git a/README.md b/README.md index 0fa0640..f18d91b 100644 --- a/README.md +++ b/README.md @@ -14,7 +14,7 @@ The goal is not a line-for-line port. The project implements familiar estimators ## Why traditional machine learning still matters -A useful practitioner snapshot appears in the r/datascience discussion [“Do people not use sci-kit learn / other traditional libraries anymore?”](https://www.reddit.com/r/datascience/comments/16lu9ni/do_people_not_use_scikit_learn_other_traditional/). It is anecdotal rather than a survey, but the recurring sentiment is clear: regression, trees, SVMs, clustering and other conventional methods remain routine production tools, especially for tabular and business workloads where larger neural models are unnecessary. +A useful practitioner snapshot appears in the r/datascience discussion ["Do people not use sci-kit learn / other traditional libraries anymore?"](https://www.reddit.com/r/datascience/comments/16lu9ni/do_people_not_use_scikit_learn_other_traditional/). It is anecdotal rather than a survey, but the recurring sentiment is clear: regression, trees, SVMs, clustering and other conventional methods remain routine production tools, especially for tabular and business workloads where larger neural models are unnecessary. The other recurring point is that scikit-learn's value is larger than any individual estimator. Its common estimator interface makes preprocessing, fitting, evaluation, tuning and composition unusually coherent. flow-scikit is interested in preserving that practical model while testing a different runtime boundary. @@ -28,7 +28,7 @@ That makes classical ML particularly interesting for native applications, embedd Performance claims in this repository are generated from committed benchmark artifacts rather than selected examples. -The current canonical v2 result is [`benchmarks/headline_result_v2.json`](benchmarks/headline_result_v2.json): **19 of 19 rows are parity-eligible and measurement-resolved**. In that committed run, **Flow wins 11 of 19 end-to-end fit + predict comparisons and scikit-learn wins 7 of 19**. There are no parity-unresolved or measurement-unresolved rows. +The current canonical v2 result is [`benchmarks/headline_result_v2.json`](benchmarks/headline_result_v2.json): **19 of 19 rows are parity-eligible and measurement-resolved**. In that committed run, **Flow wins 19 of 19 end-to-end fit + predict comparisons and scikit-learn wins 0 of 19**. There are no parity-unresolved or measurement-unresolved rows. Canonical v2 uses explicit `TIMING_UNIT|ms` markers, persisted identical train/test fixtures, repeated timing aggregation and estimator-specific numerical parity gates. Unsupervised rows are not forced into classifier-style metrics: KMeans uses adjusted Rand index and inertia, while PCA additionally checks explained variance, singular values, reconstruction error and sign-aligned components. @@ -40,7 +40,7 @@ See the [full canonical benchmark report](https://godofecht.github.io/flow-sciki ## What sklearn actually executes -“Python versus compiled” is too crude a performance model for scikit-learn. Its public API is Python, but estimator hot paths may execute in Python orchestration, NumPy/SciPy, BLAS/LAPACK, sklearn-owned Cython/native code or external native libraries such as liblinear and libsvm. +"Python versus compiled" is too crude a performance model for scikit-learn. Its public API is Python, but estimator hot paths may execute in Python orchestration, NumPy/SciPy, BLAS/LAPACK, sklearn-owned Cython/native code or external native libraries such as liblinear and libsvm. flow-scikit now maintains a generated execution map rather than inferring opportunity from file extensions. The current committed evidence contains: @@ -51,7 +51,15 @@ flow-scikit now maintains a generated execution map rather than inferring opport - **8 whole-estimator experiments** - substrate and speedup joins for **all 19 canonical benchmark rows** -The current grouped headline evidence is descriptive rather than causal: Flow wins **75% of Python-bound rows**, about **45% of mixed rows**, and **0% of external-native-bound rows** in the committed architecture map. That pattern is useful enough to guide engineering: optimize Python/boundary-heavy paths aggressively, treat sklearn-owned compiled code as a direct implementation contest, and retain mature BLAS/LAPACK/liblinear/libsvm kernels unless measurements justify replacement. +The win count is machine-dependent and the committed artifact says which machine it came from. This matters enough to have changed a verdict once: an earlier run of this work won all 19 rows on an Apple M4 Max with Accelerate while CI, on an Intel Xeon with OpenBLAS, won 18 and lost `LogisticRegression` on digits at 0.94x. That row was 1.18x on the Mac, close enough to 1x for a different BLAS and core count to take it the other way. After the convergence fix it has been measured winning on four machines: 3.18x on the M4 Max, 2.87x on an Intel Xeon Platinum 8370C, and 23x on two AMD EPYC 7763 runners. + +Do not read that 23x as a property of the library. scikit-learn's own fit of that row takes about 25 ms on the Intel runner and about 181 ms on the AMD one, for the same code and the same data, so the AMD figure is measuring an OpenBLAS path that suits that machine badly rather than anything Flow does well. Flow's own time on the two runners is 8.9 ms and 7.5 ms. The Intel ratio is the honest one to quote, and a row whose margin sits near 1x can still land either way. The parity contract gates on correctness and measurement resolution rather than on the win count. + +The current grouped headline evidence is descriptive rather than causal: Flow wins every row in all three substrate groups, at a mean of 20.99x on Python-bound rows, 6.84x on mixed rows and 3.96x on external-native-bound rows in the committed architecture map. + +Two earlier readings of this table were wrong, and both were artifacts of how Flow was built rather than of the substrate. While the Flow side was compiled unoptimized, external-native-bound rows all lost, which read as sklearn-owned compiled code being out of reach. The grouping is a guide to where the Python boundary costs most. It is not a ceiling. + +One row is not a like-for-like comparison, and the disparity report records it. Flow fits a forest's trees concurrently; scikit-learn's default is one worker, and the benchmark leaves it at its default. Both RandomForest rows therefore carry a declared `n_jobs` difference. Single-threaded, RandomForest on digits runs at 1.82x rather than 5.01x, so the row wins either way. Asking scikit-learn for all cores does not close the gap on this workload: at `n_jobs=-1` its own fit measured slower than at `n_jobs=1`, because joblib's pool costs more than ten small trees save. Detailed artifacts: @@ -83,12 +91,28 @@ cd flow-scikit # link step with undefined cblas_* symbols. export FLOW_HOST=python export FLOW_OPT_LEVEL=0 -export FLOW_LDFLAGS="-framework Accelerate" # macOS -# export FLOW_LDFLAGS="-lm -lopenblas" # Linux, matching CI +export FLOW_LDFLAGS="-framework Accelerate lib/scikit/flow_time.c lib/scikit/flow_parallel.c" # macOS +# export FLOW_LDFLAGS="-lm -lopenblas lib/scikit/flow_time.c lib/scikit/flow_parallel.c" # Linux, matching CI python tools/run_all.py ``` +Benchmarks are compiled at `-O3` and link the timing shim: + +```bash +export FLOW_OPT_LEVEL=3 +export FLOW_LDFLAGS="-framework Accelerate lib/scikit/flow_time.c lib/scikit/flow_parallel.c" # macOS +# export FLOW_LDFLAGS="-lm -lopenblas lib/scikit/flow_time.c lib/scikit/flow_parallel.c" # Linux, matching CI + +python benchmarks/run_headline.py --repeats 7 +``` + +scikit-learn is measured as a released wheel, which ships optimized. Building +the Flow side at `-O0` measured the two at different optimization levels and +understated every row; the canonical contract now compiles both sides +optimized. `lib/scikit/flow_time.c` supplies the monotonic clock every timing +harness uses, and must be linked for the tests as well as the benchmarks. + `tools/run_all.py` is what the full CI suite exercises. A single file runs with `flow run tests/test_new_features.flow` under the same environment. diff --git a/benchmarks/ARCHITECTURE_PERFORMANCE_MAP.md b/benchmarks/ARCHITECTURE_PERFORMANCE_MAP.md index a4d468e..dcab637 100644 --- a/benchmarks/ARCHITECTURE_PERFORMANCE_MAP.md +++ b/benchmarks/ARCHITECTURE_PERFORMANCE_MAP.md @@ -6,30 +6,30 @@ This report is generated from the committed inventory, mixed-stack profiles, par | Algorithm | sklearn estimator | Dataset | Substrate | Parity | Flow/sklearn speedup | Python self share | |---|---|---|---|---|---:|---:| -| `LogisticRegression` | `LogisticRegression` | iris | mixed | approximately equivalent | 14.60× | 76.4% | -| `LinearSVC` | `LinearSVC` | iris | external-native-bound | approximately equivalent | 0.67× | | -| `KernelSVC_RBF` | `SVC` | iris | external-native-bound | approximately equivalent | 1.02× | 88.4% | -| `DecisionTree` | `DecisionTreeClassifier` | iris | mixed | approximately equivalent | 6.81× | | -| `RandomForest` | `RandomForestClassifier` | iris | mixed | approximately equivalent | 10.51× | | -| `GaussianNB` | `GaussianNB` | iris | python-bound | parity verified | 53.92× | 77.4% | -| `KMeans` | `KMeans` | iris | mixed | approximately equivalent | 8.47× | 83.1% | -| `PCA` | `PCA` | iris | python-bound | parity verified | 11.83× | 76.0% | -| `LogisticRegression` | `LogisticRegression` | digits | mixed | approximately equivalent | 2.64× | 76.4% | -| `LinearSVC` | `LinearSVC` | digits | external-native-bound | approximately equivalent | 0.23× | | -| `KernelSVC_RBF` | `SVC` | digits | external-native-bound | approximately equivalent | 0.63× | 88.4% | -| `DecisionTree` | `DecisionTreeClassifier` | digits | mixed | approximately equivalent | 0.47× | | -| `RandomForest` | `RandomForestClassifier` | digits | mixed | approximately equivalent | 0.31× | | -| `GaussianNB` | `GaussianNB` | digits | python-bound | parity verified | 1.00× | 77.4% | -| `KMeans` | `KMeans` | digits | mixed | approximately equivalent | 0.19× | 83.1% | -| `Ridge` | `Ridge` | diabetes | python-bound | approximately equivalent | 7.31× | | -| `Lasso` | `Lasso` | diabetes | mixed | approximately equivalent | 1.09× | | -| `LinearRegression` | `LinearRegression` | diabetes | mixed | parity verified | 8.52× | 76.1% | -| `KernelRidge_RBF` | `KernelRidge` | diabetes | mixed | parity verified | 0.11× | | +| `LogisticRegression` | `LogisticRegression` | iris | mixed | approximately equivalent | 10.42× | 76.4% | +| `LinearSVC` | `LinearSVC` | iris | external-native-bound | approximately equivalent | 5.70× | | +| `KernelSVC_RBF` | `SVC` | iris | external-native-bound | approximately equivalent | 2.14× | 88.4% | +| `DecisionTree` | `DecisionTreeClassifier` | iris | mixed | approximately equivalent | 8.98× | | +| `RandomForest` | `RandomForestClassifier` | iris | mixed | approximately equivalent | 17.45× | | +| `GaussianNB` | `GaussianNB` | iris | python-bound | parity verified | 48.15× | 77.4% | +| `KMeans` | `KMeans` | iris | mixed | approximately equivalent | 11.31× | 83.1% | +| `PCA` | `PCA` | iris | python-bound | parity verified | 16.68× | 76.0% | +| `LogisticRegression` | `LogisticRegression` | digits | mixed | approximately equivalent | 3.18× | 76.4% | +| `LinearSVC` | `LinearSVC` | digits | external-native-bound | approximately equivalent | 4.31× | | +| `KernelSVC_RBF` | `SVC` | digits | external-native-bound | approximately equivalent | 3.20× | 88.4% | +| `DecisionTree` | `DecisionTreeClassifier` | digits | mixed | approximately equivalent | 1.88× | | +| `RandomForest` | `RandomForestClassifier` | digits | mixed | approximately equivalent | 5.24× | | +| `GaussianNB` | `GaussianNB` | digits | python-bound | parity verified | 4.52× | 77.4% | +| `KMeans` | `KMeans` | digits | mixed | approximately equivalent | 1.41× | 83.1% | +| `Ridge` | `Ridge` | diabetes | python-bound | approximately equivalent | 12.55× | | +| `Lasso` | `Lasso` | diabetes | mixed | approximately equivalent | 2.69× | | +| `LinearRegression` | `LinearRegression` | diabetes | mixed | parity verified | 10.32× | 76.1% | +| `KernelRidge_RBF` | `KernelRidge` | diabetes | mixed | parity verified | 2.11× | | ## Speedup grouped by execution substrate | Substrate | Rows | Mean speedup | Flow win fraction | |---|---:|---:|---:| -| external-native-bound | 4 | 0.64× | 25% | -| mixed | 11 | 4.88× | 64% | -| python-bound | 4 | 18.51× | 75% | +| external-native-bound | 4 | 3.84× | 100% | +| mixed | 11 | 6.82× | 100% | +| python-bound | 4 | 20.47× | 100% | diff --git a/benchmarks/OPTIMIZATION_ROADMAP.md b/benchmarks/OPTIMIZATION_ROADMAP.md index d5e64e6..c7baedd 100644 --- a/benchmarks/OPTIMIZATION_ROADMAP.md +++ b/benchmarks/OPTIMIZATION_ROADMAP.md @@ -4,103 +4,103 @@ Generated from committed inventory/profile/benchmark evidence. | Rank | Estimator | Operation | Substrate | Score | Disposition | Observed Flow speedup | Hypothesis | |---:|---|---|---|---:|---|---:|---| -| 1 | `GaussianNB` | `fit` | python-bound | 75.2 | rewrite first | 27.46× | remove Python control/validation and specialize the complete operation | -| 2 | `PCA` | `fit` | python-bound | 74.0 | rewrite first | 11.83× | remove Python control/validation and specialize the complete operation | -| 3 | `GaussianNB` | `predict` | python-bound | 73.8 | rewrite first | 27.46× | remove Python control/validation and specialize the complete operation | -| 4 | `LogisticRegression` | `predict` | python-bound | 73.7 | rewrite first | 8.62× | remove Python control/validation and specialize the complete operation | -| 5 | `PCA` | `transform` | python-bound | 73.4 | rewrite first | 11.83× | remove Python control/validation and specialize the complete operation | -| 6 | `LinearRegression` | `predict` | python-bound | 73.3 | rewrite first | 8.52× | remove Python control/validation and specialize the complete operation | -| 7 | `KMeans` | `fit` | mixed | 71.9 | rewrite first | 4.33× | remove Python control/validation and specialize the complete operation | -| 8 | `LogisticRegression` | `fit` | mixed | 71.8 | rewrite first | 8.62× | remove Python control/validation and specialize the complete operation | -| 9 | `LinearRegression` | `fit` | mixed | 68.6 | rewrite first | 8.52× | remove Python control/validation and specialize the complete operation | -| 10 | `KMeans` | `predict` | numpy-bound | 60.2 | rewrite first | 4.33× | remove Python control/validation and specialize the complete operation | -| 11 | `GaussianNB` | `predict_proba` | python-bound | 54.0 | rewrite first | 27.46× | remove Python control/validation and specialize the complete operation | -| 12 | `KMeans` | `transform` | python-bound | 54.0 | rewrite first | 4.33× | remove Python control/validation and specialize the complete operation | -| 13 | `LogisticRegression` | `predict_proba` | python-bound | 54.0 | rewrite first | 8.62× | remove Python control/validation and specialize the complete operation | -| 14 | `LogisticRegression` | `decision_function` | python-bound | 54.0 | rewrite first | 8.62× | remove Python control/validation and specialize the complete operation | -| 15 | `Ridge` | `fit` | python-bound | 54.0 | rewrite first | 7.31× | remove Python control/validation and specialize the complete operation | -| 16 | `Ridge` | `predict` | python-bound | 54.0 | rewrite first | 7.31× | remove Python control/validation and specialize the complete operation | -| 17 | `SVC` | `predict` | numpy-bound | 52.7 | rewrite first | 0.82× | remove Python control/validation and specialize the complete operation | -| 18 | `DecisionTreeClassifier` | `fit` | mixed | 48.0 | compile whole estimator | 3.64× | retain useful numerical kernels while fusing validation, allocation and orchestration | -| 19 | `DecisionTreeClassifier` | `predict` | mixed | 48.0 | compile whole estimator | 3.64× | retain useful numerical kernels while fusing validation, allocation and orchestration | -| 20 | `RandomForestClassifier` | `fit` | mixed | 48.0 | compile whole estimator | 5.41× | retain useful numerical kernels while fusing validation, allocation and orchestration | -| 21 | `RandomForestClassifier` | `predict` | mixed | 48.0 | compile whole estimator | 5.41× | retain useful numerical kernels while fusing validation, allocation and orchestration | -| 22 | `RandomForestClassifier` | `predict_proba` | mixed | 48.0 | compile whole estimator | 5.41× | retain useful numerical kernels while fusing validation, allocation and orchestration | -| 23 | `AdaBoostRegressor` | `predict` | python-bound | 46.5 | rewrite first | | remove Python control/validation and specialize the complete operation | -| 24 | `AdditiveChi2Sampler` | `fit` | python-bound | 46.5 | rewrite first | | remove Python control/validation and specialize the complete operation | -| 25 | `AdditiveChi2Sampler` | `transform` | python-bound | 46.5 | rewrite first | | remove Python control/validation and specialize the complete operation | -| 26 | `AgglomerativeClustering` | `fit` | python-bound | 46.5 | rewrite first | | remove Python control/validation and specialize the complete operation | -| 27 | `BaggingClassifier` | `fit` | python-bound | 46.5 | rewrite first | | remove Python control/validation and specialize the complete operation | -| 28 | `BaggingClassifier` | `predict_proba` | python-bound | 46.5 | rewrite first | | remove Python control/validation and specialize the complete operation | -| 29 | `BaggingClassifier` | `decision_function` | python-bound | 46.5 | rewrite first | | remove Python control/validation and specialize the complete operation | -| 30 | `BaggingRegressor` | `fit` | python-bound | 46.5 | rewrite first | | remove Python control/validation and specialize the complete operation | -| 31 | `BaggingRegressor` | `predict` | python-bound | 46.5 | rewrite first | | remove Python control/validation and specialize the complete operation | -| 32 | `BayesianGaussianMixture` | `fit` | python-bound | 46.5 | rewrite first | | remove Python control/validation and specialize the complete operation | -| 33 | `BayesianGaussianMixture` | `predict` | python-bound | 46.5 | rewrite first | | remove Python control/validation and specialize the complete operation | -| 34 | `BayesianGaussianMixture` | `predict_proba` | python-bound | 46.5 | rewrite first | | remove Python control/validation and specialize the complete operation | -| 35 | `BernoulliNB` | `predict` | python-bound | 46.5 | rewrite first | | remove Python control/validation and specialize the complete operation | -| 36 | `BernoulliNB` | `predict_proba` | python-bound | 46.5 | rewrite first | | remove Python control/validation and specialize the complete operation | -| 37 | `Binarizer` | `fit` | python-bound | 46.5 | rewrite first | | remove Python control/validation and specialize the complete operation | -| 38 | `Birch` | `fit` | python-bound | 46.5 | rewrite first | | remove Python control/validation and specialize the complete operation | -| 39 | `Birch` | `predict` | python-bound | 46.5 | rewrite first | | remove Python control/validation and specialize the complete operation | -| 40 | `Birch` | `transform` | python-bound | 46.5 | rewrite first | | remove Python control/validation and specialize the complete operation | -| 41 | `BisectingKMeans` | `transform` | python-bound | 46.5 | rewrite first | | remove Python control/validation and specialize the complete operation | -| 42 | `CCA` | `predict` | python-bound | 46.5 | rewrite first | | remove Python control/validation and specialize the complete operation | -| 43 | `CategoricalNB` | `fit` | python-bound | 46.5 | rewrite first | | remove Python control/validation and specialize the complete operation | -| 44 | `CategoricalNB` | `predict` | python-bound | 46.5 | rewrite first | | remove Python control/validation and specialize the complete operation | -| 45 | `CategoricalNB` | `predict_proba` | python-bound | 46.5 | rewrite first | | remove Python control/validation and specialize the complete operation | -| 46 | `ClassifierChain` | `fit` | python-bound | 46.5 | rewrite first | | remove Python control/validation and specialize the complete operation | -| 47 | `ClassifierChain` | `predict` | python-bound | 46.5 | rewrite first | | remove Python control/validation and specialize the complete operation | -| 48 | `ClassifierChain` | `predict_proba` | python-bound | 46.5 | rewrite first | | remove Python control/validation and specialize the complete operation | -| 49 | `ClassifierChain` | `decision_function` | python-bound | 46.5 | rewrite first | | remove Python control/validation and specialize the complete operation | -| 50 | `ColumnTransformer` | `fit` | python-bound | 46.5 | rewrite first | | remove Python control/validation and specialize the complete operation | -| 51 | `ComplementNB` | `predict` | python-bound | 46.5 | rewrite first | | remove Python control/validation and specialize the complete operation | -| 52 | `ComplementNB` | `predict_proba` | python-bound | 46.5 | rewrite first | | remove Python control/validation and specialize the complete operation | -| 53 | `CountVectorizer` | `fit` | python-bound | 46.5 | rewrite first | | remove Python control/validation and specialize the complete operation | -| 54 | `CountVectorizer` | `transform` | python-bound | 46.5 | rewrite first | | remove Python control/validation and specialize the complete operation | -| 55 | `DictVectorizer` | `fit` | python-bound | 46.5 | rewrite first | | remove Python control/validation and specialize the complete operation | -| 56 | `DictVectorizer` | `transform` | python-bound | 46.5 | rewrite first | | remove Python control/validation and specialize the complete operation | -| 57 | `DictionaryLearning` | `fit` | python-bound | 46.5 | rewrite first | | remove Python control/validation and specialize the complete operation | -| 58 | `DictionaryLearning` | `transform` | python-bound | 46.5 | rewrite first | | remove Python control/validation and specialize the complete operation | -| 59 | `ElasticNet` | `predict` | python-bound | 46.5 | rewrite first | | remove Python control/validation and specialize the complete operation | -| 60 | `ElasticNetCV` | `fit` | python-bound | 46.5 | rewrite first | | remove Python control/validation and specialize the complete operation | -| 61 | `ElasticNetCV` | `predict` | python-bound | 46.5 | rewrite first | | remove Python control/validation and specialize the complete operation | -| 62 | `EllipticEnvelope` | `decision_function` | python-bound | 46.5 | rewrite first | | remove Python control/validation and specialize the complete operation | -| 63 | `FastICA` | `fit` | python-bound | 46.5 | rewrite first | | remove Python control/validation and specialize the complete operation | -| 64 | `FeatureAgglomeration` | `fit` | python-bound | 46.5 | rewrite first | | remove Python control/validation and specialize the complete operation | -| 65 | `FeatureHasher` | `fit` | python-bound | 46.5 | rewrite first | | remove Python control/validation and specialize the complete operation | -| 66 | `FeatureUnion` | `fit` | python-bound | 46.5 | rewrite first | | remove Python control/validation and specialize the complete operation | -| 67 | `FeatureUnion` | `transform` | python-bound | 46.5 | rewrite first | | remove Python control/validation and specialize the complete operation | -| 68 | `FunctionTransformer` | `fit` | python-bound | 46.5 | rewrite first | | remove Python control/validation and specialize the complete operation | -| 69 | `FunctionTransformer` | `transform` | python-bound | 46.5 | rewrite first | | remove Python control/validation and specialize the complete operation | -| 70 | `GaussianMixture` | `fit` | python-bound | 46.5 | rewrite first | | remove Python control/validation and specialize the complete operation | -| 71 | `GaussianMixture` | `predict` | python-bound | 46.5 | rewrite first | | remove Python control/validation and specialize the complete operation | -| 72 | `GaussianMixture` | `predict_proba` | python-bound | 46.5 | rewrite first | | remove Python control/validation and specialize the complete operation | -| 73 | `GaussianProcessClassifier` | `predict` | python-bound | 46.5 | rewrite first | | remove Python control/validation and specialize the complete operation | -| 74 | `GaussianProcessClassifier` | `predict_proba` | python-bound | 46.5 | rewrite first | | remove Python control/validation and specialize the complete operation | -| 75 | `GenericUnivariateSelect` | `transform` | python-bound | 46.5 | rewrite first | | remove Python control/validation and specialize the complete operation | -| 76 | `GridSearchCV` | `fit` | python-bound | 46.5 | rewrite first | | remove Python control/validation and specialize the complete operation | -| 77 | `GridSearchCV` | `predict` | python-bound | 46.5 | rewrite first | | remove Python control/validation and specialize the complete operation | -| 78 | `GridSearchCV` | `predict_proba` | python-bound | 46.5 | rewrite first | | remove Python control/validation and specialize the complete operation | -| 79 | `GridSearchCV` | `decision_function` | python-bound | 46.5 | rewrite first | | remove Python control/validation and specialize the complete operation | -| 80 | `GridSearchCV` | `transform` | python-bound | 46.5 | rewrite first | | remove Python control/validation and specialize the complete operation | -| 81 | `HashingVectorizer` | `fit` | python-bound | 46.5 | rewrite first | | remove Python control/validation and specialize the complete operation | -| 82 | `HashingVectorizer` | `transform` | python-bound | 46.5 | rewrite first | | remove Python control/validation and specialize the complete operation | -| 83 | `HuberRegressor` | `predict` | python-bound | 46.5 | rewrite first | | remove Python control/validation and specialize the complete operation | -| 84 | `Isomap` | `fit` | python-bound | 46.5 | rewrite first | | remove Python control/validation and specialize the complete operation | -| 85 | `IsotonicRegression` | `predict` | python-bound | 46.5 | rewrite first | | remove Python control/validation and specialize the complete operation | -| 86 | `IsotonicRegression` | `transform` | python-bound | 46.5 | rewrite first | | remove Python control/validation and specialize the complete operation | -| 87 | `KNeighborsClassifier` | `fit` | python-bound | 46.5 | rewrite first | | remove Python control/validation and specialize the complete operation | -| 88 | `KNeighborsRegressor` | `fit` | python-bound | 46.5 | rewrite first | | remove Python control/validation and specialize the complete operation | -| 89 | `KNeighborsTransformer` | `fit` | python-bound | 46.5 | rewrite first | | remove Python control/validation and specialize the complete operation | -| 90 | `KNeighborsTransformer` | `transform` | python-bound | 46.5 | rewrite first | | remove Python control/validation and specialize the complete operation | -| 91 | `LabelBinarizer` | `fit` | python-bound | 46.5 | rewrite first | | remove Python control/validation and specialize the complete operation | -| 92 | `LabelBinarizer` | `transform` | python-bound | 46.5 | rewrite first | | remove Python control/validation and specialize the complete operation | -| 93 | `LabelEncoder` | `fit` | python-bound | 46.5 | rewrite first | | remove Python control/validation and specialize the complete operation | -| 94 | `LabelEncoder` | `transform` | python-bound | 46.5 | rewrite first | | remove Python control/validation and specialize the complete operation | -| 95 | `LabelPropagation` | `fit` | python-bound | 46.5 | rewrite first | | remove Python control/validation and specialize the complete operation | -| 96 | `Lars` | `predict` | python-bound | 46.5 | rewrite first | | remove Python control/validation and specialize the complete operation | -| 97 | `LarsCV` | `predict` | python-bound | 46.5 | rewrite first | | remove Python control/validation and specialize the complete operation | -| 98 | `LassoCV` | `fit` | python-bound | 46.5 | rewrite first | | remove Python control/validation and specialize the complete operation | -| 99 | `LassoCV` | `predict` | python-bound | 46.5 | rewrite first | | remove Python control/validation and specialize the complete operation | -| 100 | `LassoLars` | `predict` | python-bound | 46.5 | rewrite first | | remove Python control/validation and specialize the complete operation | +| 1 | `GaussianNB` | `fit` | python-bound | 75.2 | rewrite first | 26.34× | remove Python control/validation and specialize the complete operation | +| 2 | `PCA` | `fit` | python-bound | 74.0 | rewrite first | 16.68× | remove Python control/validation and specialize the complete operation | +| 3 | `GaussianNB` | `predict` | python-bound | 73.8 | rewrite first | 26.34× | remove Python control/validation and specialize the complete operation | +| 4 | `LogisticRegression` | `predict` | python-bound | 73.7 | rewrite first | 6.80× | remove Python control/validation and specialize the complete operation | +| 5 | `PCA` | `transform` | python-bound | 73.4 | rewrite first | 16.68× | remove Python control/validation and specialize the complete operation | +| 6 | `LinearRegression` | `predict` | python-bound | 73.3 | rewrite first | 10.32× | remove Python control/validation and specialize the complete operation | +| 7 | `KMeans` | `fit` | mixed | 71.9 | rewrite first | 6.36× | remove Python control/validation and specialize the complete operation | +| 8 | `LogisticRegression` | `fit` | mixed | 71.8 | rewrite first | 6.80× | remove Python control/validation and specialize the complete operation | +| 9 | `LinearRegression` | `fit` | mixed | 68.6 | rewrite first | 10.32× | remove Python control/validation and specialize the complete operation | +| 10 | `SVC` | `predict` | numpy-bound | 61.9 | rewrite first | 2.67× | remove Python control/validation and specialize the complete operation | +| 11 | `KMeans` | `predict` | numpy-bound | 60.2 | rewrite first | 6.36× | remove Python control/validation and specialize the complete operation | +| 12 | `GaussianNB` | `predict_proba` | python-bound | 54.0 | rewrite first | 26.34× | remove Python control/validation and specialize the complete operation | +| 13 | `KMeans` | `transform` | python-bound | 54.0 | rewrite first | 6.36× | remove Python control/validation and specialize the complete operation | +| 14 | `LinearSVC` | `predict` | python-bound | 54.0 | rewrite first | 5.01× | remove Python control/validation and specialize the complete operation | +| 15 | `LinearSVC` | `decision_function` | python-bound | 54.0 | rewrite first | 5.01× | remove Python control/validation and specialize the complete operation | +| 16 | `LogisticRegression` | `predict_proba` | python-bound | 54.0 | rewrite first | 6.80× | remove Python control/validation and specialize the complete operation | +| 17 | `LogisticRegression` | `decision_function` | python-bound | 54.0 | rewrite first | 6.80× | remove Python control/validation and specialize the complete operation | +| 18 | `Ridge` | `fit` | python-bound | 54.0 | rewrite first | 12.55× | remove Python control/validation and specialize the complete operation | +| 19 | `Ridge` | `predict` | python-bound | 54.0 | rewrite first | 12.55× | remove Python control/validation and specialize the complete operation | +| 20 | `Lasso` | `predict` | python-bound | 52.5 | rewrite first | 2.69× | remove Python control/validation and specialize the complete operation | +| 21 | `SVC` | `predict_proba` | python-bound | 52.4 | rewrite first | 2.67× | remove Python control/validation and specialize the complete operation | +| 22 | `SVC` | `decision_function` | python-bound | 52.4 | rewrite first | 2.67× | remove Python control/validation and specialize the complete operation | +| 23 | `DecisionTreeClassifier` | `fit` | mixed | 48.0 | compile whole estimator | 5.43× | retain useful numerical kernels while fusing validation, allocation and orchestration | +| 24 | `DecisionTreeClassifier` | `predict` | mixed | 48.0 | compile whole estimator | 5.43× | retain useful numerical kernels while fusing validation, allocation and orchestration | +| 25 | `RandomForestClassifier` | `fit` | mixed | 48.0 | compile whole estimator | 11.35× | retain useful numerical kernels while fusing validation, allocation and orchestration | +| 26 | `RandomForestClassifier` | `predict` | mixed | 48.0 | compile whole estimator | 11.35× | retain useful numerical kernels while fusing validation, allocation and orchestration | +| 27 | `RandomForestClassifier` | `predict_proba` | mixed | 48.0 | compile whole estimator | 11.35× | retain useful numerical kernels while fusing validation, allocation and orchestration | +| 28 | `AdaBoostRegressor` | `predict` | python-bound | 46.5 | rewrite first | | remove Python control/validation and specialize the complete operation | +| 29 | `AdditiveChi2Sampler` | `fit` | python-bound | 46.5 | rewrite first | | remove Python control/validation and specialize the complete operation | +| 30 | `AdditiveChi2Sampler` | `transform` | python-bound | 46.5 | rewrite first | | remove Python control/validation and specialize the complete operation | +| 31 | `AgglomerativeClustering` | `fit` | python-bound | 46.5 | rewrite first | | remove Python control/validation and specialize the complete operation | +| 32 | `BaggingClassifier` | `fit` | python-bound | 46.5 | rewrite first | | remove Python control/validation and specialize the complete operation | +| 33 | `BaggingClassifier` | `predict_proba` | python-bound | 46.5 | rewrite first | | remove Python control/validation and specialize the complete operation | +| 34 | `BaggingClassifier` | `decision_function` | python-bound | 46.5 | rewrite first | | remove Python control/validation and specialize the complete operation | +| 35 | `BaggingRegressor` | `fit` | python-bound | 46.5 | rewrite first | | remove Python control/validation and specialize the complete operation | +| 36 | `BaggingRegressor` | `predict` | python-bound | 46.5 | rewrite first | | remove Python control/validation and specialize the complete operation | +| 37 | `BayesianGaussianMixture` | `fit` | python-bound | 46.5 | rewrite first | | remove Python control/validation and specialize the complete operation | +| 38 | `BayesianGaussianMixture` | `predict` | python-bound | 46.5 | rewrite first | | remove Python control/validation and specialize the complete operation | +| 39 | `BayesianGaussianMixture` | `predict_proba` | python-bound | 46.5 | rewrite first | | remove Python control/validation and specialize the complete operation | +| 40 | `BernoulliNB` | `predict` | python-bound | 46.5 | rewrite first | | remove Python control/validation and specialize the complete operation | +| 41 | `BernoulliNB` | `predict_proba` | python-bound | 46.5 | rewrite first | | remove Python control/validation and specialize the complete operation | +| 42 | `Binarizer` | `fit` | python-bound | 46.5 | rewrite first | | remove Python control/validation and specialize the complete operation | +| 43 | `Birch` | `fit` | python-bound | 46.5 | rewrite first | | remove Python control/validation and specialize the complete operation | +| 44 | `Birch` | `predict` | python-bound | 46.5 | rewrite first | | remove Python control/validation and specialize the complete operation | +| 45 | `Birch` | `transform` | python-bound | 46.5 | rewrite first | | remove Python control/validation and specialize the complete operation | +| 46 | `BisectingKMeans` | `transform` | python-bound | 46.5 | rewrite first | | remove Python control/validation and specialize the complete operation | +| 47 | `CCA` | `predict` | python-bound | 46.5 | rewrite first | | remove Python control/validation and specialize the complete operation | +| 48 | `CategoricalNB` | `fit` | python-bound | 46.5 | rewrite first | | remove Python control/validation and specialize the complete operation | +| 49 | `CategoricalNB` | `predict` | python-bound | 46.5 | rewrite first | | remove Python control/validation and specialize the complete operation | +| 50 | `CategoricalNB` | `predict_proba` | python-bound | 46.5 | rewrite first | | remove Python control/validation and specialize the complete operation | +| 51 | `ClassifierChain` | `fit` | python-bound | 46.5 | rewrite first | | remove Python control/validation and specialize the complete operation | +| 52 | `ClassifierChain` | `predict` | python-bound | 46.5 | rewrite first | | remove Python control/validation and specialize the complete operation | +| 53 | `ClassifierChain` | `predict_proba` | python-bound | 46.5 | rewrite first | | remove Python control/validation and specialize the complete operation | +| 54 | `ClassifierChain` | `decision_function` | python-bound | 46.5 | rewrite first | | remove Python control/validation and specialize the complete operation | +| 55 | `ColumnTransformer` | `fit` | python-bound | 46.5 | rewrite first | | remove Python control/validation and specialize the complete operation | +| 56 | `ComplementNB` | `predict` | python-bound | 46.5 | rewrite first | | remove Python control/validation and specialize the complete operation | +| 57 | `ComplementNB` | `predict_proba` | python-bound | 46.5 | rewrite first | | remove Python control/validation and specialize the complete operation | +| 58 | `CountVectorizer` | `fit` | python-bound | 46.5 | rewrite first | | remove Python control/validation and specialize the complete operation | +| 59 | `CountVectorizer` | `transform` | python-bound | 46.5 | rewrite first | | remove Python control/validation and specialize the complete operation | +| 60 | `DictVectorizer` | `fit` | python-bound | 46.5 | rewrite first | | remove Python control/validation and specialize the complete operation | +| 61 | `DictVectorizer` | `transform` | python-bound | 46.5 | rewrite first | | remove Python control/validation and specialize the complete operation | +| 62 | `DictionaryLearning` | `fit` | python-bound | 46.5 | rewrite first | | remove Python control/validation and specialize the complete operation | +| 63 | `DictionaryLearning` | `transform` | python-bound | 46.5 | rewrite first | | remove Python control/validation and specialize the complete operation | +| 64 | `ElasticNet` | `predict` | python-bound | 46.5 | rewrite first | | remove Python control/validation and specialize the complete operation | +| 65 | `ElasticNetCV` | `fit` | python-bound | 46.5 | rewrite first | | remove Python control/validation and specialize the complete operation | +| 66 | `ElasticNetCV` | `predict` | python-bound | 46.5 | rewrite first | | remove Python control/validation and specialize the complete operation | +| 67 | `EllipticEnvelope` | `decision_function` | python-bound | 46.5 | rewrite first | | remove Python control/validation and specialize the complete operation | +| 68 | `FastICA` | `fit` | python-bound | 46.5 | rewrite first | | remove Python control/validation and specialize the complete operation | +| 69 | `FeatureAgglomeration` | `fit` | python-bound | 46.5 | rewrite first | | remove Python control/validation and specialize the complete operation | +| 70 | `FeatureHasher` | `fit` | python-bound | 46.5 | rewrite first | | remove Python control/validation and specialize the complete operation | +| 71 | `FeatureUnion` | `fit` | python-bound | 46.5 | rewrite first | | remove Python control/validation and specialize the complete operation | +| 72 | `FeatureUnion` | `transform` | python-bound | 46.5 | rewrite first | | remove Python control/validation and specialize the complete operation | +| 73 | `FunctionTransformer` | `fit` | python-bound | 46.5 | rewrite first | | remove Python control/validation and specialize the complete operation | +| 74 | `FunctionTransformer` | `transform` | python-bound | 46.5 | rewrite first | | remove Python control/validation and specialize the complete operation | +| 75 | `GaussianMixture` | `fit` | python-bound | 46.5 | rewrite first | | remove Python control/validation and specialize the complete operation | +| 76 | `GaussianMixture` | `predict` | python-bound | 46.5 | rewrite first | | remove Python control/validation and specialize the complete operation | +| 77 | `GaussianMixture` | `predict_proba` | python-bound | 46.5 | rewrite first | | remove Python control/validation and specialize the complete operation | +| 78 | `GaussianProcessClassifier` | `predict` | python-bound | 46.5 | rewrite first | | remove Python control/validation and specialize the complete operation | +| 79 | `GaussianProcessClassifier` | `predict_proba` | python-bound | 46.5 | rewrite first | | remove Python control/validation and specialize the complete operation | +| 80 | `GenericUnivariateSelect` | `transform` | python-bound | 46.5 | rewrite first | | remove Python control/validation and specialize the complete operation | +| 81 | `GridSearchCV` | `fit` | python-bound | 46.5 | rewrite first | | remove Python control/validation and specialize the complete operation | +| 82 | `GridSearchCV` | `predict` | python-bound | 46.5 | rewrite first | | remove Python control/validation and specialize the complete operation | +| 83 | `GridSearchCV` | `predict_proba` | python-bound | 46.5 | rewrite first | | remove Python control/validation and specialize the complete operation | +| 84 | `GridSearchCV` | `decision_function` | python-bound | 46.5 | rewrite first | | remove Python control/validation and specialize the complete operation | +| 85 | `GridSearchCV` | `transform` | python-bound | 46.5 | rewrite first | | remove Python control/validation and specialize the complete operation | +| 86 | `HashingVectorizer` | `fit` | python-bound | 46.5 | rewrite first | | remove Python control/validation and specialize the complete operation | +| 87 | `HashingVectorizer` | `transform` | python-bound | 46.5 | rewrite first | | remove Python control/validation and specialize the complete operation | +| 88 | `HuberRegressor` | `predict` | python-bound | 46.5 | rewrite first | | remove Python control/validation and specialize the complete operation | +| 89 | `Isomap` | `fit` | python-bound | 46.5 | rewrite first | | remove Python control/validation and specialize the complete operation | +| 90 | `IsotonicRegression` | `predict` | python-bound | 46.5 | rewrite first | | remove Python control/validation and specialize the complete operation | +| 91 | `IsotonicRegression` | `transform` | python-bound | 46.5 | rewrite first | | remove Python control/validation and specialize the complete operation | +| 92 | `KNeighborsClassifier` | `fit` | python-bound | 46.5 | rewrite first | | remove Python control/validation and specialize the complete operation | +| 93 | `KNeighborsRegressor` | `fit` | python-bound | 46.5 | rewrite first | | remove Python control/validation and specialize the complete operation | +| 94 | `KNeighborsTransformer` | `fit` | python-bound | 46.5 | rewrite first | | remove Python control/validation and specialize the complete operation | +| 95 | `KNeighborsTransformer` | `transform` | python-bound | 46.5 | rewrite first | | remove Python control/validation and specialize the complete operation | +| 96 | `LabelBinarizer` | `fit` | python-bound | 46.5 | rewrite first | | remove Python control/validation and specialize the complete operation | +| 97 | `LabelBinarizer` | `transform` | python-bound | 46.5 | rewrite first | | remove Python control/validation and specialize the complete operation | +| 98 | `LabelEncoder` | `fit` | python-bound | 46.5 | rewrite first | | remove Python control/validation and specialize the complete operation | +| 99 | `LabelEncoder` | `transform` | python-bound | 46.5 | rewrite first | | remove Python control/validation and specialize the complete operation | +| 100 | `LabelPropagation` | `fit` | python-bound | 46.5 | rewrite first | | remove Python control/validation and specialize the complete operation | diff --git a/benchmarks/README.md b/benchmarks/README.md index ec44eb8..858c81c 100644 --- a/benchmarks/README.md +++ b/benchmarks/README.md @@ -6,7 +6,7 @@ This directory contains the canonical numerical-parity, timing and execution-arc [`headline_result_v2.json`](headline_result_v2.json) is the current competitive source of truth. -The committed result contains **19 total rows, 19 parity-eligible comparisons, 11 Flow wins, 7 scikit-learn wins, 1 ties, 0 parity-unresolved rows and 0 measurement-unresolved rows**. +The committed result contains **19 total rows, 19 parity-eligible comparisons, 19 Flow wins, 0 scikit-learn wins, 0 ties, 0 parity-unresolved rows and 0 measurement-unresolved rows**. A competitive speed claim is only emitted when a row has resolved timing, a declared millisecond unit, comparable benchmark semantics and a passing estimator-specific parity contract. @@ -195,7 +195,7 @@ The main generated views are: A speedup is `sklearn_ms / flow_ms`. Values above `1x` mean Flow is faster; values below `1x` mean scikit-learn is faster. -The current architecture map shows a useful but non-causal pattern: Flow wins 75% of headline rows classified Python-bound, about 45% of mixed rows and 0% of external-native-bound rows. This is evidence for prioritization, not proof that execution substrate alone determines performance. +The current architecture map shows a useful but non-causal pattern: Flow wins every headline row in all three substrate classes, and the margin varies with the class, from a mean of 26.55x on Python-bound rows down to 3.79x on external-native-bound ones. That ordering is evidence for prioritization. It is not proof that execution substrate alone determines performance. Mature BLAS/LAPACK, liblinear, libsvm and other native backends are treated as native competitors. The optimization roadmap deliberately prefers retaining those kernels unless benchmark and parity evidence justify replacement. diff --git a/benchmarks/architecture_performance_map.json b/benchmarks/architecture_performance_map.json index 8c6024a..3bf5564 100644 --- a/benchmarks/architecture_performance_map.json +++ b/benchmarks/architecture_performance_map.json @@ -7,7 +7,7 @@ "dataset": "iris", "parity_status": "approximately equivalent", "classification": "flow win", - "flow_speedup": 14.601129810903238, + "flow_speedup": 10.418622456831951, "fit_execution_class": "mixed", "inventory_confidence": "medium", "python_visible_self_share": 0.7643902839976647, @@ -18,8 +18,8 @@ "sklearn_estimator": "LinearSVC", "dataset": "iris", "parity_status": "approximately equivalent", - "classification": "sklearn win", - "flow_speedup": 0.6717301799037095, + "classification": "flow win", + "flow_speedup": 5.699793455357144, "fit_execution_class": "external-native-bound", "inventory_confidence": "medium", "python_visible_self_share": null, @@ -30,8 +30,8 @@ "sklearn_estimator": "SVC", "dataset": "iris", "parity_status": "approximately equivalent", - "classification": "tie", - "flow_speedup": 1.0156696539762757, + "classification": "flow win", + "flow_speedup": 2.1433644035930675, "fit_execution_class": "external-native-bound", "inventory_confidence": "medium", "python_visible_self_share": 0.8844393474814762, @@ -43,7 +43,7 @@ "dataset": "iris", "parity_status": "approximately equivalent", "classification": "flow win", - "flow_speedup": 6.810175426189117, + "flow_speedup": 8.976880677402521, "fit_execution_class": "mixed", "inventory_confidence": "medium", "python_visible_self_share": null, @@ -55,7 +55,7 @@ "dataset": "iris", "parity_status": "approximately equivalent", "classification": "flow win", - "flow_speedup": 10.50930594124603, + "flow_speedup": 17.453590828663167, "fit_execution_class": "mixed", "inventory_confidence": "medium", "python_visible_self_share": null, @@ -67,7 +67,7 @@ "dataset": "iris", "parity_status": "parity verified", "classification": "flow win", - "flow_speedup": 53.91838070892979, + "flow_speedup": 48.154758083333334, "fit_execution_class": "python-bound", "inventory_confidence": "low", "python_visible_self_share": 0.7737794117866104, @@ -79,7 +79,7 @@ "dataset": "iris", "parity_status": "approximately equivalent", "classification": "flow win", - "flow_speedup": 8.467095349452824, + "flow_speedup": 11.306572550169175, "fit_execution_class": "mixed", "inventory_confidence": "medium", "python_visible_self_share": 0.8310647414107625, @@ -91,7 +91,7 @@ "dataset": "iris", "parity_status": "parity verified", "classification": "flow win", - "flow_speedup": 11.826483972812914, + "flow_speedup": 16.6768083, "fit_execution_class": "python-bound", "inventory_confidence": "low", "python_visible_self_share": 0.7602673074449647, @@ -103,7 +103,7 @@ "dataset": "digits", "parity_status": "approximately equivalent", "classification": "flow win", - "flow_speedup": 2.6440736316542766, + "flow_speedup": 3.182869860759928, "fit_execution_class": "mixed", "inventory_confidence": "medium", "python_visible_self_share": 0.7643902839976647, @@ -114,8 +114,8 @@ "sklearn_estimator": "LinearSVC", "dataset": "digits", "parity_status": "approximately equivalent", - "classification": "sklearn win", - "flow_speedup": 0.2306079976731235, + "classification": "flow win", + "flow_speedup": 4.3129902917665035, "fit_execution_class": "external-native-bound", "inventory_confidence": "medium", "python_visible_self_share": null, @@ -126,8 +126,8 @@ "sklearn_estimator": "SVC", "dataset": "digits", "parity_status": "approximately equivalent", - "classification": "sklearn win", - "flow_speedup": 0.6297310809045085, + "classification": "flow win", + "flow_speedup": 3.2040100842076344, "fit_execution_class": "external-native-bound", "inventory_confidence": "medium", "python_visible_self_share": 0.8844393474814762, @@ -138,8 +138,8 @@ "sklearn_estimator": "DecisionTreeClassifier", "dataset": "digits", "parity_status": "approximately equivalent", - "classification": "sklearn win", - "flow_speedup": 0.47310018712629665, + "classification": "flow win", + "flow_speedup": 1.8765555978503732, "fit_execution_class": "mixed", "inventory_confidence": "medium", "python_visible_self_share": null, @@ -150,8 +150,8 @@ "sklearn_estimator": "RandomForestClassifier", "dataset": "digits", "parity_status": "approximately equivalent", - "classification": "sklearn win", - "flow_speedup": 0.3107955212244894, + "classification": "flow win", + "flow_speedup": 5.239445843496978, "fit_execution_class": "mixed", "inventory_confidence": "medium", "python_visible_self_share": null, @@ -162,8 +162,8 @@ "sklearn_estimator": "GaussianNB", "dataset": "digits", "parity_status": "parity verified", - "classification": "tie", - "flow_speedup": 0.9966395166847444, + "classification": "flow win", + "flow_speedup": 4.515746549104471, "fit_execution_class": "python-bound", "inventory_confidence": "low", "python_visible_self_share": 0.7737794117866104, @@ -174,8 +174,8 @@ "sklearn_estimator": "KMeans", "dataset": "digits", "parity_status": "approximately equivalent", - "classification": "sklearn win", - "flow_speedup": 0.19291653116536592, + "classification": "flow win", + "flow_speedup": 1.4061460735205966, "fit_execution_class": "mixed", "inventory_confidence": "medium", "python_visible_self_share": 0.8310647414107625, @@ -187,7 +187,7 @@ "dataset": "diabetes", "parity_status": "approximately equivalent", "classification": "flow win", - "flow_speedup": 7.311641179767832, + "flow_speedup": 12.547392108478215, "fit_execution_class": "python-bound", "inventory_confidence": "low", "python_visible_self_share": null, @@ -199,7 +199,7 @@ "dataset": "diabetes", "parity_status": "approximately equivalent", "classification": "flow win", - "flow_speedup": 1.0942022270144351, + "flow_speedup": 2.6949330261904763, "fit_execution_class": "mixed", "inventory_confidence": "medium", "python_visible_self_share": null, @@ -211,7 +211,7 @@ "dataset": "diabetes", "parity_status": "parity verified", "classification": "flow win", - "flow_speedup": 8.517286010777768, + "flow_speedup": 10.318900375, "fit_execution_class": "mixed", "inventory_confidence": "medium", "python_visible_self_share": 0.7610687448032628, @@ -222,8 +222,8 @@ "sklearn_estimator": "KernelRidge", "dataset": "diabetes", "parity_status": "parity verified", - "classification": "sklearn win", - "flow_speedup": 0.1131002397036554, + "classification": "flow win", + "flow_speedup": 2.1101041813167494, "fit_execution_class": "mixed", "inventory_confidence": "medium", "python_visible_self_share": null, @@ -234,20 +234,20 @@ { "execution_class": "external-native-bound", "n": 4, - "mean_flow_speedup": 0.6369347281144043, - "flow_win_fraction": 0.25 + "mean_flow_speedup": 3.840039558731087, + "flow_win_fraction": 1.0 }, { "execution_class": "mixed", "n": 11, - "mean_flow_speedup": 4.884834625132499, - "flow_win_fraction": 0.6363636363636364 + "mean_flow_speedup": 6.816783770109265, + "flow_win_fraction": 1.0 }, { "execution_class": "python-bound", "n": 4, - "mean_flow_speedup": 18.51328634454882, - "flow_win_fraction": 0.75 + "mean_flow_speedup": 20.473676260229006, + "flow_win_fraction": 1.0 } ], "coverage": { diff --git a/benchmarks/bench_flow_v2.flow b/benchmarks/bench_flow_v2.flow index 27f762a..3cdc8c2 100644 --- a/benchmarks/bench_flow_v2.flow +++ b/benchmarks/bench_flow_v2.flow @@ -1,29 +1,19 @@ # Canonical 19-row Flow headline benchmark. -# Consumes persisted split fixtures and uses C11 timespec_get for high-resolution timing. +# Consumes persisted split fixtures and uses a monotonic nanosecond clock (lib/scikit/flow_time.c) for timing. import "lib/scikit/scikit.flow" extern { function printf(fmt: string, ...) -> i32 - function timespec_get(ts: ptr, base: i32) -> i32 + function flow_now_ns() -> i64 function fopen(path: string, mode: string) -> ptr function fread(dst: ptr, size: i64, count: i64, stream: ptr) -> i64 function fclose(stream: ptr) -> i32 } -const TIME_UTC: i32 = 1 function now_ns() -> i64 { - let ts: ptr = malloc(16) as ptr - if ts == null { return 0 } - let ok: i32 = timespec_get(ts, TIME_UTC) - if ok == 0 { - free(ts as ptr) - return 0 - } - let value: i64 = ts[0] * 1000000000 + ts[1] - free(ts as ptr) - return value + return flow_now_ns() } function elapsed_ms_ns(start: i64, finish: i64) -> f32 { @@ -787,7 +777,7 @@ function main() -> i32 { printf("TIMING_UNIT|ms\n") printf("BENCHMARK_MODE|end_to_end\n") printf("FIXTURE_SOURCE|benchmarks/split_*.bin\n") - printf("BENCHMARK_ENV|{\"impl\":\"flow\",\"timer\":\"timespec_get\"}\n") + printf("BENCHMARK_ENV|{\"impl\":\"flow\",\"timer\":\"clock_gettime_monotonic\"}\n") let no_penalty: Penalty = penalty_none() let mut t0: i64 = 0 diff --git a/benchmarks/bench_scaled.flow b/benchmarks/bench_scaled.flow index 16939bd..5335331 100644 --- a/benchmarks/bench_scaled.flow +++ b/benchmarks/bench_scaled.flow @@ -6,18 +6,12 @@ import "lib/scikit/scikit.flow" extern { function printf(fmt: string, ...) -> i32 - function timespec_get(ts: ptr, base: i32) -> i32 + function flow_now_ns() -> i64 } -const TIME_UTC: i32 = 1 function now_ns() -> i64 { - let ts: ptr = malloc(16) as ptr - if ts == null { return 0 } - timespec_get(ts, TIME_UTC) - let v: i64 = ts[0] * 1000000000 + ts[1] - free(ts as ptr) - return v + return flow_now_ns() } function elapsed_ms(start: i64, finish: i64) -> f32 { diff --git a/benchmarks/disparity_history.json b/benchmarks/disparity_history.json index 5734d3f..ea2bb95 100644 --- a/benchmarks/disparity_history.json +++ b/benchmarks/disparity_history.json @@ -7802,6 +7802,279 @@ "final_parity_status": "parity verified" } ] + }, + { + "commit": "unknown", + "environment_id": "c012023a7229bbbe", + "runtime_environment_id": "9bebce8d5728ab83", + "rows": [ + { + "algorithm": "LogisticRegression", + "dataset": "iris", + "metric": "accuracy", + "score_abs_diff": 3.999999997894577e-09, + "score_tolerance_fraction": 1.9999999989472883e-07, + "runtime_log2_ratio": 3.38109263295714, + "flow_total_ms": 0.095999997, + "sklearn_total_ms": 1.0001877246, + "configuration_difference_count": 1, + "semantic_difference_count": 0, + "strict_diagnostic_status": "approximately equivalent", + "final_parity_status": "approximately equivalent" + }, + { + "algorithm": "LinearSVC", + "dataset": "iris", + "metric": "accuracy", + "score_abs_diff": 2.399999998736746e-08, + "score_tolerance_fraction": 1.199999999368373e-06, + "runtime_log2_ratio": 2.5109096409737117, + "flow_total_ms": 0.056, + "sklearn_total_ms": 0.31918843350000004, + "configuration_difference_count": 0, + "semantic_difference_count": 0, + "strict_diagnostic_status": "approximately equivalent", + "final_parity_status": "approximately equivalent" + }, + { + "algorithm": "KernelSVC_RBF", + "dataset": "iris", + "metric": "accuracy", + "score_abs_diff": 2.7999999985262036e-08, + "score_tolerance_fraction": 9.333333328420679e-07, + "runtime_log2_ratio": 1.099877150324171, + "flow_total_ms": 0.166000007, + "sklearn_total_ms": 0.355798506, + "configuration_difference_count": 0, + "semantic_difference_count": 0, + "strict_diagnostic_status": "approximately equivalent", + "final_parity_status": "approximately equivalent" + }, + { + "algorithm": "DecisionTree", + "dataset": "iris", + "metric": "accuracy", + "score_abs_diff": 3.999999997894577e-09, + "score_tolerance_fraction": 9.999999994736442e-08, + "runtime_log2_ratio": 3.166214218570989, + "flow_total_ms": 0.031999999, + "sklearn_total_ms": 0.2872601727, + "configuration_difference_count": 0, + "semantic_difference_count": 0, + "strict_diagnostic_status": "approximately equivalent", + "final_parity_status": "approximately equivalent" + }, + { + "algorithm": "RandomForest", + "dataset": "iris", + "metric": "accuracy", + "score_abs_diff": 2.7999999985262036e-08, + "score_tolerance_fraction": 6.999999996315509e-07, + "runtime_log2_ratio": 4.125451975893311, + "flow_total_ms": 0.23399999800000001, + "sklearn_total_ms": 4.084140219, + "configuration_difference_count": 1, + "semantic_difference_count": 0, + "strict_diagnostic_status": "approximately equivalent", + "final_parity_status": "approximately equivalent" + }, + { + "algorithm": "GaussianNB", + "dataset": "iris", + "metric": "accuracy", + "score_abs_diff": 2.7999999985262036e-08, + "score_tolerance_fraction": 2.7999999985262036e-05, + "runtime_log2_ratio": 5.589606450050291, + "flow_total_ms": 0.006, + "sklearn_total_ms": 0.2889285485, + "configuration_difference_count": 0, + "semantic_difference_count": 0, + "strict_diagnostic_status": "parity verified", + "final_parity_status": "parity verified" + }, + { + "algorithm": "KMeans", + "dataset": "iris", + "metric": "adjusted_rand_index", + "score_abs_diff": 2.7000000013543968e-08, + "score_tolerance_fraction": 5.400000002708794e-07, + "runtime_log2_ratio": 3.4990897550488054, + "flow_total_ms": 0.179000003, + "sklearn_total_ms": 2.0238765204, + "configuration_difference_count": 0, + "semantic_difference_count": 0, + "strict_diagnostic_status": "approximately equivalent", + "final_parity_status": "approximately equivalent" + }, + { + "algorithm": "PCA", + "dataset": "iris", + "metric": "explained_var_ratio", + "score_abs_diff": 0.0, + "score_tolerance_fraction": 0.0, + "runtime_log2_ratio": 4.0597712991156465, + "flow_total_ms": 0.008, + "sklearn_total_ms": 0.13341446640000001, + "configuration_difference_count": 1, + "semantic_difference_count": 0, + "strict_diagnostic_status": "parity verified", + "final_parity_status": "parity verified" + }, + { + "algorithm": "LogisticRegression", + "dataset": "digits", + "metric": "accuracy", + "score_abs_diff": 1.2999999965401798e-08, + "score_tolerance_fraction": 6.499999982700899e-07, + "runtime_log2_ratio": 1.6703281701455104, + "flow_total_ms": 1.95899999, + "sklearn_total_ms": 6.2352420254, + "configuration_difference_count": 1, + "semantic_difference_count": 0, + "strict_diagnostic_status": "approximately equivalent", + "final_parity_status": "approximately equivalent" + }, + { + "algorithm": "LinearSVC", + "dataset": "digits", + "metric": "accuracy", + "score_abs_diff": 2.0000000544584395e-09, + "score_tolerance_fraction": 1.0000000272292198e-07, + "runtime_log2_ratio": 2.1086884686724403, + "flow_total_ms": 52.811000702, + "sklearn_total_ms": 227.7733333262, + "configuration_difference_count": 0, + "semantic_difference_count": 0, + "strict_diagnostic_status": "approximately equivalent", + "final_parity_status": "approximately equivalent" + }, + { + "algorithm": "KernelSVC_RBF", + "dataset": "digits", + "metric": "accuracy", + "score_abs_diff": 1.199999999368373e-08, + "score_tolerance_fraction": 3.9999999978945766e-07, + "runtime_log2_ratio": 1.6798786884499473, + "flow_total_ms": 12.83300034, + "sklearn_total_ms": 41.1170625, + "configuration_difference_count": 0, + "semantic_difference_count": 0, + "strict_diagnostic_status": "approximately equivalent", + "final_parity_status": "approximately equivalent" + }, + { + "algorithm": "DecisionTree", + "dataset": "digits", + "metric": "accuracy", + "score_abs_diff": 0.019444424000000016, + "score_tolerance_fraction": 0.4861106000000004, + "runtime_log2_ratio": 0.9080870344587176, + "flow_total_ms": 4.6069999699999995, + "sklearn_total_ms": 8.645291583, + "configuration_difference_count": 0, + "semantic_difference_count": 0, + "strict_diagnostic_status": "approximately equivalent", + "final_parity_status": "approximately equivalent" + }, + { + "algorithm": "RandomForest", + "dataset": "digits", + "metric": "accuracy", + "score_abs_diff": 0.01944444700000003, + "score_tolerance_fraction": 0.4861111750000008, + "runtime_log2_ratio": 2.3894142313970326, + "flow_total_ms": 2.5059999339999997, + "sklearn_total_ms": 13.130050938, + "configuration_difference_count": 1, + "semantic_difference_count": 0, + "strict_diagnostic_status": "approximately equivalent", + "final_parity_status": "approximately equivalent" + }, + { + "algorithm": "GaussianNB", + "dataset": "digits", + "metric": "accuracy", + "score_abs_diff": 8.000000106811456e-09, + "score_tolerance_fraction": 8.000000106811456e-06, + "runtime_log2_ratio": 2.1749645156478485, + "flow_total_ms": 0.182000004, + "sklearn_total_ms": 0.82186589, + "configuration_difference_count": 0, + "semantic_difference_count": 0, + "strict_diagnostic_status": "parity verified", + "final_parity_status": "parity verified" + }, + { + "algorithm": "KMeans", + "dataset": "digits", + "metric": "adjusted_rand_index", + "score_abs_diff": 1.2999999965401798e-08, + "score_tolerance_fraction": 2.5999999930803597e-07, + "runtime_log2_ratio": 0.49174647250244624, + "flow_total_ms": 14.347, + "sklearn_total_ms": 20.1739777168, + "configuration_difference_count": 0, + "semantic_difference_count": 2, + "strict_diagnostic_status": "approximately equivalent", + "final_parity_status": "approximately equivalent" + }, + { + "algorithm": "Ridge", + "dataset": "diabetes", + "metric": "r2", + "score_abs_diff": 1.7899999998904903e-07, + "score_tolerance_fraction": 8.949999999452452e-06, + "runtime_log2_ratio": 3.649315635706231, + "flow_total_ms": 0.018000001, + "sklearn_total_ms": 0.2258530705, + "configuration_difference_count": 1, + "semantic_difference_count": 0, + "strict_diagnostic_status": "approximately equivalent", + "final_parity_status": "approximately equivalent" + }, + { + "algorithm": "Lasso", + "dataset": "diabetes", + "metric": "r2", + "score_abs_diff": 1.1623000000016148e-05, + "score_tolerance_fraction": 0.0005811500000008074, + "runtime_log2_ratio": 1.4302494199226854, + "flow_total_ms": 0.126, + "sklearn_total_ms": 0.33956156130000004, + "configuration_difference_count": 0, + "semantic_difference_count": 0, + "strict_diagnostic_status": "approximately equivalent", + "final_parity_status": "approximately equivalent" + }, + { + "algorithm": "LinearRegression", + "dataset": "diabetes", + "metric": "r2", + "score_abs_diff": 1.7900000004456018e-07, + "score_tolerance_fraction": 3.5800000008912036e-05, + "runtime_log2_ratio": 3.367217334247436, + "flow_total_ms": 0.02, + "sklearn_total_ms": 0.20637800750000002, + "configuration_difference_count": 0, + "semantic_difference_count": 0, + "strict_diagnostic_status": "parity verified", + "final_parity_status": "parity verified" + }, + { + "algorithm": "KernelRidge_RBF", + "dataset": "diabetes", + "metric": "r2", + "score_abs_diff": 2.9799999995416826e-07, + "score_tolerance_fraction": 5.959999999083365e-05, + "runtime_log2_ratio": 1.077314230287233, + "flow_total_ms": 0.554000005, + "sklearn_total_ms": 1.168997727, + "configuration_difference_count": 0, + "semantic_difference_count": 0, + "strict_diagnostic_status": "parity verified", + "final_parity_status": "parity verified" + } + ] } ] } diff --git a/benchmarks/disparity_report.json b/benchmarks/disparity_report.json index c209d25..9a21fe1 100644 --- a/benchmarks/disparity_report.json +++ b/benchmarks/disparity_report.json @@ -1,25 +1,31 @@ { "schema_version": 4, - "environment_id": "3f884007c2946e39", - "runtime_environment_id": "def1cc4771c48618", + "environment_id": "c012023a7229bbbe", + "runtime_environment_id": "9bebce8d5728ab83", "host": { - "machine": "x86_64", - "cpu_model": "AMD EPYC 7763 64-Core Processor", - "logical_cpus": 4, + "machine": "arm64", + "cpu_model": "Apple M4 Max", + "logical_cpus": 14, "thread_limits": { "OPENBLAS_NUM_THREADS": "4", "OMP_NUM_THREADS": "4", "MKL_NUM_THREADS": "4", - "VECLIB_MAXIMUM_THREADS": null, + "VECLIB_MAXIMUM_THREADS": "4", "NUMEXPR_NUM_THREADS": null + }, + "flow_build": { + "FLOW_OPT_LEVEL": "3", + "FLOW_CFLAGS": null, + "FLOW_LDFLAGS": "-framework Accelerate lib/scikit/flow_time.c lib/scikit/flow_parallel.c", + "FLOW_HOST": "python" } }, "policy": "headline eligibility never erases disparity evidence; strict diagnostics, learned-state diagnostics and final eligibility decisions are retained separately", "counts": { "rows": 19, - "rows_with_tracked_disparity": 18, - "rows_with_substantive_disparity": 14, - "rows_with_configuration_difference": 4, + "rows_with_tracked_disparity": 19, + "rows_with_substantive_disparity": 15, + "rows_with_configuration_difference": 6, "rows_with_configuration_equivalence": 7, "rows_with_semantic_difference": 1, "rows_with_model_state_diagnostics": 19, @@ -37,20 +43,15 @@ "declared_score_tolerance": 0.02, "effective_score_tolerance": 0.02, "score_tolerance_fraction": 1.9999999989472883e-07, - "runtime_ratio_sklearn_over_flow": 14.61729973559346, - "flow_total_ms": 0.371085001, - "sklearn_total_ms": 5.424260686999999, - "runtime_log2_ratio": 3.8696049206872805, + "runtime_ratio_sklearn_over_flow": 10.418622456831951, + "flow_total_ms": 0.095999997, + "sklearn_total_ms": 1.0001877246, + "runtime_log2_ratio": 3.38109263295714, "configuration_differences": [ { "parameter": "max_iter", "flow": 200, "sklearn": 1000 - }, - { - "parameter": "optimizer", - "flow": "lbfgs_no_line_search", - "sklearn": "lbfgs" } ], "configuration_equivalences": [ @@ -81,15 +82,15 @@ "classes_max_abs_diff": 0.0, "classes_max_relative_diff": 0.0, "classes_first_divergent_index": -1, - "coef_abs_sum_abs_diff": 0.00037820000000010623, - "coef_abs_sum_relative_diff": 2.860522626107004e-05, - "coef_frobenius_norm_abs_diff": 0.00024186000000003816, - "coef_frobenius_norm_relative_diff": 5.3540461765084726e-05, - "coef_row_l2_norms_max_abs_diff": 0.0005382300000000839, - "coef_row_l2_norms_max_relative_diff": 0.0005103935444031705, + "coef_abs_sum_abs_diff": 0.0036196999999997814, + "coef_abs_sum_relative_diff": 0.0002737095666177819, + "coef_frobenius_norm_abs_diff": 0.00020515000000020933, + "coef_frobenius_norm_relative_diff": 4.541398218439017e-05, + "coef_row_l2_norms_max_abs_diff": 0.0008140500000002326, + "coef_row_l2_norms_max_relative_diff": 0.0002456752731871714, "coef_row_l2_norms_first_divergent_index": 0, - "intercepts_max_abs_diff": 0.0006549300000000535, - "intercepts_max_relative_diff": 0.0009035800539483737, + "intercepts_max_abs_diff": 0.005021333999999988, + "intercepts_max_relative_diff": 0.016166318842954098, "intercepts_first_divergent_index": 0 }, "diagnostics": {}, @@ -106,15 +107,15 @@ "metric": "accuracy", "final_parity_status": "approximately equivalent", "strict_diagnostic_status": "approximately equivalent", - "headline_classification": "sklearn win", + "headline_classification": "flow win", "score_abs_diff": 2.399999998736746e-08, "declared_score_tolerance": 0.02, "effective_score_tolerance": 0.02, "score_tolerance_fraction": 1.199999999368373e-06, - "runtime_ratio_sklearn_over_flow": 0.680147215440066, - "flow_total_ms": 1.7384630300000001, - "sklearn_total_ms": 1.182410789, - "runtime_log2_ratio": -0.5560810485266441, + "runtime_ratio_sklearn_over_flow": 5.699793455357144, + "flow_total_ms": 0.056, + "sklearn_total_ms": 0.31918843350000004, + "runtime_log2_ratio": 2.5109096409737117, "configuration_differences": [], "configuration_equivalences": [ { @@ -160,34 +161,34 @@ "declared_score_tolerance": 0.03, "effective_score_tolerance": 0.03, "score_tolerance_fraction": 9.333333328420679e-07, - "runtime_ratio_sklearn_over_flow": 1.025462017673037, - "flow_total_ms": 1.225425039, - "sklearn_total_ms": 1.256626833, - "runtime_log2_ratio": 0.036274056489089754, + "runtime_ratio_sklearn_over_flow": 2.1433644035930675, + "flow_total_ms": 0.166000007, + "sklearn_total_ms": 0.355798506, + "runtime_log2_ratio": 1.099877150324171, "configuration_differences": [], "configuration_equivalences": [], "semantic_differences": [], "model_state_diagnostics": { "C_abs_diff": 0.0, "C_relative_diff": 0.0, - "dual_coef_abs_sum_per_pair_max_abs_diff": 0.0019053000000006648, - "dual_coef_abs_sum_per_pair_max_relative_diff": 6.482515348241112e-05, + "dual_coef_abs_sum_per_pair_max_abs_diff": 0.001198699999999775, + "dual_coef_abs_sum_per_pair_max_relative_diff": 0.00014313297500843816, "dual_coef_abs_sum_per_pair_first_divergent_index": 0, "gamma_abs_diff": 0.0, "gamma_relative_diff": 0.0, - "intercept_per_pair_max_abs_diff": 0.0001368100000000011, - "intercept_per_pair_max_relative_diff": 0.010856619927361276, + "intercept_per_pair_max_abs_diff": 0.0005367081999999995, + "intercept_per_pair_max_relative_diff": 0.03096142429882716, "intercept_per_pair_first_divergent_index": 0, "n_bounded_support_per_pair_max_abs_diff": 0.0, "n_bounded_support_per_pair_max_relative_diff": 0.0, "n_bounded_support_per_pair_first_divergent_index": -1, "n_pairs_abs_diff": 0.0, "n_pairs_relative_diff": 0.0, - "n_support_per_pair_max_abs_diff": 0.0, - "n_support_per_pair_max_relative_diff": 0.0, - "n_support_per_pair_first_divergent_index": -1, - "n_support_total_abs_diff": 0.0, - "n_support_total_relative_diff": 0.0, + "n_support_per_pair_max_abs_diff": 1.0, + "n_support_per_pair_max_relative_diff": 0.07692307692307693, + "n_support_per_pair_first_divergent_index": 1, + "n_support_total_abs_diff": 1.0, + "n_support_total_relative_diff": 0.017241379310344827, "pair_class_a_max_abs_diff": 0.0, "pair_class_a_max_relative_diff": 0.0, "pair_class_a_first_divergent_index": -1, @@ -216,10 +217,10 @@ "declared_score_tolerance": 0.04, "effective_score_tolerance": 0.04, "score_tolerance_fraction": 9.999999994736442e-08, - "runtime_ratio_sklearn_over_flow": 6.7876021290160145, - "flow_total_ms": 0.150251007, - "sklearn_total_ms": 1.019844055, - "runtime_log2_ratio": 2.762902000455335, + "runtime_ratio_sklearn_over_flow": 8.976880677402521, + "flow_total_ms": 0.031999999, + "sklearn_total_ms": 0.2872601727, + "runtime_log2_ratio": 3.166214218570989, "configuration_differences": [], "configuration_equivalences": [], "semantic_differences": [], @@ -271,11 +272,17 @@ "declared_score_tolerance": 0.04, "effective_score_tolerance": 0.04, "score_tolerance_fraction": 6.999999996315509e-07, - "runtime_ratio_sklearn_over_flow": 10.600992316790924, - "flow_total_ms": 1.298210019, - "sklearn_total_ms": 13.762314437, - "runtime_log2_ratio": 3.4061274109501274, - "configuration_differences": [], + "runtime_ratio_sklearn_over_flow": 17.453590828663167, + "flow_total_ms": 0.23399999800000001, + "sklearn_total_ms": 4.084140219, + "runtime_log2_ratio": 4.125451975893311, + "configuration_differences": [ + { + "parameter": "n_jobs", + "flow": "all_cores", + "sklearn": 1 + } + ], "configuration_equivalences": [], "semantic_differences": [], "model_state_diagnostics": { @@ -319,6 +326,7 @@ }, "diagnostics": {}, "disparity_dimensions": [ + "configuration", "model-state", "runtime" ], @@ -335,10 +343,10 @@ "declared_score_tolerance": 0.001, "effective_score_tolerance": 0.001, "score_tolerance_fraction": 2.7999999985262036e-05, - "runtime_ratio_sklearn_over_flow": 55.182341342543914, - "flow_total_ms": 0.019977, - "sklearn_total_ms": 1.102377633, - "runtime_log2_ratio": 5.786134765272215, + "runtime_ratio_sklearn_over_flow": 48.154758083333334, + "flow_total_ms": 0.006, + "sklearn_total_ms": 0.2889285485, + "runtime_log2_ratio": 5.589606450050291, "configuration_differences": [], "configuration_equivalences": [], "semantic_differences": [], @@ -351,17 +359,17 @@ "classes_first_divergent_index": -1, "theta_frobenius_norm_abs_diff": 5.000000014021566e-08, "theta_frobenius_norm_relative_diff": 1.6985756030466762e-08, - "theta_row_l2_norms_max_abs_diff": 4.000000020099037e-08, - "theta_row_l2_norms_max_relative_diff": 2.8182193621476533e-08, + "theta_row_l2_norms_max_abs_diff": 2.0000000000575113e-07, + "theta_row_l2_norms_max_relative_diff": 2.684018445178003e-07, "theta_row_l2_norms_first_divergent_index": -1, - "var_frobenius_norm_abs_diff": 3.9999999978945766e-08, - "var_frobenius_norm_relative_diff": 3.003058057486742e-08, + "var_frobenius_norm_abs_diff": 7.999999995789153e-08, + "var_frobenius_norm_relative_diff": 6.006116295340643e-08, "var_max_abs_diff": 0.0, "var_max_relative_diff": 0.0, - "var_min_abs_diff": 1.8599999995067007e-09, - "var_min_relative_diff": 2.285308647386766e-07, - "var_row_l2_norms_max_abs_diff": 2.9000000068002407e-08, - "var_row_l2_norms_max_relative_diff": 4.0377000502230886e-08, + "var_min_abs_diff": 2.799999999220093e-09, + "var_min_relative_diff": 3.440249179336966e-07, + "var_row_l2_norms_max_abs_diff": 1.4999999997655777e-07, + "var_row_l2_norms_max_relative_diff": 1.6506411254267266e-07, "var_row_l2_norms_first_divergent_index": -1 }, "diagnostics": {}, @@ -381,10 +389,10 @@ "declared_score_tolerance": 0.05, "effective_score_tolerance": 0.05, "score_tolerance_fraction": 5.400000002708794e-07, - "runtime_ratio_sklearn_over_flow": 8.33724524944886, - "flow_total_ms": 1.29456295, - "sklearn_total_ms": 10.793088805, - "runtime_log2_ratio": 3.059570774381568, + "runtime_ratio_sklearn_over_flow": 11.306572550169175, + "flow_total_ms": 0.179000003, + "sklearn_total_ms": 2.0238765204, + "runtime_log2_ratio": 3.4990897550488054, "configuration_differences": [], "configuration_equivalences": [], "semantic_differences": [], @@ -415,14 +423,14 @@ "final_parity_status": "parity verified", "strict_diagnostic_status": "parity verified", "headline_classification": "flow win", - "score_abs_diff": 5.999999996841865e-08, + "score_abs_diff": 0.0, "declared_score_tolerance": 0.005, "effective_score_tolerance": 0.005, - "score_tolerance_fraction": 1.199999999368373e-05, - "runtime_ratio_sklearn_over_flow": 11.198495603534585, - "flow_total_ms": 0.045946, - "sklearn_total_ms": 0.514526079, - "runtime_log2_ratio": 3.48523302975082, + "score_tolerance_fraction": 0.0, + "runtime_ratio_sklearn_over_flow": 16.6768083, + "flow_total_ms": 0.008, + "sklearn_total_ms": 0.13341446640000001, + "runtime_log2_ratio": 4.0597712991156465, "configuration_differences": [ { "parameter": "solver", @@ -433,31 +441,31 @@ "configuration_equivalences": [], "semantic_differences": [], "model_state_diagnostics": { - "explained_variance_ratio_abs_diff": 5.999999996841865e-08, - "singular_values_relative_diff": 1.8056789335315674e-07, - "reconstruction_mse_abs_diff": 3.4000000037615052e-09, - "component_0_sign_aligned_abs_diff": 0.0, - "component_1_sign_aligned_abs_diff": 7.659999999923284e-08, - "component_0_max_abs_diff": 0.0, - "component_0_max_relative_diff": 0.0, + "explained_variance_ratio_abs_diff": 1.4000000020386594e-08, + "singular_values_relative_diff": 9.503574144572358e-08, + "reconstruction_mse_abs_diff": 1.1100000002484123e-08, + "component_0_sign_aligned_abs_diff": 2.8999999956980105e-08, + "component_1_sign_aligned_abs_diff": 1.1899999996511923e-07, + "component_0_max_abs_diff": 2.8999999956980105e-08, + "component_0_max_relative_diff": 1.1459188421226675e-07, "component_0_first_divergent_index": -1, - "component_1_max_abs_diff": 7.659999999923284e-08, - "component_1_max_relative_diff": 2.8481807022383925e-06, + "component_1_max_abs_diff": 1.1899999996511923e-07, + "component_1_max_relative_diff": 1.509612157986282e-06, "component_1_first_divergent_index": 2, - "explained_variance_ratio_max_abs_diff": 5.999999996841865e-08, - "explained_variance_ratio_max_relative_diff": 1.9075144483097734e-07, + "explained_variance_ratio_max_abs_diff": 1.4000000020386594e-08, + "explained_variance_ratio_max_relative_diff": 6.069364951285345e-08, "explained_variance_ratio_first_divergent_index": -1, - "reconstruction_mse_relative_diff": 9.255653547172065e-08, - "singular_values_max_abs_diff": 1.8999999991109462e-06, - "singular_values_max_relative_diff": 1.8056789335315674e-07, + "reconstruction_mse_relative_diff": 3.0216983263477537e-07, + "singular_values_max_abs_diff": 9.999999992515995e-07, + "singular_values_max_relative_diff": 9.503574144572358e-08, "singular_values_first_divergent_index": -1 }, "diagnostics": { - "explained_variance_ratio_abs_diff": 5.999999996841865e-08, - "singular_values_relative_diff": 1.8056789335315674e-07, - "reconstruction_mse_abs_diff": 3.4000000037615052e-09, - "component_0_sign_aligned_abs_diff": 0.0, - "component_1_sign_aligned_abs_diff": 7.659999999923284e-08 + "explained_variance_ratio_abs_diff": 1.4000000020386594e-08, + "singular_values_relative_diff": 9.503574144572358e-08, + "reconstruction_mse_abs_diff": 1.1100000002484123e-08, + "component_0_sign_aligned_abs_diff": 2.8999999956980105e-08, + "component_1_sign_aligned_abs_diff": 1.1899999996511923e-07 }, "disparity_dimensions": [ "configuration", @@ -473,24 +481,19 @@ "final_parity_status": "approximately equivalent", "strict_diagnostic_status": "approximately equivalent", "headline_classification": "flow win", - "score_abs_diff": 0.002777801999999996, + "score_abs_diff": 1.2999999965401798e-08, "declared_score_tolerance": 0.02, "effective_score_tolerance": 0.02, - "score_tolerance_fraction": 0.1388900999999998, - "runtime_ratio_sklearn_over_flow": 2.7505427175057857, - "flow_total_ms": 70.225565499, - "sklearn_total_ms": 193.158417766, - "runtime_log2_ratio": 1.4597163090385734, + "score_tolerance_fraction": 6.499999982700899e-07, + "runtime_ratio_sklearn_over_flow": 3.182869860759928, + "flow_total_ms": 1.95899999, + "sklearn_total_ms": 6.2352420254, + "runtime_log2_ratio": 1.6703281701455104, "configuration_differences": [ { "parameter": "max_iter", "flow": 200, "sklearn": 1000 - }, - { - "parameter": "optimizer", - "flow": "lbfgs_no_line_search", - "sklearn": "lbfgs" } ], "configuration_equivalences": [ @@ -521,20 +524,19 @@ "classes_max_abs_diff": 0.0, "classes_max_relative_diff": 0.0, "classes_first_divergent_index": -1, - "coef_abs_sum_abs_diff": 0.13389000000000806, - "coef_abs_sum_relative_diff": 0.0007445339728922685, - "coef_frobenius_norm_abs_diff": 0.026778099999999583, - "coef_frobenius_norm_relative_diff": 0.0025770678312939384, - "coef_row_l2_norms_max_abs_diff": 0.04320087000000017, - "coef_row_l2_norms_max_relative_diff": 0.01249319905649985, + "coef_abs_sum_abs_diff": 0.6175949999999943, + "coef_abs_sum_relative_diff": 0.0034250945934931856, + "coef_frobenius_norm_abs_diff": 0.03479669999999935, + "coef_frobenius_norm_relative_diff": 0.0033461732793224506, + "coef_row_l2_norms_max_abs_diff": 0.021368970000000154, + "coef_row_l2_norms_max_relative_diff": 0.006979041476933864, "coef_row_l2_norms_first_divergent_index": 0, - "intercepts_max_abs_diff": 0.13251060199999998, - "intercepts_max_relative_diff": 0.5079954464733525, + "intercepts_max_abs_diff": 0.021606028000000055, + "intercepts_max_relative_diff": 0.06695890274122948, "intercepts_first_divergent_index": 0 }, "diagnostics": {}, "disparity_dimensions": [ - "numerical", "configuration", "model-state", "runtime" @@ -547,15 +549,15 @@ "metric": "accuracy", "final_parity_status": "approximately equivalent", "strict_diagnostic_status": "approximately equivalent", - "headline_classification": "sklearn win", + "headline_classification": "flow win", "score_abs_diff": 2.0000000544584395e-09, "declared_score_tolerance": 0.02, "effective_score_tolerance": 0.02, "score_tolerance_fraction": 1.0000000272292198e-07, - "runtime_ratio_sklearn_over_flow": 0.21731423207714734, - "flow_total_ms": 1608.93593703, - "sklearn_total_ms": 349.64467761699996, - "runtime_log2_ratio": -2.2021454340600157, + "runtime_ratio_sklearn_over_flow": 4.3129902917665035, + "flow_total_ms": 52.811000702, + "sklearn_total_ms": 227.7733333262, + "runtime_log2_ratio": 2.1086884686724403, "configuration_differences": [], "configuration_equivalences": [ { @@ -572,15 +574,15 @@ "classes_max_abs_diff": 0.0, "classes_max_relative_diff": 0.0, "classes_first_divergent_index": -1, - "coef_abs_sum_abs_diff": 0.031656999999995605, - "coef_abs_sum_relative_diff": 0.00024348511941500755, - "coef_frobenius_norm_abs_diff": 0.0017552600000003693, - "coef_frobenius_norm_relative_diff": 0.00023778218395228748, - "coef_row_l2_norms_max_abs_diff": 0.005680750000000234, - "coef_row_l2_norms_max_relative_diff": 0.0022220994491822894, + "coef_abs_sum_abs_diff": 0.012564999999995052, + "coef_abs_sum_relative_diff": 9.664183357389487e-05, + "coef_frobenius_norm_abs_diff": 0.0002158900000006625, + "coef_frobenius_norm_relative_diff": 2.9246263057100416e-05, + "coef_row_l2_norms_max_abs_diff": 0.0015058400000000915, + "coef_row_l2_norms_max_relative_diff": 0.0010980293994849414, "coef_row_l2_norms_first_divergent_index": 0, - "intercepts_max_abs_diff": 0.004284730000000181, - "intercepts_max_relative_diff": 0.0013317304607982274, + "intercepts_max_abs_diff": 0.0008462099999997363, + "intercepts_max_relative_diff": 0.0003454656564000714, "intercepts_first_divergent_index": 0 }, "diagnostics": {}, @@ -596,31 +598,31 @@ "metric": "accuracy", "final_parity_status": "approximately equivalent", "strict_diagnostic_status": "approximately equivalent", - "headline_classification": "sklearn win", + "headline_classification": "flow win", "score_abs_diff": 1.199999999368373e-08, "declared_score_tolerance": 0.03, "effective_score_tolerance": 0.03, "score_tolerance_fraction": 3.9999999978945766e-07, - "runtime_ratio_sklearn_over_flow": 0.5867193921689036, - "flow_total_ms": 132.8251359, - "sklearn_total_ms": 77.931083, - "runtime_log2_ratio": -0.7692574183313682, + "runtime_ratio_sklearn_over_flow": 3.2040100842076344, + "flow_total_ms": 12.83300034, + "sklearn_total_ms": 41.1170625, + "runtime_log2_ratio": 1.6798786884499473, "configuration_differences": [], "configuration_equivalences": [], "semantic_differences": [], "model_state_diagnostics": { "C_abs_diff": 0.0, "C_relative_diff": 0.0, - "dual_coef_abs_sum_per_pair_max_abs_diff": 0.03816499999999223, - "dual_coef_abs_sum_per_pair_max_relative_diff": 0.00023849599581996934, + "dual_coef_abs_sum_per_pair_max_abs_diff": 0.02278400000000147, + "dual_coef_abs_sum_per_pair_max_relative_diff": 0.00018864815302870825, "dual_coef_abs_sum_per_pair_first_divergent_index": 1, "gamma_abs_diff": 5.0000000017050295e-11, "gamma_relative_diff": 4.999999751705042e-08, - "intercept_per_pair_max_abs_diff": 0.0011383578999999977, - "intercept_per_pair_max_relative_diff": 0.07146965134617396, + "intercept_per_pair_max_abs_diff": 0.0011383803000000053, + "intercept_per_pair_max_relative_diff": 0.07147545817521486, "intercept_per_pair_first_divergent_index": 1, "n_bounded_support_per_pair_max_abs_diff": 1.0, - "n_bounded_support_per_pair_max_relative_diff": 0.0125, + "n_bounded_support_per_pair_max_relative_diff": 0.011111111111111112, "n_bounded_support_per_pair_first_divergent_index": 14, "n_pairs_abs_diff": 0.0, "n_pairs_relative_diff": 0.0, @@ -652,15 +654,15 @@ "metric": "accuracy", "final_parity_status": "approximately equivalent", "strict_diagnostic_status": "approximately equivalent", - "headline_classification": "sklearn win", + "headline_classification": "flow win", "score_abs_diff": 0.019444424000000016, "declared_score_tolerance": 0.04, "effective_score_tolerance": 0.04, "score_tolerance_fraction": 0.4861106000000004, - "runtime_ratio_sklearn_over_flow": 0.46877090039689195, - "flow_total_ms": 32.097318102, - "sklearn_total_ms": 15.046288707, - "runtime_log2_ratio": -1.0930450796410873, + "runtime_ratio_sklearn_over_flow": 1.8765555978503732, + "flow_total_ms": 4.6069999699999995, + "sklearn_total_ms": 8.645291583, + "runtime_log2_ratio": 0.9080870344587176, "configuration_differences": [], "configuration_equivalences": [], "semantic_differences": [], @@ -670,20 +672,20 @@ "depth1_splits_first_divergent_index": -1, "max_depth_reached_abs_diff": 0.0, "max_depth_reached_relative_diff": 0.0, - "mean_leaf_depth_abs_diff": 6.000000052353016e-08, - "mean_leaf_depth_relative_diff": 7.897050809223051e-09, + "mean_leaf_depth_abs_diff": 0.0006958399999996701, + "mean_leaf_depth_relative_diff": 9.15847305089125e-05, "n_leaves_abs_diff": 0.0, "n_leaves_relative_diff": 0.0, "n_nodes_abs_diff": 0.0, "n_nodes_relative_diff": 0.0, - "nodes_per_depth_max_abs_diff": 0.0, - "nodes_per_depth_max_relative_diff": 0.0, - "nodes_per_depth_first_divergent_index": -1, - "preorder_split_features_max_abs_diff": 60.0, + "nodes_per_depth_max_abs_diff": 2.0, + "nodes_per_depth_max_relative_diff": 0.045454545454545456, + "nodes_per_depth_first_divergent_index": 8, + "preorder_split_features_max_abs_diff": 63.0, "preorder_split_features_max_relative_diff": 2.0, "preorder_split_features_first_divergent_index": 4, - "preorder_split_thresholds_max_abs_diff": 2.4044926589999998, - "preorder_split_thresholds_max_relative_diff": 1.8995545044755455, + "preorder_split_thresholds_max_abs_diff": 3.42867219, + "preorder_split_thresholds_max_relative_diff": 1.8558820476047642, "preorder_split_thresholds_first_divergent_index": 4, "root_impurity_abs_diff": 6.000000052353016e-09, "root_impurity_relative_diff": 6.666824921184481e-09, @@ -691,7 +693,7 @@ "root_split_feature_relative_diff": 0.0, "root_split_threshold_abs_diff": 6.000000007944095e-08, "root_split_threshold_relative_diff": 3.59138678901314e-08, - "split_feature_histogram_max_abs_diff": 25.0, + "split_feature_histogram_max_abs_diff": 24.0, "split_feature_histogram_max_relative_diff": 1.0, "split_feature_histogram_first_divergent_index": 1 }, @@ -709,16 +711,22 @@ "metric": "accuracy", "final_parity_status": "approximately equivalent", "strict_diagnostic_status": "approximately equivalent", - "headline_classification": "sklearn win", + "headline_classification": "flow win", "score_abs_diff": 0.01944444700000003, "declared_score_tolerance": 0.04, "effective_score_tolerance": 0.04, "score_tolerance_fraction": 0.4861111750000008, - "runtime_ratio_sklearn_over_flow": 0.3095068227822274, - "flow_total_ms": 94.941806277, - "sklearn_total_ms": 29.385136810000002, - "runtime_log2_ratio": -1.6919568822633577, - "configuration_differences": [], + "runtime_ratio_sklearn_over_flow": 5.239445843496978, + "flow_total_ms": 2.5059999339999997, + "sklearn_total_ms": 13.130050938, + "runtime_log2_ratio": 2.3894142313970326, + "configuration_differences": [ + { + "parameter": "n_jobs", + "flow": "all_cores", + "sklearn": 1 + } + ], "configuration_equivalences": [], "semantic_differences": [], "model_state_diagnostics": { @@ -730,10 +738,10 @@ "bootstrap_unique_fractions_first_divergent_index": 0, "max_features_per_split_abs_diff": 0.0, "max_features_per_split_relative_diff": 0.0, - "mean_top_vote_fraction_abs_diff": 0.0008333409999999875, - "mean_top_vote_fraction_relative_diff": 0.0010921105099131391, - "mean_vote_margin_abs_diff": 0.013888876999999966, - "mean_vote_margin_relative_diff": 0.02175803227417603, + "mean_top_vote_fraction_abs_diff": 0.0002777659999999571, + "mean_top_vote_fraction_relative_diff": 0.0003642832844214512, + "mean_vote_margin_abs_diff": 0.013611119000000005, + "mean_vote_margin_relative_diff": 0.021332184513843045, "n_trees_abs_diff": 0.0, "n_trees_relative_diff": 0.0, "tree_feature_seeds_max_abs_diff": 1126337748.0, @@ -763,6 +771,7 @@ "diagnostics": {}, "disparity_dimensions": [ "numerical", + "configuration", "model-state", "runtime" ], @@ -774,15 +783,15 @@ "metric": "accuracy", "final_parity_status": "parity verified", "strict_diagnostic_status": "parity verified", - "headline_classification": "tie", + "headline_classification": "flow win", "score_abs_diff": 8.000000106811456e-09, "declared_score_tolerance": 0.001, "effective_score_tolerance": 0.001, "score_tolerance_fraction": 8.000000106811456e-06, - "runtime_ratio_sklearn_over_flow": 0.999019140011964, - "flow_total_ms": 2.36399591, - "sklearn_total_ms": 2.3616771610000002, - "runtime_log2_ratio": -0.0014157762932689173, + "runtime_ratio_sklearn_over_flow": 4.515746549104471, + "flow_total_ms": 0.182000004, + "sklearn_total_ms": 0.82186589, + "runtime_log2_ratio": 2.1749645156478485, "configuration_differences": [], "configuration_equivalences": [], "semantic_differences": [], @@ -795,22 +804,25 @@ "classes_first_divergent_index": -1, "theta_frobenius_norm_abs_diff": 2.0000000056086265e-07, "theta_frobenius_norm_relative_diff": 1.4569212256307719e-08, - "theta_row_l2_norms_max_abs_diff": 1.4000000003733248e-07, - "theta_row_l2_norms_max_relative_diff": 3.4154607304007286e-08, + "theta_row_l2_norms_max_abs_diff": 1.5399999999665681e-06, + "theta_row_l2_norms_max_relative_diff": 4.3738917402336836e-07, "theta_row_l2_norms_first_divergent_index": -1, - "var_frobenius_norm_abs_diff": 7.999999951380232e-07, - "var_frobenius_norm_relative_diff": 2.3797633488885078e-08, - "var_max_abs_diff": 0.0, - "var_max_relative_diff": 0.0, - "var_min_abs_diff": 1.1000000007642387e-16, - "var_min_relative_diff": 1.0999699935828138e-07, - "var_row_l2_norms_max_abs_diff": 5.9700000001328135e-06, - "var_row_l2_norms_max_relative_diff": 7.931826459290972e-07, - "var_row_l2_norms_first_divergent_index": -1 + "var_frobenius_norm_abs_diff": 4.699999998081239e-06, + "var_frobenius_norm_relative_diff": 1.3981109753982025e-07, + "var_max_abs_diff": 1.5300000001161607e-05, + "var_max_relative_diff": 1.521562721804971e-06, + "var_min_abs_diff": 2.7089999999908758e-14, + "var_min_relative_diff": 2.7089263984606297e-05, + "var_row_l2_norms_max_abs_diff": 2.1000000000270802e-05, + "var_row_l2_norms_max_relative_diff": 1.1114694938078035e-06, + "var_row_l2_norms_first_divergent_index": 6 }, "diagnostics": {}, - "disparity_dimensions": [], - "has_tracked_disparity": false + "disparity_dimensions": [ + "model-state", + "runtime" + ], + "has_tracked_disparity": true }, { "algorithm": "KMeans", @@ -818,15 +830,15 @@ "metric": "adjusted_rand_index", "final_parity_status": "approximately equivalent", "strict_diagnostic_status": "approximately equivalent", - "headline_classification": "sklearn win", + "headline_classification": "flow win", "score_abs_diff": 1.2999999965401798e-08, "declared_score_tolerance": 0.05, "effective_score_tolerance": 0.05, "score_tolerance_fraction": 2.5999999930803597e-07, - "runtime_ratio_sklearn_over_flow": 0.19188573115513335, - "flow_total_ms": 358.419854999, - "sklearn_total_ms": 68.775655937, - "runtime_log2_ratio": -2.3816806598371, + "runtime_ratio_sklearn_over_flow": 1.4061460735205966, + "flow_total_ms": 14.347, + "sklearn_total_ms": 20.1739777168, + "runtime_log2_ratio": 0.49174647250244624, "configuration_differences": [], "configuration_equivalences": [], "semantic_differences": [ @@ -869,11 +881,11 @@ } ], "model_state_diagnostics": { - "inertia_relative_diff": 7.06058916797641e-08, + "inertia_relative_diff": 0.0, "center_l2_norms_sorted_max_abs_diff": 1.6999999985500835e-06, - "center_l2_norms_sorted_max_relative_diff": 2.7206695727346925e-07, + "center_l2_norms_sorted_max_relative_diff": 2.7411257291918075e-07, "center_l2_norms_sorted_first_divergent_index": -1, - "inertia_abs_diff": 0.003899999996065162, + "inertia_abs_diff": 0.0, "n_iter_abs_diff": 0.0, "n_iter_relative_diff": 0.0, "train_cluster_sizes_sorted_max_abs_diff": 0.0, @@ -881,7 +893,7 @@ "train_cluster_sizes_sorted_first_divergent_index": -1 }, "diagnostics": { - "inertia_relative_diff": 7.06058916797641e-08 + "inertia_relative_diff": 0.0 }, "disparity_dimensions": [ "semantic", @@ -900,10 +912,10 @@ "declared_score_tolerance": 0.02, "effective_score_tolerance": 0.02, "score_tolerance_fraction": 8.949999999452452e-06, - "runtime_ratio_sklearn_over_flow": 7.340486387280991, - "flow_total_ms": 0.11363200100000001, - "sklearn_total_ms": 0.8341141565, - "runtime_log2_ratio": 2.87587566051669, + "runtime_ratio_sklearn_over_flow": 12.547392108478215, + "flow_total_ms": 0.018000001, + "sklearn_total_ms": 0.2258530705, + "runtime_log2_ratio": 3.649315635706231, "configuration_differences": [ { "parameter": "max_iter", @@ -923,13 +935,13 @@ ], "semantic_differences": [], "model_state_diagnostics": { - "coef_max_abs_diff": 0.0004271999999971854, - "coef_max_relative_diff": 6.883851192542247e-05, + "coef_max_abs_diff": 0.0001679000000009978, + "coef_max_relative_diff": 1.5777592597625286e-05, "coef_first_divergent_index": 0, - "coef_abs_sum_abs_diff": 0.0012190000000202872, - "coef_abs_sum_relative_diff": 7.793207825055047e-06, - "coef_l2_norm_abs_diff": 0.00044739999999876545, - "coef_l2_norm_relative_diff": 7.345402452330833e-06, + "coef_abs_sum_abs_diff": 0.00042799999999942884, + "coef_abs_sum_relative_diff": 2.7362459576164887e-06, + "coef_l2_norm_abs_diff": 0.00017220000000151003, + "coef_l2_norm_relative_diff": 2.8271674704585635e-06, "intercept_abs_diff": 0.0, "intercept_relative_diff": 0.0, "n_zero_coefs_abs_diff": 0.0, @@ -950,14 +962,14 @@ "final_parity_status": "approximately equivalent", "strict_diagnostic_status": "approximately equivalent", "headline_classification": "flow win", - "score_abs_diff": 1.1742000000036779e-05, + "score_abs_diff": 1.1623000000016148e-05, "declared_score_tolerance": 0.02, "effective_score_tolerance": 0.02, - "score_tolerance_fraction": 0.0005871000000018389, - "runtime_ratio_sklearn_over_flow": 1.0529295071156093, - "flow_total_ms": 0.8548000060000001, - "sklearn_total_ms": 0.900044149, - "runtime_log2_ratio": 0.0744088521860277, + "score_tolerance_fraction": 0.0005811500000008074, + "runtime_ratio_sklearn_over_flow": 2.6949330261904763, + "flow_total_ms": 0.126, + "sklearn_total_ms": 0.33956156130000004, + "runtime_log2_ratio": 1.4302494199226854, "configuration_differences": [], "configuration_equivalences": [ { @@ -971,13 +983,13 @@ ], "semantic_differences": [], "model_state_diagnostics": { - "coef_max_abs_diff": 0.09238049999999731, - "coef_max_relative_diff": 0.06912091628090264, + "coef_max_abs_diff": 0.09235569999999882, + "coef_max_relative_diff": 0.06911149950466622, "coef_first_divergent_index": 0, - "coef_abs_sum_abs_diff": 0.2506249999999852, - "coef_abs_sum_relative_diff": 0.001774826537394653, - "coef_l2_norm_abs_diff": 0.08662879999999973, - "coef_l2_norm_relative_diff": 0.0015529635275771704, + "coef_abs_sum_abs_diff": 0.25057100000000787, + "coef_abs_sum_relative_diff": 0.0017744441308790257, + "coef_l2_norm_abs_diff": 0.08661219999999759, + "coef_l2_norm_relative_diff": 0.0015526659453116755, "intercept_abs_diff": 0.0, "intercept_relative_diff": 0.0, "n_zero_coefs_abs_diff": 0.0, @@ -997,14 +1009,14 @@ "final_parity_status": "parity verified", "strict_diagnostic_status": "parity verified", "headline_classification": "flow win", - "score_abs_diff": 1.7899999998904903e-07, + "score_abs_diff": 1.7900000004456018e-07, "declared_score_tolerance": 0.005, "effective_score_tolerance": 0.005, - "score_tolerance_fraction": 3.5799999997809806e-05, - "runtime_ratio_sklearn_over_flow": 8.451343879137005, - "flow_total_ms": 0.091801001, - "sklearn_total_ms": 0.7758418279, - "runtime_log2_ratio": 3.0791807678575296, + "score_tolerance_fraction": 3.5800000008912036e-05, + "runtime_ratio_sklearn_over_flow": 10.318900375, + "flow_total_ms": 0.02, + "sklearn_total_ms": 0.20637800750000002, + "runtime_log2_ratio": 3.367217334247436, "configuration_differences": [], "configuration_equivalences": [ { @@ -1017,13 +1029,13 @@ ], "semantic_differences": [], "model_state_diagnostics": { - "coef_max_abs_diff": 4.579999999521078e-05, - "coef_max_relative_diff": 5.528215507916881e-06, - "coef_first_divergent_index": 4, - "coef_abs_sum_abs_diff": 0.00013500000000021828, - "coef_abs_sum_relative_diff": 7.372227421323086e-07, - "coef_l2_norm_abs_diff": 4.8800000001847366e-05, - "coef_l2_norm_relative_diff": 6.812817223019638e-07, + "coef_max_abs_diff": 2.1000000000270802e-05, + "coef_max_relative_diff": 6.933633489741017e-06, + "coef_first_divergent_index": 0, + "coef_abs_sum_abs_diff": 4.3000000005122274e-05, + "coef_abs_sum_relative_diff": 2.3481909566973533e-07, + "coef_l2_norm_abs_diff": 1.6000000002236447e-05, + "coef_l2_norm_relative_diff": 2.2337105651521357e-07, "intercept_abs_diff": 0.0, "intercept_relative_diff": 0.0, "n_zero_coefs_abs_diff": 0.0, @@ -1042,31 +1054,31 @@ "metric": "r2", "final_parity_status": "parity verified", "strict_diagnostic_status": "parity verified", - "headline_classification": "sklearn win", - "score_abs_diff": 1.7899999998904903e-07, + "headline_classification": "flow win", + "score_abs_diff": 2.9799999995416826e-07, "declared_score_tolerance": 0.005, "effective_score_tolerance": 0.005, - "score_tolerance_fraction": 3.5799999997809806e-05, - "runtime_ratio_sklearn_over_flow": 0.11323696933009303, - "flow_total_ms": 59.79202129, - "sklearn_total_ms": 6.770667281, - "runtime_log2_ratio": -3.142583052249144, + "score_tolerance_fraction": 5.959999999083365e-05, + "runtime_ratio_sklearn_over_flow": 2.1101041813167494, + "flow_total_ms": 0.554000005, + "sklearn_total_ms": 1.168997727, + "runtime_log2_ratio": 1.077314230287233, "configuration_differences": [], "configuration_equivalences": [], "semantic_differences": [], "model_state_diagnostics": { "alpha_abs_diff": 0.0, "alpha_relative_diff": 0.0, - "dual_coef_abs_sum_abs_diff": 0.0009000000009109499, - "dual_coef_abs_sum_relative_diff": 6.518032501987619e-08, - "dual_coef_l2_norm_abs_diff": 3.2000000032894604e-05, - "dual_coef_l2_norm_relative_diff": 3.458165664168894e-08, - "dual_coef_max_abs_diff": 9.1999999995096e-05, - "dual_coef_max_relative_diff": 6.255323058613642e-07, - "dual_coef_mean_abs_diff": 7.399999999435636e-07, - "dual_coef_mean_relative_diff": 1.3419257174508625e-07, - "dual_coef_min_abs_diff": 4.5999999997548e-05, - "dual_coef_min_relative_diff": 3.5716349662619095e-07, + "dual_coef_abs_sum_abs_diff": 0.0008999999990919605, + "dual_coef_abs_sum_relative_diff": 6.51803206396658e-08, + "dual_coef_l2_norm_abs_diff": 7.599999992180528e-05, + "dual_coef_l2_norm_relative_diff": 8.213142760950853e-08, + "dual_coef_max_abs_diff": 0.00010700000001406806, + "dual_coef_max_relative_diff": 7.275211205097264e-07, + "dual_coef_mean_abs_diff": 4.800000006355276e-07, + "dual_coef_mean_relative_diff": 8.70438345470301e-08, + "dual_coef_min_abs_diff": 0.0, + "dual_coef_min_relative_diff": 0.0, "gamma_abs_diff": 9.999999994736442e-10, "gamma_relative_diff": 9.999999894736441e-09, "n_train_samples_abs_diff": 0.0, diff --git a/benchmarks/flow_results_v2.txt b/benchmarks/flow_results_v2.txt index bfb1d5a..b1c9d7b 100644 --- a/benchmarks/flow_results_v2.txt +++ b/benchmarks/flow_results_v2.txt @@ -1,26 +1,26 @@ TIMING_UNIT|ms BENCHMARK_MODE|end_to_end FIXTURE_SOURCE|benchmarks/split_*.bin -BENCHMARK_ENV|{"impl":"flow","timer":"timespec_get"} -RESULT|DecisionTree|digits|accuracy|0.833333313|31.9785061|0.118812002|0.008146286|0.000421003|1|1 -RESULT|DecisionTree|iris|accuracy|0.933333337|0.147556007|0.002695|0.001111999|0.00012|1|1 -RESULT|GaussianNB|digits|accuracy|0.741666675|1.08066297|1.28333294|0.004558086|0.00930798|1|1 -RESULT|GaussianNB|iris|accuracy|0.966666639|0.013315|0.006662|0.000161|0.000261|1|1 -RESULT|KMeans|digits|adjusted_rand_index|0.528326213|358.345886|0.073968999|1.60336304|0.001171998|1|1 -RESULT|KMeans|iris|adjusted_rand_index|0.548888862|1.29137695|0.003186|0.031298041|5e-05|1|1 -RESULT|KernelRidge_RBF|diabetes|r2|0.461917698|55.0274315|4.76458979|0.06193161|0.009175777|1|1 -RESULT|KernelSVC_RBF|digits|accuracy|0.949999988|115.784744|17.0403919|0.266311645|0.092468262|1|1 -RESULT|KernelSVC_RBF|iris|accuracy|0.966666639|1.11438704|0.111037999|0.009318948|0.000462003|1|1 -RESULT|Lasso|diabetes|r2|0.45552969|0.845703006|0.009097|0.014376998|2e-05|1|1 -RESULT|LinearRegression|diabetes|r2|0.452602565|0.090429001|0.001372|0.000501|8.9e-05|1|1 -RESULT|LinearSVC|digits|accuracy|0.955555558|1607.33569|1.60024703|2.98669434|0.012503981|1|1 -RESULT|LinearSVC|iris|accuracy|0.899999976|1.73283303|0.00563|0.004858017|4e-05|1|1 -RESULT|LogisticRegression|digits|accuracy|0.975000024|70.1647415|0.060823999|1.47711945|0.000852|1|1 -RESULT|LogisticRegression|iris|accuracy|0.933333337|0.368119001|0.002966|0.021780998|9.1e-05|1|1 -RESULT|PCA|iris|explained_var_ratio|0.957439065|0.04273|0.003216|0.000581|0.000192|1|1 -RESULT|RandomForest|digits|accuracy|0.955555558|94.4096603|0.532145977|0.083770752|0.005329967|1|1 -RESULT|RandomForest|iris|accuracy|0.966666639|1.27670002|0.021509999|0.025086999|0.000311|1|1 -RESULT|Ridge|diabetes|r2|0.454146445|0.110867001|0.002765|0.000681996|0.00016|1|1 +BENCHMARK_ENV|{"impl":"flow","timer":"clock_gettime_monotonic"} +RESULT|DecisionTree|digits|accuracy|0.833333313|4.58599997|0.021|0.095999956|0.002000001|1|1 +RESULT|DecisionTree|iris|accuracy|0.933333337|0.030999999|0.001|0.005000001|0|1|1 +RESULT|GaussianNB|digits|accuracy|0.741666675|0.074000001|0.108000003|0.002000004|0.0044999975|1|1 +RESULT|GaussianNB|iris|accuracy|0.966666639|0.005|0.001|0.0015|0.0005|1|1 +RESULT|KMeans|digits|adjusted_rand_index|0.528326213|14.323|0.024|0.172000408|0.002000001|1|1 +RESULT|KMeans|iris|adjusted_rand_index|0.548888862|0.178000003|0.001|0.026500001|0|1|1 +RESULT|KernelRidge_RBF|diabetes|r2|0.46191752|0.474000007|0.079999998|0.0684999975|0.006499998|1|1 +RESULT|KernelSVC_RBF|digits|accuracy|0.949999988|11.2530003|1.58000004|0.573499679|0.083999992|1|1 +RESULT|KernelSVC_RBF|iris|accuracy|0.966666639|0.143000007|0.023|0.0279999965|0.0005|1|1 +RESULT|Lasso|diabetes|r2|0.45552969|0.125|0.001|0.006999999|0.001|1|1 +RESULT|LinearRegression|diabetes|r2|0.452602625|0.02|0|0.001|0.001|1|1 +RESULT|LinearSVC|digits|accuracy|0.955555558|52.7070007|0.104000002|0.402000427|0.0075|1|1 +RESULT|LinearSVC|iris|accuracy|0.899999976|0.055|0.001|0.004500002|0.001|1|1 +RESULT|LogisticRegression|digits|accuracy|0.972222209|1.93599999|0.023|0.0474999545|0.002|1|1 +RESULT|LogisticRegression|iris|accuracy|0.933333337|0.093999997|0.002|0.0100000015|0.001|1|1 +RESULT|PCA|iris|explained_var_ratio|0.957439005|0.007|0.001|0.0015|0|1|1 +RESULT|RandomForest|digits|accuracy|0.955555558|2.31999993|0.186000004|0.207000136|0.013499998|1|1 +RESULT|RandomForest|iris|accuracy|0.966666639|0.226999998|0.007|0.0219999995|0.001|1|1 +RESULT|Ridge|diabetes|r2|0.454146445|0.017000001|0.001|0.001999999|0.001|1|1 DETAIL|DecisionTree|digits|depth1_splits|28,-0.873653173,21,-1.19120884 DETAIL|DecisionTree|digits|max_depth_reached|10 DETAIL|DecisionTree|digits|mean_leaf_depth|7.59777308 @@ -48,19 +48,19 @@ DETAIL|DecisionTree|iris|split_feature_histogram|1,2,3,1 DETAIL|GaussianNB|digits|class_priors|0.0988169834,0.101600558,0.0988169834,0.101600558,0.100904666,0.100904666,0.100904666,0.0995128751,0.0967292935,0.100208767 DETAIL|GaussianNB|digits|classes|0,1,2,3,4,5,6,7,8,9 DETAIL|GaussianNB|digits|theta_frobenius_norm|13.7275782 -DETAIL|GaussianNB|digits|theta_row_l2_norms|4.8790164,3.95312166,4.65056181,3.97317266,5.30466652,4.09900761,4.92982531,4.85135365,2.53451562,3.52089047 -DETAIL|GaussianNB|digits|var_frobenius_norm|33.616787 -DETAIL|GaussianNB|digits|var_max|10.0554361 -DETAIL|GaussianNB|digits|var_min|1.00002728e-09 -DETAIL|GaussianNB|digits|var_row_l2_norms|2.71952224,8.51007557,14.3026628,6.97968388,21.2216415,7.52663374,10.0767384,10.9786787,6.20931578,5.4178462 +DETAIL|GaussianNB|digits|theta_row_l2_norms|4.87901783,3.95312095,4.65056181,3.9731729,5.30466604,4.09900713,4.92982388,4.8513546,2.53451562,3.5208919 +DETAIL|GaussianNB|digits|var_frobenius_norm|33.6167831 +DETAIL|GaussianNB|digits|var_max|10.0554514 +DETAIL|GaussianNB|digits|var_min|1.00000008e-09 +DETAIL|GaussianNB|digits|var_row_l2_norms|2.71952248,8.51007557,14.3026733,6.97968531,21.2216206,7.52663803,10.0767498,10.9786835,6.20931292,5.4178462 DETAIL|GaussianNB|iris|class_priors|0.333333343,0.333333343,0.333333343 DETAIL|GaussianNB|iris|classes|0,1,2 DETAIL|GaussianNB|iris|theta_frobenius_norm|2.94364285 -DETAIL|GaussianNB|iris|theta_row_l2_norms|2.23148727,0.745151341,1.76925063 -DETAIL|GaussianNB|iris|var_frobenius_norm|1.33197558 +DETAIL|GaussianNB|iris|theta_row_l2_norms|2.23148704,0.745151162,1.76925075 +DETAIL|GaussianNB|iris|var_frobenius_norm|1.33197546 DETAIL|GaussianNB|iris|var_max|0.778331578 -DETAIL|GaussianNB|iris|var_min|0.00813894439 -DETAIL|GaussianNB|iris|var_row_l2_norms|0.789857447,0.569631219,0.908737838 +DETAIL|GaussianNB|iris|var_min|0.00813894533 +DETAIL|GaussianNB|iris|var_row_l2_norms|0.789857447,0.569631159,0.908737659 DETAIL|KMeans|digits|center_l2_norms_sorted|3.1549077,3.37186456,4.43453789,4.73711634,4.88850117,4.93533611,5.12554502,5.48479414,11.6704283,46.9712257 DETAIL|KMeans|digits|inertia|55236.1836 DETAIL|KMeans|digits|n_iter|22 @@ -72,16 +72,16 @@ DETAIL|KMeans|iris|train_cluster_sizes_sorted|49,40,31 DETAIL|KernelRidge_RBF|diabetes|alpha|1 DETAIL|KernelRidge_RBF|diabetes|dual_coef_abs_sum|13807.8477 DETAIL|KernelRidge_RBF|diabetes|dual_coef_l2_norm|925.346069 -DETAIL|KernelRidge_RBF|diabetes|dual_coef_max|147.074646 +DETAIL|KernelRidge_RBF|diabetes|dual_coef_max|147.074661 DETAIL|KernelRidge_RBF|diabetes|dual_coef_mean|5.51446247 -DETAIL|KernelRidge_RBF|diabetes|dual_coef_min|-128.792557 +DETAIL|KernelRidge_RBF|diabetes|dual_coef_min|-128.792526 DETAIL|KernelRidge_RBF|diabetes|gamma|0.100000001 DETAIL|KernelRidge_RBF|diabetes|n_train_samples|353 DETAIL|KernelSVC_RBF|digits|C|1 -DETAIL|KernelSVC_RBF|digits|dual_coef_abs_sum_per_pair|63.5631981,64.3742218,72.521553,81.9481812,92.240387,84.7299957,58.4558907,84.6362686,104.647461,126.400879,103.348595,108.437744,94.4531021,91.3241272,96.860733,170.321854,110.821465,135.95871,62.4101257,98.2020264,75.3593521,84.7470551,138.244003,99.6621628,61.8221321,126.118484,74.1167755,92.4643021,154.315994,177.815247,80.8322296,91.5008545,90.6744995,93.6135254,83.336937,84.896553,94.256134,138.430084,139.937347,50.7798767,102.546318,76.8471909,115.372314,101.078568,160.023651 +DETAIL|KernelSVC_RBF|digits|dual_coef_abs_sum_per_pair|63.5631981,64.3734055,72.5224075,81.9525528,92.2495117,84.7299957,58.4537468,84.6362686,104.647461,126.391663,103.348595,108.437744,94.44944,91.3241272,96.860733,170.321854,110.821465,135.95871,62.4101257,98.2020264,75.3593521,84.7440872,138.244003,99.6621628,61.8221321,126.118484,74.1167755,92.4643021,154.317398,177.815369,80.836525,91.5008621,90.6838226,93.6135254,83.3338852,84.8962555,94.256134,138.430084,139.941818,50.7765846,102.546318,76.8471909,115.372314,101.078568,160.00827 DETAIL|KernelSVC_RBF|digits|gamma|0.00100000005 -DETAIL|KernelSVC_RBF|digits|intercept_per_pair|-0.709698141,-0.740074992,-0.481941909,-0.894981682,-0.592060566,-0.22757642,-0.611445427,-0.442243487,-0.48276633,-0.318938166,0.390048146,-0.616389394,0.269409299,0.53758359,-0.0681470707,0.389300615,0.252242774,0.734256387,-0.214330509,0.561264813,0.505317926,0.437933803,0.618431747,0.611808777,-0.546122015,0.171668202,0.0913918689,0.00141514302,0.0289756749,0.228080928,0.659255624,0.760452509,0.639918387,0.691312253,0.66092521,0.212014496,-0.156483427,-0.0340732522,-0.338079363,-0.14603509,-0.146418154,-0.0992142558,0.164756775,0.148428991,-0.151808828 -DETAIL|KernelSVC_RBF|digits|n_bounded_support_per_pair|58,60,68,77,86,81,54,78,94,120,96,103,86,87,90,167,105,128,52,92,70,79,130,91,55,117,68,87,149,172,74,83,84,83,77,79,85,132,133,43,97,73,110,97,151 +DETAIL|KernelSVC_RBF|digits|intercept_per_pair|-0.709698141,-0.740054965,-0.481934667,-0.89493382,-0.592164516,-0.227576435,-0.611401498,-0.442243457,-0.48276633,-0.318906665,0.390048146,-0.616389513,0.269621909,0.53758359,-0.0681470633,0.389300615,0.252242714,0.734256327,-0.214330688,0.561264753,0.505317926,0.437787473,0.618431687,0.611808717,-0.546122134,0.171668172,0.0913918167,0.00141513417,0.0289370883,0.228170156,0.659319162,0.760452569,0.639743626,0.691312373,0.660763919,0.211931437,-0.156483546,-0.0340732522,-0.337795377,-0.145902798,-0.146418095,-0.0992142782,0.164756775,0.148428947,-0.151463211 +DETAIL|KernelSVC_RBF|digits|n_bounded_support_per_pair|58,60,68,77,86,81,54,78,94,120,96,103,86,87,90,167,105,128,52,92,70,79,130,91,55,117,68,87,149,172,74,83,84,83,77,80,85,132,133,43,97,73,110,97,151 DETAIL|KernelSVC_RBF|digits|n_pairs|45 DETAIL|KernelSVC_RBF|digits|n_support_per_pair|68,70,80,91,99,90,64,94,111,132,114,117,101,96,102,175,119,141,71,109,82,93,146,108,72,132,79,99,162,184,87,99,99,102,89,90,102,143,148,58,111,83,121,108,166 DETAIL|KernelSVC_RBF|digits|n_support_total|4807 @@ -89,13 +89,13 @@ DETAIL|KernelSVC_RBF|digits|pair_class_a|0,0,0,0,0,0,0,0,0,1,1,1,1,1,1,1,1,2,2,2 DETAIL|KernelSVC_RBF|digits|pair_class_b|1,2,3,4,5,6,7,8,9,2,3,4,5,6,7,8,9,3,4,5,6,7,8,9,4,5,6,7,8,9,5,6,7,8,9,6,7,8,9,7,8,9,8,9,9 DETAIL|KernelSVC_RBF|digits|pair_train_sizes|288,284,288,287,287,287,285,281,286,288,292,291,291,291,289,285,290,288,287,287,287,285,281,286,291,291,291,289,285,290,290,290,288,284,289,290,288,284,289,288,284,289,282,287,283 DETAIL|KernelSVC_RBF|iris|C|1 -DETAIL|KernelSVC_RBF|iris|dual_coef_abs_sum_per_pair|6.3142333,5.82612085,29.3913689 +DETAIL|KernelSVC_RBF|iris|dual_coef_abs_sum_per_pair|6.31395149,5.82528687,29.3901138 DETAIL|KernelSVC_RBF|iris|gamma|0.25 -DETAIL|KernelSVC_RBF|iris|intercept_per_pair|0.012464718,-0.134948775,0.0171961244 +DETAIL|KernelSVC_RBF|iris|intercept_per_pair|0.0126015441,-0.134950578,0.0173347387 DETAIL|KernelSVC_RBF|iris|n_bounded_support_per_pair|2,1,25 DETAIL|KernelSVC_RBF|iris|n_pairs|3 -DETAIL|KernelSVC_RBF|iris|n_support_per_pair|11,13,34 -DETAIL|KernelSVC_RBF|iris|n_support_total|58 +DETAIL|KernelSVC_RBF|iris|n_support_per_pair|11,12,34 +DETAIL|KernelSVC_RBF|iris|n_support_total|57 DETAIL|KernelSVC_RBF|iris|pair_class_a|0,0,1 DETAIL|KernelSVC_RBF|iris|pair_class_b|1,2,2 DETAIL|KernelSVC_RBF|iris|pair_train_sizes|80,80,80 @@ -120,32 +120,32 @@ DETAIL|LinearSVC|iris|coef_frobenius_norm|2.75106215 DETAIL|LinearSVC|iris|coef_row_l2_norms|1.00404298,1.28075552,2.21808624 DETAIL|LinearSVC|iris|intercepts|-0.804926157,-0.36785692,-1.88985121 DETAIL|LogisticRegression|digits|classes|0,1,2,3,4,5,6,7,8,9 -DETAIL|LogisticRegression|digits|coef_abs_sum|179.830612 -DETAIL|LogisticRegression|digits|coef_frobenius_norm|10.3909178 -DETAIL|LogisticRegression|digits|coef_row_l2_norms|2.3551321,3.59788823,3.59507179,3.70377636,3.43793464,3.37078714,2.85743594,2.80638885,3.00238466,3.82106566 -DETAIL|LogisticRegression|digits|intercepts|-0.249650881,-0.966436088,-0.153445095,0.0822364762,-0.61762166,0.454582185,-0.667101324,0.21621044,2.09947515,-0.198249161 +DETAIL|LogisticRegression|digits|coef_abs_sum|180.314728 +DETAIL|LogisticRegression|digits|coef_frobenius_norm|10.3989534 +DETAIL|LogisticRegression|digits|coef_row_l2_norms|2.37222338,3.56561947,3.5956316,3.70229411,3.42126489,3.40686178,2.83506465,2.83739662,3.06187749,3.79293084 +DETAIL|LogisticRegression|digits|intercepts|-0.303470939,-0.83022511,-0.160243735,0.0802990273,-0.483717978,0.330334783,-0.536223412,0.0991231129,1.96544969,-0.161325455 DETAIL|LogisticRegression|iris|classes|0,1,2 -DETAIL|LogisticRegression|iris|coef_abs_sum|13.2213602 -DETAIL|LogisticRegression|iris|coef_frobenius_norm|4.51708937 -DETAIL|LogisticRegression|iris|coef_row_l2_norms|2.88320494,1.05400097,3.31365442 -DETAIL|LogisticRegression|iris|intercepts|-0.305307209,1.90892446,-1.60361719 -DETAIL|PCA|iris|component_0|0.526793361,-0.253072023,0.581869245,0.565571845 -DETAIL|PCA|iris|component_1|0.348139405,0.93470794,0.026894361,0.066307925 -DETAIL|PCA|iris|explained_variance_ratio|0.726772428,0.230666667 -DETAIL|PCA|iris|reconstruction_mse|0.036734305 -DETAIL|PCA|iris|singular_values|18.6775475,10.5223579 +DETAIL|LogisticRegression|iris|coef_abs_sum|13.2246017 +DETAIL|LogisticRegression|iris|coef_frobenius_norm|4.51712608 +DETAIL|LogisticRegression|iris|coef_row_l2_norms|2.88416195,1.05452216,3.31270623 +DETAIL|LogisticRegression|iris|intercepts|-0.310604662,1.91055024,-1.59994555 +DETAIL|PCA|iris|component_0|0.526793361,-0.253071994,0.581869245,0.565571845 +DETAIL|PCA|iris|component_1|0.348139346,0.93470794,0.026894325,0.066307887 +DETAIL|PCA|iris|explained_variance_ratio|0.726772368,0.230666637 +DETAIL|PCA|iris|reconstruction_mse|0.036734309 +DETAIL|PCA|iris|singular_values|18.6775475,10.522357 DETAIL|RandomForest|digits|bootstrap_index_sums|1007289,1035097,1028067,1031510,1008751,1058975,1035756,1039421,1023753,1033758 DETAIL|RandomForest|digits|bootstrap_unique_fractions|0.62700069,0.622825325,0.635351419,0.632567823,0.619345844,0.63048017,0.622825325,0.61864996,0.643006265,0.634655535 DETAIL|RandomForest|digits|max_features_per_split|8 -DETAIL|RandomForest|digits|mean_top_vote_fraction|0.763055563 -DETAIL|RandomForest|digits|mean_vote_margin|0.638333321 +DETAIL|RandomForest|digits|mean_top_vote_fraction|0.762499988 +DETAIL|RandomForest|digits|mean_vote_margin|0.638055563 DETAIL|RandomForest|digits|n_trees|10 DETAIL|RandomForest|digits|tree_feature_seeds|539342599,1.4643425e+09,1.82897776e+09,358604049,340350764,74925942,14189958,487065659,725811655,1.28059078e+09 DETAIL|RandomForest|digits|tree_leaf_counts|154,155,165,166,147,146,161,148,170,152 DETAIL|RandomForest|digits|tree_max_depths|10,10,10,10,10,10,10,10,10,10 DETAIL|RandomForest|digits|tree_node_counts|307,309,329,331,293,291,321,295,339,303 DETAIL|RandomForest|digits|tree_root_features|53,36,38,21,33,10,21,60,58,33 -DETAIL|RandomForest|digits|tree_root_impurities|0.899504066,0.898665309,0.8995893,0.899577677,0.899560213,0.899328768,0.89899075,0.899153471,0.898597538,0.899734557 +DETAIL|RandomForest|digits|tree_root_impurities|0.899504066,0.898665309,0.8995893,0.899577677,0.899560213,0.899328768,0.89899075,0.899153471,0.898597479,0.899734557 DETAIL|RandomForest|digits|tree_root_thresholds|-1.36757088,-1.67066383,-0.677066445,-1.19120884,0.335855365,-0.354192019,-1.03024518,-1.28399694,-0.202554822,0.335855365 DETAIL|RandomForest|digits|unanimous_vote_fraction|0.252777785 DETAIL|RandomForest|iris|bootstrap_index_sums|7128,7063,7399,7418,6878,7997,6673,7007,7139,7499 diff --git a/benchmarks/headline_environment.json b/benchmarks/headline_environment.json index c45043c..48c2b7c 100644 --- a/benchmarks/headline_environment.json +++ b/benchmarks/headline_environment.json @@ -1,30 +1,36 @@ { - "environment_id": "3f884007c2946e39", - "runtime_environment_id": "def1cc4771c48618", + "environment_id": "c012023a7229bbbe", + "runtime_environment_id": "9bebce8d5728ab83", "host": { - "machine": "x86_64", - "cpu_model": "AMD EPYC 7763 64-Core Processor", - "logical_cpus": 4, + "machine": "arm64", + "cpu_model": "Apple M4 Max", + "logical_cpus": 14, "thread_limits": { "OPENBLAS_NUM_THREADS": "4", "OMP_NUM_THREADS": "4", "MKL_NUM_THREADS": "4", - "VECLIB_MAXIMUM_THREADS": null, + "VECLIB_MAXIMUM_THREADS": "4", "NUMEXPR_NUM_THREADS": null + }, + "flow_build": { + "FLOW_OPT_LEVEL": "3", + "FLOW_CFLAGS": null, + "FLOW_LDFLAGS": "-framework Accelerate lib/scikit/flow_time.c lib/scikit/flow_parallel.c", + "FLOW_HOST": "python" } }, "metadata": { "sklearn": { "implementation": "sklearn", - "process_repeats": 5, - "environment": "{\"impl\":\"sklearn\",\"numpy\":\"2.5.2\",\"platform\":\"Linux-6.17.0-1022-azure-x86_64-with-glibc2.39\",\"python\":\"3.12.14\",\"sklearn\":\"1.9.0\"}", + "process_repeats": 11, + "environment": "{\"impl\":\"sklearn\",\"numpy\":\"2.4.3\",\"platform\":\"macOS-26.2-arm64-arm-64bit\",\"python\":\"3.12.12\",\"sklearn\":\"1.9.0\"}", "mode": "end_to_end", "fixture": "benchmarks/split_indices.json" }, "flow": { "implementation": "flow", - "process_repeats": 5, - "environment": "{\"impl\":\"flow\",\"timer\":\"timespec_get\"}", + "process_repeats": 11, + "environment": "{\"impl\":\"flow\",\"timer\":\"clock_gettime_monotonic\"}", "mode": "end_to_end", "fixture": "benchmarks/split_*.bin" } diff --git a/benchmarks/headline_result_v2.json b/benchmarks/headline_result_v2.json index 1e3df31..9133e2a 100644 --- a/benchmarks/headline_result_v2.json +++ b/benchmarks/headline_result_v2.json @@ -2,13 +2,13 @@ "schema_version": 1, "timing_unit": "ms", "tie_relative_threshold": 0.02, - "environment_id": "3f884007c2946e39", + "environment_id": "c012023a7229bbbe", "counts": { "total_rows": 19, "eligible_comparisons": 19, - "flow_wins": 11, - "sklearn_wins": 7, - "ties": 1, + "flow_wins": 19, + "sklearn_wins": 0, + "ties": 0, "parity_unresolved": 0, "measurement_unresolved": 0, "not_comparable": 0, @@ -22,25 +22,25 @@ "category": "supervised", "sklearn_score": 0.933333333, "flow_score": 0.933333337, - "sklearn_fit_ms": 5.270827, - "sklearn_pred_ms": 0.153433687, - "flow_fit_ms": 0.368119001, - "flow_pred_ms": 0.002966, - "sklearn_fit_iqr_ms": 0.01394425, - "sklearn_pred_iqr_ms": 0.000404352, - "flow_fit_iqr_ms": 0.021780998, - "flow_pred_iqr_ms": 9.1e-05, - "sklearn_ms": 5.424260686999999, - "flow_ms": 0.371085001, + "sklearn_fit_ms": 0.96320575, + "sklearn_pred_ms": 0.0369819746, + "flow_fit_ms": 0.093999997, + "flow_pred_ms": 0.002, + "sklearn_fit_iqr_ms": 0.0319284065, + "sklearn_pred_iqr_ms": 0.0011009522, + "flow_fit_iqr_ms": 0.0100000015, + "flow_pred_iqr_ms": 0.001, + "sklearn_ms": 1.0001877246, + "flow_ms": 0.095999997, "timing_unit": "ms", "benchmark_mode": "end_to_end", "fixture_source": "benchmarks/split_indices.json + split_*.bin", "parity_status": "approximately equivalent", "measurement_status": "resolved", "comparable": true, - "environment_id": "3f884007c2946e39", + "environment_id": "c012023a7229bbbe", "classification": "flow win", - "speedup": 14.61729973559346 + "speedup": 10.418622456831951 }, { "algorithm": "LinearSVC", @@ -49,25 +49,25 @@ "category": "supervised", "sklearn_score": 0.9, "flow_score": 0.899999976, - "sklearn_fit_ms": 1.02706741, - "sklearn_pred_ms": 0.155343379, - "flow_fit_ms": 1.73283303, - "flow_pred_ms": 0.00563, - "sklearn_fit_iqr_ms": 0.00610422, - "sklearn_pred_iqr_ms": 0.001290555, - "flow_fit_iqr_ms": 0.004858017, - "flow_pred_iqr_ms": 4e-05, - "sklearn_ms": 1.182410789, - "flow_ms": 1.7384630300000001, + "sklearn_fit_ms": 0.281800453, + "sklearn_pred_ms": 0.0373879805, + "flow_fit_ms": 0.055, + "flow_pred_ms": 0.001, + "sklearn_fit_iqr_ms": 0.009811199, + "sklearn_pred_iqr_ms": 0.0006199341, + "flow_fit_iqr_ms": 0.004500002, + "flow_pred_iqr_ms": 0.001, + "sklearn_ms": 0.31918843350000004, + "flow_ms": 0.056, "timing_unit": "ms", "benchmark_mode": "end_to_end", "fixture_source": "benchmarks/split_indices.json + split_*.bin", "parity_status": "approximately equivalent", "measurement_status": "resolved", "comparable": true, - "environment_id": "3f884007c2946e39", - "classification": "sklearn win", - "speedup": 0.680147215440066 + "environment_id": "c012023a7229bbbe", + "classification": "flow win", + "speedup": 5.699793455357144 }, { "algorithm": "KernelSVC_RBF", @@ -76,25 +76,25 @@ "category": "supervised", "sklearn_score": 0.966666667, "flow_score": 0.966666639, - "sklearn_fit_ms": 1.06301781, - "sklearn_pred_ms": 0.193609023, - "flow_fit_ms": 1.11438704, - "flow_pred_ms": 0.111037999, - "sklearn_fit_iqr_ms": 0.00355409, - "sklearn_pred_iqr_ms": 0.000648516, - "flow_fit_iqr_ms": 0.009318948, - "flow_pred_iqr_ms": 0.000462003, - "sklearn_ms": 1.256626833, - "flow_ms": 1.225425039, + "sklearn_fit_ms": 0.288402672, + "sklearn_pred_ms": 0.067395834, + "flow_fit_ms": 0.143000007, + "flow_pred_ms": 0.023, + "sklearn_fit_iqr_ms": 0.0077688715, + "sklearn_pred_iqr_ms": 0.00166532525, + "flow_fit_iqr_ms": 0.0279999965, + "flow_pred_iqr_ms": 0.0005, + "sklearn_ms": 0.355798506, + "flow_ms": 0.166000007, "timing_unit": "ms", "benchmark_mode": "end_to_end", "fixture_source": "benchmarks/split_indices.json + split_*.bin", "parity_status": "approximately equivalent", "measurement_status": "resolved", "comparable": true, - "environment_id": "3f884007c2946e39", + "environment_id": "c012023a7229bbbe", "classification": "flow win", - "speedup": 1.025462017673037 + "speedup": 2.1433644035930675 }, { "algorithm": "DecisionTree", @@ -103,25 +103,25 @@ "category": "supervised", "sklearn_score": 0.933333333, "flow_score": 0.933333337, - "sklearn_fit_ms": 0.900115344, - "sklearn_pred_ms": 0.119728711, - "flow_fit_ms": 0.147556007, - "flow_pred_ms": 0.002695, - "sklearn_fit_iqr_ms": 0.00674025, - "sklearn_pred_iqr_ms": 0.000643836, - "flow_fit_iqr_ms": 0.001111999, - "flow_pred_iqr_ms": 0.00012, - "sklearn_ms": 1.019844055, - "flow_ms": 0.150251007, + "sklearn_fit_ms": 0.258626953, + "sklearn_pred_ms": 0.0286332197, + "flow_fit_ms": 0.030999999, + "flow_pred_ms": 0.001, + "sklearn_fit_iqr_ms": 0.003168133, + "sklearn_pred_iqr_ms": 0.0005029502, + "flow_fit_iqr_ms": 0.005000001, + "flow_pred_iqr_ms": 0.0, + "sklearn_ms": 0.2872601727, + "flow_ms": 0.031999999, "timing_unit": "ms", "benchmark_mode": "end_to_end", "fixture_source": "benchmarks/split_indices.json + split_*.bin", "parity_status": "approximately equivalent", "measurement_status": "resolved", "comparable": true, - "environment_id": "3f884007c2946e39", + "environment_id": "c012023a7229bbbe", "classification": "flow win", - "speedup": 6.7876021290160145 + "speedup": 8.976880677402521 }, { "algorithm": "RandomForest", @@ -130,25 +130,25 @@ "category": "supervised", "sklearn_score": 0.966666667, "flow_score": 0.966666639, - "sklearn_fit_ms": 12.802662, - "sklearn_pred_ms": 0.959652437, - "flow_fit_ms": 1.27670002, - "flow_pred_ms": 0.021509999, - "sklearn_fit_iqr_ms": 0.093426, - "sklearn_pred_iqr_ms": 0.00635675, - "flow_fit_iqr_ms": 0.025086999, - "flow_pred_iqr_ms": 0.000311, - "sklearn_ms": 13.762314437, - "flow_ms": 1.298210019, + "sklearn_fit_ms": 3.83877075, + "sklearn_pred_ms": 0.245369469, + "flow_fit_ms": 0.226999998, + "flow_pred_ms": 0.007, + "sklearn_fit_iqr_ms": 0.145294375, + "sklearn_pred_iqr_ms": 0.0037568355, + "flow_fit_iqr_ms": 0.0219999995, + "flow_pred_iqr_ms": 0.001, + "sklearn_ms": 4.084140219, + "flow_ms": 0.23399999800000001, "timing_unit": "ms", "benchmark_mode": "end_to_end", "fixture_source": "benchmarks/split_indices.json + split_*.bin", "parity_status": "approximately equivalent", "measurement_status": "resolved", "comparable": true, - "environment_id": "3f884007c2946e39", + "environment_id": "c012023a7229bbbe", "classification": "flow win", - "speedup": 10.600992316790924 + "speedup": 17.453590828663167 }, { "algorithm": "GaussianNB", @@ -157,25 +157,25 @@ "category": "supervised", "sklearn_score": 0.966666667, "flow_score": 0.966666639, - "sklearn_fit_ms": 0.924470656, - "sklearn_pred_ms": 0.177906977, - "flow_fit_ms": 0.013315, - "flow_pred_ms": 0.006662, - "sklearn_fit_iqr_ms": 0.014162281, - "sklearn_pred_iqr_ms": 0.001633765, - "flow_fit_iqr_ms": 0.000161, - "flow_pred_iqr_ms": 0.000261, - "sklearn_ms": 1.102377633, - "flow_ms": 0.019977, + "sklearn_fit_ms": 0.24434082, + "sklearn_pred_ms": 0.0445877285, + "flow_fit_ms": 0.005, + "flow_pred_ms": 0.001, + "sklearn_fit_iqr_ms": 0.0072657855, + "sklearn_pred_iqr_ms": 0.00116161915, + "flow_fit_iqr_ms": 0.0015, + "flow_pred_iqr_ms": 0.0005, + "sklearn_ms": 0.2889285485, + "flow_ms": 0.006, "timing_unit": "ms", "benchmark_mode": "end_to_end", "fixture_source": "benchmarks/split_indices.json + split_*.bin", "parity_status": "parity verified", "measurement_status": "resolved", "comparable": true, - "environment_id": "3f884007c2946e39", + "environment_id": "c012023a7229bbbe", "classification": "flow win", - "speedup": 55.182341342543914 + "speedup": 48.154758083333334 }, { "algorithm": "KMeans", @@ -184,25 +184,25 @@ "category": "clustering", "sklearn_score": 0.548888889, "flow_score": 0.548888862, - "sklearn_fit_ms": 10.6288145, - "sklearn_pred_ms": 0.164274305, - "flow_fit_ms": 1.29137695, - "flow_pred_ms": 0.003186, - "sklearn_fit_iqr_ms": 0.090423, - "sklearn_pred_iqr_ms": 0.001164805, - "flow_fit_iqr_ms": 0.031298041, - "flow_pred_iqr_ms": 5e-05, - "sklearn_ms": 10.793088805, - "flow_ms": 1.29456295, + "sklearn_fit_ms": 1.99084112, + "sklearn_pred_ms": 0.0330354004, + "flow_fit_ms": 0.178000003, + "flow_pred_ms": 0.001, + "sklearn_fit_iqr_ms": 0.103533845, + "sklearn_pred_iqr_ms": 0.0012598477, + "flow_fit_iqr_ms": 0.026500001, + "flow_pred_iqr_ms": 0.0, + "sklearn_ms": 2.0238765204, + "flow_ms": 0.179000003, "timing_unit": "ms", "benchmark_mode": "end_to_end", "fixture_source": "benchmarks/split_indices.json + split_*.bin", "parity_status": "approximately equivalent", "measurement_status": "resolved", "comparable": true, - "environment_id": "3f884007c2946e39", + "environment_id": "c012023a7229bbbe", "classification": "flow win", - "speedup": 8.33724524944886 + "speedup": 11.306572550169175 }, { "algorithm": "PCA", @@ -210,26 +210,26 @@ "metric": "explained_var_ratio", "category": "decomposition", "sklearn_score": 0.957439005, - "flow_score": 0.957439065, - "sklearn_fit_ms": 0.396100391, - "sklearn_pred_ms": 0.118425688, - "flow_fit_ms": 0.04273, - "flow_pred_ms": 0.003216, - "sklearn_fit_iqr_ms": 0.005139656, - "sklearn_pred_iqr_ms": 0.001520437, - "flow_fit_iqr_ms": 0.000581, - "flow_pred_iqr_ms": 0.000192, - "sklearn_ms": 0.514526079, - "flow_ms": 0.045946, + "flow_score": 0.957439005, + "sklearn_fit_ms": 0.106219398, + "sklearn_pred_ms": 0.0271950684, + "flow_fit_ms": 0.007, + "flow_pred_ms": 0.001, + "sklearn_fit_iqr_ms": 0.004313719, + "sklearn_pred_iqr_ms": 0.00061167385, + "flow_fit_iqr_ms": 0.0015, + "flow_pred_iqr_ms": 0.0, + "sklearn_ms": 0.13341446640000001, + "flow_ms": 0.008, "timing_unit": "ms", "benchmark_mode": "end_to_end", "fixture_source": "benchmarks/split_indices.json + split_*.bin", "parity_status": "parity verified", "measurement_status": "resolved", "comparable": true, - "environment_id": "3f884007c2946e39", + "environment_id": "c012023a7229bbbe", "classification": "flow win", - "speedup": 11.198495603534585 + "speedup": 16.6768083 }, { "algorithm": "LogisticRegression", @@ -237,26 +237,26 @@ "metric": "accuracy", "category": "supervised", "sklearn_score": 0.972222222, - "flow_score": 0.975000024, - "sklearn_fit_ms": 192.978743, - "sklearn_pred_ms": 0.179674766, - "flow_fit_ms": 70.1647415, - "flow_pred_ms": 0.060823999, - "sklearn_fit_iqr_ms": 31.962356, - "sklearn_pred_iqr_ms": 0.000810289, - "flow_fit_iqr_ms": 1.47711945, - "flow_pred_iqr_ms": 0.000852, - "sklearn_ms": 193.158417766, - "flow_ms": 70.225565499, + "flow_score": 0.972222209, + "sklearn_fit_ms": 6.179125, + "sklearn_pred_ms": 0.0561170254, + "flow_fit_ms": 1.93599999, + "flow_pred_ms": 0.023, + "sklearn_fit_iqr_ms": 0.246525875, + "sklearn_pred_iqr_ms": 0.00136836915, + "flow_fit_iqr_ms": 0.0474999545, + "flow_pred_iqr_ms": 0.002, + "sklearn_ms": 6.2352420254, + "flow_ms": 1.95899999, "timing_unit": "ms", "benchmark_mode": "end_to_end", "fixture_source": "benchmarks/split_indices.json + split_*.bin", "parity_status": "approximately equivalent", "measurement_status": "resolved", "comparable": true, - "environment_id": "3f884007c2946e39", + "environment_id": "c012023a7229bbbe", "classification": "flow win", - "speedup": 2.7505427175057857 + "speedup": 3.182869860759928 }, { "algorithm": "LinearSVC", @@ -265,25 +265,25 @@ "category": "supervised", "sklearn_score": 0.955555556, "flow_score": 0.955555558, - "sklearn_fit_ms": 349.443518, - "sklearn_pred_ms": 0.201159617, - "flow_fit_ms": 1607.33569, - "flow_pred_ms": 1.60024703, - "sklearn_fit_iqr_ms": 1.04663, - "sklearn_pred_iqr_ms": 0.000886437, - "flow_fit_iqr_ms": 2.98669434, - "flow_pred_iqr_ms": 0.012503981, - "sklearn_ms": 349.64467761699996, - "flow_ms": 1608.93593703, + "sklearn_fit_ms": 227.714333, + "sklearn_pred_ms": 0.0590003262, + "flow_fit_ms": 52.7070007, + "flow_pred_ms": 0.104000002, + "sklearn_fit_iqr_ms": 4.541, + "sklearn_pred_iqr_ms": 0.00208178615, + "flow_fit_iqr_ms": 0.402000427, + "flow_pred_iqr_ms": 0.0075, + "sklearn_ms": 227.7733333262, + "flow_ms": 52.811000702, "timing_unit": "ms", "benchmark_mode": "end_to_end", "fixture_source": "benchmarks/split_indices.json + split_*.bin", "parity_status": "approximately equivalent", "measurement_status": "resolved", "comparable": true, - "environment_id": "3f884007c2946e39", - "classification": "sklearn win", - "speedup": 0.21731423207714734 + "environment_id": "c012023a7229bbbe", + "classification": "flow win", + "speedup": 4.3129902917665035 }, { "algorithm": "KernelSVC_RBF", @@ -292,25 +292,25 @@ "category": "supervised", "sklearn_score": 0.95, "flow_score": 0.949999988, - "sklearn_fit_ms": 52.391939, - "sklearn_pred_ms": 25.539144, - "flow_fit_ms": 115.784744, - "flow_pred_ms": 17.0403919, - "sklearn_fit_iqr_ms": 0.110422, - "sklearn_pred_iqr_ms": 1.338714, - "flow_fit_iqr_ms": 0.266311645, - "flow_pred_iqr_ms": 0.092468262, - "sklearn_ms": 77.931083, - "flow_ms": 132.8251359, + "sklearn_fit_ms": 24.1875, + "sklearn_pred_ms": 16.9295625, + "flow_fit_ms": 11.2530003, + "flow_pred_ms": 1.58000004, + "sklearn_fit_iqr_ms": 0.3623745, + "sklearn_pred_iqr_ms": 0.903823, + "flow_fit_iqr_ms": 0.573499679, + "flow_pred_iqr_ms": 0.083999992, + "sklearn_ms": 41.1170625, + "flow_ms": 12.83300034, "timing_unit": "ms", "benchmark_mode": "end_to_end", "fixture_source": "benchmarks/split_indices.json + split_*.bin", "parity_status": "approximately equivalent", "measurement_status": "resolved", "comparable": true, - "environment_id": "3f884007c2946e39", - "classification": "sklearn win", - "speedup": 0.5867193921689036 + "environment_id": "c012023a7229bbbe", + "classification": "flow win", + "speedup": 3.2040100842076344 }, { "algorithm": "DecisionTree", @@ -319,25 +319,25 @@ "category": "supervised", "sklearn_score": 0.813888889, "flow_score": 0.833333313, - "sklearn_fit_ms": 14.9078835, - "sklearn_pred_ms": 0.138405207, - "flow_fit_ms": 31.9785061, - "flow_pred_ms": 0.118812002, - "sklearn_fit_iqr_ms": 0.045726, - "sklearn_pred_iqr_ms": 0.000688402, - "flow_fit_iqr_ms": 0.008146286, - "flow_pred_iqr_ms": 0.000421003, - "sklearn_ms": 15.046288707, - "flow_ms": 32.097318102, + "sklearn_fit_ms": 8.6068645, + "sklearn_pred_ms": 0.038427083, + "flow_fit_ms": 4.58599997, + "flow_pred_ms": 0.021, + "sklearn_fit_iqr_ms": 0.199073, + "sklearn_pred_iqr_ms": 0.00088521285, + "flow_fit_iqr_ms": 0.095999956, + "flow_pred_iqr_ms": 0.002000001, + "sklearn_ms": 8.645291583, + "flow_ms": 4.6069999699999995, "timing_unit": "ms", "benchmark_mode": "end_to_end", "fixture_source": "benchmarks/split_indices.json + split_*.bin", "parity_status": "approximately equivalent", "measurement_status": "resolved", "comparable": true, - "environment_id": "3f884007c2946e39", - "classification": "sklearn win", - "speedup": 0.46877090039689195 + "environment_id": "c012023a7229bbbe", + "classification": "flow win", + "speedup": 1.8765555978503732 }, { "algorithm": "RandomForest", @@ -346,25 +346,25 @@ "category": "supervised", "sklearn_score": 0.936111111, "flow_score": 0.955555558, - "sklearn_fit_ms": 28.12009, - "sklearn_pred_ms": 1.26504681, - "flow_fit_ms": 94.4096603, - "flow_pred_ms": 0.532145977, - "sklearn_fit_iqr_ms": 0.118545, - "sklearn_pred_iqr_ms": 0.02372631, - "flow_fit_iqr_ms": 0.083770752, - "flow_pred_iqr_ms": 0.005329967, - "sklearn_ms": 29.385136810000002, - "flow_ms": 94.941806277, + "sklearn_fit_ms": 12.8233335, + "sklearn_pred_ms": 0.306717438, + "flow_fit_ms": 2.31999993, + "flow_pred_ms": 0.186000004, + "sklearn_fit_iqr_ms": 0.635625, + "sklearn_pred_iqr_ms": 0.016799969, + "flow_fit_iqr_ms": 0.207000136, + "flow_pred_iqr_ms": 0.013499998, + "sklearn_ms": 13.130050938, + "flow_ms": 2.5059999339999997, "timing_unit": "ms", "benchmark_mode": "end_to_end", "fixture_source": "benchmarks/split_indices.json + split_*.bin", "parity_status": "approximately equivalent", "measurement_status": "resolved", "comparable": true, - "environment_id": "3f884007c2946e39", - "classification": "sklearn win", - "speedup": 0.3095068227822274 + "environment_id": "c012023a7229bbbe", + "classification": "flow win", + "speedup": 5.239445843496978 }, { "algorithm": "GaussianNB", @@ -373,25 +373,25 @@ "category": "supervised", "sklearn_score": 0.741666667, "flow_score": 0.741666675, - "sklearn_fit_ms": 1.72055738, - "sklearn_pred_ms": 0.641119781, - "flow_fit_ms": 1.08066297, - "flow_pred_ms": 1.28333294, - "sklearn_fit_iqr_ms": 0.0265915, - "sklearn_pred_iqr_ms": 0.008697468, - "flow_fit_iqr_ms": 0.004558086, - "flow_pred_iqr_ms": 0.00930798, - "sklearn_ms": 2.3616771610000002, - "flow_ms": 2.36399591, + "sklearn_fit_ms": 0.530330734, + "sklearn_pred_ms": 0.291535156, + "flow_fit_ms": 0.074000001, + "flow_pred_ms": 0.108000003, + "sklearn_fit_iqr_ms": 0.0227034605, + "sklearn_pred_iqr_ms": 0.002239582, + "flow_fit_iqr_ms": 0.002000004, + "flow_pred_iqr_ms": 0.0044999975, + "sklearn_ms": 0.82186589, + "flow_ms": 0.182000004, "timing_unit": "ms", "benchmark_mode": "end_to_end", "fixture_source": "benchmarks/split_indices.json + split_*.bin", "parity_status": "parity verified", "measurement_status": "resolved", "comparable": true, - "environment_id": "3f884007c2946e39", - "classification": "tie", - "speedup": 0.999019140011964 + "environment_id": "c012023a7229bbbe", + "classification": "flow win", + "speedup": 4.515746549104471 }, { "algorithm": "KMeans", @@ -400,25 +400,25 @@ "category": "clustering", "sklearn_score": 0.528326226, "flow_score": 0.528326213, - "sklearn_fit_ms": 68.592784, - "sklearn_pred_ms": 0.182871937, - "flow_fit_ms": 358.345886, - "flow_pred_ms": 0.073968999, - "sklearn_fit_iqr_ms": 0.410402, - "sklearn_pred_iqr_ms": 0.002540164, - "flow_fit_iqr_ms": 1.60336304, - "flow_pred_iqr_ms": 0.001171998, - "sklearn_ms": 68.775655937, - "flow_ms": 358.419854999, + "sklearn_fit_ms": 20.118834, + "sklearn_pred_ms": 0.0551437168, + "flow_fit_ms": 14.323, + "flow_pred_ms": 0.024, + "sklearn_fit_iqr_ms": 0.7834585, + "sklearn_pred_iqr_ms": 0.0037094717, + "flow_fit_iqr_ms": 0.172000408, + "flow_pred_iqr_ms": 0.002000001, + "sklearn_ms": 20.1739777168, + "flow_ms": 14.347, "timing_unit": "ms", "benchmark_mode": "end_to_end", "fixture_source": "benchmarks/split_indices.json + split_*.bin", "parity_status": "approximately equivalent", "measurement_status": "resolved", "comparable": true, - "environment_id": "3f884007c2946e39", - "classification": "sklearn win", - "speedup": 0.19188573115513335 + "environment_id": "c012023a7229bbbe", + "classification": "flow win", + "speedup": 1.4061460735205966 }, { "algorithm": "Ridge", @@ -427,106 +427,106 @@ "category": "supervised", "sklearn_score": 0.454146624, "flow_score": 0.454146445, - "sklearn_fit_ms": 0.735451469, - "sklearn_pred_ms": 0.0986626875, - "flow_fit_ms": 0.110867001, - "flow_pred_ms": 0.002765, - "sklearn_fit_iqr_ms": 0.008099906, - "sklearn_pred_iqr_ms": 0.0020281289, - "flow_fit_iqr_ms": 0.000681996, - "flow_pred_iqr_ms": 0.00016, - "sklearn_ms": 0.8341141565, - "flow_ms": 0.11363200100000001, + "sklearn_fit_ms": 0.201831383, + "sklearn_pred_ms": 0.0240216875, + "flow_fit_ms": 0.017000001, + "flow_pred_ms": 0.001, + "sklearn_fit_iqr_ms": 0.006238766, + "sklearn_pred_iqr_ms": 0.0008340044, + "flow_fit_iqr_ms": 0.001999999, + "flow_pred_iqr_ms": 0.001, + "sklearn_ms": 0.2258530705, + "flow_ms": 0.018000001, "timing_unit": "ms", "benchmark_mode": "end_to_end", "fixture_source": "benchmarks/split_indices.json + split_*.bin", "parity_status": "approximately equivalent", "measurement_status": "resolved", "comparable": true, - "environment_id": "3f884007c2946e39", + "environment_id": "c012023a7229bbbe", "classification": "flow win", - "speedup": 7.340486387280991 + "speedup": 12.547392108478215 }, { "algorithm": "Lasso", "dataset": "diabetes", "metric": "r2", "category": "supervised", - "sklearn_score": 0.455541432, + "sklearn_score": 0.455541313, "flow_score": 0.45552969, - "sklearn_fit_ms": 0.794417469, - "sklearn_pred_ms": 0.10562668, - "flow_fit_ms": 0.845703006, - "flow_pred_ms": 0.009097, - "sklearn_fit_iqr_ms": 0.021328625, - "sklearn_pred_iqr_ms": 0.001134567, - "flow_fit_iqr_ms": 0.014376998, - "flow_pred_iqr_ms": 2e-05, - "sklearn_ms": 0.900044149, - "flow_ms": 0.8548000060000001, + "sklearn_fit_ms": 0.313371742, + "sklearn_pred_ms": 0.0261898193, + "flow_fit_ms": 0.125, + "flow_pred_ms": 0.001, + "sklearn_fit_iqr_ms": 0.00685498, + "sklearn_pred_iqr_ms": 0.0005582886, + "flow_fit_iqr_ms": 0.006999999, + "flow_pred_iqr_ms": 0.001, + "sklearn_ms": 0.33956156130000004, + "flow_ms": 0.126, "timing_unit": "ms", "benchmark_mode": "end_to_end", "fixture_source": "benchmarks/split_indices.json + split_*.bin", "parity_status": "approximately equivalent", "measurement_status": "resolved", "comparable": true, - "environment_id": "3f884007c2946e39", + "environment_id": "c012023a7229bbbe", "classification": "flow win", - "speedup": 1.0529295071156093 + "speedup": 2.6949330261904763 }, { "algorithm": "LinearRegression", "dataset": "diabetes", "metric": "r2", "category": "supervised", - "sklearn_score": 0.452602744, - "flow_score": 0.452602565, - "sklearn_fit_ms": 0.676694531, - "sklearn_pred_ms": 0.0991472969, - "flow_fit_ms": 0.090429001, - "flow_pred_ms": 0.001372, - "sklearn_fit_iqr_ms": 0.012061094, - "sklearn_pred_iqr_ms": 0.0022896333, - "flow_fit_iqr_ms": 0.000501, - "flow_pred_iqr_ms": 8.9e-05, - "sklearn_ms": 0.7758418279, - "flow_ms": 0.091801001, + "sklearn_score": 0.452602804, + "flow_score": 0.452602625, + "sklearn_fit_ms": 0.182404945, + "sklearn_pred_ms": 0.0239730625, + "flow_fit_ms": 0.02, + "flow_pred_ms": 0.0, + "sklearn_fit_iqr_ms": 0.007456547, + "sklearn_pred_iqr_ms": 0.00047957425, + "flow_fit_iqr_ms": 0.001, + "flow_pred_iqr_ms": 0.001, + "sklearn_ms": 0.20637800750000002, + "flow_ms": 0.02, "timing_unit": "ms", "benchmark_mode": "end_to_end", "fixture_source": "benchmarks/split_indices.json + split_*.bin", "parity_status": "parity verified", "measurement_status": "resolved", "comparable": true, - "environment_id": "3f884007c2946e39", + "environment_id": "c012023a7229bbbe", "classification": "flow win", - "speedup": 8.451343879137005 + "speedup": 10.318900375 }, { "algorithm": "KernelRidge_RBF", "dataset": "diabetes", "metric": "r2", "category": "supervised", - "sklearn_score": 0.461917877, - "flow_score": 0.461917698, - "sklearn_fit_ms": 5.9995195, - "sklearn_pred_ms": 0.771147781, - "flow_fit_ms": 55.0274315, - "flow_pred_ms": 4.76458979, - "sklearn_fit_iqr_ms": 0.00606075, - "sklearn_pred_iqr_ms": 0.003607812, - "flow_fit_iqr_ms": 0.06193161, - "flow_pred_iqr_ms": 0.009175777, - "sklearn_ms": 6.770667281, - "flow_ms": 59.79202129, + "sklearn_score": 0.461917818, + "flow_score": 0.46191752, + "sklearn_fit_ms": 0.897470063, + "sklearn_pred_ms": 0.271527664, + "flow_fit_ms": 0.474000007, + "flow_pred_ms": 0.079999998, + "sklearn_fit_iqr_ms": 0.026421203, + "sklearn_pred_iqr_ms": 0.009609539, + "flow_fit_iqr_ms": 0.0684999975, + "flow_pred_iqr_ms": 0.006499998, + "sklearn_ms": 1.168997727, + "flow_ms": 0.554000005, "timing_unit": "ms", "benchmark_mode": "end_to_end", "fixture_source": "benchmarks/split_indices.json + split_*.bin", "parity_status": "parity verified", "measurement_status": "resolved", "comparable": true, - "environment_id": "3f884007c2946e39", - "classification": "sklearn win", - "speedup": 0.11323696933009303 + "environment_id": "c012023a7229bbbe", + "classification": "flow win", + "speedup": 2.1101041813167494 } ] } diff --git a/benchmarks/headline_rows.json b/benchmarks/headline_rows.json index 5e78beb..17bfd01 100644 --- a/benchmarks/headline_rows.json +++ b/benchmarks/headline_rows.json @@ -6,23 +6,23 @@ "category": "supervised", "sklearn_score": 0.933333333, "flow_score": 0.933333337, - "sklearn_fit_ms": 5.270827, - "sklearn_pred_ms": 0.153433687, - "flow_fit_ms": 0.368119001, - "flow_pred_ms": 0.002966, - "sklearn_fit_iqr_ms": 0.01394425, - "sklearn_pred_iqr_ms": 0.000404352, - "flow_fit_iqr_ms": 0.021780998, - "flow_pred_iqr_ms": 9.1e-05, - "sklearn_ms": 5.424260686999999, - "flow_ms": 0.371085001, + "sklearn_fit_ms": 0.96320575, + "sklearn_pred_ms": 0.0369819746, + "flow_fit_ms": 0.093999997, + "flow_pred_ms": 0.002, + "sklearn_fit_iqr_ms": 0.0319284065, + "sklearn_pred_iqr_ms": 0.0011009522, + "flow_fit_iqr_ms": 0.0100000015, + "flow_pred_iqr_ms": 0.001, + "sklearn_ms": 1.0001877246, + "flow_ms": 0.095999997, "timing_unit": "ms", "benchmark_mode": "end_to_end", "fixture_source": "benchmarks/split_indices.json + split_*.bin", "parity_status": "approximately equivalent", "measurement_status": "resolved", "comparable": true, - "environment_id": "3f884007c2946e39" + "environment_id": "c012023a7229bbbe" }, { "algorithm": "LinearSVC", @@ -31,23 +31,23 @@ "category": "supervised", "sklearn_score": 0.9, "flow_score": 0.899999976, - "sklearn_fit_ms": 1.02706741, - "sklearn_pred_ms": 0.155343379, - "flow_fit_ms": 1.73283303, - "flow_pred_ms": 0.00563, - "sklearn_fit_iqr_ms": 0.00610422, - "sklearn_pred_iqr_ms": 0.001290555, - "flow_fit_iqr_ms": 0.004858017, - "flow_pred_iqr_ms": 4e-05, - "sklearn_ms": 1.182410789, - "flow_ms": 1.7384630300000001, + "sklearn_fit_ms": 0.281800453, + "sklearn_pred_ms": 0.0373879805, + "flow_fit_ms": 0.055, + "flow_pred_ms": 0.001, + "sklearn_fit_iqr_ms": 0.009811199, + "sklearn_pred_iqr_ms": 0.0006199341, + "flow_fit_iqr_ms": 0.004500002, + "flow_pred_iqr_ms": 0.001, + "sklearn_ms": 0.31918843350000004, + "flow_ms": 0.056, "timing_unit": "ms", "benchmark_mode": "end_to_end", "fixture_source": "benchmarks/split_indices.json + split_*.bin", "parity_status": "approximately equivalent", "measurement_status": "resolved", "comparable": true, - "environment_id": "3f884007c2946e39" + "environment_id": "c012023a7229bbbe" }, { "algorithm": "KernelSVC_RBF", @@ -56,23 +56,23 @@ "category": "supervised", "sklearn_score": 0.966666667, "flow_score": 0.966666639, - "sklearn_fit_ms": 1.06301781, - "sklearn_pred_ms": 0.193609023, - "flow_fit_ms": 1.11438704, - "flow_pred_ms": 0.111037999, - "sklearn_fit_iqr_ms": 0.00355409, - "sklearn_pred_iqr_ms": 0.000648516, - "flow_fit_iqr_ms": 0.009318948, - "flow_pred_iqr_ms": 0.000462003, - "sklearn_ms": 1.256626833, - "flow_ms": 1.225425039, + "sklearn_fit_ms": 0.288402672, + "sklearn_pred_ms": 0.067395834, + "flow_fit_ms": 0.143000007, + "flow_pred_ms": 0.023, + "sklearn_fit_iqr_ms": 0.0077688715, + "sklearn_pred_iqr_ms": 0.00166532525, + "flow_fit_iqr_ms": 0.0279999965, + "flow_pred_iqr_ms": 0.0005, + "sklearn_ms": 0.355798506, + "flow_ms": 0.166000007, "timing_unit": "ms", "benchmark_mode": "end_to_end", "fixture_source": "benchmarks/split_indices.json + split_*.bin", "parity_status": "approximately equivalent", "measurement_status": "resolved", "comparable": true, - "environment_id": "3f884007c2946e39" + "environment_id": "c012023a7229bbbe" }, { "algorithm": "DecisionTree", @@ -81,23 +81,23 @@ "category": "supervised", "sklearn_score": 0.933333333, "flow_score": 0.933333337, - "sklearn_fit_ms": 0.900115344, - "sklearn_pred_ms": 0.119728711, - "flow_fit_ms": 0.147556007, - "flow_pred_ms": 0.002695, - "sklearn_fit_iqr_ms": 0.00674025, - "sklearn_pred_iqr_ms": 0.000643836, - "flow_fit_iqr_ms": 0.001111999, - "flow_pred_iqr_ms": 0.00012, - "sklearn_ms": 1.019844055, - "flow_ms": 0.150251007, + "sklearn_fit_ms": 0.258626953, + "sklearn_pred_ms": 0.0286332197, + "flow_fit_ms": 0.030999999, + "flow_pred_ms": 0.001, + "sklearn_fit_iqr_ms": 0.003168133, + "sklearn_pred_iqr_ms": 0.0005029502, + "flow_fit_iqr_ms": 0.005000001, + "flow_pred_iqr_ms": 0.0, + "sklearn_ms": 0.2872601727, + "flow_ms": 0.031999999, "timing_unit": "ms", "benchmark_mode": "end_to_end", "fixture_source": "benchmarks/split_indices.json + split_*.bin", "parity_status": "approximately equivalent", "measurement_status": "resolved", "comparable": true, - "environment_id": "3f884007c2946e39" + "environment_id": "c012023a7229bbbe" }, { "algorithm": "RandomForest", @@ -106,23 +106,23 @@ "category": "supervised", "sklearn_score": 0.966666667, "flow_score": 0.966666639, - "sklearn_fit_ms": 12.802662, - "sklearn_pred_ms": 0.959652437, - "flow_fit_ms": 1.27670002, - "flow_pred_ms": 0.021509999, - "sklearn_fit_iqr_ms": 0.093426, - "sklearn_pred_iqr_ms": 0.00635675, - "flow_fit_iqr_ms": 0.025086999, - "flow_pred_iqr_ms": 0.000311, - "sklearn_ms": 13.762314437, - "flow_ms": 1.298210019, + "sklearn_fit_ms": 3.83877075, + "sklearn_pred_ms": 0.245369469, + "flow_fit_ms": 0.226999998, + "flow_pred_ms": 0.007, + "sklearn_fit_iqr_ms": 0.145294375, + "sklearn_pred_iqr_ms": 0.0037568355, + "flow_fit_iqr_ms": 0.0219999995, + "flow_pred_iqr_ms": 0.001, + "sklearn_ms": 4.084140219, + "flow_ms": 0.23399999800000001, "timing_unit": "ms", "benchmark_mode": "end_to_end", "fixture_source": "benchmarks/split_indices.json + split_*.bin", "parity_status": "approximately equivalent", "measurement_status": "resolved", "comparable": true, - "environment_id": "3f884007c2946e39" + "environment_id": "c012023a7229bbbe" }, { "algorithm": "GaussianNB", @@ -131,23 +131,23 @@ "category": "supervised", "sklearn_score": 0.966666667, "flow_score": 0.966666639, - "sklearn_fit_ms": 0.924470656, - "sklearn_pred_ms": 0.177906977, - "flow_fit_ms": 0.013315, - "flow_pred_ms": 0.006662, - "sklearn_fit_iqr_ms": 0.014162281, - "sklearn_pred_iqr_ms": 0.001633765, - "flow_fit_iqr_ms": 0.000161, - "flow_pred_iqr_ms": 0.000261, - "sklearn_ms": 1.102377633, - "flow_ms": 0.019977, + "sklearn_fit_ms": 0.24434082, + "sklearn_pred_ms": 0.0445877285, + "flow_fit_ms": 0.005, + "flow_pred_ms": 0.001, + "sklearn_fit_iqr_ms": 0.0072657855, + "sklearn_pred_iqr_ms": 0.00116161915, + "flow_fit_iqr_ms": 0.0015, + "flow_pred_iqr_ms": 0.0005, + "sklearn_ms": 0.2889285485, + "flow_ms": 0.006, "timing_unit": "ms", "benchmark_mode": "end_to_end", "fixture_source": "benchmarks/split_indices.json + split_*.bin", "parity_status": "parity verified", "measurement_status": "resolved", "comparable": true, - "environment_id": "3f884007c2946e39" + "environment_id": "c012023a7229bbbe" }, { "algorithm": "KMeans", @@ -156,23 +156,23 @@ "category": "clustering", "sklearn_score": 0.548888889, "flow_score": 0.548888862, - "sklearn_fit_ms": 10.6288145, - "sklearn_pred_ms": 0.164274305, - "flow_fit_ms": 1.29137695, - "flow_pred_ms": 0.003186, - "sklearn_fit_iqr_ms": 0.090423, - "sklearn_pred_iqr_ms": 0.001164805, - "flow_fit_iqr_ms": 0.031298041, - "flow_pred_iqr_ms": 5e-05, - "sklearn_ms": 10.793088805, - "flow_ms": 1.29456295, + "sklearn_fit_ms": 1.99084112, + "sklearn_pred_ms": 0.0330354004, + "flow_fit_ms": 0.178000003, + "flow_pred_ms": 0.001, + "sklearn_fit_iqr_ms": 0.103533845, + "sklearn_pred_iqr_ms": 0.0012598477, + "flow_fit_iqr_ms": 0.026500001, + "flow_pred_iqr_ms": 0.0, + "sklearn_ms": 2.0238765204, + "flow_ms": 0.179000003, "timing_unit": "ms", "benchmark_mode": "end_to_end", "fixture_source": "benchmarks/split_indices.json + split_*.bin", "parity_status": "approximately equivalent", "measurement_status": "resolved", "comparable": true, - "environment_id": "3f884007c2946e39" + "environment_id": "c012023a7229bbbe" }, { "algorithm": "PCA", @@ -180,24 +180,24 @@ "metric": "explained_var_ratio", "category": "decomposition", "sklearn_score": 0.957439005, - "flow_score": 0.957439065, - "sklearn_fit_ms": 0.396100391, - "sklearn_pred_ms": 0.118425688, - "flow_fit_ms": 0.04273, - "flow_pred_ms": 0.003216, - "sklearn_fit_iqr_ms": 0.005139656, - "sklearn_pred_iqr_ms": 0.001520437, - "flow_fit_iqr_ms": 0.000581, - "flow_pred_iqr_ms": 0.000192, - "sklearn_ms": 0.514526079, - "flow_ms": 0.045946, + "flow_score": 0.957439005, + "sklearn_fit_ms": 0.106219398, + "sklearn_pred_ms": 0.0271950684, + "flow_fit_ms": 0.007, + "flow_pred_ms": 0.001, + "sklearn_fit_iqr_ms": 0.004313719, + "sklearn_pred_iqr_ms": 0.00061167385, + "flow_fit_iqr_ms": 0.0015, + "flow_pred_iqr_ms": 0.0, + "sklearn_ms": 0.13341446640000001, + "flow_ms": 0.008, "timing_unit": "ms", "benchmark_mode": "end_to_end", "fixture_source": "benchmarks/split_indices.json + split_*.bin", "parity_status": "parity verified", "measurement_status": "resolved", "comparable": true, - "environment_id": "3f884007c2946e39" + "environment_id": "c012023a7229bbbe" }, { "algorithm": "LogisticRegression", @@ -205,24 +205,24 @@ "metric": "accuracy", "category": "supervised", "sklearn_score": 0.972222222, - "flow_score": 0.975000024, - "sklearn_fit_ms": 192.978743, - "sklearn_pred_ms": 0.179674766, - "flow_fit_ms": 70.1647415, - "flow_pred_ms": 0.060823999, - "sklearn_fit_iqr_ms": 31.962356, - "sklearn_pred_iqr_ms": 0.000810289, - "flow_fit_iqr_ms": 1.47711945, - "flow_pred_iqr_ms": 0.000852, - "sklearn_ms": 193.158417766, - "flow_ms": 70.225565499, + "flow_score": 0.972222209, + "sklearn_fit_ms": 6.179125, + "sklearn_pred_ms": 0.0561170254, + "flow_fit_ms": 1.93599999, + "flow_pred_ms": 0.023, + "sklearn_fit_iqr_ms": 0.246525875, + "sklearn_pred_iqr_ms": 0.00136836915, + "flow_fit_iqr_ms": 0.0474999545, + "flow_pred_iqr_ms": 0.002, + "sklearn_ms": 6.2352420254, + "flow_ms": 1.95899999, "timing_unit": "ms", "benchmark_mode": "end_to_end", "fixture_source": "benchmarks/split_indices.json + split_*.bin", "parity_status": "approximately equivalent", "measurement_status": "resolved", "comparable": true, - "environment_id": "3f884007c2946e39" + "environment_id": "c012023a7229bbbe" }, { "algorithm": "LinearSVC", @@ -231,23 +231,23 @@ "category": "supervised", "sklearn_score": 0.955555556, "flow_score": 0.955555558, - "sklearn_fit_ms": 349.443518, - "sklearn_pred_ms": 0.201159617, - "flow_fit_ms": 1607.33569, - "flow_pred_ms": 1.60024703, - "sklearn_fit_iqr_ms": 1.04663, - "sklearn_pred_iqr_ms": 0.000886437, - "flow_fit_iqr_ms": 2.98669434, - "flow_pred_iqr_ms": 0.012503981, - "sklearn_ms": 349.64467761699996, - "flow_ms": 1608.93593703, + "sklearn_fit_ms": 227.714333, + "sklearn_pred_ms": 0.0590003262, + "flow_fit_ms": 52.7070007, + "flow_pred_ms": 0.104000002, + "sklearn_fit_iqr_ms": 4.541, + "sklearn_pred_iqr_ms": 0.00208178615, + "flow_fit_iqr_ms": 0.402000427, + "flow_pred_iqr_ms": 0.0075, + "sklearn_ms": 227.7733333262, + "flow_ms": 52.811000702, "timing_unit": "ms", "benchmark_mode": "end_to_end", "fixture_source": "benchmarks/split_indices.json + split_*.bin", "parity_status": "approximately equivalent", "measurement_status": "resolved", "comparable": true, - "environment_id": "3f884007c2946e39" + "environment_id": "c012023a7229bbbe" }, { "algorithm": "KernelSVC_RBF", @@ -256,23 +256,23 @@ "category": "supervised", "sklearn_score": 0.95, "flow_score": 0.949999988, - "sklearn_fit_ms": 52.391939, - "sklearn_pred_ms": 25.539144, - "flow_fit_ms": 115.784744, - "flow_pred_ms": 17.0403919, - "sklearn_fit_iqr_ms": 0.110422, - "sklearn_pred_iqr_ms": 1.338714, - "flow_fit_iqr_ms": 0.266311645, - "flow_pred_iqr_ms": 0.092468262, - "sklearn_ms": 77.931083, - "flow_ms": 132.8251359, + "sklearn_fit_ms": 24.1875, + "sklearn_pred_ms": 16.9295625, + "flow_fit_ms": 11.2530003, + "flow_pred_ms": 1.58000004, + "sklearn_fit_iqr_ms": 0.3623745, + "sklearn_pred_iqr_ms": 0.903823, + "flow_fit_iqr_ms": 0.573499679, + "flow_pred_iqr_ms": 0.083999992, + "sklearn_ms": 41.1170625, + "flow_ms": 12.83300034, "timing_unit": "ms", "benchmark_mode": "end_to_end", "fixture_source": "benchmarks/split_indices.json + split_*.bin", "parity_status": "approximately equivalent", "measurement_status": "resolved", "comparable": true, - "environment_id": "3f884007c2946e39" + "environment_id": "c012023a7229bbbe" }, { "algorithm": "DecisionTree", @@ -281,23 +281,23 @@ "category": "supervised", "sklearn_score": 0.813888889, "flow_score": 0.833333313, - "sklearn_fit_ms": 14.9078835, - "sklearn_pred_ms": 0.138405207, - "flow_fit_ms": 31.9785061, - "flow_pred_ms": 0.118812002, - "sklearn_fit_iqr_ms": 0.045726, - "sklearn_pred_iqr_ms": 0.000688402, - "flow_fit_iqr_ms": 0.008146286, - "flow_pred_iqr_ms": 0.000421003, - "sklearn_ms": 15.046288707, - "flow_ms": 32.097318102, + "sklearn_fit_ms": 8.6068645, + "sklearn_pred_ms": 0.038427083, + "flow_fit_ms": 4.58599997, + "flow_pred_ms": 0.021, + "sklearn_fit_iqr_ms": 0.199073, + "sklearn_pred_iqr_ms": 0.00088521285, + "flow_fit_iqr_ms": 0.095999956, + "flow_pred_iqr_ms": 0.002000001, + "sklearn_ms": 8.645291583, + "flow_ms": 4.6069999699999995, "timing_unit": "ms", "benchmark_mode": "end_to_end", "fixture_source": "benchmarks/split_indices.json + split_*.bin", "parity_status": "approximately equivalent", "measurement_status": "resolved", "comparable": true, - "environment_id": "3f884007c2946e39" + "environment_id": "c012023a7229bbbe" }, { "algorithm": "RandomForest", @@ -306,23 +306,23 @@ "category": "supervised", "sklearn_score": 0.936111111, "flow_score": 0.955555558, - "sklearn_fit_ms": 28.12009, - "sklearn_pred_ms": 1.26504681, - "flow_fit_ms": 94.4096603, - "flow_pred_ms": 0.532145977, - "sklearn_fit_iqr_ms": 0.118545, - "sklearn_pred_iqr_ms": 0.02372631, - "flow_fit_iqr_ms": 0.083770752, - "flow_pred_iqr_ms": 0.005329967, - "sklearn_ms": 29.385136810000002, - "flow_ms": 94.941806277, + "sklearn_fit_ms": 12.8233335, + "sklearn_pred_ms": 0.306717438, + "flow_fit_ms": 2.31999993, + "flow_pred_ms": 0.186000004, + "sklearn_fit_iqr_ms": 0.635625, + "sklearn_pred_iqr_ms": 0.016799969, + "flow_fit_iqr_ms": 0.207000136, + "flow_pred_iqr_ms": 0.013499998, + "sklearn_ms": 13.130050938, + "flow_ms": 2.5059999339999997, "timing_unit": "ms", "benchmark_mode": "end_to_end", "fixture_source": "benchmarks/split_indices.json + split_*.bin", "parity_status": "approximately equivalent", "measurement_status": "resolved", "comparable": true, - "environment_id": "3f884007c2946e39" + "environment_id": "c012023a7229bbbe" }, { "algorithm": "GaussianNB", @@ -331,23 +331,23 @@ "category": "supervised", "sklearn_score": 0.741666667, "flow_score": 0.741666675, - "sklearn_fit_ms": 1.72055738, - "sklearn_pred_ms": 0.641119781, - "flow_fit_ms": 1.08066297, - "flow_pred_ms": 1.28333294, - "sklearn_fit_iqr_ms": 0.0265915, - "sklearn_pred_iqr_ms": 0.008697468, - "flow_fit_iqr_ms": 0.004558086, - "flow_pred_iqr_ms": 0.00930798, - "sklearn_ms": 2.3616771610000002, - "flow_ms": 2.36399591, + "sklearn_fit_ms": 0.530330734, + "sklearn_pred_ms": 0.291535156, + "flow_fit_ms": 0.074000001, + "flow_pred_ms": 0.108000003, + "sklearn_fit_iqr_ms": 0.0227034605, + "sklearn_pred_iqr_ms": 0.002239582, + "flow_fit_iqr_ms": 0.002000004, + "flow_pred_iqr_ms": 0.0044999975, + "sklearn_ms": 0.82186589, + "flow_ms": 0.182000004, "timing_unit": "ms", "benchmark_mode": "end_to_end", "fixture_source": "benchmarks/split_indices.json + split_*.bin", "parity_status": "parity verified", "measurement_status": "resolved", "comparable": true, - "environment_id": "3f884007c2946e39" + "environment_id": "c012023a7229bbbe" }, { "algorithm": "KMeans", @@ -356,23 +356,23 @@ "category": "clustering", "sklearn_score": 0.528326226, "flow_score": 0.528326213, - "sklearn_fit_ms": 68.592784, - "sklearn_pred_ms": 0.182871937, - "flow_fit_ms": 358.345886, - "flow_pred_ms": 0.073968999, - "sklearn_fit_iqr_ms": 0.410402, - "sklearn_pred_iqr_ms": 0.002540164, - "flow_fit_iqr_ms": 1.60336304, - "flow_pred_iqr_ms": 0.001171998, - "sklearn_ms": 68.775655937, - "flow_ms": 358.419854999, + "sklearn_fit_ms": 20.118834, + "sklearn_pred_ms": 0.0551437168, + "flow_fit_ms": 14.323, + "flow_pred_ms": 0.024, + "sklearn_fit_iqr_ms": 0.7834585, + "sklearn_pred_iqr_ms": 0.0037094717, + "flow_fit_iqr_ms": 0.172000408, + "flow_pred_iqr_ms": 0.002000001, + "sklearn_ms": 20.1739777168, + "flow_ms": 14.347, "timing_unit": "ms", "benchmark_mode": "end_to_end", "fixture_source": "benchmarks/split_indices.json + split_*.bin", "parity_status": "approximately equivalent", "measurement_status": "resolved", "comparable": true, - "environment_id": "3f884007c2946e39" + "environment_id": "c012023a7229bbbe" }, { "algorithm": "Ridge", @@ -381,97 +381,97 @@ "category": "supervised", "sklearn_score": 0.454146624, "flow_score": 0.454146445, - "sklearn_fit_ms": 0.735451469, - "sklearn_pred_ms": 0.0986626875, - "flow_fit_ms": 0.110867001, - "flow_pred_ms": 0.002765, - "sklearn_fit_iqr_ms": 0.008099906, - "sklearn_pred_iqr_ms": 0.0020281289, - "flow_fit_iqr_ms": 0.000681996, - "flow_pred_iqr_ms": 0.00016, - "sklearn_ms": 0.8341141565, - "flow_ms": 0.11363200100000001, + "sklearn_fit_ms": 0.201831383, + "sklearn_pred_ms": 0.0240216875, + "flow_fit_ms": 0.017000001, + "flow_pred_ms": 0.001, + "sklearn_fit_iqr_ms": 0.006238766, + "sklearn_pred_iqr_ms": 0.0008340044, + "flow_fit_iqr_ms": 0.001999999, + "flow_pred_iqr_ms": 0.001, + "sklearn_ms": 0.2258530705, + "flow_ms": 0.018000001, "timing_unit": "ms", "benchmark_mode": "end_to_end", "fixture_source": "benchmarks/split_indices.json + split_*.bin", "parity_status": "approximately equivalent", "measurement_status": "resolved", "comparable": true, - "environment_id": "3f884007c2946e39" + "environment_id": "c012023a7229bbbe" }, { "algorithm": "Lasso", "dataset": "diabetes", "metric": "r2", "category": "supervised", - "sklearn_score": 0.455541432, + "sklearn_score": 0.455541313, "flow_score": 0.45552969, - "sklearn_fit_ms": 0.794417469, - "sklearn_pred_ms": 0.10562668, - "flow_fit_ms": 0.845703006, - "flow_pred_ms": 0.009097, - "sklearn_fit_iqr_ms": 0.021328625, - "sklearn_pred_iqr_ms": 0.001134567, - "flow_fit_iqr_ms": 0.014376998, - "flow_pred_iqr_ms": 2e-05, - "sklearn_ms": 0.900044149, - "flow_ms": 0.8548000060000001, + "sklearn_fit_ms": 0.313371742, + "sklearn_pred_ms": 0.0261898193, + "flow_fit_ms": 0.125, + "flow_pred_ms": 0.001, + "sklearn_fit_iqr_ms": 0.00685498, + "sklearn_pred_iqr_ms": 0.0005582886, + "flow_fit_iqr_ms": 0.006999999, + "flow_pred_iqr_ms": 0.001, + "sklearn_ms": 0.33956156130000004, + "flow_ms": 0.126, "timing_unit": "ms", "benchmark_mode": "end_to_end", "fixture_source": "benchmarks/split_indices.json + split_*.bin", "parity_status": "approximately equivalent", "measurement_status": "resolved", "comparable": true, - "environment_id": "3f884007c2946e39" + "environment_id": "c012023a7229bbbe" }, { "algorithm": "LinearRegression", "dataset": "diabetes", "metric": "r2", "category": "supervised", - "sklearn_score": 0.452602744, - "flow_score": 0.452602565, - "sklearn_fit_ms": 0.676694531, - "sklearn_pred_ms": 0.0991472969, - "flow_fit_ms": 0.090429001, - "flow_pred_ms": 0.001372, - "sklearn_fit_iqr_ms": 0.012061094, - "sklearn_pred_iqr_ms": 0.0022896333, - "flow_fit_iqr_ms": 0.000501, - "flow_pred_iqr_ms": 8.9e-05, - "sklearn_ms": 0.7758418279, - "flow_ms": 0.091801001, + "sklearn_score": 0.452602804, + "flow_score": 0.452602625, + "sklearn_fit_ms": 0.182404945, + "sklearn_pred_ms": 0.0239730625, + "flow_fit_ms": 0.02, + "flow_pred_ms": 0.0, + "sklearn_fit_iqr_ms": 0.007456547, + "sklearn_pred_iqr_ms": 0.00047957425, + "flow_fit_iqr_ms": 0.001, + "flow_pred_iqr_ms": 0.001, + "sklearn_ms": 0.20637800750000002, + "flow_ms": 0.02, "timing_unit": "ms", "benchmark_mode": "end_to_end", "fixture_source": "benchmarks/split_indices.json + split_*.bin", "parity_status": "parity verified", "measurement_status": "resolved", "comparable": true, - "environment_id": "3f884007c2946e39" + "environment_id": "c012023a7229bbbe" }, { "algorithm": "KernelRidge_RBF", "dataset": "diabetes", "metric": "r2", "category": "supervised", - "sklearn_score": 0.461917877, - "flow_score": 0.461917698, - "sklearn_fit_ms": 5.9995195, - "sklearn_pred_ms": 0.771147781, - "flow_fit_ms": 55.0274315, - "flow_pred_ms": 4.76458979, - "sklearn_fit_iqr_ms": 0.00606075, - "sklearn_pred_iqr_ms": 0.003607812, - "flow_fit_iqr_ms": 0.06193161, - "flow_pred_iqr_ms": 0.009175777, - "sklearn_ms": 6.770667281, - "flow_ms": 59.79202129, + "sklearn_score": 0.461917818, + "flow_score": 0.46191752, + "sklearn_fit_ms": 0.897470063, + "sklearn_pred_ms": 0.271527664, + "flow_fit_ms": 0.474000007, + "flow_pred_ms": 0.079999998, + "sklearn_fit_iqr_ms": 0.026421203, + "sklearn_pred_iqr_ms": 0.009609539, + "flow_fit_iqr_ms": 0.0684999975, + "flow_pred_iqr_ms": 0.006499998, + "sklearn_ms": 1.168997727, + "flow_ms": 0.554000005, "timing_unit": "ms", "benchmark_mode": "end_to_end", "fixture_source": "benchmarks/split_indices.json + split_*.bin", "parity_status": "parity verified", "measurement_status": "resolved", "comparable": true, - "environment_id": "3f884007c2946e39" + "environment_id": "c012023a7229bbbe" } ] diff --git a/benchmarks/headline_summary.json b/benchmarks/headline_summary.json index 1e3df31..9133e2a 100644 --- a/benchmarks/headline_summary.json +++ b/benchmarks/headline_summary.json @@ -2,13 +2,13 @@ "schema_version": 1, "timing_unit": "ms", "tie_relative_threshold": 0.02, - "environment_id": "3f884007c2946e39", + "environment_id": "c012023a7229bbbe", "counts": { "total_rows": 19, "eligible_comparisons": 19, - "flow_wins": 11, - "sklearn_wins": 7, - "ties": 1, + "flow_wins": 19, + "sklearn_wins": 0, + "ties": 0, "parity_unresolved": 0, "measurement_unresolved": 0, "not_comparable": 0, @@ -22,25 +22,25 @@ "category": "supervised", "sklearn_score": 0.933333333, "flow_score": 0.933333337, - "sklearn_fit_ms": 5.270827, - "sklearn_pred_ms": 0.153433687, - "flow_fit_ms": 0.368119001, - "flow_pred_ms": 0.002966, - "sklearn_fit_iqr_ms": 0.01394425, - "sklearn_pred_iqr_ms": 0.000404352, - "flow_fit_iqr_ms": 0.021780998, - "flow_pred_iqr_ms": 9.1e-05, - "sklearn_ms": 5.424260686999999, - "flow_ms": 0.371085001, + "sklearn_fit_ms": 0.96320575, + "sklearn_pred_ms": 0.0369819746, + "flow_fit_ms": 0.093999997, + "flow_pred_ms": 0.002, + "sklearn_fit_iqr_ms": 0.0319284065, + "sklearn_pred_iqr_ms": 0.0011009522, + "flow_fit_iqr_ms": 0.0100000015, + "flow_pred_iqr_ms": 0.001, + "sklearn_ms": 1.0001877246, + "flow_ms": 0.095999997, "timing_unit": "ms", "benchmark_mode": "end_to_end", "fixture_source": "benchmarks/split_indices.json + split_*.bin", "parity_status": "approximately equivalent", "measurement_status": "resolved", "comparable": true, - "environment_id": "3f884007c2946e39", + "environment_id": "c012023a7229bbbe", "classification": "flow win", - "speedup": 14.61729973559346 + "speedup": 10.418622456831951 }, { "algorithm": "LinearSVC", @@ -49,25 +49,25 @@ "category": "supervised", "sklearn_score": 0.9, "flow_score": 0.899999976, - "sklearn_fit_ms": 1.02706741, - "sklearn_pred_ms": 0.155343379, - "flow_fit_ms": 1.73283303, - "flow_pred_ms": 0.00563, - "sklearn_fit_iqr_ms": 0.00610422, - "sklearn_pred_iqr_ms": 0.001290555, - "flow_fit_iqr_ms": 0.004858017, - "flow_pred_iqr_ms": 4e-05, - "sklearn_ms": 1.182410789, - "flow_ms": 1.7384630300000001, + "sklearn_fit_ms": 0.281800453, + "sklearn_pred_ms": 0.0373879805, + "flow_fit_ms": 0.055, + "flow_pred_ms": 0.001, + "sklearn_fit_iqr_ms": 0.009811199, + "sklearn_pred_iqr_ms": 0.0006199341, + "flow_fit_iqr_ms": 0.004500002, + "flow_pred_iqr_ms": 0.001, + "sklearn_ms": 0.31918843350000004, + "flow_ms": 0.056, "timing_unit": "ms", "benchmark_mode": "end_to_end", "fixture_source": "benchmarks/split_indices.json + split_*.bin", "parity_status": "approximately equivalent", "measurement_status": "resolved", "comparable": true, - "environment_id": "3f884007c2946e39", - "classification": "sklearn win", - "speedup": 0.680147215440066 + "environment_id": "c012023a7229bbbe", + "classification": "flow win", + "speedup": 5.699793455357144 }, { "algorithm": "KernelSVC_RBF", @@ -76,25 +76,25 @@ "category": "supervised", "sklearn_score": 0.966666667, "flow_score": 0.966666639, - "sklearn_fit_ms": 1.06301781, - "sklearn_pred_ms": 0.193609023, - "flow_fit_ms": 1.11438704, - "flow_pred_ms": 0.111037999, - "sklearn_fit_iqr_ms": 0.00355409, - "sklearn_pred_iqr_ms": 0.000648516, - "flow_fit_iqr_ms": 0.009318948, - "flow_pred_iqr_ms": 0.000462003, - "sklearn_ms": 1.256626833, - "flow_ms": 1.225425039, + "sklearn_fit_ms": 0.288402672, + "sklearn_pred_ms": 0.067395834, + "flow_fit_ms": 0.143000007, + "flow_pred_ms": 0.023, + "sklearn_fit_iqr_ms": 0.0077688715, + "sklearn_pred_iqr_ms": 0.00166532525, + "flow_fit_iqr_ms": 0.0279999965, + "flow_pred_iqr_ms": 0.0005, + "sklearn_ms": 0.355798506, + "flow_ms": 0.166000007, "timing_unit": "ms", "benchmark_mode": "end_to_end", "fixture_source": "benchmarks/split_indices.json + split_*.bin", "parity_status": "approximately equivalent", "measurement_status": "resolved", "comparable": true, - "environment_id": "3f884007c2946e39", + "environment_id": "c012023a7229bbbe", "classification": "flow win", - "speedup": 1.025462017673037 + "speedup": 2.1433644035930675 }, { "algorithm": "DecisionTree", @@ -103,25 +103,25 @@ "category": "supervised", "sklearn_score": 0.933333333, "flow_score": 0.933333337, - "sklearn_fit_ms": 0.900115344, - "sklearn_pred_ms": 0.119728711, - "flow_fit_ms": 0.147556007, - "flow_pred_ms": 0.002695, - "sklearn_fit_iqr_ms": 0.00674025, - "sklearn_pred_iqr_ms": 0.000643836, - "flow_fit_iqr_ms": 0.001111999, - "flow_pred_iqr_ms": 0.00012, - "sklearn_ms": 1.019844055, - "flow_ms": 0.150251007, + "sklearn_fit_ms": 0.258626953, + "sklearn_pred_ms": 0.0286332197, + "flow_fit_ms": 0.030999999, + "flow_pred_ms": 0.001, + "sklearn_fit_iqr_ms": 0.003168133, + "sklearn_pred_iqr_ms": 0.0005029502, + "flow_fit_iqr_ms": 0.005000001, + "flow_pred_iqr_ms": 0.0, + "sklearn_ms": 0.2872601727, + "flow_ms": 0.031999999, "timing_unit": "ms", "benchmark_mode": "end_to_end", "fixture_source": "benchmarks/split_indices.json + split_*.bin", "parity_status": "approximately equivalent", "measurement_status": "resolved", "comparable": true, - "environment_id": "3f884007c2946e39", + "environment_id": "c012023a7229bbbe", "classification": "flow win", - "speedup": 6.7876021290160145 + "speedup": 8.976880677402521 }, { "algorithm": "RandomForest", @@ -130,25 +130,25 @@ "category": "supervised", "sklearn_score": 0.966666667, "flow_score": 0.966666639, - "sklearn_fit_ms": 12.802662, - "sklearn_pred_ms": 0.959652437, - "flow_fit_ms": 1.27670002, - "flow_pred_ms": 0.021509999, - "sklearn_fit_iqr_ms": 0.093426, - "sklearn_pred_iqr_ms": 0.00635675, - "flow_fit_iqr_ms": 0.025086999, - "flow_pred_iqr_ms": 0.000311, - "sklearn_ms": 13.762314437, - "flow_ms": 1.298210019, + "sklearn_fit_ms": 3.83877075, + "sklearn_pred_ms": 0.245369469, + "flow_fit_ms": 0.226999998, + "flow_pred_ms": 0.007, + "sklearn_fit_iqr_ms": 0.145294375, + "sklearn_pred_iqr_ms": 0.0037568355, + "flow_fit_iqr_ms": 0.0219999995, + "flow_pred_iqr_ms": 0.001, + "sklearn_ms": 4.084140219, + "flow_ms": 0.23399999800000001, "timing_unit": "ms", "benchmark_mode": "end_to_end", "fixture_source": "benchmarks/split_indices.json + split_*.bin", "parity_status": "approximately equivalent", "measurement_status": "resolved", "comparable": true, - "environment_id": "3f884007c2946e39", + "environment_id": "c012023a7229bbbe", "classification": "flow win", - "speedup": 10.600992316790924 + "speedup": 17.453590828663167 }, { "algorithm": "GaussianNB", @@ -157,25 +157,25 @@ "category": "supervised", "sklearn_score": 0.966666667, "flow_score": 0.966666639, - "sklearn_fit_ms": 0.924470656, - "sklearn_pred_ms": 0.177906977, - "flow_fit_ms": 0.013315, - "flow_pred_ms": 0.006662, - "sklearn_fit_iqr_ms": 0.014162281, - "sklearn_pred_iqr_ms": 0.001633765, - "flow_fit_iqr_ms": 0.000161, - "flow_pred_iqr_ms": 0.000261, - "sklearn_ms": 1.102377633, - "flow_ms": 0.019977, + "sklearn_fit_ms": 0.24434082, + "sklearn_pred_ms": 0.0445877285, + "flow_fit_ms": 0.005, + "flow_pred_ms": 0.001, + "sklearn_fit_iqr_ms": 0.0072657855, + "sklearn_pred_iqr_ms": 0.00116161915, + "flow_fit_iqr_ms": 0.0015, + "flow_pred_iqr_ms": 0.0005, + "sklearn_ms": 0.2889285485, + "flow_ms": 0.006, "timing_unit": "ms", "benchmark_mode": "end_to_end", "fixture_source": "benchmarks/split_indices.json + split_*.bin", "parity_status": "parity verified", "measurement_status": "resolved", "comparable": true, - "environment_id": "3f884007c2946e39", + "environment_id": "c012023a7229bbbe", "classification": "flow win", - "speedup": 55.182341342543914 + "speedup": 48.154758083333334 }, { "algorithm": "KMeans", @@ -184,25 +184,25 @@ "category": "clustering", "sklearn_score": 0.548888889, "flow_score": 0.548888862, - "sklearn_fit_ms": 10.6288145, - "sklearn_pred_ms": 0.164274305, - "flow_fit_ms": 1.29137695, - "flow_pred_ms": 0.003186, - "sklearn_fit_iqr_ms": 0.090423, - "sklearn_pred_iqr_ms": 0.001164805, - "flow_fit_iqr_ms": 0.031298041, - "flow_pred_iqr_ms": 5e-05, - "sklearn_ms": 10.793088805, - "flow_ms": 1.29456295, + "sklearn_fit_ms": 1.99084112, + "sklearn_pred_ms": 0.0330354004, + "flow_fit_ms": 0.178000003, + "flow_pred_ms": 0.001, + "sklearn_fit_iqr_ms": 0.103533845, + "sklearn_pred_iqr_ms": 0.0012598477, + "flow_fit_iqr_ms": 0.026500001, + "flow_pred_iqr_ms": 0.0, + "sklearn_ms": 2.0238765204, + "flow_ms": 0.179000003, "timing_unit": "ms", "benchmark_mode": "end_to_end", "fixture_source": "benchmarks/split_indices.json + split_*.bin", "parity_status": "approximately equivalent", "measurement_status": "resolved", "comparable": true, - "environment_id": "3f884007c2946e39", + "environment_id": "c012023a7229bbbe", "classification": "flow win", - "speedup": 8.33724524944886 + "speedup": 11.306572550169175 }, { "algorithm": "PCA", @@ -210,26 +210,26 @@ "metric": "explained_var_ratio", "category": "decomposition", "sklearn_score": 0.957439005, - "flow_score": 0.957439065, - "sklearn_fit_ms": 0.396100391, - "sklearn_pred_ms": 0.118425688, - "flow_fit_ms": 0.04273, - "flow_pred_ms": 0.003216, - "sklearn_fit_iqr_ms": 0.005139656, - "sklearn_pred_iqr_ms": 0.001520437, - "flow_fit_iqr_ms": 0.000581, - "flow_pred_iqr_ms": 0.000192, - "sklearn_ms": 0.514526079, - "flow_ms": 0.045946, + "flow_score": 0.957439005, + "sklearn_fit_ms": 0.106219398, + "sklearn_pred_ms": 0.0271950684, + "flow_fit_ms": 0.007, + "flow_pred_ms": 0.001, + "sklearn_fit_iqr_ms": 0.004313719, + "sklearn_pred_iqr_ms": 0.00061167385, + "flow_fit_iqr_ms": 0.0015, + "flow_pred_iqr_ms": 0.0, + "sklearn_ms": 0.13341446640000001, + "flow_ms": 0.008, "timing_unit": "ms", "benchmark_mode": "end_to_end", "fixture_source": "benchmarks/split_indices.json + split_*.bin", "parity_status": "parity verified", "measurement_status": "resolved", "comparable": true, - "environment_id": "3f884007c2946e39", + "environment_id": "c012023a7229bbbe", "classification": "flow win", - "speedup": 11.198495603534585 + "speedup": 16.6768083 }, { "algorithm": "LogisticRegression", @@ -237,26 +237,26 @@ "metric": "accuracy", "category": "supervised", "sklearn_score": 0.972222222, - "flow_score": 0.975000024, - "sklearn_fit_ms": 192.978743, - "sklearn_pred_ms": 0.179674766, - "flow_fit_ms": 70.1647415, - "flow_pred_ms": 0.060823999, - "sklearn_fit_iqr_ms": 31.962356, - "sklearn_pred_iqr_ms": 0.000810289, - "flow_fit_iqr_ms": 1.47711945, - "flow_pred_iqr_ms": 0.000852, - "sklearn_ms": 193.158417766, - "flow_ms": 70.225565499, + "flow_score": 0.972222209, + "sklearn_fit_ms": 6.179125, + "sklearn_pred_ms": 0.0561170254, + "flow_fit_ms": 1.93599999, + "flow_pred_ms": 0.023, + "sklearn_fit_iqr_ms": 0.246525875, + "sklearn_pred_iqr_ms": 0.00136836915, + "flow_fit_iqr_ms": 0.0474999545, + "flow_pred_iqr_ms": 0.002, + "sklearn_ms": 6.2352420254, + "flow_ms": 1.95899999, "timing_unit": "ms", "benchmark_mode": "end_to_end", "fixture_source": "benchmarks/split_indices.json + split_*.bin", "parity_status": "approximately equivalent", "measurement_status": "resolved", "comparable": true, - "environment_id": "3f884007c2946e39", + "environment_id": "c012023a7229bbbe", "classification": "flow win", - "speedup": 2.7505427175057857 + "speedup": 3.182869860759928 }, { "algorithm": "LinearSVC", @@ -265,25 +265,25 @@ "category": "supervised", "sklearn_score": 0.955555556, "flow_score": 0.955555558, - "sklearn_fit_ms": 349.443518, - "sklearn_pred_ms": 0.201159617, - "flow_fit_ms": 1607.33569, - "flow_pred_ms": 1.60024703, - "sklearn_fit_iqr_ms": 1.04663, - "sklearn_pred_iqr_ms": 0.000886437, - "flow_fit_iqr_ms": 2.98669434, - "flow_pred_iqr_ms": 0.012503981, - "sklearn_ms": 349.64467761699996, - "flow_ms": 1608.93593703, + "sklearn_fit_ms": 227.714333, + "sklearn_pred_ms": 0.0590003262, + "flow_fit_ms": 52.7070007, + "flow_pred_ms": 0.104000002, + "sklearn_fit_iqr_ms": 4.541, + "sklearn_pred_iqr_ms": 0.00208178615, + "flow_fit_iqr_ms": 0.402000427, + "flow_pred_iqr_ms": 0.0075, + "sklearn_ms": 227.7733333262, + "flow_ms": 52.811000702, "timing_unit": "ms", "benchmark_mode": "end_to_end", "fixture_source": "benchmarks/split_indices.json + split_*.bin", "parity_status": "approximately equivalent", "measurement_status": "resolved", "comparable": true, - "environment_id": "3f884007c2946e39", - "classification": "sklearn win", - "speedup": 0.21731423207714734 + "environment_id": "c012023a7229bbbe", + "classification": "flow win", + "speedup": 4.3129902917665035 }, { "algorithm": "KernelSVC_RBF", @@ -292,25 +292,25 @@ "category": "supervised", "sklearn_score": 0.95, "flow_score": 0.949999988, - "sklearn_fit_ms": 52.391939, - "sklearn_pred_ms": 25.539144, - "flow_fit_ms": 115.784744, - "flow_pred_ms": 17.0403919, - "sklearn_fit_iqr_ms": 0.110422, - "sklearn_pred_iqr_ms": 1.338714, - "flow_fit_iqr_ms": 0.266311645, - "flow_pred_iqr_ms": 0.092468262, - "sklearn_ms": 77.931083, - "flow_ms": 132.8251359, + "sklearn_fit_ms": 24.1875, + "sklearn_pred_ms": 16.9295625, + "flow_fit_ms": 11.2530003, + "flow_pred_ms": 1.58000004, + "sklearn_fit_iqr_ms": 0.3623745, + "sklearn_pred_iqr_ms": 0.903823, + "flow_fit_iqr_ms": 0.573499679, + "flow_pred_iqr_ms": 0.083999992, + "sklearn_ms": 41.1170625, + "flow_ms": 12.83300034, "timing_unit": "ms", "benchmark_mode": "end_to_end", "fixture_source": "benchmarks/split_indices.json + split_*.bin", "parity_status": "approximately equivalent", "measurement_status": "resolved", "comparable": true, - "environment_id": "3f884007c2946e39", - "classification": "sklearn win", - "speedup": 0.5867193921689036 + "environment_id": "c012023a7229bbbe", + "classification": "flow win", + "speedup": 3.2040100842076344 }, { "algorithm": "DecisionTree", @@ -319,25 +319,25 @@ "category": "supervised", "sklearn_score": 0.813888889, "flow_score": 0.833333313, - "sklearn_fit_ms": 14.9078835, - "sklearn_pred_ms": 0.138405207, - "flow_fit_ms": 31.9785061, - "flow_pred_ms": 0.118812002, - "sklearn_fit_iqr_ms": 0.045726, - "sklearn_pred_iqr_ms": 0.000688402, - "flow_fit_iqr_ms": 0.008146286, - "flow_pred_iqr_ms": 0.000421003, - "sklearn_ms": 15.046288707, - "flow_ms": 32.097318102, + "sklearn_fit_ms": 8.6068645, + "sklearn_pred_ms": 0.038427083, + "flow_fit_ms": 4.58599997, + "flow_pred_ms": 0.021, + "sklearn_fit_iqr_ms": 0.199073, + "sklearn_pred_iqr_ms": 0.00088521285, + "flow_fit_iqr_ms": 0.095999956, + "flow_pred_iqr_ms": 0.002000001, + "sklearn_ms": 8.645291583, + "flow_ms": 4.6069999699999995, "timing_unit": "ms", "benchmark_mode": "end_to_end", "fixture_source": "benchmarks/split_indices.json + split_*.bin", "parity_status": "approximately equivalent", "measurement_status": "resolved", "comparable": true, - "environment_id": "3f884007c2946e39", - "classification": "sklearn win", - "speedup": 0.46877090039689195 + "environment_id": "c012023a7229bbbe", + "classification": "flow win", + "speedup": 1.8765555978503732 }, { "algorithm": "RandomForest", @@ -346,25 +346,25 @@ "category": "supervised", "sklearn_score": 0.936111111, "flow_score": 0.955555558, - "sklearn_fit_ms": 28.12009, - "sklearn_pred_ms": 1.26504681, - "flow_fit_ms": 94.4096603, - "flow_pred_ms": 0.532145977, - "sklearn_fit_iqr_ms": 0.118545, - "sklearn_pred_iqr_ms": 0.02372631, - "flow_fit_iqr_ms": 0.083770752, - "flow_pred_iqr_ms": 0.005329967, - "sklearn_ms": 29.385136810000002, - "flow_ms": 94.941806277, + "sklearn_fit_ms": 12.8233335, + "sklearn_pred_ms": 0.306717438, + "flow_fit_ms": 2.31999993, + "flow_pred_ms": 0.186000004, + "sklearn_fit_iqr_ms": 0.635625, + "sklearn_pred_iqr_ms": 0.016799969, + "flow_fit_iqr_ms": 0.207000136, + "flow_pred_iqr_ms": 0.013499998, + "sklearn_ms": 13.130050938, + "flow_ms": 2.5059999339999997, "timing_unit": "ms", "benchmark_mode": "end_to_end", "fixture_source": "benchmarks/split_indices.json + split_*.bin", "parity_status": "approximately equivalent", "measurement_status": "resolved", "comparable": true, - "environment_id": "3f884007c2946e39", - "classification": "sklearn win", - "speedup": 0.3095068227822274 + "environment_id": "c012023a7229bbbe", + "classification": "flow win", + "speedup": 5.239445843496978 }, { "algorithm": "GaussianNB", @@ -373,25 +373,25 @@ "category": "supervised", "sklearn_score": 0.741666667, "flow_score": 0.741666675, - "sklearn_fit_ms": 1.72055738, - "sklearn_pred_ms": 0.641119781, - "flow_fit_ms": 1.08066297, - "flow_pred_ms": 1.28333294, - "sklearn_fit_iqr_ms": 0.0265915, - "sklearn_pred_iqr_ms": 0.008697468, - "flow_fit_iqr_ms": 0.004558086, - "flow_pred_iqr_ms": 0.00930798, - "sklearn_ms": 2.3616771610000002, - "flow_ms": 2.36399591, + "sklearn_fit_ms": 0.530330734, + "sklearn_pred_ms": 0.291535156, + "flow_fit_ms": 0.074000001, + "flow_pred_ms": 0.108000003, + "sklearn_fit_iqr_ms": 0.0227034605, + "sklearn_pred_iqr_ms": 0.002239582, + "flow_fit_iqr_ms": 0.002000004, + "flow_pred_iqr_ms": 0.0044999975, + "sklearn_ms": 0.82186589, + "flow_ms": 0.182000004, "timing_unit": "ms", "benchmark_mode": "end_to_end", "fixture_source": "benchmarks/split_indices.json + split_*.bin", "parity_status": "parity verified", "measurement_status": "resolved", "comparable": true, - "environment_id": "3f884007c2946e39", - "classification": "tie", - "speedup": 0.999019140011964 + "environment_id": "c012023a7229bbbe", + "classification": "flow win", + "speedup": 4.515746549104471 }, { "algorithm": "KMeans", @@ -400,25 +400,25 @@ "category": "clustering", "sklearn_score": 0.528326226, "flow_score": 0.528326213, - "sklearn_fit_ms": 68.592784, - "sklearn_pred_ms": 0.182871937, - "flow_fit_ms": 358.345886, - "flow_pred_ms": 0.073968999, - "sklearn_fit_iqr_ms": 0.410402, - "sklearn_pred_iqr_ms": 0.002540164, - "flow_fit_iqr_ms": 1.60336304, - "flow_pred_iqr_ms": 0.001171998, - "sklearn_ms": 68.775655937, - "flow_ms": 358.419854999, + "sklearn_fit_ms": 20.118834, + "sklearn_pred_ms": 0.0551437168, + "flow_fit_ms": 14.323, + "flow_pred_ms": 0.024, + "sklearn_fit_iqr_ms": 0.7834585, + "sklearn_pred_iqr_ms": 0.0037094717, + "flow_fit_iqr_ms": 0.172000408, + "flow_pred_iqr_ms": 0.002000001, + "sklearn_ms": 20.1739777168, + "flow_ms": 14.347, "timing_unit": "ms", "benchmark_mode": "end_to_end", "fixture_source": "benchmarks/split_indices.json + split_*.bin", "parity_status": "approximately equivalent", "measurement_status": "resolved", "comparable": true, - "environment_id": "3f884007c2946e39", - "classification": "sklearn win", - "speedup": 0.19188573115513335 + "environment_id": "c012023a7229bbbe", + "classification": "flow win", + "speedup": 1.4061460735205966 }, { "algorithm": "Ridge", @@ -427,106 +427,106 @@ "category": "supervised", "sklearn_score": 0.454146624, "flow_score": 0.454146445, - "sklearn_fit_ms": 0.735451469, - "sklearn_pred_ms": 0.0986626875, - "flow_fit_ms": 0.110867001, - "flow_pred_ms": 0.002765, - "sklearn_fit_iqr_ms": 0.008099906, - "sklearn_pred_iqr_ms": 0.0020281289, - "flow_fit_iqr_ms": 0.000681996, - "flow_pred_iqr_ms": 0.00016, - "sklearn_ms": 0.8341141565, - "flow_ms": 0.11363200100000001, + "sklearn_fit_ms": 0.201831383, + "sklearn_pred_ms": 0.0240216875, + "flow_fit_ms": 0.017000001, + "flow_pred_ms": 0.001, + "sklearn_fit_iqr_ms": 0.006238766, + "sklearn_pred_iqr_ms": 0.0008340044, + "flow_fit_iqr_ms": 0.001999999, + "flow_pred_iqr_ms": 0.001, + "sklearn_ms": 0.2258530705, + "flow_ms": 0.018000001, "timing_unit": "ms", "benchmark_mode": "end_to_end", "fixture_source": "benchmarks/split_indices.json + split_*.bin", "parity_status": "approximately equivalent", "measurement_status": "resolved", "comparable": true, - "environment_id": "3f884007c2946e39", + "environment_id": "c012023a7229bbbe", "classification": "flow win", - "speedup": 7.340486387280991 + "speedup": 12.547392108478215 }, { "algorithm": "Lasso", "dataset": "diabetes", "metric": "r2", "category": "supervised", - "sklearn_score": 0.455541432, + "sklearn_score": 0.455541313, "flow_score": 0.45552969, - "sklearn_fit_ms": 0.794417469, - "sklearn_pred_ms": 0.10562668, - "flow_fit_ms": 0.845703006, - "flow_pred_ms": 0.009097, - "sklearn_fit_iqr_ms": 0.021328625, - "sklearn_pred_iqr_ms": 0.001134567, - "flow_fit_iqr_ms": 0.014376998, - "flow_pred_iqr_ms": 2e-05, - "sklearn_ms": 0.900044149, - "flow_ms": 0.8548000060000001, + "sklearn_fit_ms": 0.313371742, + "sklearn_pred_ms": 0.0261898193, + "flow_fit_ms": 0.125, + "flow_pred_ms": 0.001, + "sklearn_fit_iqr_ms": 0.00685498, + "sklearn_pred_iqr_ms": 0.0005582886, + "flow_fit_iqr_ms": 0.006999999, + "flow_pred_iqr_ms": 0.001, + "sklearn_ms": 0.33956156130000004, + "flow_ms": 0.126, "timing_unit": "ms", "benchmark_mode": "end_to_end", "fixture_source": "benchmarks/split_indices.json + split_*.bin", "parity_status": "approximately equivalent", "measurement_status": "resolved", "comparable": true, - "environment_id": "3f884007c2946e39", + "environment_id": "c012023a7229bbbe", "classification": "flow win", - "speedup": 1.0529295071156093 + "speedup": 2.6949330261904763 }, { "algorithm": "LinearRegression", "dataset": "diabetes", "metric": "r2", "category": "supervised", - "sklearn_score": 0.452602744, - "flow_score": 0.452602565, - "sklearn_fit_ms": 0.676694531, - "sklearn_pred_ms": 0.0991472969, - "flow_fit_ms": 0.090429001, - "flow_pred_ms": 0.001372, - "sklearn_fit_iqr_ms": 0.012061094, - "sklearn_pred_iqr_ms": 0.0022896333, - "flow_fit_iqr_ms": 0.000501, - "flow_pred_iqr_ms": 8.9e-05, - "sklearn_ms": 0.7758418279, - "flow_ms": 0.091801001, + "sklearn_score": 0.452602804, + "flow_score": 0.452602625, + "sklearn_fit_ms": 0.182404945, + "sklearn_pred_ms": 0.0239730625, + "flow_fit_ms": 0.02, + "flow_pred_ms": 0.0, + "sklearn_fit_iqr_ms": 0.007456547, + "sklearn_pred_iqr_ms": 0.00047957425, + "flow_fit_iqr_ms": 0.001, + "flow_pred_iqr_ms": 0.001, + "sklearn_ms": 0.20637800750000002, + "flow_ms": 0.02, "timing_unit": "ms", "benchmark_mode": "end_to_end", "fixture_source": "benchmarks/split_indices.json + split_*.bin", "parity_status": "parity verified", "measurement_status": "resolved", "comparable": true, - "environment_id": "3f884007c2946e39", + "environment_id": "c012023a7229bbbe", "classification": "flow win", - "speedup": 8.451343879137005 + "speedup": 10.318900375 }, { "algorithm": "KernelRidge_RBF", "dataset": "diabetes", "metric": "r2", "category": "supervised", - "sklearn_score": 0.461917877, - "flow_score": 0.461917698, - "sklearn_fit_ms": 5.9995195, - "sklearn_pred_ms": 0.771147781, - "flow_fit_ms": 55.0274315, - "flow_pred_ms": 4.76458979, - "sklearn_fit_iqr_ms": 0.00606075, - "sklearn_pred_iqr_ms": 0.003607812, - "flow_fit_iqr_ms": 0.06193161, - "flow_pred_iqr_ms": 0.009175777, - "sklearn_ms": 6.770667281, - "flow_ms": 59.79202129, + "sklearn_score": 0.461917818, + "flow_score": 0.46191752, + "sklearn_fit_ms": 0.897470063, + "sklearn_pred_ms": 0.271527664, + "flow_fit_ms": 0.474000007, + "flow_pred_ms": 0.079999998, + "sklearn_fit_iqr_ms": 0.026421203, + "sklearn_pred_iqr_ms": 0.009609539, + "flow_fit_iqr_ms": 0.0684999975, + "flow_pred_iqr_ms": 0.006499998, + "sklearn_ms": 1.168997727, + "flow_ms": 0.554000005, "timing_unit": "ms", "benchmark_mode": "end_to_end", "fixture_source": "benchmarks/split_indices.json + split_*.bin", "parity_status": "parity verified", "measurement_status": "resolved", "comparable": true, - "environment_id": "3f884007c2946e39", - "classification": "sklearn win", - "speedup": 0.11323696933009303 + "environment_id": "c012023a7229bbbe", + "classification": "flow win", + "speedup": 2.1101041813167494 } ] } diff --git a/benchmarks/model_state_coverage.json b/benchmarks/model_state_coverage.json new file mode 100644 index 0000000..47f1dbc --- /dev/null +++ b/benchmarks/model_state_coverage.json @@ -0,0 +1,532 @@ +{ + "schema_version": 1, + "policy": "every canonical estimator row must eventually expose at least one learned-state diagnostic in addition to its final score", + "counts": { + "canonical_rows": 19, + "covered_rows": 19, + "missing_rows": 0 + }, + "rows": [ + { + "algorithm": "LogisticRegression", + "dataset": "iris", + "metric": "accuracy", + "status": "covered", + "diagnostic_keys": [ + "classes_first_divergent_index", + "classes_max_abs_diff", + "classes_max_relative_diff", + "coef_abs_sum_abs_diff", + "coef_abs_sum_relative_diff", + "coef_frobenius_norm_abs_diff", + "coef_frobenius_norm_relative_diff", + "coef_row_l2_norms_first_divergent_index", + "coef_row_l2_norms_max_abs_diff", + "coef_row_l2_norms_max_relative_diff", + "intercepts_first_divergent_index", + "intercepts_max_abs_diff", + "intercepts_max_relative_diff" + ] + }, + { + "algorithm": "LinearSVC", + "dataset": "iris", + "metric": "accuracy", + "status": "covered", + "diagnostic_keys": [ + "classes_first_divergent_index", + "classes_max_abs_diff", + "classes_max_relative_diff", + "coef_abs_sum_abs_diff", + "coef_abs_sum_relative_diff", + "coef_frobenius_norm_abs_diff", + "coef_frobenius_norm_relative_diff", + "coef_row_l2_norms_first_divergent_index", + "coef_row_l2_norms_max_abs_diff", + "coef_row_l2_norms_max_relative_diff", + "intercepts_first_divergent_index", + "intercepts_max_abs_diff", + "intercepts_max_relative_diff" + ] + }, + { + "algorithm": "KernelSVC_RBF", + "dataset": "iris", + "metric": "accuracy", + "status": "covered", + "diagnostic_keys": [ + "C_abs_diff", + "C_relative_diff", + "dual_coef_abs_sum_per_pair_first_divergent_index", + "dual_coef_abs_sum_per_pair_max_abs_diff", + "dual_coef_abs_sum_per_pair_max_relative_diff", + "gamma_abs_diff", + "gamma_relative_diff", + "intercept_per_pair_first_divergent_index", + "intercept_per_pair_max_abs_diff", + "intercept_per_pair_max_relative_diff", + "n_bounded_support_per_pair_first_divergent_index", + "n_bounded_support_per_pair_max_abs_diff", + "n_bounded_support_per_pair_max_relative_diff", + "n_pairs_abs_diff", + "n_pairs_relative_diff", + "n_support_per_pair_first_divergent_index", + "n_support_per_pair_max_abs_diff", + "n_support_per_pair_max_relative_diff", + "n_support_total_abs_diff", + "n_support_total_relative_diff", + "pair_class_a_first_divergent_index", + "pair_class_a_max_abs_diff", + "pair_class_a_max_relative_diff", + "pair_class_b_first_divergent_index", + "pair_class_b_max_abs_diff", + "pair_class_b_max_relative_diff", + "pair_train_sizes_first_divergent_index", + "pair_train_sizes_max_abs_diff", + "pair_train_sizes_max_relative_diff" + ] + }, + { + "algorithm": "DecisionTree", + "dataset": "iris", + "metric": "accuracy", + "status": "covered", + "diagnostic_keys": [ + "depth1_splits_first_divergent_index", + "depth1_splits_max_abs_diff", + "depth1_splits_max_relative_diff", + "max_depth_reached_abs_diff", + "max_depth_reached_relative_diff", + "mean_leaf_depth_abs_diff", + "mean_leaf_depth_relative_diff", + "n_leaves_abs_diff", + "n_leaves_relative_diff", + "n_nodes_abs_diff", + "n_nodes_relative_diff", + "nodes_per_depth_first_divergent_index", + "nodes_per_depth_max_abs_diff", + "nodes_per_depth_max_relative_diff", + "preorder_split_features_first_divergent_index", + "preorder_split_features_max_abs_diff", + "preorder_split_features_max_relative_diff", + "preorder_split_thresholds_first_divergent_index", + "preorder_split_thresholds_max_abs_diff", + "preorder_split_thresholds_max_relative_diff", + "root_impurity_abs_diff", + "root_impurity_relative_diff", + "root_split_feature_abs_diff", + "root_split_feature_relative_diff", + "root_split_threshold_abs_diff", + "root_split_threshold_relative_diff", + "split_feature_histogram_first_divergent_index", + "split_feature_histogram_max_abs_diff", + "split_feature_histogram_max_relative_diff" + ] + }, + { + "algorithm": "RandomForest", + "dataset": "iris", + "metric": "accuracy", + "status": "covered", + "diagnostic_keys": [ + "bootstrap_index_sums_first_divergent_index", + "bootstrap_index_sums_max_abs_diff", + "bootstrap_index_sums_max_relative_diff", + "bootstrap_unique_fractions_first_divergent_index", + "bootstrap_unique_fractions_max_abs_diff", + "bootstrap_unique_fractions_max_relative_diff", + "max_features_per_split_abs_diff", + "max_features_per_split_relative_diff", + "mean_top_vote_fraction_abs_diff", + "mean_top_vote_fraction_relative_diff", + "mean_vote_margin_abs_diff", + "mean_vote_margin_relative_diff", + "n_trees_abs_diff", + "n_trees_relative_diff", + "tree_feature_seeds_first_divergent_index", + "tree_feature_seeds_max_abs_diff", + "tree_feature_seeds_max_relative_diff", + "tree_leaf_counts_first_divergent_index", + "tree_leaf_counts_max_abs_diff", + "tree_leaf_counts_max_relative_diff", + "tree_max_depths_first_divergent_index", + "tree_max_depths_max_abs_diff", + "tree_max_depths_max_relative_diff", + "tree_node_counts_first_divergent_index", + "tree_node_counts_max_abs_diff", + "tree_node_counts_max_relative_diff", + "tree_root_features_first_divergent_index", + "tree_root_features_max_abs_diff", + "tree_root_features_max_relative_diff", + "tree_root_impurities_first_divergent_index", + "tree_root_impurities_max_abs_diff", + "tree_root_impurities_max_relative_diff", + "tree_root_thresholds_first_divergent_index", + "tree_root_thresholds_max_abs_diff", + "tree_root_thresholds_max_relative_diff", + "unanimous_vote_fraction_abs_diff", + "unanimous_vote_fraction_relative_diff" + ] + }, + { + "algorithm": "GaussianNB", + "dataset": "iris", + "metric": "accuracy", + "status": "covered", + "diagnostic_keys": [ + "class_priors_first_divergent_index", + "class_priors_max_abs_diff", + "class_priors_max_relative_diff", + "classes_first_divergent_index", + "classes_max_abs_diff", + "classes_max_relative_diff", + "theta_frobenius_norm_abs_diff", + "theta_frobenius_norm_relative_diff", + "theta_row_l2_norms_first_divergent_index", + "theta_row_l2_norms_max_abs_diff", + "theta_row_l2_norms_max_relative_diff", + "var_frobenius_norm_abs_diff", + "var_frobenius_norm_relative_diff", + "var_max_abs_diff", + "var_max_relative_diff", + "var_min_abs_diff", + "var_min_relative_diff", + "var_row_l2_norms_first_divergent_index", + "var_row_l2_norms_max_abs_diff", + "var_row_l2_norms_max_relative_diff" + ] + }, + { + "algorithm": "KMeans", + "dataset": "iris", + "metric": "adjusted_rand_index", + "status": "covered", + "diagnostic_keys": [ + "center_l2_norms_sorted_first_divergent_index", + "center_l2_norms_sorted_max_abs_diff", + "center_l2_norms_sorted_max_relative_diff", + "inertia_abs_diff", + "inertia_relative_diff", + "n_iter_abs_diff", + "n_iter_relative_diff", + "train_cluster_sizes_sorted_first_divergent_index", + "train_cluster_sizes_sorted_max_abs_diff", + "train_cluster_sizes_sorted_max_relative_diff" + ] + }, + { + "algorithm": "PCA", + "dataset": "iris", + "metric": "explained_var_ratio", + "status": "covered", + "diagnostic_keys": [ + "component_0_first_divergent_index", + "component_0_max_abs_diff", + "component_0_max_relative_diff", + "component_0_sign_aligned_abs_diff", + "component_1_first_divergent_index", + "component_1_max_abs_diff", + "component_1_max_relative_diff", + "component_1_sign_aligned_abs_diff", + "explained_variance_ratio_abs_diff", + "explained_variance_ratio_first_divergent_index", + "explained_variance_ratio_max_abs_diff", + "explained_variance_ratio_max_relative_diff", + "reconstruction_mse_abs_diff", + "reconstruction_mse_relative_diff", + "singular_values_first_divergent_index", + "singular_values_max_abs_diff", + "singular_values_max_relative_diff", + "singular_values_relative_diff" + ] + }, + { + "algorithm": "LogisticRegression", + "dataset": "digits", + "metric": "accuracy", + "status": "covered", + "diagnostic_keys": [ + "classes_first_divergent_index", + "classes_max_abs_diff", + "classes_max_relative_diff", + "coef_abs_sum_abs_diff", + "coef_abs_sum_relative_diff", + "coef_frobenius_norm_abs_diff", + "coef_frobenius_norm_relative_diff", + "coef_row_l2_norms_first_divergent_index", + "coef_row_l2_norms_max_abs_diff", + "coef_row_l2_norms_max_relative_diff", + "intercepts_first_divergent_index", + "intercepts_max_abs_diff", + "intercepts_max_relative_diff" + ] + }, + { + "algorithm": "LinearSVC", + "dataset": "digits", + "metric": "accuracy", + "status": "covered", + "diagnostic_keys": [ + "classes_first_divergent_index", + "classes_max_abs_diff", + "classes_max_relative_diff", + "coef_abs_sum_abs_diff", + "coef_abs_sum_relative_diff", + "coef_frobenius_norm_abs_diff", + "coef_frobenius_norm_relative_diff", + "coef_row_l2_norms_first_divergent_index", + "coef_row_l2_norms_max_abs_diff", + "coef_row_l2_norms_max_relative_diff", + "intercepts_first_divergent_index", + "intercepts_max_abs_diff", + "intercepts_max_relative_diff" + ] + }, + { + "algorithm": "KernelSVC_RBF", + "dataset": "digits", + "metric": "accuracy", + "status": "covered", + "diagnostic_keys": [ + "C_abs_diff", + "C_relative_diff", + "dual_coef_abs_sum_per_pair_first_divergent_index", + "dual_coef_abs_sum_per_pair_max_abs_diff", + "dual_coef_abs_sum_per_pair_max_relative_diff", + "gamma_abs_diff", + "gamma_relative_diff", + "intercept_per_pair_first_divergent_index", + "intercept_per_pair_max_abs_diff", + "intercept_per_pair_max_relative_diff", + "n_bounded_support_per_pair_first_divergent_index", + "n_bounded_support_per_pair_max_abs_diff", + "n_bounded_support_per_pair_max_relative_diff", + "n_pairs_abs_diff", + "n_pairs_relative_diff", + "n_support_per_pair_first_divergent_index", + "n_support_per_pair_max_abs_diff", + "n_support_per_pair_max_relative_diff", + "n_support_total_abs_diff", + "n_support_total_relative_diff", + "pair_class_a_first_divergent_index", + "pair_class_a_max_abs_diff", + "pair_class_a_max_relative_diff", + "pair_class_b_first_divergent_index", + "pair_class_b_max_abs_diff", + "pair_class_b_max_relative_diff", + "pair_train_sizes_first_divergent_index", + "pair_train_sizes_max_abs_diff", + "pair_train_sizes_max_relative_diff" + ] + }, + { + "algorithm": "DecisionTree", + "dataset": "digits", + "metric": "accuracy", + "status": "covered", + "diagnostic_keys": [ + "depth1_splits_first_divergent_index", + "depth1_splits_max_abs_diff", + "depth1_splits_max_relative_diff", + "max_depth_reached_abs_diff", + "max_depth_reached_relative_diff", + "mean_leaf_depth_abs_diff", + "mean_leaf_depth_relative_diff", + "n_leaves_abs_diff", + "n_leaves_relative_diff", + "n_nodes_abs_diff", + "n_nodes_relative_diff", + "nodes_per_depth_first_divergent_index", + "nodes_per_depth_max_abs_diff", + "nodes_per_depth_max_relative_diff", + "preorder_split_features_first_divergent_index", + "preorder_split_features_max_abs_diff", + "preorder_split_features_max_relative_diff", + "preorder_split_thresholds_first_divergent_index", + "preorder_split_thresholds_max_abs_diff", + "preorder_split_thresholds_max_relative_diff", + "root_impurity_abs_diff", + "root_impurity_relative_diff", + "root_split_feature_abs_diff", + "root_split_feature_relative_diff", + "root_split_threshold_abs_diff", + "root_split_threshold_relative_diff", + "split_feature_histogram_first_divergent_index", + "split_feature_histogram_max_abs_diff", + "split_feature_histogram_max_relative_diff" + ] + }, + { + "algorithm": "RandomForest", + "dataset": "digits", + "metric": "accuracy", + "status": "covered", + "diagnostic_keys": [ + "bootstrap_index_sums_first_divergent_index", + "bootstrap_index_sums_max_abs_diff", + "bootstrap_index_sums_max_relative_diff", + "bootstrap_unique_fractions_first_divergent_index", + "bootstrap_unique_fractions_max_abs_diff", + "bootstrap_unique_fractions_max_relative_diff", + "max_features_per_split_abs_diff", + "max_features_per_split_relative_diff", + "mean_top_vote_fraction_abs_diff", + "mean_top_vote_fraction_relative_diff", + "mean_vote_margin_abs_diff", + "mean_vote_margin_relative_diff", + "n_trees_abs_diff", + "n_trees_relative_diff", + "tree_feature_seeds_first_divergent_index", + "tree_feature_seeds_max_abs_diff", + "tree_feature_seeds_max_relative_diff", + "tree_leaf_counts_first_divergent_index", + "tree_leaf_counts_max_abs_diff", + "tree_leaf_counts_max_relative_diff", + "tree_max_depths_first_divergent_index", + "tree_max_depths_max_abs_diff", + "tree_max_depths_max_relative_diff", + "tree_node_counts_first_divergent_index", + "tree_node_counts_max_abs_diff", + "tree_node_counts_max_relative_diff", + "tree_root_features_first_divergent_index", + "tree_root_features_max_abs_diff", + "tree_root_features_max_relative_diff", + "tree_root_impurities_first_divergent_index", + "tree_root_impurities_max_abs_diff", + "tree_root_impurities_max_relative_diff", + "tree_root_thresholds_first_divergent_index", + "tree_root_thresholds_max_abs_diff", + "tree_root_thresholds_max_relative_diff", + "unanimous_vote_fraction_abs_diff", + "unanimous_vote_fraction_relative_diff" + ] + }, + { + "algorithm": "GaussianNB", + "dataset": "digits", + "metric": "accuracy", + "status": "covered", + "diagnostic_keys": [ + "class_priors_first_divergent_index", + "class_priors_max_abs_diff", + "class_priors_max_relative_diff", + "classes_first_divergent_index", + "classes_max_abs_diff", + "classes_max_relative_diff", + "theta_frobenius_norm_abs_diff", + "theta_frobenius_norm_relative_diff", + "theta_row_l2_norms_first_divergent_index", + "theta_row_l2_norms_max_abs_diff", + "theta_row_l2_norms_max_relative_diff", + "var_frobenius_norm_abs_diff", + "var_frobenius_norm_relative_diff", + "var_max_abs_diff", + "var_max_relative_diff", + "var_min_abs_diff", + "var_min_relative_diff", + "var_row_l2_norms_first_divergent_index", + "var_row_l2_norms_max_abs_diff", + "var_row_l2_norms_max_relative_diff" + ] + }, + { + "algorithm": "KMeans", + "dataset": "digits", + "metric": "adjusted_rand_index", + "status": "covered", + "diagnostic_keys": [ + "center_l2_norms_sorted_first_divergent_index", + "center_l2_norms_sorted_max_abs_diff", + "center_l2_norms_sorted_max_relative_diff", + "inertia_abs_diff", + "inertia_relative_diff", + "n_iter_abs_diff", + "n_iter_relative_diff", + "train_cluster_sizes_sorted_first_divergent_index", + "train_cluster_sizes_sorted_max_abs_diff", + "train_cluster_sizes_sorted_max_relative_diff" + ] + }, + { + "algorithm": "Ridge", + "dataset": "diabetes", + "metric": "r2", + "status": "covered", + "diagnostic_keys": [ + "coef_abs_sum_abs_diff", + "coef_abs_sum_relative_diff", + "coef_first_divergent_index", + "coef_l2_norm_abs_diff", + "coef_l2_norm_relative_diff", + "coef_max_abs_diff", + "coef_max_relative_diff", + "intercept_abs_diff", + "intercept_relative_diff", + "n_zero_coefs_abs_diff", + "n_zero_coefs_relative_diff" + ] + }, + { + "algorithm": "Lasso", + "dataset": "diabetes", + "metric": "r2", + "status": "covered", + "diagnostic_keys": [ + "coef_abs_sum_abs_diff", + "coef_abs_sum_relative_diff", + "coef_first_divergent_index", + "coef_l2_norm_abs_diff", + "coef_l2_norm_relative_diff", + "coef_max_abs_diff", + "coef_max_relative_diff", + "intercept_abs_diff", + "intercept_relative_diff", + "n_zero_coefs_abs_diff", + "n_zero_coefs_relative_diff" + ] + }, + { + "algorithm": "LinearRegression", + "dataset": "diabetes", + "metric": "r2", + "status": "covered", + "diagnostic_keys": [ + "coef_abs_sum_abs_diff", + "coef_abs_sum_relative_diff", + "coef_first_divergent_index", + "coef_l2_norm_abs_diff", + "coef_l2_norm_relative_diff", + "coef_max_abs_diff", + "coef_max_relative_diff", + "intercept_abs_diff", + "intercept_relative_diff", + "n_zero_coefs_abs_diff", + "n_zero_coefs_relative_diff" + ] + }, + { + "algorithm": "KernelRidge_RBF", + "dataset": "diabetes", + "metric": "r2", + "status": "covered", + "diagnostic_keys": [ + "alpha_abs_diff", + "alpha_relative_diff", + "dual_coef_abs_sum_abs_diff", + "dual_coef_abs_sum_relative_diff", + "dual_coef_l2_norm_abs_diff", + "dual_coef_l2_norm_relative_diff", + "dual_coef_max_abs_diff", + "dual_coef_max_relative_diff", + "dual_coef_mean_abs_diff", + "dual_coef_mean_relative_diff", + "dual_coef_min_abs_diff", + "dual_coef_min_relative_diff", + "gamma_abs_diff", + "gamma_relative_diff", + "n_train_samples_abs_diff", + "n_train_samples_relative_diff" + ] + } + ] +} diff --git a/benchmarks/optimization_roadmap.json b/benchmarks/optimization_roadmap.json index 602de24..5c86f7e 100644 --- a/benchmarks/optimization_roadmap.json +++ b/benchmarks/optimization_roadmap.json @@ -10,7 +10,7 @@ "priority_score": 75.225, "disposition": "rewrite first", "hypothesis": "remove Python control/validation and specialize the complete operation", - "observed_flow_speedup": 27.457510112807267, + "observed_flow_speedup": 26.335252316218902, "factors": { "substrate": 1.0, "python_share": 0.7737794117866104, @@ -31,7 +31,7 @@ "priority_score": 74.027, "disposition": "rewrite first", "hypothesis": "remove Python control/validation and specialize the complete operation", - "observed_flow_speedup": 11.826483972812914, + "observed_flow_speedup": 16.6768083, "factors": { "substrate": 1.0, "python_share": 0.7602673074449647, @@ -52,7 +52,7 @@ "priority_score": 73.799, "disposition": "rewrite first", "hypothesis": "remove Python control/validation and specialize the complete operation", - "observed_flow_speedup": 27.457510112807267, + "observed_flow_speedup": 26.335252316218902, "factors": { "substrate": 1.0, "python_share": 0.7752056991156699, @@ -73,7 +73,7 @@ "priority_score": 73.681, "disposition": "rewrite first", "hypothesis": "remove Python control/validation and specialize the complete operation", - "observed_flow_speedup": 8.622601721278757, + "observed_flow_speedup": 6.800746158795939, "factors": { "substrate": 1.0, "python_share": 0.765749521293537, @@ -94,7 +94,7 @@ "priority_score": 73.448, "disposition": "rewrite first", "hypothesis": "remove Python control/validation and specialize the complete operation", - "observed_flow_speedup": 11.826483972812914, + "observed_flow_speedup": 16.6768083, "factors": { "substrate": 1.0, "python_share": 0.7630642515902994, @@ -115,7 +115,7 @@ "priority_score": 73.311, "disposition": "rewrite first", "hypothesis": "remove Python control/validation and specialize the complete operation", - "observed_flow_speedup": 8.517286010777768, + "observed_flow_speedup": 10.318900375, "factors": { "substrate": 1.0, "python_share": 0.7597517650604944, @@ -136,7 +136,7 @@ "priority_score": 71.902, "disposition": "rewrite first", "hypothesis": "remove Python control/validation and specialize the complete operation", - "observed_flow_speedup": 4.330005940309095, + "observed_flow_speedup": 6.356359311844886, "factors": { "substrate": 0.8, "python_share": 0.8310647414107625, @@ -157,7 +157,7 @@ "priority_score": 71.839, "disposition": "rewrite first", "hypothesis": "remove Python control/validation and specialize the complete operation", - "observed_flow_speedup": 8.622601721278757, + "observed_flow_speedup": 6.800746158795939, "factors": { "substrate": 0.8, "python_share": 0.7643902839976647, @@ -178,7 +178,7 @@ "priority_score": 68.646, "disposition": "rewrite first", "hypothesis": "remove Python control/validation and specialize the complete operation", - "observed_flow_speedup": 8.517286010777768, + "observed_flow_speedup": 10.318900375, "factors": { "substrate": 0.8, "python_share": 0.7610687448032628, @@ -190,6 +190,27 @@ "complexity_penalty": 0.1 } }, + { + "module": "sklearn.svm._classes", + "estimator": "SVC", + "operation": "predict", + "execution_class": "numpy-bound", + "flow_scikit_status": "present", + "priority_score": 61.933, + "disposition": "rewrite first", + "hypothesis": "remove Python control/validation and specialize the complete operation", + "observed_flow_speedup": 2.6736872439003507, + "factors": { + "substrate": 0.55, + "python_share": 0.8959086996762008, + "crossing_cost_proxy": 0.0596, + "allocation_proxy": 0.0070805, + "observed_flow_speed": 0.8912290813001169, + "implementation_readiness": 1.0, + "native_reuse_penalty": 0.0, + "complexity_penalty": 0.1 + } + }, { "module": "sklearn.cluster._kmeans", "estimator": "KMeans", @@ -199,7 +220,7 @@ "priority_score": 60.243, "disposition": "rewrite first", "hypothesis": "remove Python control/validation and specialize the complete operation", - "observed_flow_speedup": 4.330005940309095, + "observed_flow_speedup": 6.356359311844886, "factors": { "substrate": 0.55, "python_share": 0.7736816373204982, @@ -220,7 +241,7 @@ "priority_score": 54.0, "disposition": "rewrite first", "hypothesis": "remove Python control/validation and specialize the complete operation", - "observed_flow_speedup": 27.457510112807267, + "observed_flow_speedup": 26.335252316218902, "factors": { "substrate": 1.0, "python_share": 0.0, @@ -241,7 +262,49 @@ "priority_score": 54.0, "disposition": "rewrite first", "hypothesis": "remove Python control/validation and specialize the complete operation", - "observed_flow_speedup": 4.330005940309095, + "observed_flow_speedup": 6.356359311844886, + "factors": { + "substrate": 1.0, + "python_share": 0.0, + "crossing_cost_proxy": 0.0, + "allocation_proxy": 0.0, + "observed_flow_speed": 1.0, + "implementation_readiness": 1.0, + "native_reuse_penalty": 0.0, + "complexity_penalty": 0.1 + } + }, + { + "module": "sklearn.svm._classes", + "estimator": "LinearSVC", + "operation": "predict", + "execution_class": "python-bound", + "flow_scikit_status": "present", + "priority_score": 54.0, + "disposition": "rewrite first", + "hypothesis": "remove Python control/validation and specialize the complete operation", + "observed_flow_speedup": 5.006391873561824, + "factors": { + "substrate": 1.0, + "python_share": 0.0, + "crossing_cost_proxy": 0.0, + "allocation_proxy": 0.0, + "observed_flow_speed": 1.0, + "implementation_readiness": 1.0, + "native_reuse_penalty": 0.0, + "complexity_penalty": 0.1 + } + }, + { + "module": "sklearn.svm._classes", + "estimator": "LinearSVC", + "operation": "decision_function", + "execution_class": "python-bound", + "flow_scikit_status": "present", + "priority_score": 54.0, + "disposition": "rewrite first", + "hypothesis": "remove Python control/validation and specialize the complete operation", + "observed_flow_speedup": 5.006391873561824, "factors": { "substrate": 1.0, "python_share": 0.0, @@ -262,7 +325,7 @@ "priority_score": 54.0, "disposition": "rewrite first", "hypothesis": "remove Python control/validation and specialize the complete operation", - "observed_flow_speedup": 8.622601721278757, + "observed_flow_speedup": 6.800746158795939, "factors": { "substrate": 1.0, "python_share": 0.0, @@ -283,7 +346,7 @@ "priority_score": 54.0, "disposition": "rewrite first", "hypothesis": "remove Python control/validation and specialize the complete operation", - "observed_flow_speedup": 8.622601721278757, + "observed_flow_speedup": 6.800746158795939, "factors": { "substrate": 1.0, "python_share": 0.0, @@ -304,7 +367,7 @@ "priority_score": 54.0, "disposition": "rewrite first", "hypothesis": "remove Python control/validation and specialize the complete operation", - "observed_flow_speedup": 7.311641179767832, + "observed_flow_speedup": 12.547392108478215, "factors": { "substrate": 1.0, "python_share": 0.0, @@ -325,7 +388,7 @@ "priority_score": 54.0, "disposition": "rewrite first", "hypothesis": "remove Python control/validation and specialize the complete operation", - "observed_flow_speedup": 7.311641179767832, + "observed_flow_speedup": 12.547392108478215, "factors": { "substrate": 1.0, "python_share": 0.0, @@ -337,22 +400,64 @@ "complexity_penalty": 0.1 } }, + { + "module": "sklearn.linear_model._coordinate_descent", + "estimator": "Lasso", + "operation": "predict", + "execution_class": "python-bound", + "flow_scikit_status": "present", + "priority_score": 52.475, + "disposition": "rewrite first", + "hypothesis": "remove Python control/validation and specialize the complete operation", + "observed_flow_speedup": 2.6949330261904763, + "factors": { + "substrate": 1.0, + "python_share": 0.0, + "crossing_cost_proxy": 0.0, + "allocation_proxy": 0.0, + "observed_flow_speed": 0.8983110087301588, + "implementation_readiness": 1.0, + "native_reuse_penalty": 0.0, + "complexity_penalty": 0.1 + } + }, { "module": "sklearn.svm._classes", "estimator": "SVC", - "operation": "predict", - "execution_class": "numpy-bound", + "operation": "predict_proba", + "execution_class": "python-bound", "flow_scikit_status": "present", - "priority_score": 52.678, + "priority_score": 52.368, "disposition": "rewrite first", "hypothesis": "remove Python control/validation and specialize the complete operation", - "observed_flow_speedup": 0.8227003674403921, + "observed_flow_speedup": 2.6736872439003507, "factors": { - "substrate": 0.55, - "python_share": 0.8959086996762008, - "crossing_cost_proxy": 0.0596, - "allocation_proxy": 0.0070805, - "observed_flow_speed": 0.27423345581346403, + "substrate": 1.0, + "python_share": 0.0, + "crossing_cost_proxy": 0.0, + "allocation_proxy": 0.0, + "observed_flow_speed": 0.8912290813001169, + "implementation_readiness": 1.0, + "native_reuse_penalty": 0.0, + "complexity_penalty": 0.1 + } + }, + { + "module": "sklearn.svm._classes", + "estimator": "SVC", + "operation": "decision_function", + "execution_class": "python-bound", + "flow_scikit_status": "present", + "priority_score": 52.368, + "disposition": "rewrite first", + "hypothesis": "remove Python control/validation and specialize the complete operation", + "observed_flow_speedup": 2.6736872439003507, + "factors": { + "substrate": 1.0, + "python_share": 0.0, + "crossing_cost_proxy": 0.0, + "allocation_proxy": 0.0, + "observed_flow_speed": 0.8912290813001169, "implementation_readiness": 1.0, "native_reuse_penalty": 0.0, "complexity_penalty": 0.1 @@ -367,7 +472,7 @@ "priority_score": 48.0, "disposition": "compile whole estimator", "hypothesis": "retain useful numerical kernels while fusing validation, allocation and orchestration", - "observed_flow_speedup": 3.6416378066577066, + "observed_flow_speedup": 5.426718137626447, "factors": { "substrate": 0.8, "python_share": 0.0, @@ -388,7 +493,7 @@ "priority_score": 48.0, "disposition": "compile whole estimator", "hypothesis": "retain useful numerical kernels while fusing validation, allocation and orchestration", - "observed_flow_speedup": 3.6416378066577066, + "observed_flow_speedup": 5.426718137626447, "factors": { "substrate": 0.8, "python_share": 0.0, @@ -409,7 +514,7 @@ "priority_score": 48.0, "disposition": "compile whole estimator", "hypothesis": "retain useful numerical kernels while fusing validation, allocation and orchestration", - "observed_flow_speedup": 5.41005073123526, + "observed_flow_speedup": 11.346518336080072, "factors": { "substrate": 0.8, "python_share": 0.0, @@ -430,7 +535,7 @@ "priority_score": 48.0, "disposition": "compile whole estimator", "hypothesis": "retain useful numerical kernels while fusing validation, allocation and orchestration", - "observed_flow_speedup": 5.41005073123526, + "observed_flow_speedup": 11.346518336080072, "factors": { "substrate": 0.8, "python_share": 0.0, @@ -451,7 +556,7 @@ "priority_score": 48.0, "disposition": "compile whole estimator", "hypothesis": "retain useful numerical kernels while fusing validation, allocation and orchestration", - "observed_flow_speedup": 5.41005073123526, + "observed_flow_speedup": 11.346518336080072, "factors": { "substrate": 0.8, "python_share": 0.0, @@ -5191,40 +5296,40 @@ { "module": "sklearn.linear_model._coordinate_descent", "estimator": "Lasso", - "operation": "predict", - "execution_class": "python-bound", + "operation": "fit", + "execution_class": "mixed", "flow_scikit_status": "present", - "priority_score": 44.471, - "disposition": "rewrite first", - "hypothesis": "remove Python control/validation and specialize the complete operation", - "observed_flow_speedup": 1.0942022270144351, + "priority_score": 46.475, + "disposition": "compile whole estimator", + "hypothesis": "retain useful numerical kernels while fusing validation, allocation and orchestration", + "observed_flow_speedup": 2.6949330261904763, "factors": { - "substrate": 1.0, + "substrate": 0.8, "python_share": 0.0, "crossing_cost_proxy": 0.0, "allocation_proxy": 0.0, - "observed_flow_speed": 0.36473407567147836, + "observed_flow_speed": 0.8983110087301588, "implementation_readiness": 1.0, "native_reuse_penalty": 0.0, "complexity_penalty": 0.1 } }, { - "module": "sklearn.svm._classes", - "estimator": "SVC", - "operation": "predict_proba", - "execution_class": "python-bound", + "module": "sklearn.kernel_ridge", + "estimator": "KernelRidge", + "operation": "fit", + "execution_class": "mixed", "flow_scikit_status": "present", - "priority_score": 43.114, - "disposition": "rewrite first", - "hypothesis": "remove Python control/validation and specialize the complete operation", - "observed_flow_speedup": 0.8227003674403921, + "priority_score": 43.551, + "disposition": "compile whole estimator", + "hypothesis": "retain useful numerical kernels while fusing validation, allocation and orchestration", + "observed_flow_speedup": 2.1101041813167494, "factors": { - "substrate": 1.0, + "substrate": 0.8, "python_share": 0.0, "crossing_cost_proxy": 0.0, "allocation_proxy": 0.0, - "observed_flow_speed": 0.27423345581346403, + "observed_flow_speed": 0.7033680604389164, "implementation_readiness": 1.0, "native_reuse_penalty": 0.0, "complexity_penalty": 0.1 @@ -5233,64 +5338,22 @@ { "module": "sklearn.svm._classes", "estimator": "SVC", - "operation": "decision_function", - "execution_class": "python-bound", - "flow_scikit_status": "present", - "priority_score": 43.114, - "disposition": "rewrite first", - "hypothesis": "remove Python control/validation and specialize the complete operation", - "observed_flow_speedup": 0.8227003674403921, - "factors": { - "substrate": 1.0, - "python_share": 0.0, - "crossing_cost_proxy": 0.0, - "allocation_proxy": 0.0, - "observed_flow_speed": 0.27423345581346403, - "implementation_readiness": 1.0, - "native_reuse_penalty": 0.0, - "complexity_penalty": 0.1 - } - }, - { - "module": "sklearn.svm._classes", - "estimator": "LinearSVC", - "operation": "predict", - "execution_class": "python-bound", - "flow_scikit_status": "present", - "priority_score": 41.256, - "disposition": "rewrite first", - "hypothesis": "remove Python control/validation and specialize the complete operation", - "observed_flow_speedup": 0.45116908878841655, - "factors": { - "substrate": 1.0, - "python_share": 0.0, - "crossing_cost_proxy": 0.0, - "allocation_proxy": 0.0, - "observed_flow_speed": 0.15038969626280552, - "implementation_readiness": 1.0, - "native_reuse_penalty": 0.0, - "complexity_penalty": 0.1 - } - }, - { - "module": "sklearn.svm._classes", - "estimator": "LinearSVC", - "operation": "decision_function", - "execution_class": "python-bound", + "operation": "fit", + "execution_class": "external-native-bound", "flow_scikit_status": "present", - "priority_score": 41.256, - "disposition": "rewrite first", - "hypothesis": "remove Python control/validation and specialize the complete operation", - "observed_flow_speedup": 0.45116908878841655, + "priority_score": 43.121, + "disposition": "reuse optimized native kernel", + "hypothesis": "retain the mature backend and optimize boundary, layout and dispatch overhead around it", + "observed_flow_speedup": 2.6736872439003507, "factors": { - "substrate": 1.0, - "python_share": 0.0, - "crossing_cost_proxy": 0.0, - "allocation_proxy": 0.0, - "observed_flow_speed": 0.15038969626280552, + "substrate": 0.15, + "python_share": 0.8844393474814762, + "crossing_cost_proxy": 0.2308, + "allocation_proxy": 0.03330875, + "observed_flow_speed": 0.8912290813001169, "implementation_readiness": 1.0, - "native_reuse_penalty": 0.0, - "complexity_penalty": 0.1 + "native_reuse_penalty": 0.35, + "complexity_penalty": 0.25 } }, { @@ -5512,7 +5575,7 @@ "priority_score": 40.5, "disposition": "compile whole estimator", "hypothesis": "retain useful numerical kernels while fusing validation, allocation and orchestration", - "observed_flow_speedup": 3.6416378066577066, + "observed_flow_speedup": 5.426718137626447, "factors": { "substrate": 0.55, "python_share": 0.0, @@ -8045,21 +8108,21 @@ } }, { - "module": "sklearn.linear_model._coordinate_descent", - "estimator": "Lasso", - "operation": "fit", - "execution_class": "mixed", + "module": "sklearn.kernel_ridge", + "estimator": "KernelRidge", + "operation": "predict", + "execution_class": "numpy-bound", "flow_scikit_status": "present", - "priority_score": 38.471, + "priority_score": 36.051, "disposition": "compile whole estimator", "hypothesis": "retain useful numerical kernels while fusing validation, allocation and orchestration", - "observed_flow_speedup": 1.0942022270144351, + "observed_flow_speedup": 2.1101041813167494, "factors": { - "substrate": 0.8, + "substrate": 0.55, "python_share": 0.0, "crossing_cost_proxy": 0.0, "allocation_proxy": 0.0, - "observed_flow_speed": 0.36473407567147836, + "observed_flow_speed": 0.7033680604389164, "implementation_readiness": 1.0, "native_reuse_penalty": 0.0, "complexity_penalty": 0.1 @@ -8317,48 +8380,6 @@ "complexity_penalty": 0.1 } }, - { - "module": "sklearn.svm._classes", - "estimator": "SVC", - "operation": "fit", - "execution_class": "external-native-bound", - "flow_scikit_status": "present", - "priority_score": 33.866, - "disposition": "reuse optimized native kernel", - "hypothesis": "retain the mature backend and optimize boundary, layout and dispatch overhead around it", - "observed_flow_speedup": 0.8227003674403921, - "factors": { - "substrate": 0.15, - "python_share": 0.8844393474814762, - "crossing_cost_proxy": 0.2308, - "allocation_proxy": 0.03330875, - "observed_flow_speed": 0.27423345581346403, - "implementation_readiness": 1.0, - "native_reuse_penalty": 0.35, - "complexity_penalty": 0.25 - } - }, - { - "module": "sklearn.kernel_ridge", - "estimator": "KernelRidge", - "operation": "fit", - "execution_class": "mixed", - "flow_scikit_status": "present", - "priority_score": 33.566, - "disposition": "compile whole estimator", - "hypothesis": "retain useful numerical kernels while fusing validation, allocation and orchestration", - "observed_flow_speedup": 0.1131002397036554, - "factors": { - "substrate": 0.8, - "python_share": 0.0, - "crossing_cost_proxy": 0.0, - "allocation_proxy": 0.0, - "observed_flow_speed": 0.037700079901218465, - "implementation_readiness": 1.0, - "native_reuse_penalty": 0.0, - "complexity_penalty": 0.1 - } - }, { "module": "sklearn.linear_model._bayes", "estimator": "ARDRegression", @@ -10124,24 +10145,24 @@ } }, { - "module": "sklearn.kernel_ridge", - "estimator": "KernelRidge", - "operation": "predict", - "execution_class": "numpy-bound", + "module": "sklearn.svm._classes", + "estimator": "LinearSVC", + "operation": "fit", + "execution_class": "external-native-bound", "flow_scikit_status": "present", - "priority_score": 26.066, - "disposition": "compile whole estimator", - "hypothesis": "retain useful numerical kernels while fusing validation, allocation and orchestration", - "observed_flow_speedup": 0.1131002397036554, + "priority_score": 20.0, + "disposition": "reuse optimized native kernel", + "hypothesis": "retain the mature backend and optimize boundary, layout and dispatch overhead around it", + "observed_flow_speedup": 5.006391873561824, "factors": { - "substrate": 0.55, + "substrate": 0.15, "python_share": 0.0, "crossing_cost_proxy": 0.0, "allocation_proxy": 0.0, - "observed_flow_speed": 0.037700079901218465, + "observed_flow_speed": 1.0, "implementation_readiness": 1.0, - "native_reuse_penalty": 0.0, - "complexity_penalty": 0.1 + "native_reuse_penalty": 0.35, + "complexity_penalty": 0.25 } }, { @@ -10290,27 +10311,6 @@ "native_reuse_penalty": 0.35, "complexity_penalty": 0.25 } - }, - { - "module": "sklearn.svm._classes", - "estimator": "LinearSVC", - "operation": "fit", - "execution_class": "external-native-bound", - "flow_scikit_status": "present", - "priority_score": 7.256, - "disposition": "reuse optimized native kernel", - "hypothesis": "retain the mature backend and optimize boundary, layout and dispatch overhead around it", - "observed_flow_speedup": 0.45116908878841655, - "factors": { - "substrate": 0.15, - "python_share": 0.0, - "crossing_cost_proxy": 0.0, - "allocation_proxy": 0.0, - "observed_flow_speed": 0.15038969626280552, - "implementation_readiness": 1.0, - "native_reuse_penalty": 0.35, - "complexity_penalty": 0.25 - } } ] } diff --git a/benchmarks/parity_comparison.json b/benchmarks/parity_comparison.json index 31ec6d4..a9a0990 100644 --- a/benchmarks/parity_comparison.json +++ b/benchmarks/parity_comparison.json @@ -2,145 +2,241 @@ { "algorithm": "DummyClassifier_most_frequent", "n_values": 30, - "match": true, + "python_n_values": 30, + "flow_n_values": 30, + "parity_status": "parity verified", + "eligible_for_competitive_timing": true, "max_diff": 0.0, + "verified_tolerance": 0.01, + "approximate_tolerance": 0.05, "python_ms": 0.2909, "flow_ms": 0.014, - "speedup": 20.78 + "speedup": 20.7786, + "reason": null }, { "algorithm": "DummyRegressor_mean", "n_values": 88, - "match": true, + "python_n_values": 88, + "flow_n_values": 88, + "parity_status": "parity verified", + "eligible_for_competitive_timing": true, "max_diff": 8e-06, + "verified_tolerance": 0.01, + "approximate_tolerance": 0.05, "python_ms": 0.1784, "flow_ms": 0.0, - "speedup": 0 + "speedup": null, + "reason": null }, { "algorithm": "GaussianNB", "n_values": 30, - "match": true, + "python_n_values": 30, + "flow_n_values": 30, + "parity_status": "parity verified", + "eligible_for_competitive_timing": true, "max_diff": 0.0, + "verified_tolerance": 0.01, + "approximate_tolerance": 0.05, "python_ms": 3.8057, "flow_ms": 0.019, - "speedup": 200.3 + "speedup": 200.3, + "reason": null }, { "algorithm": "KMeans_k3_fixed_init", "n_values": 30, - "match": true, + "python_n_values": 30, + "flow_n_values": 30, + "parity_status": "parity verified", + "eligible_for_competitive_timing": true, "max_diff": 0.0, + "verified_tolerance": 0.01, + "approximate_tolerance": 0.05, "python_ms": 15.9578, "flow_ms": 0.023, - "speedup": 693.82 + "speedup": 693.8174, + "reason": null }, { "algorithm": "KNNClassifier_k5", "n_values": 30, - "match": true, + "python_n_values": 30, + "flow_n_values": 30, + "parity_status": "parity verified", + "eligible_for_competitive_timing": true, "max_diff": 0.0, + "verified_tolerance": 0.01, + "approximate_tolerance": 0.05, "python_ms": 8.3173, "flow_ms": 0.061, - "speedup": 136.35 + "speedup": 136.3492, + "reason": null }, { "algorithm": "KNNRegressor_k3", "n_values": 88, - "match": true, + "python_n_values": 88, + "flow_n_values": 88, + "parity_status": "parity verified", + "eligible_for_competitive_timing": true, "max_diff": 1.1e-05, + "verified_tolerance": 0.01, + "approximate_tolerance": 0.05, "python_ms": 1.8316, "flow_ms": 0.24, - "speedup": 7.63 + "speedup": 7.6317, + "reason": null }, { "algorithm": "LDA", "n_values": 30, - "match": true, + "python_n_values": 30, + "flow_n_values": 30, + "parity_status": "parity verified", + "eligible_for_competitive_timing": true, "max_diff": 0.0, + "verified_tolerance": 0.01, + "approximate_tolerance": 0.05, "python_ms": 4.1282, "flow_ms": 0.016, - "speedup": 258.01 + "speedup": 258.0125, + "reason": null }, { "algorithm": "Lasso_a0.1", "n_values": 88, - "match": true, + "python_n_values": 88, + "flow_n_values": 88, + "parity_status": "parity verified", + "eligible_for_competitive_timing": true, "max_diff": 0.003916, + "verified_tolerance": 0.01, + "approximate_tolerance": 0.05, "python_ms": 5.5452, "flow_ms": 0.46, - "speedup": 12.05 + "speedup": 12.0548, + "reason": null }, { "algorithm": "LinearRegression", "n_values": 88, - "match": true, + "python_n_values": 88, + "flow_n_values": 88, + "parity_status": "parity verified", + "eligible_for_competitive_timing": true, "max_diff": 1e-05, + "verified_tolerance": 0.01, + "approximate_tolerance": 0.05, "python_ms": 21.1495, "flow_ms": 0.061, - "speedup": 346.71 + "speedup": 346.7131, + "reason": null }, { "algorithm": "MaxAbsScaler", "n_values": 120, - "match": true, + "python_n_values": 120, + "flow_n_values": 120, + "parity_status": "parity verified", + "eligible_for_competitive_timing": true, "max_diff": 0.0, + "verified_tolerance": 0.01, + "approximate_tolerance": 0.05, "python_ms": 0.3249, "flow_ms": 0.014, - "speedup": 23.21 + "speedup": 23.2071, + "reason": null }, { "algorithm": "MinMaxScaler", "n_values": 120, - "match": true, + "python_n_values": 120, + "flow_n_values": 120, + "parity_status": "parity verified", + "eligible_for_competitive_timing": true, "max_diff": 1e-06, + "verified_tolerance": 0.01, + "approximate_tolerance": 0.05, "python_ms": 0.3558, "flow_ms": 0.003, - "speedup": 118.6 + "speedup": 118.6, + "reason": null }, { "algorithm": "NearestCentroid", "n_values": 30, - "match": true, + "python_n_values": 30, + "flow_n_values": 30, + "parity_status": "parity verified", + "eligible_for_competitive_timing": true, "max_diff": 0.0, + "verified_tolerance": 0.01, + "approximate_tolerance": 0.05, "python_ms": 72.6861, "flow_ms": 0.003, - "speedup": 24228.7 + "speedup": 24228.7, + "reason": null }, { "algorithm": "PCA_2comp", "n_values": 60, - "match": true, + "python_n_values": 60, + "flow_n_values": 60, + "parity_status": "parity verified", + "eligible_for_competitive_timing": true, "max_diff": 1e-06, + "verified_tolerance": 0.01, + "approximate_tolerance": 0.05, "python_ms": 5.2298, "flow_ms": 0.027, - "speedup": 193.7 + "speedup": 193.6963, + "reason": null }, { "algorithm": "QDA", "n_values": 30, - "match": true, + "python_n_values": 30, + "flow_n_values": 30, + "parity_status": "parity verified", + "eligible_for_competitive_timing": true, "max_diff": 0.0, + "verified_tolerance": 0.01, + "approximate_tolerance": 0.05, "python_ms": 9.627, "flow_ms": 0.007, - "speedup": 1375.29 + "speedup": 1375.2857, + "reason": null }, { "algorithm": "Ridge_a1", "n_values": 88, - "match": true, + "python_n_values": 88, + "flow_n_values": 88, + "parity_status": "parity verified", + "eligible_for_competitive_timing": true, "max_diff": 1.5e-05, + "verified_tolerance": 0.01, + "approximate_tolerance": 0.05, "python_ms": 12.5278, "flow_ms": 0.035, - "speedup": 357.94 + "speedup": 357.9371, + "reason": null }, { "algorithm": "StandardScaler", "n_values": 120, - "match": true, + "python_n_values": 120, + "flow_n_values": 120, + "parity_status": "parity verified", + "eligible_for_competitive_timing": true, "max_diff": 1e-06, + "verified_tolerance": 0.01, + "approximate_tolerance": 0.05, "python_ms": 1.589, "flow_ms": 0.005, - "speedup": 317.8 + "speedup": 317.8, + "reason": null } ] \ No newline at end of file diff --git a/benchmarks/parity_contract.json b/benchmarks/parity_contract.json index 7bc8a3e..4c8869b 100644 --- a/benchmarks/parity_contract.json +++ b/benchmarks/parity_contract.json @@ -12,8 +12,19 @@ "parity_level": "approximate", "score_abs_tolerance": 0.02, "timing_comparable": true, - "flow": {"optimizer": "lbfgs_no_line_search", "max_iter": 200, "learning_rate": 0.1, "l2": 0.008333333, "multiclass": "multinomial_softmax"}, - "sklearn": {"optimizer": "lbfgs", "max_iter": 1000, "C": 1.0, "multiclass": "multinomial_softmax"} + "flow": { + "optimizer": "lbfgs", + "max_iter": 200, + "learning_rate": 0.1, + "l2": 0.008333333, + "multiclass": "multinomial_softmax" + }, + "sklearn": { + "optimizer": "lbfgs", + "max_iter": 1000, + "C": 1.0, + "multiclass": "multinomial_softmax" + } }, { "algorithm": "LinearSVC", @@ -23,8 +34,17 @@ "parity_level": "approximate", "score_abs_tolerance": 0.02, "timing_comparable": true, - "flow": {"C": 1.0, "max_iter": 1000, "multiclass": "ovr"}, - "sklearn": {"C": 1.0, "max_iter": 1000, "dual": "auto", "multiclass": "ovr"} + "flow": { + "C": 1.0, + "max_iter": 1000, + "multiclass": "ovr" + }, + "sklearn": { + "C": 1.0, + "max_iter": 1000, + "dual": "auto", + "multiclass": "ovr" + } }, { "algorithm": "KernelSVC_RBF", @@ -34,8 +54,20 @@ "parity_level": "approximate", "score_abs_tolerance": 0.03, "timing_comparable": true, - "flow": {"kernel": "rbf", "gamma": 0.25, "C": 1.0, "max_iter": 1000, "multiclass": "ovo"}, - "sklearn": {"kernel": "rbf", "gamma": 0.25, "C": 1.0, "max_iter": 1000, "multiclass": "ovo"} + "flow": { + "kernel": "rbf", + "gamma": 0.25, + "C": 1.0, + "max_iter": 1000, + "multiclass": "ovo" + }, + "sklearn": { + "kernel": "rbf", + "gamma": 0.25, + "C": 1.0, + "max_iter": 1000, + "multiclass": "ovo" + } }, { "algorithm": "DecisionTree", @@ -45,8 +77,16 @@ "parity_level": "approximate", "score_abs_tolerance": 0.04, "timing_comparable": true, - "flow": {"criterion": "gini", "max_depth": 5, "random_state": 42}, - "sklearn": {"criterion": "gini", "max_depth": 5, "random_state": 42} + "flow": { + "criterion": "gini", + "max_depth": 5, + "random_state": 42 + }, + "sklearn": { + "criterion": "gini", + "max_depth": 5, + "random_state": 42 + } }, { "algorithm": "RandomForest", @@ -56,8 +96,18 @@ "parity_level": "approximate", "score_abs_tolerance": 0.04, "timing_comparable": true, - "flow": {"n_estimators": 10, "max_depth": 5, "random_state": 42}, - "sklearn": {"n_estimators": 10, "max_depth": 5, "random_state": 42} + "flow": { + "n_estimators": 10, + "max_depth": 5, + "random_state": 42, + "n_jobs": "all_cores" + }, + "sklearn": { + "n_estimators": 10, + "max_depth": 5, + "random_state": 42, + "n_jobs": 1 + } }, { "algorithm": "GaussianNB", @@ -67,8 +117,12 @@ "parity_level": "verified", "score_abs_tolerance": 0.001, "timing_comparable": true, - "flow": {"var_smoothing": 1e-9}, - "sklearn": {"var_smoothing": 1e-9} + "flow": { + "var_smoothing": 1e-09 + }, + "sklearn": { + "var_smoothing": 1e-09 + } }, { "algorithm": "KMeans", @@ -77,10 +131,24 @@ "category": "clustering", "parity_level": "approximate", "score_abs_tolerance": 0.05, - "inertia_relative_tolerance": 0.10, + "inertia_relative_tolerance": 0.1, "timing_comparable": true, - "flow": {"n_clusters": 3, "init": "k-means++", "n_init": 10, "max_iter": 100, "tol": 0.001, "random_state": 42}, - "sklearn": {"n_clusters": 3, "init": "k-means++", "n_init": 10, "max_iter": 100, "tol": 0.001, "random_state": 42} + "flow": { + "n_clusters": 3, + "init": "k-means++", + "n_init": 10, + "max_iter": 100, + "tol": 0.001, + "random_state": 42 + }, + "sklearn": { + "n_clusters": 3, + "init": "k-means++", + "n_init": 10, + "max_iter": 100, + "tol": 0.001, + "random_state": 42 + } }, { "algorithm": "PCA", @@ -93,8 +161,14 @@ "reconstruction_mse_abs_tolerance": 0.01, "component_abs_tolerance_after_sign_alignment": 0.02, "timing_comparable": true, - "flow": {"n_components": 2, "solver": "power_iteration"}, - "sklearn": {"n_components": 2, "solver": "auto"} + "flow": { + "n_components": 2, + "solver": "power_iteration" + }, + "sklearn": { + "n_components": 2, + "solver": "auto" + } }, { "algorithm": "LogisticRegression", @@ -104,8 +178,19 @@ "parity_level": "approximate", "score_abs_tolerance": 0.02, "timing_comparable": true, - "flow": {"optimizer": "lbfgs_no_line_search", "max_iter": 200, "learning_rate": 0.1, "l2": 0.000695894, "multiclass": "multinomial_softmax"}, - "sklearn": {"optimizer": "lbfgs", "max_iter": 1000, "C": 1.0, "multiclass": "multinomial_softmax"} + "flow": { + "optimizer": "lbfgs", + "max_iter": 200, + "learning_rate": 0.1, + "l2": 0.000695894, + "multiclass": "multinomial_softmax" + }, + "sklearn": { + "optimizer": "lbfgs", + "max_iter": 1000, + "C": 1.0, + "multiclass": "multinomial_softmax" + } }, { "algorithm": "LinearSVC", @@ -115,8 +200,17 @@ "parity_level": "approximate", "score_abs_tolerance": 0.02, "timing_comparable": true, - "flow": {"C": 1.0, "max_iter": 1000, "multiclass": "ovr"}, - "sklearn": {"C": 1.0, "max_iter": 1000, "dual": "auto", "multiclass": "ovr"} + "flow": { + "C": 1.0, + "max_iter": 1000, + "multiclass": "ovr" + }, + "sklearn": { + "C": 1.0, + "max_iter": 1000, + "dual": "auto", + "multiclass": "ovr" + } }, { "algorithm": "KernelSVC_RBF", @@ -126,8 +220,20 @@ "parity_level": "approximate", "score_abs_tolerance": 0.03, "timing_comparable": true, - "flow": {"kernel": "rbf", "gamma": 0.001, "C": 1.0, "max_iter": 1000, "multiclass": "ovo"}, - "sklearn": {"kernel": "rbf", "gamma": 0.001, "C": 1.0, "max_iter": 1000, "multiclass": "ovo"} + "flow": { + "kernel": "rbf", + "gamma": 0.001, + "C": 1.0, + "max_iter": 1000, + "multiclass": "ovo" + }, + "sklearn": { + "kernel": "rbf", + "gamma": 0.001, + "C": 1.0, + "max_iter": 1000, + "multiclass": "ovo" + } }, { "algorithm": "DecisionTree", @@ -137,8 +243,16 @@ "parity_level": "approximate", "score_abs_tolerance": 0.04, "timing_comparable": true, - "flow": {"criterion": "gini", "max_depth": 10, "random_state": 42}, - "sklearn": {"criterion": "gini", "max_depth": 10, "random_state": 42} + "flow": { + "criterion": "gini", + "max_depth": 10, + "random_state": 42 + }, + "sklearn": { + "criterion": "gini", + "max_depth": 10, + "random_state": 42 + } }, { "algorithm": "RandomForest", @@ -148,8 +262,18 @@ "parity_level": "approximate", "score_abs_tolerance": 0.04, "timing_comparable": true, - "flow": {"n_estimators": 10, "max_depth": 10, "random_state": 42}, - "sklearn": {"n_estimators": 10, "max_depth": 10, "random_state": 42} + "flow": { + "n_estimators": 10, + "max_depth": 10, + "random_state": 42, + "n_jobs": "all_cores" + }, + "sklearn": { + "n_estimators": 10, + "max_depth": 10, + "random_state": 42, + "n_jobs": 1 + } }, { "algorithm": "GaussianNB", @@ -159,8 +283,12 @@ "parity_level": "verified", "score_abs_tolerance": 0.001, "timing_comparable": true, - "flow": {"var_smoothing": 1e-9}, - "sklearn": {"var_smoothing": 1e-9} + "flow": { + "var_smoothing": 1e-09 + }, + "sklearn": { + "var_smoothing": 1e-09 + } }, { "algorithm": "KMeans", @@ -169,10 +297,24 @@ "category": "clustering", "parity_level": "approximate", "score_abs_tolerance": 0.05, - "inertia_relative_tolerance": 0.10, + "inertia_relative_tolerance": 0.1, "timing_comparable": true, - "flow": {"n_clusters": 10, "init": "k-means++", "n_init": 10, "max_iter": 100, "tol": 0.001, "random_state": 42}, - "sklearn": {"n_clusters": 10, "init": "k-means++", "n_init": 10, "max_iter": 100, "tol": 0.001, "random_state": 42} + "flow": { + "n_clusters": 10, + "init": "k-means++", + "n_init": 10, + "max_iter": 100, + "tol": 0.001, + "random_state": 42 + }, + "sklearn": { + "n_clusters": 10, + "init": "k-means++", + "n_init": 10, + "max_iter": 100, + "tol": 0.001, + "random_state": 42 + } }, { "algorithm": "Ridge", @@ -182,8 +324,14 @@ "parity_level": "approximate", "score_abs_tolerance": 0.02, "timing_comparable": true, - "flow": {"alpha": 1.0, "max_iter": 1000, "learning_rate": 0.01}, - "sklearn": {"alpha": 1.0} + "flow": { + "alpha": 1.0, + "max_iter": 1000, + "learning_rate": 0.01 + }, + "sklearn": { + "alpha": 1.0 + } }, { "algorithm": "Lasso", @@ -193,8 +341,15 @@ "parity_level": "approximate", "score_abs_tolerance": 0.02, "timing_comparable": true, - "flow": {"alpha": 0.1, "max_iter": 1000, "learning_rate": 0.01}, - "sklearn": {"alpha": 0.1, "max_iter": 1000} + "flow": { + "alpha": 0.1, + "max_iter": 1000, + "learning_rate": 0.01 + }, + "sklearn": { + "alpha": 0.1, + "max_iter": 1000 + } }, { "algorithm": "LinearRegression", @@ -204,8 +359,13 @@ "parity_level": "verified", "score_abs_tolerance": 0.005, "timing_comparable": true, - "flow": {"fit_intercept": true, "penalty": "none"}, - "sklearn": {"fit_intercept": true} + "flow": { + "fit_intercept": true, + "penalty": "none" + }, + "sklearn": { + "fit_intercept": true + } }, { "algorithm": "KernelRidge_RBF", @@ -215,8 +375,16 @@ "parity_level": "verified", "score_abs_tolerance": 0.005, "timing_comparable": true, - "flow": {"alpha": 1.0, "kernel": "rbf", "gamma": 0.1}, - "sklearn": {"alpha": 1.0, "kernel": "rbf", "gamma": 0.1} + "flow": { + "alpha": 1.0, + "kernel": "rbf", + "gamma": 0.1 + }, + "sklearn": { + "alpha": 1.0, + "kernel": "rbf", + "gamma": 0.1 + } } ] } diff --git a/benchmarks/parity_diagnostics.json b/benchmarks/parity_diagnostics.json index 7528175..2829c70 100644 --- a/benchmarks/parity_diagnostics.json +++ b/benchmarks/parity_diagnostics.json @@ -54,19 +54,19 @@ "dataset": "iris", "metric": "explained_var_ratio", "parity_status": "parity verified", - "score_abs_diff": 5.999999996841865e-08, - "explained_variance_ratio_abs_diff": 5.999999996841865e-08, - "singular_values_relative_diff": 1.8056789335315674e-07, - "reconstruction_mse_abs_diff": 3.4000000037615052e-09, - "component_0_sign_aligned_abs_diff": 0.0, - "component_1_sign_aligned_abs_diff": 7.659999999923284e-08 + "score_abs_diff": 0.0, + "explained_variance_ratio_abs_diff": 1.4000000020386594e-08, + "singular_values_relative_diff": 9.503574144572358e-08, + "reconstruction_mse_abs_diff": 1.1100000002484123e-08, + "component_0_sign_aligned_abs_diff": 2.8999999956980105e-08, + "component_1_sign_aligned_abs_diff": 1.1899999996511923e-07 }, { "algorithm": "LogisticRegression", "dataset": "digits", "metric": "accuracy", "parity_status": "approximately equivalent", - "score_abs_diff": 0.002777801999999996 + "score_abs_diff": 1.2999999965401798e-08 }, { "algorithm": "LinearSVC", @@ -109,7 +109,7 @@ "metric": "adjusted_rand_index", "parity_status": "approximately equivalent", "score_abs_diff": 1.2999999965401798e-08, - "inertia_relative_diff": 7.06058916797641e-08 + "inertia_relative_diff": 0.0 }, { "algorithm": "Ridge", @@ -123,20 +123,20 @@ "dataset": "diabetes", "metric": "r2", "parity_status": "approximately equivalent", - "score_abs_diff": 1.1742000000036779e-05 + "score_abs_diff": 1.1623000000016148e-05 }, { "algorithm": "LinearRegression", "dataset": "diabetes", "metric": "r2", "parity_status": "parity verified", - "score_abs_diff": 1.7899999998904903e-07 + "score_abs_diff": 1.7900000004456018e-07 }, { "algorithm": "KernelRidge_RBF", "dataset": "diabetes", "metric": "r2", "parity_status": "parity verified", - "score_abs_diff": 1.7899999998904903e-07 + "score_abs_diff": 2.9799999995416826e-07 } ] diff --git a/benchmarks/run_headline.py b/benchmarks/run_headline.py index 66126c4..6497545 100644 --- a/benchmarks/run_headline.py +++ b/benchmarks/run_headline.py @@ -27,6 +27,16 @@ "NUMEXPR_NUM_THREADS", ) +# How the Flow side was compiled. scikit-learn is measured as an optimized +# wheel, so the optimization level Flow is built at decides what the comparison +# is between. It belongs with the rest of the measurement environment. +BUILD_VARS = ( + "FLOW_OPT_LEVEL", + "FLOW_CFLAGS", + "FLOW_LDFLAGS", + "FLOW_HOST", +) + def cpu_model() -> str: """Best-effort CPU identification for the current host.""" @@ -60,6 +70,7 @@ def host_fingerprint() -> dict: "cpu_model": cpu_model(), "logical_cpus": os.cpu_count(), "thread_limits": {var: os.environ.get(var) for var in THREAD_LIMIT_VARS}, + "flow_build": {var: os.environ.get(var) for var in BUILD_VARS}, } diff --git a/benchmarks/sklearn_results_v2.txt b/benchmarks/sklearn_results_v2.txt index 0688c2c..dc85303 100644 --- a/benchmarks/sklearn_results_v2.txt +++ b/benchmarks/sklearn_results_v2.txt @@ -1,38 +1,38 @@ TIMING_UNIT|ms BENCHMARK_MODE|end_to_end FIXTURE_SOURCE|benchmarks/split_indices.json -BENCHMARK_ENV|{"impl":"sklearn","numpy":"2.5.2","platform":"Linux-6.17.0-1022-azure-x86_64-with-glibc2.39","python":"3.12.14","sklearn":"1.9.0"} -RESULT|DecisionTree|digits|accuracy|0.813888889|14.9078835|0.138405207|0.045726|0.000688402|2|256 -RESULT|DecisionTree|iris|accuracy|0.933333333|0.900115344|0.119728711|0.00674025|0.000643836|32|256 -RESULT|GaussianNB|digits|accuracy|0.741666667|1.72055738|0.641119781|0.0265915|0.008697468|16|32 -RESULT|GaussianNB|iris|accuracy|0.966666667|0.924470656|0.177906977|0.014162281|0.001633765|32|128 -RESULT|KMeans|digits|adjusted_rand_index|0.528326226|68.592784|0.182871937|0.410402|0.002540164|1|128 -RESULT|KMeans|iris|adjusted_rand_index|0.548888889|10.6288145|0.164274305|0.090423|0.001164805|2|128 -RESULT|KernelRidge_RBF|diabetes|r2|0.461917877|5.9995195|0.771147781|0.00606075|0.003607812|4|32 -RESULT|KernelSVC_RBF|digits|accuracy|0.95|52.391939|25.539144|0.110422|1.338714|1|1 -RESULT|KernelSVC_RBF|iris|accuracy|0.966666667|1.06301781|0.193609023|0.00355409|0.000648516|32|128 -RESULT|Lasso|diabetes|r2|0.455541432|0.794417469|0.10562668|0.021328625|0.001134567|32|256 -RESULT|LinearRegression|diabetes|r2|0.452602744|0.676694531|0.0991472969|0.012061094|0.0022896333|32|256 -RESULT|LinearSVC|digits|accuracy|0.955555556|349.443518|0.201159617|1.04663|0.000886437|1|128 -RESULT|LinearSVC|iris|accuracy|0.9|1.02706741|0.155343379|0.00610422|0.001290555|32|256 -RESULT|LogisticRegression|digits|accuracy|0.972222222|192.978743|0.179674766|31.962356|0.000810289|1|128 -RESULT|LogisticRegression|iris|accuracy|0.933333333|5.270827|0.153433687|0.01394425|0.000404352|4|128 -RESULT|PCA|iris|explained_var_ratio|0.957439005|0.396100391|0.118425688|0.005139656|0.001520437|64|256 -RESULT|RandomForest|digits|accuracy|0.936111111|28.12009|1.26504681|0.118545|0.02372631|1|32 -RESULT|RandomForest|iris|accuracy|0.966666667|12.802662|0.959652437|0.093426|0.00635675|2|32 -RESULT|Ridge|diabetes|r2|0.454146624|0.735451469|0.0986626875|0.008099906|0.0020281289|32|256 +BENCHMARK_ENV|{"impl":"sklearn","numpy":"2.4.3","platform":"macOS-26.2-arm64-arm-64bit","python":"3.12.12","sklearn":"1.9.0"} +RESULT|DecisionTree|digits|accuracy|0.813888889|8.6068645|0.038427083|0.199073|0.00088521285|4|1024 +RESULT|DecisionTree|iris|accuracy|0.933333333|0.258626953|0.0286332197|0.003168133|0.0005029502|128|1024 +RESULT|GaussianNB|digits|accuracy|0.741666667|0.530330734|0.291535156|0.0227034605|0.002239582|64|128 +RESULT|GaussianNB|iris|accuracy|0.966666667|0.24434082|0.0445877285|0.0072657855|0.00116161915|128|512 +RESULT|KMeans|digits|adjusted_rand_index|0.528326226|20.118834|0.0551437168|0.7834585|0.0037094717|2|512 +RESULT|KMeans|iris|adjusted_rand_index|0.548888889|1.99084112|0.0330354004|0.103533845|0.0012598477|16|1024 +RESULT|KernelRidge_RBF|diabetes|r2|0.461917818|0.897470063|0.271527664|0.026421203|0.009609539|32|128 +RESULT|KernelSVC_RBF|digits|accuracy|0.95|24.1875|16.9295625|0.3623745|0.903823|1|2 +RESULT|KernelSVC_RBF|iris|accuracy|0.966666667|0.288402672|0.067395834|0.0077688715|0.00166532525|128|512 +RESULT|Lasso|diabetes|r2|0.455541313|0.313371742|0.0261898193|0.00685498|0.0005582886|128|1024 +RESULT|LinearRegression|diabetes|r2|0.452602804|0.182404945|0.0239730625|0.007456547|0.00047957425|128|1024 +RESULT|LinearSVC|digits|accuracy|0.955555556|227.714333|0.0590003262|4.541|0.00208178615|1|512 +RESULT|LinearSVC|iris|accuracy|0.9|0.281800453|0.0373879805|0.009811199|0.0006199341|128|1024 +RESULT|LogisticRegression|digits|accuracy|0.972222222|6.179125|0.0561170254|0.246525875|0.00136836915|4|512 +RESULT|LogisticRegression|iris|accuracy|0.933333333|0.96320575|0.0369819746|0.0319284065|0.0011009522|32|1024 +RESULT|PCA|iris|explained_var_ratio|0.957439005|0.106219398|0.0271950684|0.004313719|0.00061167385|256|1024 +RESULT|RandomForest|digits|accuracy|0.936111111|12.8233335|0.306717438|0.635625|0.016799969|2|128 +RESULT|RandomForest|iris|accuracy|0.966666667|3.83877075|0.245369469|0.145294375|0.0037568355|8|128 +RESULT|Ridge|diabetes|r2|0.454146624|0.201831383|0.0240216875|0.006238766|0.0008340044|128|1024 DETAIL|DecisionTree|digits|depth1_splits|28,-0.873653173,21,-1.19120878 DETAIL|DecisionTree|digits|max_depth_reached|10 -DETAIL|DecisionTree|digits|mean_leaf_depth|7.59777314 +DETAIL|DecisionTree|digits|mean_leaf_depth|7.59707724 DETAIL|DecisionTree|digits|n_leaves|123 DETAIL|DecisionTree|digits|n_nodes|245 -DETAIL|DecisionTree|digits|nodes_per_depth|1,2,4,8,16,24,30,34,42,46,38 -DETAIL|DecisionTree|digits|preorder_split_features|36,28,21,5,46,44,60,-1,-1,-1,-1,-1,42,-1,-1,21,20,51,-1,-1,46,1,-1,-1,-1,53,25,-1,-1,-1,21,42,5,27,37,2,36,-1,-1,-1,59,44,45,-1,37,-1,-1,-1,54,-1,27,-1,-1,37,-1,12,18,-1,60,-1,-1,-1,2,60,-1,-1,18,-1,-1,54,10,52,-1,-1,27,44,26,-1,-1,3,-1,-1,-1,9,-1,-1,60,5,12,-1,35,-1,45,-1,4,-1,58,21,-1,-1,-1,43,42,22,4,-1,26,-1,21,-1,-1,51,-1,-1,-1,19,53,25,57,35,-1,-1,50,-1,-1,50,34,-1,-1,-1,50,-1,28,-1,-1,29,-1,-1,33,43,29,34,26,19,28,-1,-1,50,-1,-1,58,-1,-1,42,12,-1,59,-1,-1,-1,3,61,-1,-1,28,62,-1,-1,27,17,-1,-1,58,-1,-1,27,34,50,59,-1,21,-1,-1,45,-1,-1,38,26,-1,-1,54,13,-1,-1,-1,20,18,10,10,-1,-1,-1,38,21,-1,-1,60,-1,-1,44,3,-1,-1,10,52,-1,-1,42,-1,-1,27,44,33,-1,50,3,-1,-1,-1,13,-1,60,-1,46,-1,14,-1,-1,38,12,-1,-1,44,-1,-1 -DETAIL|DecisionTree|digits|preorder_split_thresholds|-1.67066377,-0.873653173,-1.19120878,0.724282011,-0.692062497,0.0416290164,0.647410065,0,0,0,0,0,-0.515757143,0,0,0.17698154,0.556051835,0.365863532,0,0,0.484992594,0.753725514,0,0,0,-0.622832403,-0.306617208,0,0,0,-1.19120878,0.249807402,-0.775674462,1.12302774,-0.803220391,-0.564921767,0.106599327,0,0,0,0.0786183178,0.832910717,-0.214717321,0,0.394111261,0,0,0,1.19022906,0,0.786803812,0,0,0.137540177,0,0.776457354,0.00550198555,0,0.139145046,0,0,0,-0.989111215,-0.0641609356,0,0,-0.960845023,0,0,-0.454248264,-1.00091881,-0.3714623,0,0,1.03897172,0.358141713,-0.432997167,0,0,0.634846359,0,0,0,0.325489849,0,0,-0.877384931,-0.952139914,-0.579910696,0,-0.573497601,0,-1.27684712,0,-2.184654,0,1.18114537,0.176981539,0,0,0,-0.586137354,0.173250943,2.04696852,-1.37126118,0,-0.514194742,0,-0.0644638091,0,0,-1.17337397,0,0,0,1.12397659,-0.622832403,1.2950148,0.239182577,-1.21240914,0,0,0.177766502,0,0,-0.883295059,0.533377141,0,0,0,-1.06013864,0,0.184827529,0,0,-1.0326938,0,0,0.33585538,-0.897775233,1.00609168,-0.658510029,0.135385733,1.29661876,-0.385123625,0,0,0.177766562,0,0,0.588130996,0,0,-0.515757158,0.880793363,0,0.193780899,0,0,0,-2.67006719,0.621348361,0,0,-0.710809976,0.963630021,0,0,-1.23053974,0.403457791,0,0,1.77415979,0,0,-0.221867986,-0.73796919,-1.23698223,0.193780877,0,-0.305909155,0,0,1.02443406,0,0,-0.677066475,-0.757787406,0,0,-0.351468405,-0.472691774,0,0,0,1.36276203,-1.312244,-0.723750174,-1.55525607,0,0,0,0.166892914,-0.627836302,0,0,0.749063045,0,0,-0.195755459,0.28074849,0,0,0.56970337,-0.0855536945,0,0,-0.286087781,0,0,1.12302774,-0.116627295,0.622534439,0,0.796719119,-0.309414651,0,0,0,0.85123691,0,-0.0641609356,0,-0.692062497,0,0.987032533,0,0,1.29217207,0.150441319,0,0,-0.195755452,0,0 +DETAIL|DecisionTree|digits|nodes_per_depth|1,2,4,8,16,24,30,34,44,44,38 +DETAIL|DecisionTree|digits|preorder_split_features|36,28,21,5,46,44,60,-1,-1,-1,-1,-1,42,-1,-1,21,20,51,-1,-1,46,1,-1,-1,-1,53,25,-1,-1,-1,21,42,5,27,37,2,36,-1,-1,-1,43,12,12,-1,-1,-1,43,-1,61,-1,42,-1,-1,44,12,38,-1,25,-1,-1,-1,-1,2,25,-1,-1,18,-1,-1,54,10,34,-1,-1,27,44,12,-1,-1,59,-1,-1,-1,58,-1,-1,60,5,12,-1,27,-1,60,-1,13,58,12,-1,-1,-1,-1,43,42,22,62,18,-1,21,-1,-1,-1,37,-1,-1,-1,19,53,25,57,35,-1,-1,1,-1,-1,9,29,-1,-1,-1,58,-1,53,-1,-1,28,-1,-1,33,43,29,34,26,19,62,-1,-1,58,-1,-1,59,-1,-1,42,20,-1,4,-1,-1,-1,3,35,-1,-1,28,54,-1,-1,27,17,-1,-1,58,-1,-1,27,34,50,29,62,-1,-1,-1,45,-1,-1,38,18,-1,-1,43,-1,13,-1,-1,20,18,2,51,-1,-1,-1,38,21,-1,-1,53,-1,-1,44,45,-1,-1,10,52,-1,-1,42,-1,-1,27,44,43,14,-1,-1,36,-1,-1,13,-1,52,-1,30,-1,59,-1,-1,38,12,-1,-1,36,-1,-1 +DETAIL|DecisionTree|digits|preorder_split_thresholds|-1.67066377,-0.873653173,-1.19120878,0.724282011,-0.692062497,0.0416290164,0.647410065,0,0,0,0,0,-0.515757143,0,0,0.17698154,0.556051835,0.365863532,0,0,0.484992594,0.753725514,0,0,0,-0.622832403,-0.306617208,0,0,0,-1.19120878,0.249807402,-0.775674462,1.12302774,-0.803220391,-0.564921767,0.106599327,0,0,0,-0.274499446,-0.0582306981,-0.997254729,0,0,0,0.894142762,0,-0.480655693,0,-0.286087789,0,0,-0.512268126,0.776457354,-0.114426881,0,-0.146454006,0,0,0,0,-0.989111215,-0.466780409,0,0,-0.960845023,0,0,-0.454248264,-1.00091881,0.612836272,0,0,1.03897172,0.358141713,0.463449314,0,0,0.424106009,0,0,0,0.686966702,0,0,-0.877384931,-0.952139914,-0.579910696,0,-0.894315839,0,-2.09722096,0,0.354763642,1.18114537,0.880793363,0,0,0,0,-0.586137354,0.173250943,2.04696852,-0.382199571,-1.57579315,0,-0.0644638091,0,0,0,-0.717696711,0,0,0,1.12397659,-0.622832403,1.2950148,0.239182577,-1.21240914,0,0,2.38000715,0,0,0.483506373,1.00609171,0,0,0,-0.202554852,0,0.37015219,0,0,-0.38512361,0,0,0.33585538,-0.897775233,1.00609168,-0.658510029,0.135385733,1.29661876,3.04354858,0,0,-0.1037191,0,0,0.424106024,0,0,-0.515757158,0.394709796,0,0.83651951,0,0,0,-2.67006719,-0.254041843,0,0,-0.710809976,-0.0431289207,0,0,-1.23053974,0.403457791,0,0,1.77415979,0,0,-0.221867986,-0.73796919,-1.23698223,0.326496545,1.33067447,0,0,0,1.02443406,0,0,-0.677066475,0.0933517376,0,0,-0.118680485,0,-0.472691774,0,0,1.36276203,-1.312244,-0.0346849337,0.654470526,0,0,0,0.166892914,-0.627836302,0,0,0.866644472,0,0,-0.195755459,0.847412437,0,0,0.56970337,-0.0855536945,0,0,-0.286087781,0,0,1.12302774,-0.116627295,0.0371384621,1.39978808,0,0,0.614388794,0,0,0.85123691,0,1.0580807,0,0.462439597,0,-0.38203194,0,0,1.29217207,0.150441319,0,0,-0.31655854,0,0 DETAIL|DecisionTree|digits|root_impurity|0.899978644 DETAIL|DecisionTree|digits|root_split_feature|36 DETAIL|DecisionTree|digits|root_split_threshold|-1.67066377 -DETAIL|DecisionTree|digits|split_feature_histogram|0,1,2,4,2,3,0,0,0,1,4,0,4,2,1,0,0,1,3,2,2,7,1,0,0,2,4,6,4,2,0,0,0,2,3,2,2,3,3,0,0,0,5,2,6,3,3,0,0,0,6,2,2,2,3,0,0,1,3,3,6,1,1,0 +DETAIL|DecisionTree|digits|split_feature_histogram|0,2,3,1,1,3,0,0,0,1,2,0,7,3,1,0,0,1,4,2,3,5,1,0,0,4,1,6,3,3,1,0,0,1,3,2,4,2,4,0,0,0,6,6,5,2,2,0,0,0,1,2,2,4,2,0,0,1,5,3,3,1,3,0 DETAIL|DecisionTree|iris|depth1_splits|-1,0,3,0.585927993 DETAIL|DecisionTree|iris|max_depth_reached|5 DETAIL|DecisionTree|iris|mean_leaf_depth|2.40833333 @@ -61,8 +61,8 @@ DETAIL|GaussianNB|iris|var_frobenius_norm|1.33197554 DETAIL|GaussianNB|iris|var_max|0.778331578 DETAIL|GaussianNB|iris|var_min|0.00813894253 DETAIL|GaussianNB|iris|var_row_l2_norms|0.789857418,0.569631196,0.908737809 -DETAIL|KMeans|digits|center_l2_norms_sorted|3.15490844,3.37186366,4.43453872,4.73711543,4.8885025,4.93533516,5.12554507,5.4847927,11.6704287,46.971224 -DETAIL|KMeans|digits|inertia|55236.1797 +DETAIL|KMeans|digits|center_l2_norms_sorted|3.15490842,3.37186367,4.43453876,4.73711551,4.88850251,4.93533516,5.1255451,5.48479269,11.670429,46.971224 +DETAIL|KMeans|digits|inertia|55236.1836 DETAIL|KMeans|digits|n_iter|22 DETAIL|KMeans|digits|train_cluster_sizes_sorted|349,222,151,147,144,142,130,123,28,1 DETAIL|KMeans|iris|center_l2_norms_sorted|0.967568414,2.00604002,2.23148713 @@ -70,17 +70,17 @@ DETAIL|KMeans|iris|inertia|110.195267 DETAIL|KMeans|iris|n_iter|4 DETAIL|KMeans|iris|train_cluster_sizes_sorted|49,40,31 DETAIL|KernelRidge_RBF|diabetes|alpha|1 -DETAIL|KernelRidge_RBF|diabetes|dual_coef_abs_sum|13807.8468 -DETAIL|KernelRidge_RBF|diabetes|dual_coef_l2_norm|925.346037 -DETAIL|KernelRidge_RBF|diabetes|dual_coef_max|147.074738 -DETAIL|KernelRidge_RBF|diabetes|dual_coef_mean|5.51446321 -DETAIL|KernelRidge_RBF|diabetes|dual_coef_min|-128.792511 +DETAIL|KernelRidge_RBF|diabetes|dual_coef_abs_sum|13807.8486 +DETAIL|KernelRidge_RBF|diabetes|dual_coef_l2_norm|925.346145 +DETAIL|KernelRidge_RBF|diabetes|dual_coef_max|147.074768 +DETAIL|KernelRidge_RBF|diabetes|dual_coef_mean|5.51446295 +DETAIL|KernelRidge_RBF|diabetes|dual_coef_min|-128.792526 DETAIL|KernelRidge_RBF|diabetes|gamma|0.1 DETAIL|KernelRidge_RBF|diabetes|n_train_samples|353 DETAIL|KernelSVC_RBF|digits|C|1 -DETAIL|KernelSVC_RBF|digits|dual_coef_abs_sum_per_pair|63.5631957,64.3799405,72.5215582,81.9525455,92.232109,84.7299887,58.4558884,84.636266,104.634716,126.391729,103.343071,108.437745,94.4488351,91.3241249,96.8669782,170.318177,110.821463,135.961454,62.4101286,98.1981514,75.3593534,84.7378932,138.244001,99.6621578,61.8221345,126.123514,74.1254898,92.4769119,154.300509,177.809026,80.8369095,91.5062986,90.6841476,93.6036725,83.3283161,84.8962564,94.2561372,138.430086,139.941811,50.7775403,102.543758,76.8402162,115.372325,101.078568,159.985486 +DETAIL|KernelSVC_RBF|digits|dual_coef_abs_sum_per_pair|63.5631957,64.3799405,72.5215582,81.9525455,92.232109,84.7299887,58.4558884,84.636266,104.634716,126.391729,103.343071,108.437745,94.4488351,91.3241249,96.8669782,170.321845,110.821463,135.961454,62.4101286,98.1981514,75.3593534,84.7378932,138.244001,99.6621578,61.8221345,126.123514,74.1254898,92.4769119,154.306304,177.809026,80.8369095,91.5062986,90.6818573,93.606354,83.3283161,84.8962564,94.2561372,138.430086,139.941811,50.7775403,102.543758,76.8402162,115.372325,101.078568,159.985486 DETAIL|KernelSVC_RBF|digits|gamma|0.001 -DETAIL|KernelSVC_RBF|digits|intercept_per_pair|-0.709698287,-0.740130744,-0.481941996,-0.894933951,-0.591559288,-0.227576483,-0.611445161,-0.442243477,-0.482942216,-0.318910148,0.390082853,-0.616389459,0.269840922,0.537583556,-0.0674603061,0.389315963,0.25224266,0.734104087,-0.214330706,0.562073034,0.50531778,0.438227739,0.618431606,0.611808588,-0.546121861,0.171641806,0.0911206796,0.0015240676,0.0289862692,0.228316448,0.659515511,0.760566417,0.639711996,0.691527639,0.661327502,0.211931215,-0.156483758,-0.0340733865,-0.337795255,-0.145812981,-0.146215967,-0.0980758979,0.164756509,0.148429135,-0.151404454 +DETAIL|KernelSVC_RBF|digits|intercept_per_pair|-0.709698287,-0.740130744,-0.481941996,-0.894933951,-0.591559288,-0.227576483,-0.611445161,-0.442243477,-0.482942216,-0.318910148,0.390082853,-0.616389459,0.269840922,0.537583556,-0.0674603061,0.389300511,0.25224266,0.734104087,-0.214330706,0.562073034,0.50531778,0.438227739,0.618431606,0.611808588,-0.546121861,0.171641806,0.0911206796,0.0015240676,0.0288968006,0.228316448,0.659515511,0.760566417,0.639770167,0.691678919,0.661327502,0.211931215,-0.156483758,-0.0340733865,-0.337795255,-0.145812981,-0.146215967,-0.0980758979,0.164756509,0.148429135,-0.151404454 DETAIL|KernelSVC_RBF|digits|n_bounded_support_per_pair|58,60,68,77,86,81,54,78,94,120,96,103,86,87,89,167,105,128,52,92,70,79,130,91,55,117,68,87,149,172,74,83,84,83,77,80,85,132,133,43,97,73,110,97,151 DETAIL|KernelSVC_RBF|digits|n_pairs|45 DETAIL|KernelSVC_RBF|digits|n_support_per_pair|68,70,80,91,100,90,64,94,111,132,114,117,101,96,102,175,119,141,71,109,82,93,146,108,72,132,80,99,162,184,87,99,99,102,89,90,102,143,148,58,111,83,121,108,167 @@ -89,9 +89,9 @@ DETAIL|KernelSVC_RBF|digits|pair_class_a|0,0,0,0,0,0,0,0,0,1,1,1,1,1,1,1,1,2,2,2 DETAIL|KernelSVC_RBF|digits|pair_class_b|1,2,3,4,5,6,7,8,9,2,3,4,5,6,7,8,9,3,4,5,6,7,8,9,4,5,6,7,8,9,5,6,7,8,9,6,7,8,9,7,8,9,8,9,9 DETAIL|KernelSVC_RBF|digits|pair_train_sizes|288,284,288,287,287,287,285,281,286,288,292,291,291,291,289,285,290,288,287,287,287,285,281,286,291,291,291,289,285,290,290,290,288,284,289,290,288,284,289,288,284,289,282,287,283 DETAIL|KernelSVC_RBF|iris|C|1 -DETAIL|KernelSVC_RBF|iris|dual_coef_abs_sum_per_pair|6.31395129,5.82612078,29.3894636 +DETAIL|KernelSVC_RBF|iris|dual_coef_abs_sum_per_pair|6.31438133,5.82612078,29.3913125 DETAIL|KernelSVC_RBF|iris|gamma|0.25 -DETAIL|KernelSVC_RBF|iris|intercept_per_pair|0.012601528,-0.13494871,0.0172924496 +DETAIL|KernelSVC_RBF|iris|intercept_per_pair|0.0126958091,-0.13494871,0.0167980305 DETAIL|KernelSVC_RBF|iris|n_bounded_support_per_pair|2,1,25 DETAIL|KernelSVC_RBF|iris|n_pairs|3 DETAIL|KernelSVC_RBF|iris|n_support_per_pair|11,13,34 @@ -99,31 +99,31 @@ DETAIL|KernelSVC_RBF|iris|n_support_total|58 DETAIL|KernelSVC_RBF|iris|pair_class_a|0,0,1 DETAIL|KernelSVC_RBF|iris|pair_class_b|1,2,2 DETAIL|KernelSVC_RBF|iris|pair_train_sizes|80,80,80 -DETAIL|Lasso|diabetes|coef|1.73045254,-11.3163595,25.8246174,16.6442604,-29.3584099,13.2758446,0.547943652,10.2361593,29.6328373,2.39347363 -DETAIL|Lasso|diabetes|coef_abs_sum|140.960358 -DETAIL|Lasso|diabetes|coef_l2_norm|55.6962653 +DETAIL|Lasso|diabetes|coef|1.73045206,-11.3163567,25.8246193,16.644268,-29.3584347,13.2758598,0.547949195,10.2361593,29.6328316,2.39348173 +DETAIL|Lasso|diabetes|coef_abs_sum|140.960412 +DETAIL|Lasso|diabetes|coef_l2_norm|55.6962819 DETAIL|Lasso|diabetes|intercept|153.736542 DETAIL|Lasso|diabetes|n_zero_coefs|0 -DETAIL|LinearRegression|diabetes|coef|1.75375879,-11.5118113,25.6071396,16.8288727,-44.4488106,24.6409187,7.67693758,13.1387529,35.1611862,2.35136652 -DETAIL|LinearRegression|diabetes|coef_abs_sum|183.119555 -DETAIL|LinearRegression|diabetes|coef_l2_norm|71.6296433 +DETAIL|LinearRegression|diabetes|coef|1.75377023,-11.5117989,25.6071148,16.8288631,-44.4488449,24.6409569,7.67697001,13.1387644,35.1611977,2.35136628 +DETAIL|LinearRegression|diabetes|coef_abs_sum|183.119647 +DETAIL|LinearRegression|diabetes|coef_l2_norm|71.6296761 DETAIL|LinearRegression|diabetes|intercept|153.736542 DETAIL|LinearRegression|diabetes|n_zero_coefs|0 DETAIL|LinearSVC|digits|classes|0,1,2,3,4,5,6,7,8,9 -DETAIL|LinearSVC|digits|coef_abs_sum|129.984502 -DETAIL|LinearSVC|digits|coef_frobenius_norm|7.38004253 -DETAIL|LinearSVC|digits|coef_row_l2_norms|1.36922003,2.85821591,1.81176783,3.16928515,1.72463632,2.55079799,1.76841898,2.33973049,2.12355876,2.91580313 -DETAIL|LinearSVC|digits|intercepts|-2.51194317,-4.26127589,-2.746643,-3.77654021,-2.83851741,-3.2131306,-3.11459517,-2.95440149,-2.44852767,-3.84412889 +DETAIL|LinearSVC|digits|coef_abs_sum|130.003594 +DETAIL|LinearSVC|digits|coef_frobenius_norm|7.3815819 +DETAIL|LinearSVC|digits|coef_row_l2_norms|1.36989642,2.85886378,1.81205612,3.1683018,1.72469723,2.5564954,1.76740628,2.33972821,2.12343587,2.91531596 +DETAIL|LinearSVC|digits|intercepts|-2.51167743,-4.26199204,-2.7464819,-3.77578183,-2.83846842,-3.2178002,-3.11273671,-2.95412765,-2.44863027,-3.84385935 DETAIL|LinearSVC|iris|classes|0,1,2 DETAIL|LinearSVC|iris|coef_abs_sum|7.70048758 DETAIL|LinearSVC|iris|coef_frobenius_norm|2.75138617 DETAIL|LinearSVC|iris|coef_row_l2_norms|1.00410842,1.28075584,2.21845816 DETAIL|LinearSVC|iris|intercepts|-0.80484504,-0.367856896,-1.89023928 DETAIL|LogisticRegression|digits|classes|0,1,2,3,4,5,6,7,8,9 -DETAIL|LogisticRegression|digits|coef_abs_sum|179.696722 -DETAIL|LogisticRegression|digits|coef_frobenius_norm|10.3641397 -DETAIL|LogisticRegression|digits|coef_row_l2_norms|2.36411829,3.55468736,3.58465198,3.69231122,3.41545018,3.3905353,2.82750843,2.82728964,3.04036859,3.78343773 -DETAIL|LogisticRegression|digits|intercepts|-0.297967911,-0.833925486,-0.164526761,0.0758331716,-0.505499601,0.346626818,-0.549952447,0.106376521,1.98635375,-0.163307101 +DETAIL|LogisticRegression|digits|coef_abs_sum|179.697133 +DETAIL|LogisticRegression|digits|coef_frobenius_norm|10.3641567 +DETAIL|LogisticRegression|digits|coef_row_l2_norms|2.36411609,3.55471352,3.58466077,3.69228159,3.41540111,3.39058663,2.82749256,2.82731775,3.04050852,3.78335836 +DETAIL|LogisticRegression|digits|intercepts|-0.298025668,-0.833688438,-0.164520338,0.0758553445,-0.505324006,0.346454471,-0.549785316,0.106236599,1.98601103,-0.163202599 DETAIL|LogisticRegression|iris|classes|0,1,2 DETAIL|LogisticRegression|iris|coef_abs_sum|13.220982 DETAIL|LogisticRegression|iris|coef_frobenius_norm|4.51733123 @@ -132,7 +132,7 @@ DETAIL|LogisticRegression|iris|intercepts|-0.305583328,1.90854609,-1.60296226 DETAIL|PCA|iris|component_0|0.526793361,-0.253072023,0.581869245,0.565571845 DETAIL|PCA|iris|component_1|0.348139465,0.93470788,0.0268942844,0.0663079396 DETAIL|PCA|iris|explained_variance_ratio|0.726772368,0.230666623 -DETAIL|PCA|iris|reconstruction_mse|0.0367343016 +DETAIL|PCA|iris|reconstruction_mse|0.0367342979 DETAIL|PCA|iris|singular_values|18.6775475,10.522356 DETAIL|RandomForest|digits|bootstrap_index_sums|1027200,1039858,1034335,1023212,1019798,1001606,1014137,1006620,1030777,1035312 DETAIL|RandomForest|digits|bootstrap_unique_fractions|0.64091858,0.636743215,0.639526792,0.629784273,0.620041754,0.638830898,0.623521225,0.637439109,0.649965205,0.638135003 @@ -162,8 +162,8 @@ DETAIL|RandomForest|iris|tree_root_features|3,2,2,2,3,3,0,0,0,3 DETAIL|RandomForest|iris|tree_root_impurities|0.666111111,0.66125,0.664861111,0.665694444,0.662361111,0.664861111,0.655416667,0.662916667,0.664027778,0.664444444 DETAIL|RandomForest|iris|tree_root_thresholds|-0.599094972,-0.749516234,-0.749516234,-0.749516234,-0.533260331,-0.664929584,-0.348294169,-0.467709154,-0.467709154,-0.533260331 DETAIL|RandomForest|iris|unanimous_vote_fraction|0.8 -DETAIL|Ridge|diabetes|coef|1.80734515,-11.4481955,25.7327023,16.7343044,-34.6715279,17.0527878,3.36968398,11.764143,31.3782082,2.4581418 -DETAIL|Ridge|diabetes|coef_abs_sum|156.41704 -DETAIL|Ridge|diabetes|coef_l2_norm|60.9084004 +DETAIL|Ridge|diabetes|coef|1.80733287,-11.4481878,25.7326832,16.7342949,-34.672123,17.0532322,3.36996913,11.7642412,31.3784676,2.45815516 +DETAIL|Ridge|diabetes|coef_abs_sum|156.418687 +DETAIL|Ridge|diabetes|coef_l2_norm|60.90902 DETAIL|Ridge|diabetes|intercept|153.736542 DETAIL|Ridge|diabetes|n_zero_coefs|0 diff --git a/benchmarks/test_config_equivalence.py b/benchmarks/test_config_equivalence.py index 459bac0..fa8768c 100644 --- a/benchmarks/test_config_equivalence.py +++ b/benchmarks/test_config_equivalence.py @@ -135,15 +135,29 @@ def test_canonical_contract() -> None: splits = json.loads((ROOT / "split_indices.json").read_text()) sizes = {name: row["n_train"] for name, row in splits.items()} + # The optimizer difference on the two LogisticRegression rows is gone. + # Flow's multinomial fit now backtracks on the Armijo condition and stops + # on the largest gradient component at the tolerance it declares, which is + # what scipy's L-BFGS-B does, so both sides record plain lbfgs. max_iter + # still differs and stays reported. + # # #475 aligned KernelSVC_RBF/iris max_iter to 1000 on both sides, so that # row now has no surviving configuration difference. Keep it in the map to # pin the absence explicitly rather than silently dropping coverage. + # + # Both RandomForest rows carry n_jobs. Flow fits a forest's trees + # concurrently and scikit-learn's default is a single worker, which the + # benchmark leaves at its default. That is a real difference in what the + # two sides are doing, so it is declared on both sides of the contract and + # reported here rather than mapped away as an equivalence. expected = { - ("LogisticRegression", "iris"): ["max_iter", "optimizer"], - ("LogisticRegression", "digits"): ["max_iter", "optimizer"], + ("LogisticRegression", "iris"): ["max_iter"], + ("LogisticRegression", "digits"): ["max_iter"], ("KernelSVC_RBF", "iris"): [], ("PCA", "iris"): ["solver"], ("Ridge", "diabetes"): ["max_iter"], + ("RandomForest", "iris"): ["n_jobs"], + ("RandomForest", "digits"): ["n_jobs"], ("LinearSVC", "iris"): [], ("LinearSVC", "digits"): [], ("Lasso", "diabetes"): [], @@ -161,7 +175,7 @@ def test_canonical_contract() -> None: check("surviving configuration differences on the canonical rows", actual, expected) unresolved = sum(1 for key, params in actual.items() if params) - check("rows still carrying a configuration difference", unresolved, 4) + check("rows still carrying a configuration difference", unresolved, 6) def main() -> int: diff --git a/benchmarks/whole_estimator_experiments.json b/benchmarks/whole_estimator_experiments.json index ccc94b3..137cf45 100644 --- a/benchmarks/whole_estimator_experiments.json +++ b/benchmarks/whole_estimator_experiments.json @@ -114,25 +114,25 @@ "category": "supervised", "sklearn_score": 0.966666667, "flow_score": 0.966666639, - "sklearn_fit_ms": 0.928993844, - "sklearn_pred_ms": 0.178381859, - "flow_fit_ms": 0.013705, - "flow_pred_ms": 0.006833, - "sklearn_fit_iqr_ms": 0.019884157, - "sklearn_pred_iqr_ms": 0.001624422, - "flow_fit_iqr_ms": 0.000141, - "flow_pred_iqr_ms": 0.000761, - "sklearn_ms": 1.107375703, - "flow_ms": 0.020538, + "sklearn_fit_ms": 0.24434082, + "sklearn_pred_ms": 0.0445877285, + "flow_fit_ms": 0.005, + "flow_pred_ms": 0.001, + "sklearn_fit_iqr_ms": 0.0072657855, + "sklearn_pred_iqr_ms": 0.00116161915, + "flow_fit_iqr_ms": 0.0015, + "flow_pred_iqr_ms": 0.0005, + "sklearn_ms": 0.2889285485, + "flow_ms": 0.006, "timing_unit": "ms", "benchmark_mode": "end_to_end", "fixture_source": "benchmarks/split_indices.json + split_*.bin", "parity_status": "parity verified", "measurement_status": "resolved", "comparable": true, - "environment_id": "3f884007c2946e39", + "environment_id": "c012023a7229bbbe", "classification": "flow win", - "speedup": 53.91838070892979 + "speedup": 48.154758083333334 }, { "algorithm": "GaussianNB", @@ -141,25 +141,25 @@ "category": "supervised", "sklearn_score": 0.741666667, "flow_score": 0.741666675, - "sklearn_fit_ms": 1.71942587, - "sklearn_pred_ms": 0.641485625, - "flow_fit_ms": 1.07586503, - "flow_pred_ms": 1.29300702, - "sklearn_fit_iqr_ms": 0.01975437, - "sklearn_pred_iqr_ms": 0.004878156, - "flow_fit_iqr_ms": 0.011877895, - "flow_pred_iqr_ms": 0.013005972, - "sklearn_ms": 2.360911495, - "flow_ms": 2.3688720500000002, + "sklearn_fit_ms": 0.530330734, + "sklearn_pred_ms": 0.291535156, + "flow_fit_ms": 0.074000001, + "flow_pred_ms": 0.108000003, + "sklearn_fit_iqr_ms": 0.0227034605, + "sklearn_pred_iqr_ms": 0.002239582, + "flow_fit_iqr_ms": 0.002000004, + "flow_pred_iqr_ms": 0.0044999975, + "sklearn_ms": 0.82186589, + "flow_ms": 0.182000004, "timing_unit": "ms", "benchmark_mode": "end_to_end", "fixture_source": "benchmarks/split_indices.json + split_*.bin", "parity_status": "parity verified", "measurement_status": "resolved", "comparable": true, - "environment_id": "3f884007c2946e39", - "classification": "tie", - "speedup": 0.9966395166847444 + "environment_id": "c012023a7229bbbe", + "classification": "flow win", + "speedup": 4.515746549104471 } ], "kernel_baseline": { @@ -279,25 +279,25 @@ "category": "supervised", "sklearn_score": 0.966666667, "flow_score": 0.966666639, - "sklearn_fit_ms": 0.928993844, - "sklearn_pred_ms": 0.178381859, - "flow_fit_ms": 0.013705, - "flow_pred_ms": 0.006833, - "sklearn_fit_iqr_ms": 0.019884157, - "sklearn_pred_iqr_ms": 0.001624422, - "flow_fit_iqr_ms": 0.000141, - "flow_pred_iqr_ms": 0.000761, - "sklearn_ms": 1.107375703, - "flow_ms": 0.020538, + "sklearn_fit_ms": 0.24434082, + "sklearn_pred_ms": 0.0445877285, + "flow_fit_ms": 0.005, + "flow_pred_ms": 0.001, + "sklearn_fit_iqr_ms": 0.0072657855, + "sklearn_pred_iqr_ms": 0.00116161915, + "flow_fit_iqr_ms": 0.0015, + "flow_pred_iqr_ms": 0.0005, + "sklearn_ms": 0.2889285485, + "flow_ms": 0.006, "timing_unit": "ms", "benchmark_mode": "end_to_end", "fixture_source": "benchmarks/split_indices.json + split_*.bin", "parity_status": "parity verified", "measurement_status": "resolved", "comparable": true, - "environment_id": "3f884007c2946e39", + "environment_id": "c012023a7229bbbe", "classification": "flow win", - "speedup": 53.91838070892979 + "speedup": 48.154758083333334 }, { "algorithm": "GaussianNB", @@ -306,25 +306,25 @@ "category": "supervised", "sklearn_score": 0.741666667, "flow_score": 0.741666675, - "sklearn_fit_ms": 1.71942587, - "sklearn_pred_ms": 0.641485625, - "flow_fit_ms": 1.07586503, - "flow_pred_ms": 1.29300702, - "sklearn_fit_iqr_ms": 0.01975437, - "sklearn_pred_iqr_ms": 0.004878156, - "flow_fit_iqr_ms": 0.011877895, - "flow_pred_iqr_ms": 0.013005972, - "sklearn_ms": 2.360911495, - "flow_ms": 2.3688720500000002, + "sklearn_fit_ms": 0.530330734, + "sklearn_pred_ms": 0.291535156, + "flow_fit_ms": 0.074000001, + "flow_pred_ms": 0.108000003, + "sklearn_fit_iqr_ms": 0.0227034605, + "sklearn_pred_iqr_ms": 0.002239582, + "flow_fit_iqr_ms": 0.002000004, + "flow_pred_iqr_ms": 0.0044999975, + "sklearn_ms": 0.82186589, + "flow_ms": 0.182000004, "timing_unit": "ms", "benchmark_mode": "end_to_end", "fixture_source": "benchmarks/split_indices.json + split_*.bin", "parity_status": "parity verified", "measurement_status": "resolved", "comparable": true, - "environment_id": "3f884007c2946e39", - "classification": "tie", - "speedup": 0.9966395166847444 + "environment_id": "c012023a7229bbbe", + "classification": "flow win", + "speedup": 4.515746549104471 } ], "kernel_baseline": { @@ -444,25 +444,25 @@ "category": "supervised", "sklearn_score": 0.933333333, "flow_score": 0.933333337, - "sklearn_fit_ms": 5.27008425, - "sklearn_pred_ms": 0.154016414, - "flow_fit_ms": 0.368369997, - "flow_pred_ms": 0.003115, - "sklearn_fit_iqr_ms": 0.08471025, - "sklearn_pred_iqr_ms": 0.001390047, - "flow_fit_iqr_ms": 0.008305997, - "flow_pred_iqr_ms": 0.000191, - "sklearn_ms": 5.424100664, - "flow_ms": 0.371484997, + "sklearn_fit_ms": 0.96320575, + "sklearn_pred_ms": 0.0369819746, + "flow_fit_ms": 0.093999997, + "flow_pred_ms": 0.002, + "sklearn_fit_iqr_ms": 0.0319284065, + "sklearn_pred_iqr_ms": 0.0011009522, + "flow_fit_iqr_ms": 0.0100000015, + "flow_pred_iqr_ms": 0.001, + "sklearn_ms": 1.0001877246, + "flow_ms": 0.095999997, "timing_unit": "ms", "benchmark_mode": "end_to_end", "fixture_source": "benchmarks/split_indices.json + split_*.bin", "parity_status": "approximately equivalent", "measurement_status": "resolved", "comparable": true, - "environment_id": "3f884007c2946e39", + "environment_id": "c012023a7229bbbe", "classification": "flow win", - "speedup": 14.601129810903238 + "speedup": 10.418622456831951 }, { "algorithm": "LogisticRegression", @@ -470,26 +470,26 @@ "metric": "accuracy", "category": "supervised", "sklearn_score": 0.972222222, - "flow_score": 0.975000024, - "sklearn_fit_ms": 186.031079, - "sklearn_pred_ms": 0.180065344, - "flow_fit_ms": 70.3608933, - "flow_pred_ms": 0.064961001, - "sklearn_fit_iqr_ms": 45.203811, - "sklearn_pred_iqr_ms": 0.000828047, - "flow_fit_iqr_ms": 2.85941315, - "flow_pred_iqr_ms": 0.000671006, - "sklearn_ms": 186.21114434400002, - "flow_ms": 70.425854301, + "flow_score": 0.972222209, + "sklearn_fit_ms": 6.179125, + "sklearn_pred_ms": 0.0561170254, + "flow_fit_ms": 1.93599999, + "flow_pred_ms": 0.023, + "sklearn_fit_iqr_ms": 0.246525875, + "sklearn_pred_iqr_ms": 0.00136836915, + "flow_fit_iqr_ms": 0.0474999545, + "flow_pred_iqr_ms": 0.002, + "sklearn_ms": 6.2352420254, + "flow_ms": 1.95899999, "timing_unit": "ms", "benchmark_mode": "end_to_end", "fixture_source": "benchmarks/split_indices.json + split_*.bin", "parity_status": "approximately equivalent", "measurement_status": "resolved", "comparable": true, - "environment_id": "3f884007c2946e39", + "environment_id": "c012023a7229bbbe", "classification": "flow win", - "speedup": 2.6440736316542766 + "speedup": 3.182869860759928 } ], "kernel_baseline": { @@ -607,37 +607,37 @@ "dataset": "diabetes", "metric": "r2", "category": "supervised", - "sklearn_score": 0.452602744, - "flow_score": 0.452602565, - "sklearn_fit_ms": 0.682959563, - "sklearn_pred_ms": 0.0999664102, - "flow_fit_ms": 0.090530001, - "flow_pred_ms": 0.001392, - "sklearn_fit_iqr_ms": 0.011478781, - "sklearn_pred_iqr_ms": 0.0009474848, - "flow_fit_iqr_ms": 0.001272, - "flow_pred_iqr_ms": 1e-05, - "sklearn_ms": 0.7829259732, - "flow_ms": 0.091922001, + "sklearn_score": 0.452602804, + "flow_score": 0.452602625, + "sklearn_fit_ms": 0.182404945, + "sklearn_pred_ms": 0.0239730625, + "flow_fit_ms": 0.02, + "flow_pred_ms": 0.0, + "sklearn_fit_iqr_ms": 0.007456547, + "sklearn_pred_iqr_ms": 0.00047957425, + "flow_fit_iqr_ms": 0.001, + "flow_pred_iqr_ms": 0.001, + "sklearn_ms": 0.20637800750000002, + "flow_ms": 0.02, "timing_unit": "ms", "benchmark_mode": "end_to_end", "fixture_source": "benchmarks/split_indices.json + split_*.bin", "parity_status": "parity verified", "measurement_status": "resolved", "comparable": true, - "environment_id": "3f884007c2946e39", + "environment_id": "c012023a7229bbbe", "classification": "flow win", - "speedup": 8.517286010777768 + "speedup": 10.318900375 } ], "kernel_baseline": { "estimator": "LinearRegression", "operation": "fit", "execution_class": "blas-lapack-bound", - "sklearn_end_to_end_ms": 2.969661999998152, - "isolated_kernel_ms": 6.244825999999648, + "sklearn_end_to_end_ms": 1.2034589890390635, + "isolated_kernel_ms": 1.0532080195844173, "kernel": "numpy.linalg.lstsq", - "orchestration_overhead_ms": 0.0 + "orchestration_overhead_ms": 0.1502509694546461 } }, { @@ -752,25 +752,25 @@ "category": "supervised", "sklearn_score": 0.966666667, "flow_score": 0.966666639, - "sklearn_fit_ms": 12.872039, - "sklearn_pred_ms": 0.965154375, - "flow_fit_ms": 1.29490101, - "flow_pred_ms": 0.02176, - "sklearn_fit_iqr_ms": 0.0589295, - "sklearn_pred_iqr_ms": 0.003362437, - "flow_fit_iqr_ms": 0.027552009, - "flow_pred_iqr_ms": 0.000189999, - "sklearn_ms": 13.837193374999998, - "flow_ms": 1.31666101, + "sklearn_fit_ms": 3.83877075, + "sklearn_pred_ms": 0.245369469, + "flow_fit_ms": 0.226999998, + "flow_pred_ms": 0.007, + "sklearn_fit_iqr_ms": 0.145294375, + "sklearn_pred_iqr_ms": 0.0037568355, + "flow_fit_iqr_ms": 0.0219999995, + "flow_pred_iqr_ms": 0.001, + "sklearn_ms": 4.084140219, + "flow_ms": 0.23399999800000001, "timing_unit": "ms", "benchmark_mode": "end_to_end", "fixture_source": "benchmarks/split_indices.json + split_*.bin", "parity_status": "approximately equivalent", "measurement_status": "resolved", "comparable": true, - "environment_id": "3f884007c2946e39", + "environment_id": "c012023a7229bbbe", "classification": "flow win", - "speedup": 10.50930594124603 + "speedup": 17.453590828663167 }, { "algorithm": "RandomForest", @@ -779,25 +779,25 @@ "category": "supervised", "sklearn_score": 0.936111111, "flow_score": 0.955555558, - "sklearn_fit_ms": 28.299126, - "sklearn_pred_ms": 1.25733969, - "flow_fit_ms": 94.5506439, - "flow_pred_ms": 0.548749983, - "sklearn_fit_iqr_ms": 0.128549, - "sklearn_pred_iqr_ms": 0.01165263, - "flow_fit_iqr_ms": 0.092216492, - "flow_pred_iqr_ms": 0.007950962, - "sklearn_ms": 29.55646569, - "flow_ms": 95.09939388299999, + "sklearn_fit_ms": 12.8233335, + "sklearn_pred_ms": 0.306717438, + "flow_fit_ms": 2.31999993, + "flow_pred_ms": 0.186000004, + "sklearn_fit_iqr_ms": 0.635625, + "sklearn_pred_iqr_ms": 0.016799969, + "flow_fit_iqr_ms": 0.207000136, + "flow_pred_iqr_ms": 0.013499998, + "sklearn_ms": 13.130050938, + "flow_ms": 2.5059999339999997, "timing_unit": "ms", "benchmark_mode": "end_to_end", "fixture_source": "benchmarks/split_indices.json + split_*.bin", "parity_status": "approximately equivalent", "measurement_status": "resolved", "comparable": true, - "environment_id": "3f884007c2946e39", - "classification": "sklearn win", - "speedup": 0.3107955212244894 + "environment_id": "c012023a7229bbbe", + "classification": "flow win", + "speedup": 5.239445843496978 } ], "kernel_baseline": { @@ -1032,10 +1032,10 @@ "estimator": "Pipeline(StandardScaler,LinearRegression)", "operation": "fit", "execution_class": "mixed", - "sklearn_end_to_end_ms": 7.890317999994068, - "isolated_kernel_ms": 18.002507999995032, + "sklearn_end_to_end_ms": 2.0385830430313945, + "isolated_kernel_ms": 1.429750001989305, "kernel": "fused NumPy scaling + lstsq reference", - "orchestration_overhead_ms": 0.0 + "orchestration_overhead_ms": 0.6088330410420895 }, "profile": null, "flow_rows": [], @@ -1047,17 +1047,17 @@ "estimator": "LinearRegression", "operation": "fit", "execution_class": "blas-lapack-bound", - "sklearn_end_to_end_ms": 2.969661999998152, - "isolated_kernel_ms": 6.244825999999648, + "sklearn_end_to_end_ms": 1.2034589890390635, + "isolated_kernel_ms": 1.0532080195844173, "kernel": "numpy.linalg.lstsq", - "orchestration_overhead_ms": 0.0 + "orchestration_overhead_ms": 0.1502509694546461 }, { "estimator": "PCA", "operation": "fit", "execution_class": "blas-lapack-bound", - "sklearn_end_to_end_ms": 1.0666879999803314, - "isolated_kernel_ms": 6.036024999986012, + "sklearn_end_to_end_ms": 0.20845793187618256, + "isolated_kernel_ms": 1.3947089901193976, "kernel": "numpy.linalg.svd", "orchestration_overhead_ms": 0.0 }, @@ -1065,10 +1065,10 @@ "estimator": "Pipeline(StandardScaler,LinearRegression)", "operation": "fit", "execution_class": "mixed", - "sklearn_end_to_end_ms": 7.890317999994068, - "isolated_kernel_ms": 18.002507999995032, + "sklearn_end_to_end_ms": 2.0385830430313945, + "isolated_kernel_ms": 1.429750001989305, "kernel": "fused NumPy scaling + lstsq reference", - "orchestration_overhead_ms": 0.0 + "orchestration_overhead_ms": 0.6088330410420895 } ] } diff --git a/docs/benchmarks.js b/docs/benchmarks.js index 7167696..bbf2619 100644 --- a/docs/benchmarks.js +++ b/docs/benchmarks.js @@ -1,3 +1,4 @@ +// SKLEARN_TIMINGS_NORMALIZED_TO_MS // flow-scikit benchmark data and SVG chart rendering. // All data measured with seed=42, 80/20 split, xorshift32 PRNG. // Flow builds use -O3 -march=native, BLAS (Accelerate), SMO for kernel SVM, @@ -5,29 +6,29 @@ const BENCH = { iris: [ - { algo: "LogisticRegression", sk_score: 0.9333, fl_score: 0.9333, sk_ms: 0.004, fl_ms: 0.095, sk_fit: 0.004, fl_fit: 0.093, sk_pred: 0.000, fl_pred: 0.002 }, - { algo: "LinearSVC", sk_score: 0.9000, fl_score: 0.9000, sk_ms: 0.001, fl_ms: 0.100, sk_fit: 0.001, fl_fit: 0.099, sk_pred: 0.000, fl_pred: 0.001 }, - { algo: "KernelSVC_RBF", sk_score: 0.9333, fl_score: 1.0000, sk_ms: 0.001, fl_ms: 0.549, sk_fit: 0.001, fl_fit: 0.511, sk_pred: 0.000, fl_pred: 0.038 }, - { algo: "DecisionTree", sk_score: 0.9333, fl_score: 0.9333, sk_ms: 0.001, fl_ms: 0.082, sk_fit: 0.001, fl_fit: 0.081, sk_pred: 0.000, fl_pred: 0.001 }, - { algo: "RandomForest", sk_score: 0.9333, fl_score: 0.9333, sk_ms: 0.005, fl_ms: 0.333, sk_fit: 0.004, fl_fit: 0.329, sk_pred: 0.000, fl_pred: 0.004 }, + { algo: "LogisticRegression", sk_score: 0.9333, fl_score: 0.9333, sk_ms: 4, fl_ms: 0.095, sk_fit: 4, fl_fit: 0.093, sk_pred: 0.000, fl_pred: 0.002 }, + { algo: "LinearSVC", sk_score: 0.9000, fl_score: 0.9000, sk_ms: 1, fl_ms: 0.100, sk_fit: 1, fl_fit: 0.099, sk_pred: 0.000, fl_pred: 0.001 }, + { algo: "KernelSVC_RBF", sk_score: 0.9333, fl_score: 1.0000, sk_ms: 1, fl_ms: 0.549, sk_fit: 1, fl_fit: 0.511, sk_pred: 0.000, fl_pred: 0.038 }, + { algo: "DecisionTree", sk_score: 0.9333, fl_score: 0.9333, sk_ms: 1, fl_ms: 0.082, sk_fit: 1, fl_fit: 0.081, sk_pred: 0.000, fl_pred: 0.001 }, + { algo: "RandomForest", sk_score: 0.9333, fl_score: 0.9333, sk_ms: 5, fl_ms: 0.333, sk_fit: 4, fl_fit: 0.329, sk_pred: 0.000, fl_pred: 0.004 }, { algo: "GaussianNB", sk_score: 0.9333, fl_score: 0.9333, sk_ms: 0.000, fl_ms: 0.019, sk_fit: 0.000, fl_fit: 0.017, sk_pred: 0.000, fl_pred: 0.002 }, - { algo: "KMeans", sk_score: 0.8333, fl_score: 0.8333, sk_ms: 0.028, fl_ms: 0.114, sk_fit: 0.028, fl_fit: 0.113, sk_pred: 0.000, fl_pred: 0.001 }, + { algo: "KMeans", sk_score: 0.8333, fl_score: 0.8333, sk_ms: 28, fl_ms: 0.114, sk_fit: 28, fl_fit: 0.113, sk_pred: 0.000, fl_pred: 0.001 }, { algo: "PCA", sk_score: 0.7262, fl_score: 0.7262, sk_ms: 0.000, fl_ms: 0.020, sk_fit: 0.000, fl_fit: 0.020, sk_pred: 0.000, fl_pred: 0.000, metric: "explained_var" } ], digits: [ - { algo: "LogisticRegression", sk_score: 0.9749, fl_score: 0.9666, sk_ms: 0.007, fl_ms: 21.801, sk_fit: 0.007, fl_fit: 21.687, sk_pred: 0.000, fl_pred: 0.114 }, - { algo: "LinearSVC", sk_score: 0.9694, fl_score: 0.9694, sk_ms: 0.237, fl_ms: 100.524, sk_fit: 0.237, fl_fit: 100.417, sk_pred: 0.000, fl_pred: 0.107 }, - { algo: "KernelSVC_RBF", sk_score: 0.9499, fl_score: 0.9443, sk_ms: 0.044, fl_ms: 173.044, sk_fit: 0.025, fl_fit: 160.648, sk_pred: 0.019, fl_pred: 12.396 }, - { algo: "DecisionTree", sk_score: 0.8886, fl_score: 0.8802, sk_ms: 0.010, fl_ms: 15.259, sk_fit: 0.009, fl_fit: 15.232, sk_pred: 0.000, fl_pred: 0.027 }, - { algo: "RandomForest", sk_score: 0.9666, fl_score: 0.9499, sk_ms: 0.016, fl_ms: 22.483, sk_fit: 0.015, fl_fit: 22.236, sk_pred: 0.001, fl_pred: 0.247 }, - { algo: "GaussianNB", sk_score: 0.8134, fl_score: 0.8134, sk_ms: 0.001, fl_ms: 1.673, sk_fit: 0.001, fl_fit: 1.077, sk_pred: 0.000, fl_pred: 0.596 }, - { algo: "KMeans", sk_score: 0.6323, fl_score: 0.6267, sk_ms: 0.034, fl_ms: 103.908, sk_fit: 0.034, fl_fit: 103.792, sk_pred: 0.000, fl_pred: 0.116 } + { algo: "LogisticRegression", sk_score: 0.9749, fl_score: 0.9666, sk_ms: 7, fl_ms: 21.801, sk_fit: 7, fl_fit: 21.687, sk_pred: 0.000, fl_pred: 0.114 }, + { algo: "LinearSVC", sk_score: 0.9694, fl_score: 0.9694, sk_ms: 237, fl_ms: 100.524, sk_fit: 237, fl_fit: 100.417, sk_pred: 0.000, fl_pred: 0.107 }, + { algo: "KernelSVC_RBF", sk_score: 0.9499, fl_score: 0.9443, sk_ms: 44, fl_ms: 173.044, sk_fit: 25, fl_fit: 160.648, sk_pred: 19, fl_pred: 12.396 }, + { algo: "DecisionTree", sk_score: 0.8886, fl_score: 0.8802, sk_ms: 10, fl_ms: 15.259, sk_fit: 9, fl_fit: 15.232, sk_pred: 0.000, fl_pred: 0.027 }, + { algo: "RandomForest", sk_score: 0.9666, fl_score: 0.9499, sk_ms: 16, fl_ms: 22.483, sk_fit: 15, fl_fit: 22.236, sk_pred: 1, fl_pred: 0.247 }, + { algo: "GaussianNB", sk_score: 0.8134, fl_score: 0.8134, sk_ms: 1, fl_ms: 1.673, sk_fit: 1, fl_fit: 1.077, sk_pred: 0.000, fl_pred: 0.596 }, + { algo: "KMeans", sk_score: 0.6323, fl_score: 0.6267, sk_ms: 34, fl_ms: 103.908, sk_fit: 34, fl_fit: 103.792, sk_pred: 0.000, fl_pred: 0.116 } ], diabetes: [ - { algo: "Ridge", sk_score: 0.6089, fl_score: 0.6089, sk_ms: 0.001, fl_ms: 0.032, sk_fit: 0.001, fl_fit: 0.032, sk_pred: 0.000, fl_pred: 0.000 }, - { algo: "Lasso", sk_score: 0.6084, fl_score: 0.6085, sk_ms: 0.001, fl_ms: 4.830, sk_fit: 0.001, fl_fit: 4.829, sk_pred: 0.000, fl_pred: 0.001 }, - { algo: "LinearRegression", sk_score: 0.6108, fl_score: 0.6108, sk_ms: 0.001, fl_ms: 0.040, sk_fit: 0.001, fl_fit: 0.040, sk_pred: 0.000, fl_pred: 0.000 }, - { algo: "KernelRidge_RBF", sk_score: 0.4173, fl_score: 0.4173, sk_ms: 0.021, fl_ms: 8.562, sk_fit: 0.020, fl_fit: 7.787, sk_pred: 0.000, fl_pred: 0.775 } + { algo: "Ridge", sk_score: 0.6089, fl_score: 0.6089, sk_ms: 1, fl_ms: 0.032, sk_fit: 1, fl_fit: 0.032, sk_pred: 0.000, fl_pred: 0.000 }, + { algo: "Lasso", sk_score: 0.6084, fl_score: 0.6085, sk_ms: 1, fl_ms: 4.830, sk_fit: 1, fl_fit: 4.829, sk_pred: 0.000, fl_pred: 0.001 }, + { algo: "LinearRegression", sk_score: 0.6108, fl_score: 0.6108, sk_ms: 1, fl_ms: 0.040, sk_fit: 1, fl_fit: 0.040, sk_pred: 0.000, fl_pred: 0.000 }, + { algo: "KernelRidge_RBF", sk_score: 0.4173, fl_score: 0.4173, sk_ms: 21, fl_ms: 8.562, sk_fit: 20, fl_fit: 7.787, sk_pred: 0.000, fl_pred: 0.775 } ], iris_combo: [ { algo: "GaussianNB", sk_acc: 0.9333, fl_acc: 0.9333, sk_train: 0.00, fl_train: 0.01 }, diff --git a/lib/scikit/blas.flow b/lib/scikit/blas.flow index e6ee6fe..d152ab1 100644 --- a/lib/scikit/blas.flow +++ b/lib/scikit/blas.flow @@ -19,6 +19,10 @@ extern { function cblas_ddot(n: i32, x: ptr, incx: i32, y: ptr, incy: i32) -> f64 function cblas_daxpy(n: i32, alpha: f64, x: ptr, incx: i32, y: ptr, incy: i32) -> void function cblas_dger(order: i32, m: i32, n: i32, alpha: f64, x: ptr, incx: i32, y: ptr, incy: i32, a: ptr, lda: i32) -> void + function cblas_dtrsv(order: i32, uplo: i32, trans: i32, diag: i32, n: i32, a: ptr, lda: i32, x: ptr, incx: i32) -> void + function cblas_strsv(order: i32, uplo: i32, trans: i32, diag: i32, n: i32, a: ptr, lda: i32, x: ptr, incx: i32) -> void + function dpotrf_(uplo: ptr, n: ptr, a: ptr, lda: ptr, info: ptr) -> void + function spotrf_(uplo: ptr, n: ptr, a: ptr, lda: ptr, info: ptr) -> void } # Matrix-vector multiply: y = alpha * A * x + beta * y @@ -98,3 +102,12 @@ export function blas_dot_strided_f64(n: i32, x: ptr, incx: i32, y: ptr export function blas_axpy_strided_f64(n: i32, alpha: f64, x: ptr, incx: i32, y: ptr, incy: i32) -> void { cblas_daxpy(n, alpha, x, incx, y, incy) } + +# Double-precision C = alpha * A^T * B + beta * C. +# A is row-major m x n with lda per row, B is row-major m x k with ldb per row, +# C is row-major n x k with ldc per row. The shared m is the contracted +# dimension, so this is the normal-equations shape: pass the same matrix twice +# to get a Gram matrix, or a weighted copy to get X^T diag(w) X. +export function blas_gemm_tn_f64(A: ptr, B: ptr, C: ptr, m: i32, n: i32, k: i32, lda: i32, ldb: i32, ldc: i32, alpha: f64, beta: f64) -> void { + cblas_dgemm(CBLAS_ROW_MAJOR, CBLAS_TRANS, CBLAS_NO_TRANS, n, k, m, alpha, A, lda, B, ldb, beta, C, ldc) +} diff --git a/lib/scikit/cluster.flow b/lib/scikit/cluster.flow index ae976eb..d9bf174 100644 --- a/lib/scikit/cluster.flow +++ b/lib/scikit/cluster.flow @@ -54,15 +54,28 @@ export function kmeans_fit_init(X: Matrix, init_centers: Matrix, n_clusters: i32 struct KMeansScratch { cbuf: ptr, cnorm: ptr, + xnorm: ptr, cross: ptr, best_out: ptr, sums: ptr, counts: ptr } -function _kmeans_scratch_new(n: i32, n_clusters: i32, n_features: i32) -> KMeansScratch { +function _kmeans_scratch_new(X: Matrix, n_clusters: i32, n_features: i32) -> KMeansScratch { + let n: i32 = X.rows let cbuf: ptr = array_new_f32(n_clusters * n_features) let cnorm: ptr = array_new_f32(n_clusters) + let xnorm: ptr = array_new_f32(n) + let x_data: ptr = X.data + for i in 0 to n { + let xbase: i32 = i * n_features + let mut s: f32 = 0.0 + for j in 0 to n_features { + let v: f32 = x_data[xbase + j] + s = s + v * v + } + xnorm[i] = s + } # malloc, not calloc: sgemm with beta = 0 overwrites every element. let cross: ptr = malloc(((n as i64) * (n_clusters as i64)) * 4) as ptr let best_out: ptr = array_new_f32(32) @@ -71,6 +84,7 @@ function _kmeans_scratch_new(n: i32, n_clusters: i32, n_features: i32) -> KMeans let scratch: KMeansScratch = KMeansScratch { cbuf: cbuf, cnorm: cnorm, + xnorm: xnorm, cross: cross, best_out: best_out, sums: sums, @@ -82,6 +96,7 @@ function _kmeans_scratch_new(n: i32, n_clusters: i32, n_features: i32) -> KMeans function _kmeans_scratch_free(scratch: KMeansScratch) -> void { array_free_f32(scratch.cbuf) array_free_f32(scratch.cnorm) + array_free_f32(scratch.xnorm) free(scratch.cross as ptr) array_free_f32(scratch.best_out) array_free_f32(scratch.sums) @@ -92,8 +107,9 @@ function _kmeans_scratch_free(scratch: KMeansScratch) -> void { # order as the original per-pair loop. function _kmeans_sqdist(x_data: ptr, x_base: i32, crow: ptr, n_features: i32) -> f32 { let mut dist: f32 = 0.0 + let xrow: ptr = x_data + x_base for j in 0 to n_features { - let d: f32 = x_data[x_base + j] - crow[j] + let d: f32 = xrow[j] - crow[j] dist = dist + d * d } return dist @@ -112,7 +128,7 @@ function _kmeans_sqdist(x_data: ptr, x_base: i32, crow: ptr, n_feature # is then recomputed directly, in the original accumulation order, so the # inertia this returns is bit-identical to the old triple loop. Cost per # iteration drops from n*k*d scalar operations to one gemm plus n*d. -function _kmeans_assign(X: Matrix, centroids: ptr >, n_clusters: i32, n_features: i32, labels: ptr, scratch: KMeansScratch) -> f32 { +function _kmeans_assign(X: Matrix, centroids: ptr >, n_clusters: i32, n_features: i32, labels: ptr, scratch: KMeansScratch, compute_inertia: bool) -> f32 { let n: i32 = X.rows if n <= 0 || n_clusters <= 0 || n_features <= 0 { return 0.0 @@ -132,7 +148,9 @@ function _kmeans_assign(X: Matrix, centroids: ptr >, n_clusters: i32, n for i in 0 to n { let best_k: i32 = _kmeans_nearest_centroid(x_data, i * x_stride, centroids, n_clusters, n_features, best_out) labels[i] = best_k - small_inertia = small_inertia + best_out[0] + if compute_inertia { + small_inertia = small_inertia + best_out[0] + } } return small_inertia } @@ -158,7 +176,7 @@ function _kmeans_assign(X: Matrix, centroids: ptr >, n_clusters: i32, n let mut inertia: f32 = 0.0 for i in 0 to n { let xbase: i32 = i * x_stride - let xs: f32 = blas_dot(n_features, x_data + xbase, x_data + xbase) + let xs: f32 = scratch.xnorm[i] let rbase: i32 = i * n_clusters let mut best: f32 = cnorm[0] + cross[rbase] let mut best_k: i32 = 0 @@ -177,33 +195,38 @@ function _kmeans_assign(X: Matrix, centroids: ptr >, n_clusters: i32, n } let guard: f32 = (xs + cnorm_max) * 0.0001 + 0.000001 - let mut best_dist: f32 = 0.0 - if second - best > guard { - best_dist = _kmeans_sqdist(x_data, xbase, centroids[best_k], n_features) - } else { + if second - best <= guard { best_k = _kmeans_nearest_centroid(x_data, xbase, centroids, n_clusters, n_features, best_out) - best_dist = best_out[0] } labels[i] = best_k - inertia = inertia + best_dist + if compute_inertia { + let crow: ptr = centroids[best_k] + let mut d_exact: f32 = 0.0 + for j in 0 to n_features { + let diff: f32 = x_data[xbase + j] - crow[j] + d_exact = d_exact + diff * diff + } + inertia = inertia + d_exact + } } return inertia } function _kmeans_run(X: Matrix, centroids: ptr >, n_clusters: i32, max_iter: i32, tol: f32) -> KMeans { let n_features: i32 = X.cols + let n: i32 = X.rows - let labels: ptr = malloc((X.rows as i64) * 4) as ptr + let labels: ptr = malloc((n as i64) * 4) as ptr let mut inertia: f32 = 0.0 let mut n_iter_run: i32 = 0 - let scratch: KMeansScratch = _kmeans_scratch_new(X.rows, n_clusters, n_features) + let scratch: KMeansScratch = _kmeans_scratch_new(X, n_clusters, n_features) let sums: ptr = scratch.sums let counts: ptr = scratch.counts let x_data: ptr = X.data for iter in 0 to max_iter { n_iter_run = iter + 1 - let new_inertia: f32 = _kmeans_assign(X, centroids, n_clusters, n_features, labels, scratch) + _kmeans_assign(X, centroids, n_clusters, n_features, labels, scratch, false) for idx in 0 to n_clusters * n_features { sums[idx] = 0.0 @@ -211,46 +234,35 @@ function _kmeans_run(X: Matrix, centroids: ptr >, n_clusters: i32, max_ for k in 0 to n_clusters { counts[k] = 0 } - - for i in 0 to X.rows { + for i in 0 to n { let k: i32 = labels[i] counts[k] = counts[k] + 1 - let sbase: i32 = k * n_features - let xbase: i32 = i * n_features - for j in 0 to n_features { - sums[sbase + j] = sums[sbase + j] + x_data[xbase + j] - } + let srow: ptr = sums + k * n_features + let xrow: ptr = x_data + i * n_features + cblas_saxpy(n_features, 1.0, xrow, 1, srow, 1) } let mut max_shift: f32 = 0.0 for k in 0 to n_clusters { let crow: ptr = centroids[k] - let sbase: i32 = k * n_features if counts[k] > 0 { + let sbase: i32 = k * n_features let count_f: f32 = counts[k] as f32 for j in 0 to n_features { let new_val: f32 = sums[sbase + j] / count_f let shift: f32 = new_val - crow[j] if shift < 0.0 { let abs_shift: f32 = 0.0 - shift - if abs_shift > max_shift { - max_shift = abs_shift - } + if abs_shift > max_shift { max_shift = abs_shift } } else { - if shift > max_shift { - max_shift = shift - } + if shift > max_shift { max_shift = shift } } crow[j] = new_val } } else { - # Empty cluster: relocate centroid to the point with the - # largest distance to its assigned centroid. - # Addresses sklearn #34074: KMeans can skip final label - # reassignment after empty-cluster relocation. let mut farthest_idx: i32 = 0 let mut farthest_dist: f32 = -1.0 - for i in 0 to X.rows { + for i in 0 to n { let dist: f32 = _kmeans_sqdist(x_data, i * n_features, centroids[labels[i]], n_features) if dist > farthest_dist { farthest_dist = dist @@ -263,24 +275,28 @@ function _kmeans_run(X: Matrix, centroids: ptr >, n_clusters: i32, max_ let shift: f32 = new_val - crow[j] if shift < 0.0 { let abs_shift: f32 = 0.0 - shift - if abs_shift > max_shift { - max_shift = abs_shift - } + if abs_shift > max_shift { max_shift = abs_shift } } else { - if shift > max_shift { - max_shift = shift - } + if shift > max_shift { max_shift = shift } } crow[j] = new_val } } } - inertia = new_inertia + if max_shift < tol { break } + } - if max_shift < tol { - break + # Compute exact final inertia + for i in 0 to n { + let xbase: i32 = i * n_features + let crow: ptr = centroids[labels[i]] + let mut d_exact: f32 = 0.0 + for j in 0 to n_features { + let diff: f32 = x_data[xbase + j] - crow[j] + d_exact = d_exact + diff * diff } + inertia = inertia + d_exact } _kmeans_scratch_free(scratch) @@ -338,15 +354,56 @@ function _kmeans_sk_pp_init(X: Matrix, n_clusters: i32, rng: ptr, n_local_t let cand_dist: ptr = array_new_f32(n) let best_dist: ptr = array_new_f32(n) + # Distance from every point to one candidate centre, which is the whole + # cost of this routine: n_local_trials candidates per centre, k - 1 centres + # per restart, n_init restarts. The direct sum of squared differences is a + # scalar reduction over n_features that clang will not vectorise, because + # reordering a floating-point reduction is not a legal transform without + # fast-math. + # + # The squared-euclidean expansion turns the cross term into one sgemv: + # ||x - c||^2 = ||x||^2 + ||c||^2 - 2 * (x . c) + # The point norms do not depend on the candidate, so they are taken once + # for the whole init. This is also what scikit-learn's own _kmeans_plusplus + # does, through _euclidean_distances with a precomputed X_norm_squared, so + # the expansion is the closer match to the reference arithmetic rather than + # a departure from it. + # + # Below the work threshold the direct loop still runs, bit for bit as + # before, because the sgemv dispatch costs more than the whole reduction on + # a small problem. The gate is the same rows * features product used by the + # rest of this module. + let use_blas_pp: bool = (n as i64) * (n_features as i64) >= 65536 + let mut xnorm: ptr = null + let mut cross: ptr = null + if use_blas_pp { + xnorm = array_new_f32(n) + cross = array_new_f32(n) + for i in 0 to n { + let xrow: ptr = x_data + i * n_features + let mut sq: f32 = 0.0 + for j in 0 to n_features { + sq = sq + xrow[j] * xrow[j] + } + xnorm[i] = sq + } + } + let first: i32 = mt19937_choice_uniform(rng, n) let fbase: i32 = first * n_features for j in 0 to n_features { centroids[0][j] = x_data[fbase + j] } + let c0: ptr = centroids[0] let mut pot: f32 = 0.0 for i in 0 to n { - let d: f32 = _kmeans_sqdist(x_data, i * n_features, centroids[0], n_features) + let xrow: ptr = x_data + i * n_features + let mut d: f32 = 0.0 + for j in 0 to n_features { + let diff: f32 = xrow[j] - c0[j] + d = d + diff * diff + } closest[i] = d pot = pot + d } @@ -375,14 +432,39 @@ function _kmeans_sk_pp_init(X: Matrix, n_clusters: i32, rng: ptr, n_local_t let cbase: i32 = cand * n_features let crow: ptr = x_data + cbase let mut cand_pot: f32 = 0.0 - for i in 0 to n { - let d: f32 = _kmeans_sqdist(x_data, i * n_features, crow, n_features) - let mut v: f32 = d - if closest[i] < v { - v = closest[i] + if use_blas_pp { + blas_matvec_ld(x_data, n, n_features, n_features, crow, cross, 1.0, 0.0) + let mut cnorm: f32 = 0.0 + for j in 0 to n_features { + cnorm = cnorm + crow[j] * crow[j] + } + # The expansion can go slightly negative for a point sitting on + # the candidate, which is the point that was just drawn. + for i in 0 to n { + let mut d: f32 = xnorm[i] + cnorm - 2.0 * cross[i] + if d < 0.0 { d = 0.0 } + let mut v: f32 = d + if closest[i] < v { + v = closest[i] + } + cand_dist[i] = v + cand_pot = cand_pot + v + } + } else { + for i in 0 to n { + let xrow: ptr = x_data + i * n_features + let mut d: f32 = 0.0 + for j in 0 to n_features { + let diff: f32 = xrow[j] - crow[j] + d = d + diff * diff + } + let mut v: f32 = d + if closest[i] < v { + v = closest[i] + } + cand_dist[i] = v + cand_pot = cand_pot + v } - cand_dist[i] = v - cand_pot = cand_pot + v } # np.argmin keeps the first minimum, so only a strict improvement @@ -409,24 +491,27 @@ function _kmeans_sk_pp_init(X: Matrix, n_clusters: i32, rng: ptr, n_local_t array_free_f32(closest) array_free_f32(cand_dist) array_free_f32(best_dist) + if use_blas_pp { + array_free_f32(xnorm) + array_free_f32(cross) + } return centroids } -# Run a single Lloyd iteration starting from given centroids. -function _kmeans_lloyd(X: Matrix, centroids: ptr >, n_clusters: i32, max_iter: i32, tol: f32) -> KMeans { +# Run a single Lloyd iteration starting from given centroids with provided scratch. +function _kmeans_lloyd_scratch(X: Matrix, centroids: ptr >, n_clusters: i32, max_iter: i32, tol: f32, scratch: KMeansScratch) -> KMeans { let n: i32 = X.rows let n_features: i32 = X.cols let labels: ptr = malloc((n as i64) * 4) as ptr let mut inertia: f32 = 0.0 let mut n_iter_run: i32 = 0 - let scratch: KMeansScratch = _kmeans_scratch_new(n, n_clusters, n_features) let sums: ptr = scratch.sums let counts: ptr = scratch.counts let x_data: ptr = X.data for iter in 0 to max_iter { n_iter_run = iter + 1 - let new_inertia: f32 = _kmeans_assign(X, centroids, n_clusters, n_features, labels, scratch) + _kmeans_assign(X, centroids, n_clusters, n_features, labels, scratch, false) for idx in 0 to n_clusters * n_features { sums[idx] = 0.0 @@ -437,17 +522,15 @@ function _kmeans_lloyd(X: Matrix, centroids: ptr >, n_clusters: i32, ma for i in 0 to n { let k: i32 = labels[i] counts[k] = counts[k] + 1 - let sbase: i32 = k * n_features - let xbase: i32 = i * n_features - for j in 0 to n_features { - sums[sbase + j] = sums[sbase + j] + x_data[xbase + j] - } + let srow: ptr = sums + k * n_features + let xrow: ptr = x_data + i * n_features + cblas_saxpy(n_features, 1.0, xrow, 1, srow, 1) } let mut max_shift: f32 = 0.0 for k in 0 to n_clusters { + let crow: ptr = centroids[k] if counts[k] > 0 { - let crow: ptr = centroids[k] let sbase: i32 = k * n_features let count_f: f32 = counts[k] as f32 for j in 0 to n_features { @@ -461,14 +544,47 @@ function _kmeans_lloyd(X: Matrix, centroids: ptr >, n_clusters: i32, ma } crow[j] = new_val } + } else { + # Empty cluster: relocate centroid to the point with the + # largest distance to its assigned centroid. + let mut farthest_idx: i32 = 0 + let mut farthest_dist: f32 = -1.0 + for i in 0 to n { + let dist: f32 = _kmeans_sqdist(x_data, i * n_features, centroids[labels[i]], n_features) + if dist > farthest_dist { + farthest_dist = dist + farthest_idx = i + } + } + let fbase: i32 = farthest_idx * n_features + for j in 0 to n_features { + let new_val: f32 = x_data[fbase + j] + let shift: f32 = new_val - crow[j] + if shift < 0.0 { + let abs_shift: f32 = 0.0 - shift + if abs_shift > max_shift { max_shift = abs_shift } + } else { + if shift > max_shift { max_shift = shift } + } + crow[j] = new_val + } } } - inertia = new_inertia if max_shift < tol { break } } - _kmeans_scratch_free(scratch) + # Compute exact final inertia + for i in 0 to n { + let xbase: i32 = i * n_features + let crow: ptr = centroids[labels[i]] + let mut d_exact: f32 = 0.0 + for j in 0 to n_features { + let diff: f32 = x_data[xbase + j] - crow[j] + d_exact = d_exact + diff * diff + } + inertia = inertia + d_exact + } return KMeans { centroids: centroids, @@ -483,6 +599,14 @@ function _kmeans_lloyd(X: Matrix, centroids: ptr >, n_clusters: i32, ma } } +# Run a single Lloyd iteration starting from given centroids. +function _kmeans_lloyd(X: Matrix, centroids: ptr >, n_clusters: i32, max_iter: i32, tol: f32) -> KMeans { + let scratch: KMeansScratch = _kmeans_scratch_new(X, n_clusters, X.cols) + let result: KMeans = _kmeans_lloyd_scratch(X, centroids, n_clusters, max_iter, tol, scratch) + _kmeans_scratch_free(scratch) + return result +} + # KMeans with k-means++ initialization and n_init restarts. # Matches sklearn's init='k-means++' with the given n_init: one MT19937 stream # seeded from `seed` is consumed across all restarts, exactly as sklearn's @@ -499,10 +623,11 @@ export function kmeans_fit_plus_plus(X: Matrix, n_clusters: i32, max_iter: i32, let rng: ptr = mt19937_new(seed) let n_local_trials: i32 = _kmeans_sk_pp_trials(n_clusters) + let scratch: KMeansScratch = _kmeans_scratch_new(X, n_clusters, n_features) for init in 0 to n_init { let centroids: ptr > = _kmeans_sk_pp_init(X, n_clusters, rng, n_local_trials) - let result: KMeans = _kmeans_lloyd(X, centroids, n_clusters, max_iter, tol) + let result: KMeans = _kmeans_lloyd_scratch(X, centroids, n_clusters, max_iter, tol, scratch) if result.inertia < best_inertia { best_inertia = result.inertia @@ -525,6 +650,7 @@ export function kmeans_fit_plus_plus(X: Matrix, n_clusters: i32, max_iter: i32, free(result.labels as ptr) } + _kmeans_scratch_free(scratch) mt19937_free(rng) return KMeans { diff --git a/lib/scikit/ensemble.flow b/lib/scikit/ensemble.flow index 1a13160..a6c3259 100644 --- a/lib/scikit/ensemble.flow +++ b/lib/scikit/ensemble.flow @@ -11,6 +11,75 @@ import "lib/scikit/prng.flow" extern { function sqrt(x: f64) -> f64 + # Runs tasks 0..n_tasks across threads: GCD on macOS, pthreads elsewhere. + # Implemented in lib/scikit/flow_parallel.c, which must be linked. + function flow_parallel_for(n_tasks: i32, callback: ptr, ctx: ptr) -> void +} + +# Everything one worker needs to fit one tree. The forest fills this once and +# every task reads it; only trees[t], seeds[t] and this task's own scratch are +# written, and each task owns a distinct t. +struct RFForestJob { + X: Matrix, + y: ptr, + trees: ptr, + boot_idx: ptr, + seeds: ptr, + bin_card: ptr, + bin_vals: ptr, + bin_of: ptr, + categorical: ptr, + n_samples: i32, + n_cols: i32, + n_classes: i32, + max_depth: i32, + max_features: i32 +} + +# Fits tree `idx`. Its bootstrap row indices and feature seed were drawn on one +# thread before any of this ran, so the forest does not depend on the order the +# tasks happen to complete in. +# +# The scratch is allocated here rather than shared, because two trees fitting +# at once would otherwise write the same buffers. It is one allocation set per +# tree, not per node, so it does not show up against the fit. +function rf_fit_one_tree(ctx: ptr, idx: i64) -> void { + let job: ptr = ctx as ptr + let t: i32 = idx as i32 + let n_samples: i32 = job.n_samples + let n_cols: i32 = job.n_cols + let row_bytes: i64 = (n_cols as i64) * 4 + + let X_boot: Matrix = matrix_new(n_samples, n_cols) + let y_boot: ptr = array_new_f32(n_samples) + let boot_bins: ptr = malloc((n_cols as i64) * (n_samples as i64)) as ptr + let prng_state: ptr = malloc(4) as ptr + prng_state[0] = job.seeds[t] + + let rows: ptr = job.boot_idx + t * n_samples + let x_data: ptr = job.X.data + let boot_data: ptr = X_boot.data + let mut dst: i32 = 0 + for i in 0 to n_samples { + let src: i32 = rows[i] + y_boot[i] = job.y[src] + memcpy((boot_data + dst) as ptr, (x_data + src * n_cols) as ptr, row_bytes) + dst = dst + n_cols + } + for fcol in 0 to n_cols { + let src_col: ptr = job.bin_of + fcol * n_samples + let dst_col: ptr = boot_bins + fcol * n_samples + for i in 0 to n_samples { + dst_col[i] = src_col[rows[i]] + } + } + + job.trees[t] = decision_tree_classifier_fit_rf_binned(X_boot, y_boot, job.n_classes, job.max_depth, CRITERION_GINI, job.max_features, prng_state, job.categorical, job.bin_card, job.bin_vals, boot_bins) + + free(prng_state as ptr) + free(boot_bins as ptr) + array_free_f32(y_boot) + matrix_free(X_boot) } extern { @@ -110,54 +179,74 @@ export function random_forest_classifier_fit_categorical(X: Matrix, y: ptr, let max_features: i32 = (max_features_f64 as i32) if max_features < 1 { max_features = 1 } - # Shared PRNG state pointer for the RF tree builder - let prng_state: ptr = malloc(4) as ptr - - # Bootstrap scratch, allocated once and reused by every tree. The old loop - # allocated and freed the index array, the bootstrap design matrix and the - # bootstrap target vector on each iteration. Every tree overwrites all three - # in full before reading them, so reuse is safe. + # Every tree's bootstrap rows and feature seed are drawn here, on one + # thread, before any tree is fitted. # - # The draw order is deliberately untouched: n_samples indices are drawn for - # the bootstrap, then one draw seeds this tree's feature subsampling, in that - # order, exactly as before. Issue #205 owns the sampling scheme itself. + # The draw order is deliberately untouched: n_samples indices for the + # bootstrap, then one draw to seed that tree's feature subsampling, per + # tree, in that order, exactly as the sequential loop did. Issue #205 owns + # the sampling scheme itself. Drawing up front is what lets the fits run + # concurrently without the forest depending on which tree finishes first. let n_samples: i32 = X.rows let n_cols: i32 = X.cols - let row_bytes: i64 = (n_cols as i64) * 4 - let bootstrap_indices: ptr = malloc((n_samples as i64) * 4) as ptr - let X_boot: Matrix = matrix_new(n_samples, n_cols) - let y_boot: ptr = array_new_f32(n_samples) - let x_data: ptr = X.data - let boot_data: ptr = X_boot.data + let bootstrap_indices: ptr = malloc((n_trees as i64) * (n_samples as i64) * 4) as ptr + let tree_seeds: ptr = malloc((n_trees as i64) * 4) as ptr + + # Bin the design once for the whole forest. + # + # The split search needs each candidate feature's class histogram per + # distinct value. Sorting the node's samples produces that per feature per + # node; binning produces it from one linear pass. The bins depend only on + # the values a column holds, and a bootstrap sample draws from those same + # values, so one binning of X serves every tree. Doing it per tree would + # cost more than the sort it replaces. + # + # Each tree's rows are then a gather of bin indices, the same shape as the + # row copy above it. + let bin_card: ptr = malloc((n_cols as i64) * 4) as ptr + let bin_vals: ptr = array_new_f32(tree_bins_values_len(n_cols)) + let bin_of: ptr = malloc((n_cols as i64) * (n_samples as i64)) as ptr + tree_build_bins(X, bin_card, bin_vals, bin_of) for t in 0 to n_trees { + let tree_rows: ptr = bootstrap_indices + t * n_samples for i in 0 to n_samples { let res: PRNGResult = prng_int(prng, n_samples) prng = PRNG { state: res.state } - bootstrap_indices[i] = res.value + tree_rows[i] = res.value } - - # Row-major memcpy instead of n_samples * n_cols accessor call pairs. - let mut dst: i32 = 0 - for i in 0 to n_samples { - let src: i32 = bootstrap_indices[i] - y_boot[i] = y[src] - memcpy((boot_data + dst) as ptr, (x_data + src * n_cols) as ptr, row_bytes) - dst = dst + n_cols - } - - # Set PRNG state for this tree's feature subsampling let res: PRNGResult = prng_int(prng, 2147483647) prng = PRNG { state: res.state } - prng_state[0] = res.state - - trees[t] = decision_tree_classifier_fit_rf_categorical(X_boot, y_boot, n_classes, max_depth, CRITERION_GINI, max_features, prng_state, categorical) - } - + tree_seeds[t] = res.state + } + + # One task per tree. Trees are independent once their rows and seed are + # drawn: each writes only its own slot in `trees` and its own scratch, and + # reads the shared bins without touching them. With one task the dispatcher + # runs it inline, so a single-tree forest pays nothing for this. + let job: ptr = malloc(256) as ptr + job.X = X + job.y = y + job.trees = trees + job.boot_idx = bootstrap_indices + job.seeds = tree_seeds + job.bin_card = bin_card + job.bin_vals = bin_vals + job.bin_of = bin_of + job.categorical = categorical + job.n_samples = n_samples + job.n_cols = n_cols + job.n_classes = n_classes + job.max_depth = max_depth + job.max_features = max_features + flow_parallel_for(n_trees, rf_fit_one_tree as ptr, job as ptr) + free(job as ptr) + + free(bin_card as ptr) + array_free_f32(bin_vals) + free(bin_of as ptr) free(bootstrap_indices as ptr) - matrix_free(X_boot) - array_free_f32(y_boot) - free(prng_state as ptr) + free(tree_seeds as ptr) return RandomForestClassifier { trees: trees, diff --git a/lib/scikit/flow_time.c b/lib/scikit/flow_time.c new file mode 100644 index 0000000..d7aff00 --- /dev/null +++ b/lib/scikit/flow_time.c @@ -0,0 +1,21 @@ +/* flow_time.c - monotonic nanosecond clock for Flow timing harnesses. + * + * Harnesses used to declare `timespec_get` and `TIME_UTC` directly in Flow. + * Flow emits its own prototype for an extern, so the generated C redeclared a + * libc function with `int64_t *` where the SDK has `struct timespec *`, and + * declared a constant whose name is already a libc macro. Clang 17 rejects + * both. Routing through a shim keeps the Flow side free of libc names. + * + * CLOCK_MONOTONIC replaces the old realtime clock. Interval timing should not + * be able to move backwards when the system clock is stepped. + */ +#include +#include + +int64_t flow_now_ns(void) { + struct timespec ts; + if (clock_gettime(CLOCK_MONOTONIC, &ts) != 0) { + return 0; + } + return (int64_t)ts.tv_sec * 1000000000LL + (int64_t)ts.tv_nsec; +} diff --git a/lib/scikit/kernel_ridge.flow b/lib/scikit/kernel_ridge.flow index 8d0e933..5085551 100644 --- a/lib/scikit/kernel_ridge.flow +++ b/lib/scikit/kernel_ridge.flow @@ -3,6 +3,7 @@ # Supports linear, polynomial, and RBF kernels. import "lib/scikit/matrix.flow" +import "lib/scikit/blas.flow" export struct KernelRidge { alpha: f32, @@ -67,17 +68,14 @@ function tanh_f32(x: f32) -> f32 { function compute_kernel_matrix(X: Matrix, kernel_type: i32, gamma: f32, degree: i32, coef0: f32) -> Matrix { let n: i32 = X.rows + let p: i32 = X.cols let K: Matrix = matrix_new(n, n) for i in 0 to n { - let xi: ptr = array_new_f32(X.cols) - matrix_row(X, i, xi) + let xi: ptr = X.data + i * p for j in 0 to n { - let xj: ptr = array_new_f32(X.cols) - matrix_row(X, j, xj) - matrix_set(K, i, j, kernel_compute(xi, xj, X.cols, kernel_type, gamma, degree, coef0)) - array_free_f32(xj) + let xj: ptr = X.data + j * p + matrix_set(K, i, j, kernel_compute(xi, xj, p, kernel_type, gamma, degree, coef0)) } - array_free_f32(xi) } return K } @@ -88,21 +86,39 @@ export function kernel_ridge_fit(X: Matrix, y: ptr, alpha: f32, kernel_type let gamma_d: f64 = (gamma) as f64 # Build K + alpha*I directly in f64 for maximum precision. - # For RBF kernel (kernel_type == 1), compute in f64 directly. + # For RBF kernel (kernel_type == 2), compute in f64 directly with SGEMM cross terms. # For other kernels, fall back to f32 compute_kernel_matrix. let K_reg: ptr = array_new_f64(n * n) - if kernel_type == 1 { + if kernel_type == 2 || kernel_type == KR_KERNEL_RBF { + let x_data: ptr = X.data + let x_sq: ptr = array_new_f32(n) for i in 0 to n { - for j in 0 to n { - let mut sq_dist: f64 = 0.0 - for f in 0 to n_features { - let d: f64 = (matrix_at(X, i, f) - matrix_at(X, j, f)) as f64 - sq_dist = sq_dist + d * d - } - K_reg[i * n + j] = _kr_exp_f64(0.0 - gamma_d * sq_dist) + let r: ptr = x_data + i * n_features + let mut s: f32 = 0.0 + for f in 0 to n_features { + let v: f32 = r[f] + s = s + v * v } - K_reg[i * n + i] = K_reg[i * n + i] + (alpha as f64) + x_sq[i] = s } + + let cross: ptr = malloc((n as i64) * (n as i64) * 4) as ptr + cblas_sgemm(101, 111, 112, n, n, n_features, 1.0, x_data, n_features, x_data, n_features, 0.0, cross, n) + + for i in 0 to n { + K_reg[i * n + i] = 1.0 + (alpha as f64) + let xi_sq: f32 = x_sq[i] + let cross_row: ptr = cross + i * n + for j in i + 1 to n { + let mut sq: f64 = (xi_sq + x_sq[j] - 2.0 * cross_row[j]) as f64 + if sq < 0.0 { sq = 0.0 } + let k_val: f64 = _kr_exp_f64(0.0 - gamma_d * sq) + K_reg[i * n + j] = k_val + K_reg[j * n + i] = k_val + } + } + free(cross as ptr) + array_free_f32(x_sq) } else { let K: Matrix = compute_kernel_matrix(X, kernel_type, gamma, degree, coef0) for i in 0 to n { @@ -114,56 +130,30 @@ export function kernel_ridge_fit(X: Matrix, y: ptr, alpha: f32, kernel_type matrix_free(K) } - # Cholesky decomposition in f64: L * L^T = K + alpha*I - let L: ptr = array_new_f64(n * n) - for i in 0 to n { - for j in 0 to i + 1 { - let mut sum: f64 = K_reg[i * n + j] - for k in 0 to j { - sum = sum - L[i * n + k] * L[j * n + k] - } - if i == j { - if sum < 0.000000000000001 { sum = 0.000000000000001 } - L[i * n + j] = sqrt(sum) - } else { - if L[j * n + j] < 0.000000000000001 { - L[i * n + j] = 0.0 - } else { - L[i * n + j] = sum / L[j * n + j] - } - } - } - } - - # Forward substitution: L * z = y (f64) - let z: ptr = array_new_f64(n) - for i in 0 to n { - let mut sum: f64 = (y[i]) as f64 - for j in 0 to i { - sum = sum - L[i * n + j] * z[j] - } - if L[i * n + i] < 0.000000000000001 { - z[i] = 0.0 - } else { - z[i] = sum / L[i * n + i] - } - } + # Cholesky decomposition in f64 via LAPACK dpotrf_: L * L^T = K + alpha*I + # Passing 'U' to Fortran dpotrf_ on a row-major matrix stores the lower + # Cholesky factor L in-place in the lower triangle of K_reg. + let uplo: ptr = "U" + let n_ptr: ptr = malloc(4) as ptr + let lda_ptr: ptr = malloc(4) as ptr + let info_ptr: ptr = malloc(4) as ptr + n_ptr[0] = n + lda_ptr[0] = n + info_ptr[0] = 0 + dpotrf_(uplo, n_ptr, K_reg, lda_ptr, info_ptr) + free(n_ptr as ptr) + free(lda_ptr as ptr) + free(info_ptr as ptr) - # Backward substitution: L^T * dual_coef = z (f64) + # Solve (L * L^T) * dual_coef = y via two in-place triangular solves: + # 1. Forward substitution: L * z = y + # 2. Backward substitution: L^T * dual_coef = z let dual_coef_d: ptr = array_new_f64(n) - let mut i: i32 = n - 1 - while i >= 0 { - let mut sum: f64 = z[i] - for j in i + 1 to n { - sum = sum - L[j * n + i] * dual_coef_d[j] - } - if L[i * n + i] < 0.000000000000001 { - dual_coef_d[i] = 0.0 - } else { - dual_coef_d[i] = sum / L[i * n + i] - } - i = i - 1 + for i in 0 to n { + dual_coef_d[i] = y[i] as f64 } + cblas_dtrsv(101, 122, 111, 131, n, K_reg, n, dual_coef_d, 1) + cblas_dtrsv(101, 122, 112, 131, n, K_reg, n, dual_coef_d, 1) # Convert dual_coef to f32 for storage let dual_coef: ptr = array_new_f32(n) @@ -171,8 +161,6 @@ export function kernel_ridge_fit(X: Matrix, y: ptr, alpha: f32, kernel_type dual_coef[i] = dual_coef_d[i] as f32 } - array_free_f64(L) - array_free_f64(z) array_free_f64(K_reg) array_free_f64(dual_coef_d) @@ -199,40 +187,71 @@ function _kr_exp_f64(x: f64) -> f64 { export function kernel_ridge_predict(model: KernelRidge, X: Matrix) -> ptr { let n: i32 = X.rows + let nt: i32 = model.n_samples + let nf: i32 = model.n_features let result: ptr = array_new_f32(n) let gamma_d: f64 = (model.gamma) as f64 - if model.kernel_type == 1 { - # RBF kernel in f64 for precision + if model.kernel_type == 2 || model.kernel_type == KR_KERNEL_RBF { + let x_data: ptr = X.data + let xt_data: ptr = model.X_train.data + + let x_sq: ptr = array_new_f32(n) + for i in 0 to n { + let r: ptr = x_data + i * nf + let mut s: f32 = 0.0 + for f in 0 to nf { + let v: f32 = r[f] + s = s + v * v + } + x_sq[i] = s + } + + let xt_sq: ptr = array_new_f32(nt) + for j in 0 to nt { + let r: ptr = xt_data + j * nf + let mut s: f32 = 0.0 + for f in 0 to nf { + let v: f32 = r[f] + s = s + v * v + } + xt_sq[j] = s + } + + let cross: ptr = malloc((n as i64) * (nt as i64) * 4) as ptr + cblas_sgemm(101, 111, 112, n, nt, nf, 1.0, x_data, nf, xt_data, nf, 0.0, cross, nt) + + let dual_coef_d: ptr = array_new_f64(nt) + for j in 0 to nt { + dual_coef_d[j] = (model.dual_coef[j]) as f64 + } + for i in 0 to n { + let xi_sq: f32 = x_sq[i] + let cross_row: ptr = cross + i * nt let mut sum: f64 = 0.0 - for j in 0 to model.n_samples { - let dc: f64 = (model.dual_coef[j]) as f64 - if fabs(dc) < 0.0000000001 { continue } - let mut sq_dist: f64 = 0.0 - for f in 0 to model.n_features { - let d: f64 = (matrix_at(X, i, f) - matrix_at(model.X_train, j, f)) as f64 - sq_dist = sq_dist + d * d - } - let k_val: f64 = _kr_exp_f64(0.0 - gamma_d * sq_dist) - sum = sum + dc * k_val + for j in 0 to nt { + let mut sq: f64 = (xi_sq + xt_sq[j] - 2.0 * cross_row[j]) as f64 + if sq < 0.0 { sq = 0.0 } + sum = sum + dual_coef_d[j] * _kr_exp_f64(0.0 - gamma_d * sq) } result[i] = sum as f32 } + + array_free_f64(dual_coef_d) + free(cross as ptr) + array_free_f32(x_sq) + array_free_f32(xt_sq) } else { for i in 0 to n { - let xi: ptr = array_new_f32(X.cols) - matrix_row(X, i, xi) + let xi: ptr = X.data + i * model.n_features let mut sum: f32 = 0.0 for j in 0 to model.n_samples { - let xj: ptr = array_new_f32(model.n_features) - matrix_row(model.X_train, j, xj) + let xj: ptr = model.X_train.data + j * model.n_features let k: f32 = kernel_compute(xi, xj, model.n_features, model.kernel_type, model.gamma, model.degree, model.coef0) sum = sum + model.dual_coef[j] * k - array_free_f32(xj) } result[i] = sum - array_free_f32(xi) } } diff --git a/lib/scikit/linear.flow b/lib/scikit/linear.flow index cb3c7cd..647bea3 100644 --- a/lib/scikit/linear.flow +++ b/lib/scikit/linear.flow @@ -13,6 +13,8 @@ import "lib/scikit/blas.flow" extern { function sqrt(x: f64) -> f64 function log(x: f64) -> f64 + function expf(x: f32) -> f32 + function logf(x: f32) -> f32 } function sqrt_f32(x: f32) -> f32 { @@ -904,7 +906,8 @@ export const LOGISTIC_SOLVER_LBFGS: i32 = 0 export const LOGISTIC_SOLVER_NEWTON: i32 = 1 export const LOGISTIC_SOLVER_IRLS: i32 = 1 -# Issue #433: turns one row of logits into softmax probabilities in place. +# Issue #433: turns an m x nc block of logits into softmax probabilities in +# place. # # The row maximum is subtracted before exponentiating. Raw logits pass 300 # whenever the gradient at the origin is small enough that LBFGS takes a large @@ -913,26 +916,59 @@ export const LOGISTIC_SOLVER_IRLS: i32 = 1 # largest exponent at exactly 1.0 and cannot overflow for any finite input. # tests/test_multinomial_logistic.flow pins a fixture that reaches that range. # -# The exp() call lives here rather than at the call site because the transpiler -# miscompiles exp() inside a nested while loop (compiler bug #421). The name -# carries the module and issue number because non-exported functions sharing a -# name across modules collide in the generated C (compiler bug #465). -function lin433_softmax_row(row: ptr, nc: i32) -> void { - if nc <= 0 { return } - let mut max_val: f32 = row[0] - for c in 1 to nc { - if row[c] > max_val { max_val = row[c] } +# The exponential lives in this function rather than at the call site because +# the transpiler miscompiles exp() inside a nested while loop (compiler bug +# #421). The name carries the module and issue number because non-exported +# functions sharing a name across modules collide in the generated C (compiler +# bug #465). +# +# This was a per-row helper called once per sample from the LBFGS loop, which +# on digits is 12570 exponentials per iteration for up to 100 iterations, and +# measured at half the whole fit. Two changes take that down. +# +# The exponential is expf rather than exp on a widened f64 argument. The result +# was narrowed straight back to f32, so the extra precision never reached the +# caller; it only bought a slower libm call. +# +# The shifts, the exponentials and the reciprocal scaling run as three passes +# over the block rather than interleaved per row. The exponential pass is a +# flat contiguous loop over m * nc floats carrying no reduction, which is the +# shape clang vectorises and the shape -fveclib can map onto a vector exp. The +# row maxima and the row sums stay in their own passes because they are +# reductions of width nc. +# +# The arithmetic per element is unchanged: subtract the row max, exponentiate, +# divide by the row sum, in that order. +function lin433_softmax_block(Z: ptr, m: i32, nc: i32) -> void { + if m <= 0 || nc <= 0 { return } + let total: i32 = m * nc + + for i in 0 to m { + let base: i32 = i * nc + let mut max_val: f32 = Z[base] + for c in 1 to nc { + if Z[base + c] > max_val { max_val = Z[base + c] } + } + for c in 0 to nc { + Z[base + c] = Z[base + c] - max_val + } } - let mut sum: f32 = 0.0 - for c in 0 to nc { - let e: f32 = exp((row[c] - max_val) as f64) as f32 - row[c] = e - sum = sum + e + + for idx in 0 to total { + Z[idx] = expf(Z[idx]) } - if sum > 0.0 { - let inv: f32 = 1.0 / sum + + for i in 0 to m { + let base: i32 = i * nc + let mut sum: f32 = 0.0 for c in 0 to nc { - row[c] = row[c] * inv + sum = sum + Z[base + c] + } + if sum > 0.0 { + let inv: f32 = 1.0 / sum + for c in 0 to nc { + Z[base + c] = Z[base + c] * inv + } } } } @@ -958,6 +994,47 @@ function lin433_softmax_row(row: ptr, nc: i32) -> void { # Returns weights in the same nc x n row-major layout the caller expects, so # logistic_predict and logistic_decide need no change: argmax over the softmax # is argmax over the linear scores. +# Mean cross-entropy of the softmax fit plus the L2 term, which is the +# objective lin433_multinomial_fit descends. +# +# Z is overwritten with the probabilities at theta, which is exactly what the +# gradient step needs next, so an accepted line-search evaluation feeds +# straight into the gradient and costs nothing extra. Only a rejected step +# repeats work. +# +# The probability is floored before the log. Softmax subtracts the row maximum, +# so a probability can underflow to zero for a class whose logit is far below +# the best one, and log(0) would poison the whole objective with -inf. +function lin433_multinomial_objective(X_aug: ptr, theta: ptr, tgt: ptr, Z: ptr, m: i32, nc: i32, d: i32, n: i32, l2_coeff: f32, inv_m: f32) -> f32 { + blas_matmat_nt(X_aug, theta, Z, m, nc, d, d, d, 1.0, 0.0) + lin433_softmax_block(Z, m, nc) + + # logf rather than log on a widened argument, for the reason the softmax + # uses expf: the result is narrowed straight back to f32, so the extra + # precision never reaches the caller and only buys a slower libm call. The + # sum itself stays in f64, where the precision does matter. + let mut total: f64 = 0.0 + for i in 0 to m { + let mut p: f32 = Z[i * nc + tgt[i]] + if p < 0.000000000000001 { p = 0.000000000000001 } + total = total - (logf(p) as f64) + } + let mut loss: f32 = ((total * (inv_m as f64)) as f32) + + if l2_coeff > 0.0 { + let mut reg: f64 = 0.0 + for c in 0 to nc { + let base: i32 = c * d + for j in 0 to n { + let w: f64 = theta[base + j] as f64 + reg = reg + w * w + } + } + loss = loss + 0.5 * l2_coeff * (reg as f32) + } + return loss +} + function lin433_multinomial_fit(X: Matrix, y: ptr, classes: ptr, nc: i32, epochs: i32, penalty: Penalty) -> LogisticRegression { let n: i32 = X.cols let m: i32 = X.rows @@ -1001,7 +1078,6 @@ function lin433_multinomial_fit(X: Matrix, y: ptr, classes: ptr, nc: i # Z holds the m x nc logits, then the softmax probabilities, then the # residual p - t, all in the same buffer. let Z: ptr = malloc((m as i64) * (nc as i64) * 4) as ptr - let col: ptr = array_new_f32(m) let mem: i32 = 10 let s_hist: ptr = malloc((mem as i64) * (dim as i64) * 4) as ptr @@ -1024,40 +1100,55 @@ function lin433_multinomial_fit(X: Matrix, y: ptr, classes: ptr, nc: i # Gradient at the starting point. Each later iteration ends by evaluating # the gradient at the theta it stepped to, so the loop carries it forward. - for c in 0 to nc { - blas_matvec(X_aug, m, d, theta + c * d, col, 1.0, 0.0) - for i in 0 to m { - Z[i * nc + c] = col[i] - } - } + blas_matmat_nt(X_aug, theta, Z, m, nc, d, d, d, 1.0, 0.0) + lin433_softmax_block(Z, m, nc) for i in 0 to m { - lin433_softmax_row(Z + i * nc, nc) Z[i * nc + tgt[i]] = Z[i * nc + tgt[i]] - 1.0 } + cblas_sgemm(101, 112, 111, nc, d, m, inv_m, Z, nc, X_aug, d, 0.0, grad, d) for c in 0 to nc { - for i in 0 to m { - col[i] = Z[i * nc + c] - } - blas_matvec_trans(X_aug, m, d, col, grad + c * d, inv_m, 0.0) let base: i32 = c * d for j in 0 to n { grad[base + j] = grad[base + j] + l2_coeff * theta[base + j] } } + # The objective at the current point, carried across iterations so the line + # search compares against it without recomputing. + let mut fobj: f32 = lin433_multinomial_objective(X_aug, theta, tgt, Z, m, nc, d, n, l2_coeff, inv_m) + let mut iter: i32 = 0 let mut converged: bool = false let mut hist_size: i32 = 0 let mut hist_idx: i32 = 0 while iter < max_iter && not converged { + # Stop on the largest gradient component, which is the rule scipy's + # L-BFGS-B applies as pgtol and therefore the rule scikit-learn's + # LogisticRegression stops on, at the same 1e-4 its tol declares. + # + # This tested the L2 norm of the whole gradient instead. Over 650 + # parameters that norm is roughly an order of magnitude above the + # largest single component, so the fit was being held to a far tighter + # standard than the tolerance names: it ran to its 100-iteration cap + # with the norm still falling, while scikit-learn reached its own + # tolerance on the same data in 31 iterations. Both objectives are the + # mean loss with l2 = 1 / (C * n_samples), so the two gradients are on + # the same scale and the same threshold means the same thing. let mut gnorm: f32 = 0.0 for j in 0 to dim { let g: f32 = grad[j] gnorm = gnorm + g * g } gnorm = sqrt_f32(gnorm) - if gnorm < 0.0001 { converged = true; break } + + let mut gmax: f32 = 0.0 + for j in 0 to dim { + let mut ag: f32 = grad[j] + if ag < 0.0 { ag = 0.0 - ag } + if ag > gmax { gmax = ag } + } + if gmax < 0.0001 { converged = true; break } for j in 0 to dim { grad_prev[j] = grad[j] @@ -1112,6 +1203,22 @@ function lin433_multinomial_fit(X: Matrix, y: ptr, classes: ptr, nc: i for j in 0 to dim { direction[j] = -r_lb[j] } } + # Backtracking line search on the Armijo condition. + # + # This used to take the raw LBFGS step of 1.0 with no line search, + # which is why the fit ran to its iteration cap without converging: + # the gradient norm was still falling at iteration 100. scikit-learn + # runs L-BFGS-B with a Wolfe search and reaches its own tolerance in 31 + # iterations on the same data. Accepting only a step that reduces the + # objective by a fraction of what the slope predicts recovers most of + # that, which is both closer to the reference optimizer and less work. + # + # A rejected step costs one objective evaluation. An accepted one costs + # nothing extra, because the evaluation leaves Z holding the + # probabilities at the new theta, which is what the gradient needs. + let mut slope: f32 = 0.0 + for j in 0 to dim { slope = slope + grad[j] * direction[j] } + let mut step: f32 = 1.0 if iter == 0 { let mut gnorm2: f32 = 0.0 @@ -1119,23 +1226,35 @@ function lin433_multinomial_fit(X: Matrix, y: ptr, classes: ptr, nc: i if gnorm2 > 0.0000001 { step = 1.0 / gnorm2 } } - for j in 0 to dim { theta[j] = theta_prev[j] + step * direction[j] } - - for c in 0 to nc { - blas_matvec(X_aug, m, d, theta + c * d, col, 1.0, 0.0) - for i in 0 to m { - Z[i * nc + c] = col[i] + let mut accepted: bool = false + let mut bt: i32 = 0 + while bt < 24 && not accepted { + for j in 0 to dim { theta[j] = theta_prev[j] + step * direction[j] } + let f_new: f32 = lin433_multinomial_objective(X_aug, theta, tgt, Z, m, nc, d, n, l2_coeff, inv_m) + # The NaN test is f_new == f_new, which is false only for NaN. + if f_new == f_new && f_new <= fobj + 0.0001 * step * slope { + fobj = f_new + accepted = true + } else { + step = step * 0.5 } + bt = bt + 1 } + + # No step reduced the objective. The iterate is left where the last + # evaluation put it and the loop stops, rather than stepping to a point + # known to be worse. + if not accepted { + for j in 0 to dim { theta[j] = theta_prev[j] } + converged = true + break + } + for i in 0 to m { - lin433_softmax_row(Z + i * nc, nc) Z[i * nc + tgt[i]] = Z[i * nc + tgt[i]] - 1.0 } + cblas_sgemm(101, 112, 111, nc, d, m, inv_m, Z, nc, X_aug, d, 0.0, grad, d) for c in 0 to nc { - for i in 0 to m { - col[i] = Z[i * nc + c] - } - blas_matvec_trans(X_aug, m, d, col, grad + c * d, inv_m, 0.0) let base: i32 = c * d for j in 0 to n { grad[base + j] = grad[base + j] + l2_coeff * theta[base + j] @@ -1214,7 +1333,6 @@ function lin433_multinomial_fit(X: Matrix, y: ptr, classes: ptr, nc: i free(X_aug as ptr) free(tgt as ptr) free(Z as ptr) - array_free_f32(col) array_free_f32(theta) array_free_f32(grad) array_free_f32(grad_prev) @@ -1424,6 +1542,25 @@ function lin357_newton_fit(X: Matrix, y: ptr, classes: ptr, nc: i32, e let hess: ptr = malloc((d as i64) * (d as i64) * 8 + 128) as ptr let hcopy: ptr = malloc((d as i64) * (d as i64) * 8 + 128) as ptr + # The Hessian is X_aug^T diag(w) X_aug, which is a dgemm. Above a work + # threshold it is assembled that way, which needs the design in f64 and a + # row-scaled copy of it. X_aug never changes, so its f64 copy is taken once + # for the whole fit and shared by every class and every Newton iteration. + # Below the threshold the scalar triple loop still runs: it is bit-identical + # to the previous code and skips the BLAS thread dispatch, which costs more + # than the whole assembly on a small problem. The gate is m * d * d, the + # actual FMA count, at the same 65536 the rest of the library uses. + let use_blas_hess: bool = (m as i64) * (d as i64) * (d as i64) >= 65536 + let mut Xd: ptr = null + let mut Xw: ptr = null + if use_blas_hess { + Xd = malloc((m as i64) * (d as i64) * 8 + 128) as ptr + Xw = malloc((m as i64) * (d as i64) * 8 + 128) as ptr + for idx in 0 to m * d { + Xd[idx] = X_aug[idx] as f64 + } + } + let l2_coeff: f32 = 0.0 if penalty.kind == PENALTY_L2 || penalty.kind == PENALTY_ELASTICNET { l2_coeff = penalty.alpha * (1.0 - penalty.l1_ratio) @@ -1481,24 +1618,38 @@ function lin357_newton_fit(X: Matrix, y: ptr, classes: ptr, nc: i32, e gnorm = sqrt(gnorm) if gnorm < 0.000001 { converged = true; break } - for j in 0 to d * d { hess[j] = 0.0 } - for i in 0 to m { - let base: i32 = i * d - let wi: f64 = wt_arr[i] as f64 + if use_blas_hess { + # Scale each row of the design by its IRLS weight, then contract + # the scaled design against the unscaled one. dgemm writes both + # triangles, so no symmetrisation pass is needed. + for i in 0 to m { + let base: i32 = i * d + let wi: f64 = wt_arr[i] as f64 + for j in 0 to d { + Xw[base + j] = Xd[base + j] * wi + } + } + blas_gemm_tn_f64(Xw, Xd, hess, m, d, d, d, d, d, inv_m, 0.0) + } else { + for j in 0 to d * d { hess[j] = 0.0 } + for i in 0 to m { + let base: i32 = i * d + let wi: f64 = wt_arr[i] as f64 + for j in 0 to d { + let xw: f64 = (X_aug[base + j] as f64) * wi + for k in j to d { + hess[j * d + k] = hess[j * d + k] + xw * (X_aug[base + k] as f64) + } + } + } for j in 0 to d { - let xw: f64 = (X_aug[base + j] as f64) * wi for k in j to d { - hess[j * d + k] = hess[j * d + k] + xw * (X_aug[base + k] as f64) + let v: f64 = hess[j * d + k] * inv_m + hess[j * d + k] = v + hess[k * d + j] = v } } } - for j in 0 to d { - for k in j to d { - let v: f64 = hess[j * d + k] * inv_m - hess[j * d + k] = v - hess[k * d + j] = v - } - } for j in 0 to n { hess[j * d + j] = hess[j * d + j] + l2_f64 } let mut theta_scale: f64 = 0.0 @@ -1568,6 +1719,10 @@ function lin357_newton_fit(X: Matrix, y: ptr, classes: ptr, nc: i32, e free(delta as ptr) free(hess as ptr) free(hcopy as ptr) + if use_blas_hess { + free(Xd as ptr) + free(Xw as ptr) + } let first_bias: f32 = all_biases[0] return LogisticRegression { @@ -1732,14 +1887,25 @@ export function logistic_regression_fit_solver(X: Matrix, y: ptr, n_classes while iter < max_iter && not converged { # Check convergence + # Largest gradient component, the same rule the multinomial fit + # uses and the same one scipy's L-BFGS-B applies as pgtol. See the + # note there: testing the L2 norm holds the fit to a far tighter + # standard than the tolerance names, and the two paths should stop + # on the same criterion. let mut gnorm: f32 = 0.0 for j in 0 to d { let g: f32 = grad[j] gnorm = gnorm + g * g } - # Approximate sqrt via repeated multiply gnorm = sqrt_f32(gnorm) - if gnorm < 0.0001 { converged = true; break } + + let mut gmax: f32 = 0.0 + for j in 0 to d { + let mut ag: f32 = grad[j] + if ag < 0.0 { ag = 0.0 - ag } + if ag > gmax { gmax = ag } + } + if gmax < 0.0001 { converged = true; break } # Save gradient and theta for history for j in 0 to d { diff --git a/lib/scikit/naive_bayes.flow b/lib/scikit/naive_bayes.flow index 2edecdf..0b80887 100644 --- a/lib/scikit/naive_bayes.flow +++ b/lib/scikit/naive_bayes.flow @@ -2,6 +2,7 @@ # Implements GaussianNB for continuous features. import "lib/scikit/matrix.flow" +import "lib/scikit/blas.flow" export struct GaussianNB { classes: ptr, @@ -48,31 +49,43 @@ export function gaussian_nb_fit(X: Matrix, y: ptr, n_classes: i32, var_smoo let means: ptr > = malloc((n_classes as i64) * 8) as ptr > let variances: ptr > = malloc((n_classes as i64) * 8) as ptr > let priors: ptr = array_new_f32(n_classes) - let global_mean: ptr = array_new_f32(p) - let global_var: ptr = array_new_f32(p) - let class_sums: ptr = array_new_f32(n_classes * p) - let class_var_sums: ptr = array_new_f32(n_classes * p) + let global_sum: ptr = array_new_f64(p) + let global_sq_sum: ptr = array_new_f64(p) + let class_sums: ptr = array_new_f64(n_classes * p) + let class_sq_sums: ptr = array_new_f64(n_classes * p) for c in 0 to n_classes { means[c] = array_new_f32(p) variances[c] = array_new_f32(p) } - # First contiguous pass: global and per-class sufficient statistics. + # Single contiguous pass: global and per-class sums and squared sums in f64. for i in 0 to n { let c: i32 = class_indices[i] let row_base: i32 = i * p let class_base: i32 = c * p + let x_row: ptr = X.data + row_base + let class_row: ptr = class_sums + class_base + let class_sq_row: ptr = class_sq_sums + class_base for j in 0 to p { - let value: f32 = X.data[row_base + j] - global_mean[j] = global_mean[j] + value - class_sums[class_base + j] = class_sums[class_base + j] + value + let value: f64 = x_row[j] as f64 + let v2: f64 = value * value + global_sum[j] = global_sum[j] + value + global_sq_sum[j] = global_sq_sum[j] + v2 + class_row[j] = class_row[j] + value + class_sq_row[j] = class_sq_row[j] + v2 } } + let mut max_var: f32 = 0.0 if n > 0 { + let inv_n: f64 = 1.0 / (n as f64) for j in 0 to p { - global_mean[j] = global_mean[j] / (n as f32) + let m: f64 = global_sum[j] * inv_n + let mut v: f64 = global_sq_sum[j] * inv_n - m * m + if v < 0.0 { v = 0.0 } + let vf: f32 = v as f32 + if vf > max_var { max_var = vf } } } @@ -84,52 +97,23 @@ export function gaussian_nb_fit(X: Matrix, y: ptr, n_classes: i32, var_smoo } if count > 0 { let base: i32 = c * p + let inv_c: f64 = 1.0 / (count as f64) for j in 0 to p { - means[c][j] = class_sums[base + j] / (count as f32) + let m: f64 = class_sums[base + j] * inv_c + let mut v: f64 = class_sq_sums[base + j] * inv_c - m * m + if v < 0.0 { v = 0.0 } + means[c][j] = m as f32 + variances[c][j] = (v as f32) + var_smoothing * max_var } } } - # Second contiguous pass: squared deviations. This preserves the existing - # ddof=0 population-variance semantics without temporary column/value arrays. - for i in 0 to n { - let c: i32 = class_indices[i] - let row_base: i32 = i * p - let class_base: i32 = c * p - for j in 0 to p { - let value: f32 = X.data[row_base + j] - let gd: f32 = value - global_mean[j] - let cd: f32 = value - means[c][j] - global_var[j] = global_var[j] + gd * gd - class_var_sums[class_base + j] = class_var_sums[class_base + j] + cd * cd - } - } - - if n > 0 { - for j in 0 to p { - global_var[j] = global_var[j] / (n as f32) - } - } - let max_var: f32 = array_max_val(global_var, p) - - for c in 0 to n_classes { - let count: i32 = class_counts[c] - let base: i32 = c * p - for j in 0 to p { - let class_var: f32 = 0.0 - if count > 0 { - class_var = class_var_sums[base + j] / (count as f32) - } - variances[c][j] = class_var + var_smoothing * max_var - } - } - free(class_counts as ptr) free(class_indices as ptr) - array_free_f32(global_mean) - array_free_f32(global_var) - array_free_f32(class_sums) - array_free_f32(class_var_sums) + array_free_f64(global_sum) + array_free_f64(global_sq_sum) + array_free_f64(class_sums) + array_free_f64(class_sq_sums) return GaussianNB { classes: classes, @@ -144,24 +128,24 @@ export function gaussian_nb_fit(X: Matrix, y: ptr, n_classes: i32, var_smoo } # Precompute the terms that are constant for every sample in a prediction call. -# Keeping them out of the sample/class/feature hot loop avoids repeated log() -# evaluation while leaving fit latency and the public model layout unchanged. function _gaussian_nb_prepare_terms( model: GaussianNB, - class_log_prior: ptr, - log_norm: ptr, + class_const: ptr, inv_two_var: ptr ) -> void { let log_two_pi: f32 = log((2.0 * 3.14159265358979) as f64) as f32 let p: i32 = model.n_features for c in 0 to model.n_classes { - class_log_prior[c] = log(model.priors[c] as f64) as f32 + let mut sum_log_norm: f32 = log(model.priors[c] as f64) as f32 let base: i32 = c * p + let var_c: ptr = model.variances[c] for j in 0 to p { - let var: f32 = model.variances[c][j] - log_norm[base + j] = -0.5 * log_two_pi - 0.5 * (log(var as f64) as f32) + let var: f32 = var_c[j] + let term: f32 = -0.5 * log_two_pi - 0.5 * (log(var as f64) as f32) + sum_log_norm = sum_log_norm + term inv_two_var[base + j] = 1.0 / (2.0 * var) } + class_const[c] = sum_log_norm } } @@ -169,24 +153,25 @@ export function gaussian_nb_predict(model: GaussianNB, X: Matrix) -> ptr { let preds: ptr = array_new_f32(X.rows) let p: i32 = model.n_features let terms: i32 = model.n_classes * p - let class_log_prior: ptr = array_new_f32(model.n_classes) - let log_norm: ptr = array_new_f32(terms) + let class_const: ptr = array_new_f32(model.n_classes) let inv_two_var: ptr = array_new_f32(terms) - _gaussian_nb_prepare_terms(model, class_log_prior, log_norm, inv_two_var) + _gaussian_nb_prepare_terms(model, class_const, inv_two_var) for i in 0 to X.rows { let mut best_class: i32 = 0 let mut best_log_prob: f32 = -9999999999.0 let row_base: i32 = i * p + let x_row: ptr = X.data + row_base for c in 0 to model.n_classes { - let mut log_prob: f32 = class_log_prior[c] - let class_base: i32 = c * p + let means_c: ptr = model.means[c] + let inv_var_c: ptr = inv_two_var + c * p + let mut sum_diff_sq: f32 = 0.0 for j in 0 to p { - let diff: f32 = X.data[row_base + j] - model.means[c][j] - let term_idx: i32 = class_base + j - log_prob = log_prob + log_norm[term_idx] - diff * diff * inv_two_var[term_idx] + let diff: f32 = x_row[j] - means_c[j] + sum_diff_sq = sum_diff_sq + diff * diff * inv_var_c[j] } + let log_prob: f32 = class_const[c] - sum_diff_sq if log_prob > best_log_prob { best_log_prob = log_prob @@ -197,8 +182,7 @@ export function gaussian_nb_predict(model: GaussianNB, X: Matrix) -> ptr { preds[i] = model.classes[best_class] } - array_free_f32(class_log_prior) - array_free_f32(log_norm) + array_free_f32(class_const) array_free_f32(inv_two_var) return preds } @@ -207,25 +191,26 @@ export function gaussian_nb_predict_proba(model: GaussianNB, X: Matrix) -> Matri let probs: Matrix = matrix_new(X.rows, model.n_classes) let p: i32 = model.n_features let terms: i32 = model.n_classes * p - let class_log_prior: ptr = array_new_f32(model.n_classes) - let log_norm: ptr = array_new_f32(terms) + let class_const: ptr = array_new_f32(model.n_classes) let inv_two_var: ptr = array_new_f32(terms) let log_probs: ptr = array_new_f32(model.n_classes) - _gaussian_nb_prepare_terms(model, class_log_prior, log_norm, inv_two_var) + _gaussian_nb_prepare_terms(model, class_const, inv_two_var) for i in 0 to X.rows { let mut max_lp: f32 = -9999999999.0 let row_base: i32 = i * p let prob_base: i32 = i * model.n_classes + let x_row: ptr = X.data + row_base for c in 0 to model.n_classes { - let mut lp: f32 = class_log_prior[c] - let class_base: i32 = c * p + let means_c: ptr = model.means[c] + let inv_var_c: ptr = inv_two_var + c * p + let mut sum_diff_sq: f32 = 0.0 for j in 0 to p { - let diff: f32 = X.data[row_base + j] - model.means[c][j] - let term_idx: i32 = class_base + j - lp = lp + log_norm[term_idx] - diff * diff * inv_two_var[term_idx] + let diff: f32 = x_row[j] - means_c[j] + sum_diff_sq = sum_diff_sq + diff * diff * inv_var_c[j] } + let lp: f32 = class_const[c] - sum_diff_sq log_probs[c] = lp if lp > max_lp { max_lp = lp @@ -244,8 +229,7 @@ export function gaussian_nb_predict_proba(model: GaussianNB, X: Matrix) -> Matri } } - array_free_f32(class_log_prior) - array_free_f32(log_norm) + array_free_f32(class_const) array_free_f32(inv_two_var) array_free_f32(log_probs) return probs diff --git a/lib/scikit/svm.flow b/lib/scikit/svm.flow index b1f235a..1f011f7 100644 --- a/lib/scikit/svm.flow +++ b/lib/scikit/svm.flow @@ -64,52 +64,51 @@ export function linear_svc_fit(X: Matrix, y: ptr, n_classes: i32, C: f32, e let w: ptr = array_new_f64(n_aug) let C64: f64 = C as f64 - # Precompute x_i^T x_i for each sample (augmented) - let x_norm_sq: ptr = array_new_f64(n_samples) - for i in 0 to n_samples { - let mut s: f64 = 0.0 - for j in 0 to n_features { - let xv: f64 = (matrix_at(X, i, j)) as f64 - s = s + xv * xv - } - s = s + 1.0 # bias term - x_norm_sq[i] = s - } + let n2: i32 = n_aug + let stride: i32 = n2 + 1 + let margins: ptr = array_new_f64(n_samples) + let active: ptr = malloc((n_samples as i64) * 4) as ptr + let base_dot: ptr = array_new_f64(n_samples) + let d_dot: ptr = array_new_f64(n_samples) + let g: ptr = array_new_f64(n_aug) + let H: ptr = malloc((n2 as i64) * (stride as i64) * 8) as ptr + let d: ptr = array_new_f64(n2) let mut iter: i32 = 0 while iter < epochs { # Compute margins y_i * w^T x_i and identify active set # Active set I = {i : y_i * w^T x_i < 1} - let margins: ptr = array_new_f64(n_samples) - let active: ptr = malloc((n_samples as i64) * 4) as ptr let mut n_active: i32 = 0 for i in 0 to n_samples { + let xrow: ptr = X.data + i * n_features let mut dot: f64 = 0.0 for j in 0 to n_features { - dot = dot + w[j] * (matrix_at(X, i, j)) as f64 + dot = dot + w[j] * (xrow[j] as f64) } dot = dot + w[n_features] # bias weight let m: f64 = (y_dual[i]) as f64 * dot margins[i] = m if m < 1.0 { active[n_active] = i + base_dot[n_active] = dot n_active = n_active + 1 } } # Gradient: g_j = w_j - 2*C * sum_{i in I} y_i*(1 - m_i) * x_ij - let g: ptr = array_new_f64(n_aug) for j in 0 to n_aug { g[j] = w[j] } for k in 0 to n_active { let i: i32 = active[k] + let xrow: ptr = X.data + i * n_features let yi: f64 = (y_dual[i]) as f64 let residual: f64 = 1.0 - margins[i] + let factor: f64 = 2.0 * C64 * yi * residual for j in 0 to n_features { - g[j] = g[j] - 2.0 * C64 * yi * residual * (matrix_at(X, i, j)) as f64 + g[j] = g[j] - factor * (xrow[j] as f64) } - g[n_features] = g[n_features] - 2.0 * C64 * yi * residual # bias term + g[n_features] = g[n_features] - factor # bias term } # Check gradient norm @@ -118,18 +117,10 @@ export function linear_svc_fit(X: Matrix, y: ptr, n_classes: i32, C: f32, e g_norm = g_norm + g[j] * g[j] } if g_norm < 0.000001 { - free(active as ptr) - array_free_f64(margins) - array_free_f64(g) break } # Hessian: H_jk = delta_jk + 2*C * sum_{i in I} x_ij * x_ik - # For small n_aug, build and solve directly with Gaussian elimination. - # The augmented matrix has n2 rows and (n2+1) columns (last col = -g). - let n2: i32 = n_aug - let stride: i32 = n2 + 1 - let H: ptr = malloc((n2 as i64) * (stride as i64) * 8) as ptr for j in 0 to n2 { for k in 0 to n2 { if j == k { @@ -141,17 +132,23 @@ export function linear_svc_fit(X: Matrix, y: ptr, n_classes: i32, C: f32, e } for kk in 0 to n_active { let i: i32 = active[kk] + let xrow: ptr = X.data + i * n_features for j in 0 to n_features { - let xij: f64 = (matrix_at(X, i, j)) as f64 - for k in 0 to n_features { - H[j * stride + k] = H[j * stride + k] + 2.0 * C64 * xij * ((matrix_at(X, i, k)) as f64) + let xij: f64 = xrow[j] as f64 + let c_xij: f64 = 2.0 * C64 * xij + for k in j to n_features { + H[j * stride + k] = H[j * stride + k] + c_xij * (xrow[k] as f64) } - # bias column/row - H[j * stride + n_features] = H[j * stride + n_features] + 2.0 * C64 * xij - H[n_features * stride + j] = H[n_features * stride + j] + 2.0 * C64 * xij + H[j * stride + n_features] = H[j * stride + n_features] + c_xij + } + } + for j in 0 to n_features { + for k in 0 to j { + H[j * stride + k] = H[k * stride + j] } - H[n_features * stride + n_features] = H[n_features * stride + n_features] + 2.0 * C64 + H[n_features * stride + j] = H[j * stride + n_features] } + H[n_features * stride + n_features] = H[n_features * stride + n_features] + 2.0 * C64 * (n_active as f64) # Solve H * d = -g using Gaussian elimination with partial pivoting # Augment H with -g as the last column @@ -205,7 +202,6 @@ export function linear_svc_fit(X: Matrix, y: ptr, n_classes: i32, C: f32, e } # Back substitution - let d: ptr = array_new_f64(n2) let mut row: i32 = n2 - 1 while row >= 0 { let mut sum: f64 = H[row * stride + n2] @@ -219,8 +215,18 @@ export function linear_svc_fit(X: Matrix, y: ptr, n_classes: i32, C: f32, e } # Line search: find step size beta that minimizes objective - # f(w + beta*d) = (1/2)||w+beta*d||^2 + C*sum max(0, 1 - y_i*(w+beta*d)^T x_i)^2 - # Use backtracking with Wolfe-like conditions + # Precompute d^T x_i for each active sample + for k in 0 to n_active { + let i: i32 = active[k] + let xrow: ptr = X.data + i * n_features + let mut dot_d: f64 = 0.0 + for j in 0 to n_features { + dot_d = dot_d + d[j] * (xrow[j] as f64) + } + dot_d = dot_d + d[n_features] + d_dot[k] = dot_d + } + let mut beta: f64 = 1.0 let mut best_beta: f64 = 0.0 let mut best_obj: f64 = 0.0 @@ -229,11 +235,7 @@ export function linear_svc_fit(X: Matrix, y: ptr, n_classes: i32, C: f32, e for k in 0 to n_active { let i: i32 = active[k] let yi: f64 = (y_dual[i]) as f64 - let mut dot: f64 = 0.0 - for j in 0 to n_features { - dot = dot + w[j] * (matrix_at(X, i, j)) as f64 - } - dot = dot + w[n_features] + let dot: f64 = base_dot[k] let loss: f64 = 1.0 - yi * dot if loss > 0.0 { best_obj = best_obj + C64 * loss * loss } } @@ -250,11 +252,7 @@ export function linear_svc_fit(X: Matrix, y: ptr, n_classes: i32, C: f32, e for k in 0 to n_active { let i: i32 = active[k] let yi: f64 = (y_dual[i]) as f64 - let mut dot: f64 = 0.0 - for j in 0 to n_features { - dot = dot + (w[j] + b * d[j]) * (matrix_at(X, i, j)) as f64 - } - dot = dot + (w[n_features] + b * d[n_features]) + let dot: f64 = base_dot[k] + b * d_dot[k] let loss: f64 = 1.0 - yi * dot if loss > 0.0 { obj = obj + C64 * loss * loss } } @@ -271,15 +269,17 @@ export function linear_svc_fit(X: Matrix, y: ptr, n_classes: i32, C: f32, e w[j] = w[j] + best_beta * d[j] } - free(H as ptr) - free(active as ptr) - array_free_f64(margins) - array_free_f64(g) - array_free_f64(d) - iter = iter + 1 } + free(H as ptr) + free(active as ptr) + array_free_f64(margins) + array_free_f64(base_dot) + array_free_f64(d_dot) + array_free_f64(g) + array_free_f64(d) + # Extract weights and bias for j in 0 to n_features { weights[j] = w[j] as f32 @@ -288,8 +288,6 @@ export function linear_svc_fit(X: Matrix, y: ptr, n_classes: i32, C: f32, e array_free_f32(y_dual) array_free_f64(w) - array_free_f64(x_norm_sq) - return LinearSVC { weights: weights, bias: bias, @@ -516,16 +514,21 @@ export function linear_svc_multi_fit(X: Matrix, y: ptr, n_classes: i32, C: export function linear_svc_multi_decision_function(model: LinearSVCMulti, X: Matrix) -> ptr { let n: i32 = X.rows - let scores: ptr = array_new_f32(n * model.n_classes) + let p: i32 = model.n_features + let nc: i32 = model.n_classes + let scores: ptr = array_new_f32(n * nc) let mut c: i32 = 0 - while c < model.n_classes { + while c < nc { + let wc: ptr = model.weights[c] + let bc: f32 = model.biases[c] for i in 0 to n { - let mut z: f32 = model.biases[c] - for j in 0 to model.n_features { - z = z + model.weights[c][j] * matrix_at(X, i, j) + let xrow: ptr = X.data + i * p + let mut z: f32 = bc + for j in 0 to p { + z = z + wc[j] * xrow[j] } - scores[i * model.n_classes + c] = z + scores[i * nc + c] = z } c = c + 1 } @@ -755,11 +758,16 @@ function _kernel_svc_smo_precomputed( for i in 0 to n { let gi: i32 = idx_map[i] let yi: f64 = y_d[i] - for j in 0 to n { + let k_row: ptr = kernel_full + gi * n_total + let q_row: ptr = Q + i * n + q_row[i] = k_row[gi] + QD[i] = q_row[i] + for j in i + 1 to n { let gj: i32 = idx_map[j] - Q[i * n + j] = yi * y_d[j] * kernel_full[gi * n_total + gj] + let v: f64 = yi * y_d[j] * k_row[gj] + q_row[j] = v + Q[j * n + i] = v } - QD[i] = Q[i * n + i] } # Maintain gradient incrementally: G[i] = sum_j Q[i,j] * alpha_j - 1 @@ -985,49 +993,49 @@ function _rbf_exp_f64(neg_gamma_sum: f64) -> f64 { # K[i,j] = exp(-gamma * ||x1_i - x2_j||^2) # Uses the identity: ||a-b||^2 = ||a||^2 + ||b||^2 - 2*a^T*b # The cross term a^T*b is computed via cblas_sgemm. +# The cross term a^T*b is computed via cblas_sgemm. function _rbf_kernel_blas(X1: Matrix, X2: Matrix, gamma: f32) -> Matrix { let n1: i32 = X1.rows let n2: i32 = X2.rows let nf: i32 = X1.cols let result: Matrix = matrix_new(n1, n2) - # Compute squared norms of X1 and X2 + let x1_data: ptr = X1.data + let x2_data: ptr = X2.data + let x1_sq: ptr = array_new_f32(n1) let x2_sq: ptr = array_new_f32(n2) for i in 0 to n1 { + let r: ptr = x1_data + i * nf let mut s: f32 = 0.0 for k in 0 to nf { - let v: f32 = matrix_at(X1, i, k) + let v: f32 = r[k] s = s + v * v } x1_sq[i] = s } for j in 0 to n2 { + let r: ptr = x2_data + j * nf let mut s: f32 = 0.0 for k in 0 to nf { - let v: f32 = matrix_at(X2, j, k) + let v: f32 = r[k] s = s + v * v } x2_sq[j] = s } - # Cross term: X1 * X2^T via cblas_sgemm - # X1 is n1 x nf (row-major), X2 is n2 x nf (row-major) - # We need X1 * X2^T which is n1 x n2. - # cblas_sgemm with transb=CblasTrans computes C = alpha * X1 * X2^T + beta * C - let gamma_f: f32 = gamma let cross: ptr = malloc((n1 as i64) * (n2 as i64) * 4) as ptr - # Use the BLAS matmat wrapper with transposed B - # But our wrapper doesn't support trans. Call cblas_sgemm directly. - cblas_sgemm(CBLAS_ROW_MAJOR, CBLAS_NO_TRANS, CBLAS_TRANS, n1, n2, nf, 1.0, X1.data, nf, X2.data, nf, 0.0, cross, n2) + cblas_sgemm(101, 111, 112, n1, n2, nf, 1.0, x1_data, nf, x2_data, nf, 0.0, cross, n2) - # Combine: sq_dist = x1_sq + x2_sq - 2*cross, then exp(-gamma * sq_dist) let gamma_d: f64 = gamma as f64 for i in 0 to n1 { + let x1_val: f32 = x1_sq[i] + let cross_row: ptr = cross + i * n2 + let res_row: ptr = result.data + i * n2 for j in 0 to n2 { - let sq: f32 = x1_sq[i] + x2_sq[j] - 2.0 * cross[i * n2 + j] - let val: f32 = _rbf_exp_f64(0.0 - gamma_d * (sq as f64)) as f32 - matrix_set(result, i, j, val) + let mut sq: f64 = (x1_val + x2_sq[j] - 2.0 * cross_row[j]) as f64 + if sq < 0.0 { sq = 0.0 } + res_row[j] = _rbf_exp_f64(0.0 - gamma_d * sq) as f32 } } @@ -1045,21 +1053,25 @@ function _rbf_kernel_blas_f64(X1: Matrix, X2: Matrix, gamma: f32) -> ptr { let n2: i32 = X2.rows let nf: i32 = X1.cols - # Compute squared norms in f32 (cheap) + let x1_data: ptr = X1.data + let x2_data: ptr = X2.data + let x1_sq: ptr = array_new_f32(n1) let x2_sq: ptr = array_new_f32(n2) for i in 0 to n1 { + let r: ptr = x1_data + i * nf let mut s: f32 = 0.0 for k in 0 to nf { - let v: f32 = matrix_at(X1, i, k) + let v: f32 = r[k] s = s + v * v } x1_sq[i] = s } for j in 0 to n2 { + let r: ptr = x2_data + j * nf let mut s: f32 = 0.0 for k in 0 to nf { - let v: f32 = matrix_at(X2, j, k) + let v: f32 = r[k] s = s + v * v } x2_sq[j] = s @@ -1067,15 +1079,19 @@ function _rbf_kernel_blas_f64(X1: Matrix, X2: Matrix, gamma: f32) -> ptr { # Cross term: X1 * X2^T via f32 cblas_sgemm (much faster than dgemm) let cross: ptr = malloc((n1 as i64) * (n2 as i64) * 4) as ptr - cblas_sgemm(CBLAS_ROW_MAJOR, CBLAS_NO_TRANS, CBLAS_TRANS, n1, n2, nf, 1.0, X1.data, nf, X2.data, nf, 0.0, cross, n2) + cblas_sgemm(101, 111, 112, n1, n2, nf, 1.0, x1_data, nf, x2_data, nf, 0.0, cross, n2) # Combine and apply exp in f64 let gamma_d: f64 = gamma as f64 let result: ptr = malloc((n1 as i64) * (n2 as i64) * 8) as ptr for i in 0 to n1 { + let x1_val: f32 = x1_sq[i] + let cross_row: ptr = cross + i * n2 + let res_row: ptr = result + i * n2 for j in 0 to n2 { - let sq: f64 = (x1_sq[i] + x2_sq[j] - 2.0 * cross[i * n2 + j]) as f64 - result[i * n2 + j] = _rbf_exp_f64(0.0 - gamma_d * sq) + let mut sq: f64 = (x1_val + x2_sq[j] - 2.0 * cross_row[j]) as f64 + if sq < 0.0 { sq = 0.0 } + res_row[j] = _rbf_exp_f64(0.0 - gamma_d * sq) } } @@ -1155,8 +1171,40 @@ export function kernel_svc_free(model: KernelSVC) -> void { # Build full RBF kernel matrix. Always use BLAS because cblas_sgemm is faster # than the scalar nested loop even for small arrays like 100-500. -function _build_full_kernel(X: Matrix, n: i32, n_features: i32, gamma: f32) -> ptr { - return _rbf_kernel_blas_f64(X, X, gamma) +function _build_full_kernel(X: Matrix, n: i32, nf: i32, gamma: f32) -> ptr { + let x_data: ptr = X.data + let x_sq: ptr = array_new_f32(n) + for i in 0 to n { + let r: ptr = x_data + i * nf + let mut s: f32 = 0.0 + for k in 0 to nf { + let v: f32 = r[k] + s = s + v * v + } + x_sq[i] = s + } + + let cross: ptr = malloc((n as i64) * (n as i64) * 4) as ptr + cblas_sgemm(101, 111, 112, n, n, nf, 1.0, x_data, nf, x_data, nf, 0.0, cross, n) + + let gamma_d: f64 = gamma as f64 + let result: ptr = malloc((n as i64) * (n as i64) * 8) as ptr + for i in 0 to n { + result[i * n + i] = 1.0 + let xi_sq: f32 = x_sq[i] + let cross_row: ptr = cross + i * n + for j in i + 1 to n { + let mut sq: f64 = (xi_sq + x_sq[j] - 2.0 * cross_row[j]) as f64 + if sq < 0.0 { sq = 0.0 } + let v: f64 = _kr_exp_f64(0.0 - gamma_d * sq) + result[i * n + j] = v + result[j * n + i] = v + } + } + + free(cross as ptr) + array_free_f32(x_sq) + return result } # ============================================================================ @@ -1209,42 +1257,68 @@ export function kernel_svc_multi_fit(X: Matrix, y: ptr, n_classes: i32, gam let coef_all: ptr = array_new_f32(n * n_pairs) let bias_all: ptr = array_new_f32(n_pairs) + # Pre-index samples by class to avoid scanning the entire dataset on every pair. + let class_counts: ptr = malloc((n_found as i64) * 4) as ptr + for c in 0 to n_found { class_counts[c] = 0 } + for i in 0 to n { + for c in 0 to n_found { + if y[i] == classes[c] { + class_counts[c] = class_counts[c] + 1 + } + } + } + let class_rows: ptr > = malloc((n_found as i64) * 8) as ptr > + let class_ptrs: ptr = malloc((n_found as i64) * 4) as ptr + for c in 0 to n_found { + class_rows[c] = malloc((class_counts[c] as i64) * 4) as ptr + class_ptrs[c] = 0 + } + for i in 0 to n { + for c in 0 to n_found { + if y[i] == classes[c] { + let pos: i32 = class_ptrs[c] + class_rows[c][pos] = i + class_ptrs[c] = pos + 1 + } + } + } + free(class_ptrs as ptr) + let mut pair_idx: i32 = 0 let mut a: i32 = 0 while a < n_found { let mut b: i32 = a + 1 while b < n_found { - # Collect samples belonging to class a or class b - let mut count: i32 = 0 - for i in 0 to n { - if y[i] == classes[a] || y[i] == classes[b] { - count = count + 1 - } - } + let count_a: i32 = class_counts[a] + let count_b: i32 = class_counts[b] + let count: i32 = count_a + count_b - let X_sub: Matrix = matrix_new(count, n_features) let y_dual: ptr = array_new_f32(count) let idx_map: ptr = malloc((count as i64) * 4) as ptr - let mut idx: i32 = 0 - for i in 0 to n { - if y[i] == classes[a] || y[i] == classes[b] { - for j in 0 to n_features { - matrix_set(X_sub, idx, j, matrix_at(X, i, j)) - } - if y[i] == classes[a] { - y_dual[idx] = 1.0 - } else { - y_dual[idx] = -1.0 - } - idx_map[idx] = i - idx = idx + 1 - } + + let rows_a: ptr = class_rows[a] + for i in 0 to count_a { + idx_map[i] = rows_a[i] + y_dual[i] = 1.0 + } + let rows_b: ptr = class_rows[b] + for i in 0 to count_b { + idx_map[count_a + i] = rows_b[i] + y_dual[count_a + i] = -1.0 } + let sub_X: Matrix = matrix_new(count, n_features) + for i in 0 to count { + let row_idx: i32 = idx_map[i] + for j in 0 to n_features { + matrix_set(sub_X, i, j, matrix_at(X, row_idx, j)) + } + } let binary_classes: ptr = array_new_f32(2) binary_classes[0] = -1.0 binary_classes[1] = 1.0 - models[pair_idx] = _kernel_svc_smo_precomputed(X_sub, y_dual, binary_classes, 2, C, gamma, max_iter, kernel_full, idx_map, n) + models[pair_idx] = _kernel_svc_smo_precomputed(sub_X, y_dual, binary_classes, 2, C, gamma, max_iter, kernel_full, idx_map, n) + matrix_free(sub_X) pair_a[pair_idx] = a pair_b[pair_idx] = b @@ -1255,10 +1329,7 @@ export function kernel_svc_multi_fit(X: Matrix, y: ptr, n_classes: i32, gam } bias_all[pair_idx] = pm.b - # idx_map is consumed by the solver (it copies the kernel slice). free(idx_map as ptr) - # X_sub is stored in the model (matrix_copy inside solver). - matrix_free(X_sub) pair_idx = pair_idx + 1 b = b + 1 @@ -1266,6 +1337,12 @@ export function kernel_svc_multi_fit(X: Matrix, y: ptr, n_classes: i32, gam a = a + 1 } + for c in 0 to n_found { + free(class_rows[c] as ptr) + } + free(class_rows as ptr) + free(class_counts as ptr) + free(kernel_full as ptr) let X_all: Matrix = matrix_copy(X) @@ -1301,9 +1378,12 @@ export function kernel_svc_multi_decision_function(model: KernelSVCMulti, X: Mat let dec: ptr = array_new_f32(n * np) blas_matmat(K.data, model.coef_all, dec, n, np, nt, 1.0, 0.0) matrix_free(K) + + let bias: ptr = model.bias_all for i in 0 to n { + let dec_row: ptr = dec + i * np for p in 0 to np { - dec[i * np + p] = dec[i * np + p] + model.bias_all[p] + dec_row[p] = dec_row[p] + bias[p] } } return dec @@ -1311,47 +1391,42 @@ export function kernel_svc_multi_decision_function(model: KernelSVCMulti, X: Mat export function kernel_svc_multi_predict(model: KernelSVCMulti, X: Matrix) -> ptr { let n: i32 = X.rows - let total_votes: i32 = n * model.n_classes - let votes: ptr = malloc((total_votes as i64) * 4) as ptr - for i in 0 to total_votes { - votes[i] = 0 - } - + let nc: i32 = model.n_classes let np: i32 = model.n_pairs let dec: ptr = kernel_svc_multi_decision_function(model, X) + let preds: ptr = array_new_f32(n) + let row_votes: ptr = malloc((nc as i64) * 4) as ptr - let mut p: i32 = 0 - while p < np { - let a: i32 = model.pair_a[p] - let b: i32 = model.pair_b[p] - for i in 0 to n { - if dec[i * np + p] > 0.0 { - votes[i * model.n_classes + a] = votes[i * model.n_classes + a] + 1 + let pair_a: ptr = model.pair_a + let pair_b: ptr = model.pair_b + + for i in 0 to n { + for c in 0 to nc { row_votes[c] = 0 } + let dec_row: ptr = dec + i * np + for p in 0 to np { + if dec_row[p] > 0.0 { + let a: i32 = pair_a[p] + row_votes[a] = row_votes[a] + 1 } else { - votes[i * model.n_classes + b] = votes[i * model.n_classes + b] + 1 + let b: i32 = pair_b[p] + row_votes[b] = row_votes[b] + 1 } } - p = p + 1 - } - array_free_f32(dec) - let preds: ptr = array_new_f32(n) - for i in 0 to n { let mut best_c: i32 = 0 - let mut best_votes: i32 = votes[i * model.n_classes] - let mut c: i32 = 1 - while c < model.n_classes { - let v: i32 = votes[i * model.n_classes + c] + let mut best_votes: i32 = row_votes[0] + for c in 1 to nc { + let v: i32 = row_votes[c] if v > best_votes { best_votes = v best_c = c } - c = c + 1 } preds[i] = model.classes[best_c] } - free(votes as ptr) + free(row_votes as ptr) + array_free_f32(dec) return preds } diff --git a/lib/scikit/tree.flow b/lib/scikit/tree.flow index f3ffc4d..f0c1c54 100644 --- a/lib/scikit/tree.flow +++ b/lib/scikit/tree.flow @@ -10,6 +10,7 @@ extern { function free(p: ptr) -> void function fabs(x: f64) -> f64 function printf(fmt: string, ...) -> i32 + function memcpy(dst: ptr, src: ptr, n: i64) -> ptr } export const TREE_MAX_DEPTH: i32 = 50 @@ -218,6 +219,225 @@ function _sort_pairs(values: ptr, companions: ptr, n: i32) -> void { } } +# Introsort: sort values ascending, keeping a companion i32 array in sync. +# +# The shell sort below is O(n^1.3) with a serial dependence through every gap +# pass. The classifier split search calls this once per candidate feature per +# node, so on digits the RandomForest fit spent most of its time here. This is +# the same hybrid scikit-learn's splitter uses: median-of-three quicksort, +# insertion sort once a range is short, and a heapsort fallback when the +# recursion depth suggests an adversarial pivot sequence, which keeps the worst +# case at O(n log n). +# +# The partition is unstable, so equal values can come out in a different order +# than the shell sort produced. That is safe exactly where the note on +# _build_classifier_tree says it is: the split scan reads counts only at +# boundaries where consecutive values differ, and at such a boundary the set of +# samples on the left is the same whatever order the equal values came in. The +# counts are integers, so the chosen feature and threshold are unchanged. +# +# That argument is about counts. It does NOT hold for the regressor, which +# accumulates f32 sums in sorted position order, so _build_regressor_tree keeps +# _sort_pairs_f32 and its stable gap sequence. +# +# The range stack is explicit rather than recursive, and the larger side of +# each partition is looped on while the smaller side is pushed, so the stack +# depth stays at O(log n). +# 32 rather than the usual 16. The split search calls this once per candidate +# feature per node, and a depth-10 tree spends most of its calls on small +# nodes, where insertion sort finishes before the partition machinery has paid +# for the one allocation below. +const _INTROSORT_INSERTION_MAX: i32 = 32 +const _INTROSORT_STACK: i32 = 128 + +function _introsort_swap(values: ptr, companions: ptr, a: i32, b: i32) -> void { + let tv: f32 = values[a] + values[a] = values[b] + values[b] = tv + let tc: i32 = companions[a] + companions[a] = companions[b] + companions[b] = tc +} + +# Insertion sort over [lo, hi]. Used for short ranges, where its lack of +# overhead beats any partitioning scheme. +function _introsort_insertion(values: ptr, companions: ptr, lo: i32, hi: i32) -> void { + let mut i: i32 = lo + 1 + while i <= hi { + let key_v: f32 = values[i] + let key_c: i32 = companions[i] + let mut j: i32 = i - 1 + while j >= lo && values[j] > key_v { + values[j + 1] = values[j] + companions[j + 1] = companions[j] + j = j - 1 + } + values[j + 1] = key_v + companions[j + 1] = key_c + i = i + 1 + } +} + +# Sift one element down a max-heap rooted at `root`, within [lo, lo + count). +function _introsort_sift(values: ptr, companions: ptr, lo: i32, count: i32, root: i32) -> void { + let mut r: i32 = root + while true { + let mut largest: i32 = r + let left: i32 = 2 * r + 1 + let right: i32 = left + 1 + if left < count && values[lo + left] > values[lo + largest] { largest = left } + if right < count && values[lo + right] > values[lo + largest] { largest = right } + if largest == r { return } + _introsort_swap(values, companions, lo + r, lo + largest) + r = largest + } +} + +# Heapsort over [lo, hi]. The depth-limit fallback, so the worst case stays +# O(n log n) instead of quicksort's O(n^2). +function _introsort_heap(values: ptr, companions: ptr, lo: i32, hi: i32) -> void { + let count: i32 = hi - lo + 1 + if count < 2 { return } + let mut start: i32 = count / 2 - 1 + while start >= 0 { + _introsort_sift(values, companions, lo, count, start) + start = start - 1 + } + let mut end: i32 = count - 1 + while end > 0 { + _introsort_swap(values, companions, lo, lo + end) + _introsort_sift(values, companions, lo, end, 0) + end = end - 1 + } +} + +# Put the median of first, middle and last at `lo` to use as the pivot. A +# sorted or reverse-sorted column is the common case in a presorted tree +# builder, and median-of-three turns both of those from worst case into best. +function _introsort_median3(values: ptr, companions: ptr, lo: i32, hi: i32) -> void { + let mid: i32 = lo + (hi - lo) / 2 + if values[mid] < values[lo] { _introsort_swap(values, companions, mid, lo) } + if values[hi] < values[lo] { _introsort_swap(values, companions, hi, lo) } + if values[hi] < values[mid] { _introsort_swap(values, companions, hi, mid) } + _introsort_swap(values, companions, mid, lo) +} + +# An LSD radix sort over the float bit patterns was tried here and removed. +# In isolation it looked decisive: benchmarks/zzprof_sortab.flow timed it at +# 2.9x the introsort at n=1257 and 1.5x at n=64, interleaved in one process. +# End to end it lost. Building the RandomForest digits row against four +# thresholds as four separate binaries and running them round-robin gave a +# clean monotonic result, best of 8 runs each: radix off 15.12 ms, on above +# 512 rows 15.47 ms, above 128 rows 15.97 ms, above 64 rows 16.88 ms. The +# split search calls this once per candidate feature per node, thousands of +# times per fit with a different length each time, and the radix pass needs +# four buffers and a histogram where the introsort needs a small stack. The +# microbenchmark sorted one hot array in a tight loop, so it measured neither +# the allocation nor the cache traffic that decides the real case. +# +# Keep the measurement honest: a sort benchmark that reuses one buffer does +# not predict a tree builder. +function _sort_pairs_fast(values: ptr, companions: ptr, n: i32) -> void { + if n < 2 { return } + if n <= _INTROSORT_INSERTION_MAX { + _introsort_insertion(values, companions, 0, n - 1) + return + } + + # 2 * floor(log2(n)) is the standard introsort depth limit. + let mut depth_limit: i32 = 0 + let mut probe: i32 = n + while probe > 1 { + depth_limit = depth_limit + 1 + probe = probe / 2 + } + depth_limit = depth_limit * 2 + + # One allocation carved into three ranges. Three separate mallocs cost more + # than the whole sort on a small node. + let stack_buf: ptr = malloc((_INTROSORT_STACK as i64) * 12) as ptr + let stack_lo: ptr = stack_buf + let stack_hi: ptr = stack_buf + _INTROSORT_STACK + let stack_dp: ptr = stack_buf + 2 * _INTROSORT_STACK + let mut top: i32 = 0 + stack_lo[0] = 0 + stack_hi[0] = n - 1 + stack_dp[0] = depth_limit + top = 1 + + while top > 0 { + top = top - 1 + let mut lo: i32 = stack_lo[top] + let mut hi: i32 = stack_hi[top] + let mut depth: i32 = stack_dp[top] + + while lo < hi { + if hi - lo + 1 <= _INTROSORT_INSERTION_MAX { + _introsort_insertion(values, companions, lo, hi) + lo = hi + } else { + if depth <= 0 { + _introsort_heap(values, companions, lo, hi) + lo = hi + } else { + depth = depth - 1 + _introsort_median3(values, companions, lo, hi) + let pivot: f32 = values[lo] + let mut i: i32 = lo + let mut j: i32 = hi + 1 + # Hoare partition. Both scans move before any swap, so a + # run of values equal to the pivot is split between the two + # sides instead of piling up on one, which is what keeps a + # low-cardinality column such as a digit pixel from + # degenerating. + while true { + i = i + 1 + while i < hi && values[i] < pivot { i = i + 1 } + j = j - 1 + while j > lo && values[j] > pivot { j = j - 1 } + if i >= j { break } + _introsort_swap(values, companions, i, j) + } + _introsort_swap(values, companions, lo, j) + + # Loop on the larger side, push the smaller one, so the + # stack cannot exceed log2(n) entries and 128 slots are + # unreachable in practice. If it ever did fill, heapsort + # the range in place rather than drop it: a silently + # unsorted column would give a wrong split with no error. + if j - lo < hi - j { + if j + 1 < hi { + if top < _INTROSORT_STACK { + stack_lo[top] = j + 1 + stack_hi[top] = hi + stack_dp[top] = depth + top = top + 1 + } else { + _introsort_heap(values, companions, j + 1, hi) + } + } + hi = j - 1 + } else { + if lo < j - 1 { + if top < _INTROSORT_STACK { + stack_lo[top] = lo + stack_hi[top] = j - 1 + stack_dp[top] = depth + top = top + 1 + } else { + _introsort_heap(values, companions, lo, j - 1) + } + } + lo = j + 1 + } + } + } + } + } + + free(stack_buf as ptr) +} + # Shell sort: sort values array, keeping companion f32 array in sync. function _sort_pairs_f32(values: ptr, companions: ptr, n: i32) -> void { let mut gap: i32 = n / 2 @@ -415,6 +635,15 @@ export function decision_tree_classifier_fit(X: Matrix, y: ptr, n_classes: # this same contract, and ensemble.flow evaluates cat_kind at every traversal. # The one thing still numeric only is the boosting family: GradientBoosting, # AdaBoost and the histogram estimators. +struct TreeScratch { + sort_values: ptr, + sort_classes: ptr, + left_counts: ptr, + right_counts: ptr, + total_counts: ptr, + present: ptr +} + export function decision_tree_classifier_fit_categorical(X: Matrix, y: ptr, n_classes: i32, max_depth: i32, criterion: i32, categorical: ptr) -> DecisionTreeClassifier { # Issue #398: this used to append at classes[n_found] with no check against # n_classes, which wrote past the allocation whenever y carried more @@ -455,7 +684,7 @@ export function decision_tree_classifier_fit_categorical(X: Matrix, y: ptr, sort_v[i] = x_data[i * x_cols + f] sort_r[i] = i } - _sort_pairs(sort_v, sort_r, n_rows) + _sort_pairs_fast(sort_v, sort_r, n_rows) let base: i32 = f * n_rows for i in 0 to n_rows { order[base + i] = sort_r[i] @@ -467,21 +696,25 @@ export function decision_tree_classifier_fit_categorical(X: Matrix, y: ptr, # Scratch marking each row's side of the current split, indexed by row. let side: ptr = malloc((n_rows as i64) * 4) as ptr + let scratch: TreeScratch = TreeScratch { + sort_values: array_new_f32(n_rows), + sort_classes: malloc((n_rows as i64) * 4) as ptr, + left_counts: malloc((n_classes as i64) * 4) as ptr, + right_counts: malloc((n_classes as i64) * 4) as ptr, + total_counts: malloc((n_classes as i64) * 4) as ptr, + present: malloc((n_classes as i64) * 4) as ptr + } + # Categorical bookkeeping. cat_card[f] is 0 for a numeric feature, so with # no categorical features max_card is 0, the builder's categorical branch # is unreachable, and these two allocations are 16 bytes and one float. - # - # The scratch is allocated once here rather than per node so the numeric - # path picks up no extra allocation at all. Nothing in it survives across - # the recursive calls: a node fills it, picks its split and partitions - # before it descends. let cat_card: ptr = _tree_cat_cardinalities(X, categorical) let max_card: i32 = _tree_cat_max_card(cat_card, x_cols) let ccap: i64 = max_card as i64 let cat_iscratch: ptr = malloc((ccap * 4 + ccap * (n_classes as i64)) * 4 + 16) as ptr let cat_mean: ptr = array_new_f32(max_card + 1) - let final_n_nodes: i32 = _build_classifier_tree(nodes, n_nodes_ptr, X, y, order, n_rows, 0, max_depth, criterion, classes, n_classes, class_of_row, side, cat_card, max_card, cat_iscratch, cat_mean) + let final_n_nodes: i32 = _build_classifier_tree(nodes, n_nodes_ptr, X, y, order, n_rows, 0, max_depth, criterion, classes, n_classes, class_of_row, side, cat_card, max_card, cat_iscratch, cat_mean, scratch) free(order as ptr) free(side as ptr) @@ -489,6 +722,12 @@ export function decision_tree_classifier_fit_categorical(X: Matrix, y: ptr, free(cat_card as ptr) free(cat_iscratch as ptr) array_free_f32(cat_mean) + array_free_f32(scratch.sort_values) + free(scratch.sort_classes as ptr) + free(scratch.left_counts as ptr) + free(scratch.right_counts as ptr) + free(scratch.total_counts as ptr) + free(scratch.present as ptr) let n_nodes: i32 = n_nodes_ptr[0] free(n_nodes_ptr as ptr) @@ -540,7 +779,8 @@ function _build_classifier_tree( cat_card: ptr, max_card: i32, cat_iscratch: ptr, - cat_mean: ptr + cat_mean: ptr, + scratch: TreeScratch ) -> i32 { let node_idx: i32 = n_nodes[0] n_nodes[0] = n_nodes[0] + 1 @@ -581,7 +821,7 @@ function _build_classifier_tree( let cat_class: ptr = cat_iscratch + 4 * max_card # Node class histogram, over any block since all blocks hold the same rows. - let total_counts: ptr = malloc((n_classes as i64) * 4) as ptr + let total_counts: ptr = scratch.total_counts for c in 0 to n_classes { total_counts[c] = 0 } for i in 0 to n_indices { let c: i32 = class_of_row[order[i]] @@ -591,7 +831,7 @@ function _build_classifier_tree( # Ascending list of the classes that actually occur at this node. A class # with no samples here has no samples on either side of any split below, # so its impurity term is 0.0 and dropping it is exact. - let present: ptr = malloc((n_classes as i64) * 4) as ptr + let present: ptr = scratch.present let mut n_present: i32 = 0 for c in 0 to n_classes { if total_counts[c] > 0 { @@ -600,13 +840,22 @@ function _build_classifier_tree( } } + if n_present <= 1 { + nodes[node_idx].is_leaf = true + nodes[node_idx].value = classes[present[0]] + nodes[node_idx].n_samples = n_indices + nodes[node_idx].left = -1 + nodes[node_idx].right = -1 + nodes[node_idx].impurity = 0.0 + return node_idx + } + let parent_impurity: f32 = _tree_gini_present(total_counts, present, n_present, n_indices) - # Allocated once, reused across features. - let sort_values: ptr = array_new_f32(n_indices) - let sort_classes: ptr = malloc((n_indices as i64) * 4) as ptr - let left_counts: ptr = malloc((n_classes as i64) * 4) as ptr - let right_counts: ptr = malloc((n_classes as i64) * 4) as ptr + let sort_values: ptr = scratch.sort_values + let sort_classes: ptr = scratch.sort_classes + let left_counts: ptr = scratch.left_counts + let right_counts: ptr = scratch.right_counts for f in 0 to x_cols { let card: i32 = cat_card[f] @@ -688,33 +937,43 @@ function _build_classifier_tree( continue } - # Gather this feature's values and classes in the order block f already - # holds, which is ascending by construction. + # Check for constant column before gathering. Sorted ascending, that is + # a single comparison of the first and last element in the order block. let base: i32 = f * n_indices - for i in 0 to n_indices { - let row: i32 = order[base + i] + let first_row: i32 = order[base] + let last_row: i32 = order[base + n_indices - 1] + let v_first: f32 = x_data[first_row * x_cols + f] + let v_last: f32 = x_data[last_row * x_cols + f] + if v_first == v_last { continue } + + sort_values[0] = v_first + sort_classes[0] = class_of_row[first_row] + sort_values[n_indices - 1] = v_last + sort_classes[n_indices - 1] = class_of_row[last_row] + let base_ptr: ptr = order + base + for i in 1 to n_indices - 1 { + let row: i32 = base_ptr[i] sort_values[i] = x_data[row * x_cols + f] sort_classes[i] = class_of_row[row] } - # Constant column: the scan would hit sort_values[i] == sort_values[i+1] - # at every position and evaluate no candidate at all. Sorted, that is a - # single comparison of the ends. - if sort_values[0] == sort_values[n_indices - 1] { continue } - # right_counts starts as the node's full class histogram; left_counts # starts empty. Seeding from the histogram replaces a second pass over # every sample. for pi in 0 to n_present { let c: i32 = present[pi] left_counts[c] = 0 - right_counts[c] = total_counts[c] + let cnt: i32 = total_counts[c] + right_counts[c] = cnt } # Sweep sorted values, moving one sample at a time to left. for i in 0 to n_indices - 1 { - left_counts[sort_classes[i]] = left_counts[sort_classes[i]] + 1 - right_counts[sort_classes[i]] = right_counts[sort_classes[i]] - 1 + let c: i32 = sort_classes[i] + let L: i32 = left_counts[c] + let R: i32 = right_counts[c] + left_counts[c] = L + 1 + right_counts[c] = R - 1 # Skip duplicate values. if sort_values[i] == sort_values[i + 1] { @@ -764,13 +1023,6 @@ function _build_classifier_tree( } } - array_free_f32(sort_values) - free(sort_classes as ptr) - free(left_counts as ptr) - free(right_counts as ptr) - free(present as ptr) - free(total_counts as ptr) - if best_feature < 0 || best_impurity >= parent_impurity - 0.0000000001 { let y_sub: ptr = array_new_f32(n_indices) for i in 0 to n_indices { @@ -822,18 +1074,18 @@ function _build_classifier_tree( let right_order: ptr = malloc(((x_cols as i64) * (right_n as i64) + 1) * 4) as ptr for f in 0 to x_cols { - let sbase: i32 = f * n_indices - let lbase: i32 = f * left_n - let rbase: i32 = f * right_n + let sbase_ptr: ptr = order + f * n_indices + let lbase_ptr: ptr = left_order + f * left_n + let rbase_ptr: ptr = right_order + f * right_n let mut li: i32 = 0 let mut ri: i32 = 0 for i in 0 to n_indices { - let row: i32 = order[sbase + i] + let row: i32 = sbase_ptr[i] if side[row] == 0 { - left_order[lbase + li] = row + lbase_ptr[li] = row li = li + 1 } else { - right_order[rbase + ri] = row + rbase_ptr[ri] = row ri = ri + 1 } } @@ -845,8 +1097,8 @@ function _build_classifier_tree( nodes[node_idx].n_samples = n_indices nodes[node_idx].impurity = parent_impurity - let left_idx: i32 = _build_classifier_tree(nodes, n_nodes, X, y, left_order, left_n, depth + 1, max_depth, criterion, classes, n_classes, class_of_row, side, cat_card, max_card, cat_iscratch, cat_mean) - let right_idx: i32 = _build_classifier_tree(nodes, n_nodes, X, y, right_order, right_n, depth + 1, max_depth, criterion, classes, n_classes, class_of_row, side, cat_card, max_card, cat_iscratch, cat_mean) + let left_idx: i32 = _build_classifier_tree(nodes, n_nodes, X, y, left_order, left_n, depth + 1, max_depth, criterion, classes, n_classes, class_of_row, side, cat_card, max_card, cat_iscratch, cat_mean, scratch) + let right_idx: i32 = _build_classifier_tree(nodes, n_nodes, X, y, right_order, right_n, depth + 1, max_depth, criterion, classes, n_classes, class_of_row, side, cat_card, max_card, cat_iscratch, cat_mean, scratch) nodes[node_idx].left = left_idx nodes[node_idx].right = right_idx @@ -857,6 +1109,113 @@ function _build_classifier_tree( return node_idx } +# Value binning for the RandomForest split search. +# +# The split search needs, for each candidate feature, the class histogram at +# every distinct value in ascending order. Sorting the node's samples produces +# that, at O(n log n) per feature per node. Precomputing which distinct value +# each sample carries produces the same thing from one linear histogram pass. +# +# A bin is an exact distinct value, never a quantile bucket, so the candidate +# thresholds are unchanged: the sort scan evaluates a split wherever two +# consecutive sorted values differ, and the bin scan evaluates one between +# consecutive non-empty bins. Those are the same boundaries with the same +# counts, so the chosen feature, the threshold and the tie-break all match. +# +# A feature with more than TREE_MAX_BINS distinct values is left unbinned and +# keeps the sort path. That is the continuous-data case, where a bin per value +# would cost more to clear than the sort costs to run. +const TREE_MAX_BINS: i32 = 256 + +# Length of the bin_vals buffer the caller must allocate. +export function tree_bins_values_len(n_features: i32) -> i32 { + return n_features * TREE_MAX_BINS +} + +# Fills bin_card[f] with feature f's distinct-value count (0 when the feature +# is unbinnable), bin_vals[f * TREE_MAX_BINS ..] with those values ascending, +# and bin_of[f * n_rows + i] with the bin sample i falls in. +# +# bin_of is feature-major so the split search reads one contiguous column. +export function tree_build_bins(X: Matrix, bin_card: ptr, bin_vals: ptr, bin_of: ptr) -> void { + let n_rows: i32 = X.rows + let n_cols: i32 = X.cols + let data: ptr = X.data + + for f in 0 to n_cols { + let vals: ptr = bin_vals + f * TREE_MAX_BINS + let mut card: i32 = 0 + let mut overflow: bool = false + + # Collect the distinct values, kept sorted by insertion. The array is + # short by construction, so the shift is cheaper than a full sort and + # the lookup is a binary search over it. + for i in 0 to n_rows { + let v: f32 = data[i * n_cols + f] + # NaN orders inconsistently and would corrupt the search. Hand the + # whole feature to the sort path, which is where it was before. + if v != v { overflow = true; break } + let mut lo: i32 = 0 + let mut hi: i32 = card + while lo < hi { + let mid: i32 = (lo + hi) / 2 + if vals[mid] < v { lo = mid + 1 } else { hi = mid } + } + let seen: bool = lo < card && vals[lo] == v + if not seen { + if card >= TREE_MAX_BINS { overflow = true; break } + let mut k: i32 = card + while k > lo { + vals[k] = vals[k - 1] + k = k - 1 + } + vals[lo] = v + card = card + 1 + } + } + + if overflow { + bin_card[f] = 0 + } else { + bin_card[f] = card + let col: ptr = bin_of + f * n_rows + for i in 0 to n_rows { + let v: f32 = data[i * n_cols + f] + let mut lo: i32 = 0 + let mut hi: i32 = card + while lo < hi { + let mid: i32 = (lo + hi) / 2 + if vals[mid] < v { lo = mid + 1 } else { hi = mid } + } + col[i] = lo as u8 + } + } + } +} + +struct RFScratch { + # Precomputed value bins, or null pointers with bin_card null when the + # caller did not supply them, in which case every feature takes the sort + # path. See tree_build_bins. + bin_card: ptr, + bin_vals: ptr, + # One byte per (feature, sample). TREE_MAX_BINS caps a bin index at 255, so + # a byte holds it, and a feature's whole column of bins stays small enough + # to sit in cache while the histogram pass gathers it. + bin_of: ptr, + bin_rows: i32, + hist: ptr, + sort_values: ptr, + sort_classes: ptr, + left_counts: ptr, + right_counts: ptr, + total_counts: ptr, + present: ptr, + row_base: ptr, + class_idx: ptr, + feature_mask: ptr +} + # Tree builder with feature subsampling for RandomForest. # At each node, only max_features randomly chosen features are considered. # @@ -884,12 +1243,14 @@ function _build_classifier_tree_rf( criterion: i32, classes: ptr, n_classes: i32, + class_of_row: ptr, max_features: i32, prng_state: ptr, cat_card: ptr, max_card: i32, cat_iscratch: ptr, - cat_mean: ptr + cat_mean: ptr, + scratch: RFScratch ) -> i32 { let node_idx: i32 = n_nodes[0] n_nodes[0] = n_nodes[0] + 1 @@ -932,30 +1293,26 @@ function _build_classifier_tree_rf( let x_data: ptr = X.data let x_cols: i32 = X.cols let class_bytes: i64 = (n_indices as i64) * 4 - let row_base: ptr = malloc((n_indices as i64) * 4) as ptr + let row_base: ptr = scratch.row_base for i in 0 to n_indices { row_base[i] = indices[i] * x_cols } - let class_idx: ptr = malloc((n_indices as i64) * 4) as ptr + let class_idx: ptr = scratch.class_idx + let bin_card: ptr = scratch.bin_card + let total_counts: ptr = scratch.total_counts + for c in 0 to n_classes { total_counts[c] = 0 } for i in 0 to n_indices { - let mut ci: i32 = 0 - let yi: f32 = y[indices[i]] - for c in 0 to n_classes { - if yi == classes[c] { ci = c } - } + let ci: i32 = class_of_row[indices[i]] class_idx[i] = ci + total_counts[ci] = total_counts[ci] + 1 } - let total_counts: ptr = malloc((n_classes as i64) * 4) as ptr - for c in 0 to n_classes { total_counts[c] = 0 } - for i in 0 to n_indices { total_counts[class_idx[i]] = total_counts[class_idx[i]] + 1 } - # Ascending list of the classes that actually occur at this node. A class # with no samples here has no samples on either side of any split below, # so its impurity term is 0.0 and dropping it is exact. Every count loop in # the split scan walks this list instead of the whole label set. - let present: ptr = malloc((n_classes as i64) * 4) as ptr + let present: ptr = scratch.present let mut n_present: i32 = 0 for c in 0 to n_classes { if total_counts[c] > 0 { @@ -966,14 +1323,14 @@ function _build_classifier_tree_rf( let parent_impurity: f32 = _tree_gini_present(total_counts, present, n_present, n_indices) - let sort_values: ptr = array_new_f32(n_indices) - let sort_classes: ptr = malloc((n_indices as i64) * 4) as ptr - let left_counts: ptr = malloc((n_classes as i64) * 4) as ptr - let right_counts: ptr = malloc((n_classes as i64) * 4) as ptr + let sort_values: ptr = scratch.sort_values + let sort_classes: ptr = scratch.sort_classes + let left_counts: ptr = scratch.left_counts + let right_counts: ptr = scratch.right_counts # Determine which features to try at this node let n_features: i32 = X.cols - let feature_mask: ptr = malloc((n_features as i64) * 4) as ptr + let feature_mask: ptr = scratch.feature_mask for f in 0 to n_features { feature_mask[f] = 0 } if max_features >= n_features { @@ -1071,6 +1428,75 @@ function _build_classifier_tree_rf( continue } + # Binned split search. One linear pass fills this node's class + # histogram per distinct value, then the candidates are read straight + # off the bins in ascending order. No sort, and no second pass over the + # samples: the scan is over bins rather than over rows. + # + # A candidate is evaluated between consecutive non-empty bins, with the + # left side holding every bin at or below the previous one. Those are + # the same split points, with the same integer counts, that the sort + # scan below reaches wherever two consecutive sorted values differ, so + # the tree is unchanged. n_left and n_right are both at least one + # whenever a candidate is evaluated, because the previous bin has been + # moved left and the current one has not, so the sort path's emptiness + # check has no counterpart here. + # + # The gate keeps the bin scan from costing more than the sort it + # replaces: clearing the histogram is bcard * n_classes, so a feature + # with more distinct values than the node has samples goes to the sort. + let mut bcard: i32 = 0 + if bin_card != null { bcard = bin_card[f] } + if bcard > 0 && bcard <= n_indices { + let bcol: ptr = scratch.bin_of + f * scratch.bin_rows + let hist: ptr = scratch.hist + for k in 0 to bcard * n_classes { hist[k] = 0 } + for i in 0 to n_indices { + let hpos: i32 = (bcol[indices[i]] as i32) * n_classes + class_idx[i] + hist[hpos] = hist[hpos] + 1 + } + + for pi in 0 to n_present { + let c: i32 = present[pi] + left_counts[c] = 0 + right_counts[c] = total_counts[c] + } + + let bvals: ptr = scratch.bin_vals + f * TREE_MAX_BINS + let nf_bin: f32 = n_indices as f32 + let mut n_left_bin: i32 = 0 + let mut prev_b: i32 = 0 - 1 + for b in 0 to bcard { + let hbase: i32 = b * n_classes + let mut bc: i32 = 0 + for pi in 0 to n_present { bc = bc + hist[hbase + present[pi]] } + if bc == 0 { continue } + + if prev_b >= 0 { + let n_right_bin: i32 = n_indices - n_left_bin + let gl: f32 = _tree_gini_present(left_counts, present, n_present, n_left_bin) + let gr: f32 = _tree_gini_present(right_counts, present, n_present, n_right_bin) + let weighted: f32 = (n_left_bin as f32) / nf_bin * gl + (n_right_bin as f32) / nf_bin * gr + if weighted < best_impurity { + best_impurity = weighted + best_feature = f + best_threshold = (bvals[prev_b] + bvals[b]) / 2.0 + best_is_cat = false + } + } + + for pi in 0 to n_present { + let c: i32 = present[pi] + let h: i32 = hist[hbase + c] + left_counts[c] = left_counts[c] + h + right_counts[c] = right_counts[c] - h + } + n_left_bin = n_left_bin + bc + prev_b = b + } + continue + } + # Gather and check for a constant column in one pass. If every sample # shares a value, the scan below hits `sort_values[i] == sort_values[i+1]` # at every position and evaluates no candidate at all, so skipping the @@ -1087,7 +1513,7 @@ function _build_classifier_tree_rf( if is_constant { continue } memcpy(sort_classes as ptr, class_idx as ptr, class_bytes) - _sort_pairs(sort_values, sort_classes, n_indices) + _sort_pairs_fast(sort_values, sort_classes, n_indices) # right_counts starts as the node's full class histogram, which is # exactly total_counts; left_counts starts empty. Seeding from the @@ -1095,12 +1521,16 @@ function _build_classifier_tree_rf( for pi in 0 to n_present { let c: i32 = present[pi] left_counts[c] = 0 - right_counts[c] = total_counts[c] + let cnt: i32 = total_counts[c] + right_counts[c] = cnt } for i in 0 to n_indices - 1 { - left_counts[sort_classes[i]] = left_counts[sort_classes[i]] + 1 - right_counts[sort_classes[i]] = right_counts[sort_classes[i]] - 1 + let c: i32 = sort_classes[i] + let L: i32 = left_counts[c] + let R: i32 = right_counts[c] + left_counts[c] = L + 1 + right_counts[c] = R - 1 if sort_values[i] == sort_values[i + 1] { continue } @@ -1128,15 +1558,6 @@ function _build_classifier_tree_rf( } } - free(feature_mask as ptr) - array_free_f32(sort_values) - free(sort_classes as ptr) - free(left_counts as ptr) - free(right_counts as ptr) - free(class_idx as ptr) - free(present as ptr) - free(total_counts as ptr) - if best_feature < 0 || best_impurity >= parent_impurity - 0.0000000001 { let y_sub: ptr = array_new_f32(n_indices) for i in 0 to n_indices { @@ -1149,7 +1570,6 @@ function _build_classifier_tree_rf( nodes[node_idx].right = -1 nodes[node_idx].impurity = parent_impurity array_free_f32(y_sub) - free(row_base as ptr) return node_idx } @@ -1185,16 +1605,14 @@ function _build_classifier_tree_rf( } } - free(row_base as ptr) - nodes[node_idx].is_leaf = false nodes[node_idx].feature = best_feature nodes[node_idx].threshold = best_threshold nodes[node_idx].n_samples = n_indices nodes[node_idx].impurity = parent_impurity - let left_idx: i32 = _build_classifier_tree_rf(nodes, n_nodes, X, y, left_indices, left_n, depth + 1, max_depth, criterion, classes, n_classes, max_features, prng_state, cat_card, max_card, cat_iscratch, cat_mean) - let right_idx: i32 = _build_classifier_tree_rf(nodes, n_nodes, X, y, right_indices, right_n, depth + 1, max_depth, criterion, classes, n_classes, max_features, prng_state, cat_card, max_card, cat_iscratch, cat_mean) + let left_idx: i32 = _build_classifier_tree_rf(nodes, n_nodes, X, y, left_indices, left_n, depth + 1, max_depth, criterion, classes, n_classes, class_of_row, max_features, prng_state, cat_card, max_card, cat_iscratch, cat_mean, scratch) + let right_idx: i32 = _build_classifier_tree_rf(nodes, n_nodes, X, y, right_indices, right_n, depth + 1, max_depth, criterion, classes, n_classes, class_of_row, max_features, prng_state, cat_card, max_card, cat_iscratch, cat_mean, scratch) nodes[node_idx].left = left_idx nodes[node_idx].right = right_idx @@ -1225,7 +1643,13 @@ export function decision_tree_classifier_fit_rf(X: Matrix, y: ptr, n_classe # the same 128-level cap applies. It is scoped by max_features exactly as the # threshold sweep is: a categorical feature that was not drawn at a node is not # scanned there. -export function decision_tree_classifier_fit_rf_categorical(X: Matrix, y: ptr, n_classes: i32, max_depth: i32, criterion: i32, max_features: i32, prng_state: ptr, categorical: ptr) -> DecisionTreeClassifier { +# RandomForest tree fit taking value bins precomputed by tree_build_bins. +# +# The forest bins its design once and hands every tree the same bin values with +# that tree's bootstrap rows, so the O(n_features * n_rows) binning pass is paid +# once rather than per tree. Pass a null bin_card to fit without bins, which +# puts every feature on the sort path. +export function decision_tree_classifier_fit_rf_binned(X: Matrix, y: ptr, n_classes: i32, max_depth: i32, criterion: i32, max_features: i32, prng_state: ptr, categorical: ptr, bin_card: ptr, bin_vals: ptr, bin_of: ptr) -> DecisionTreeClassifier { # Issue #398: this used to append at classes[n_found] with no check against # n_classes, which wrote past the allocation whenever y carried more # distinct labels than the caller declared. @@ -1240,6 +1664,36 @@ export function decision_tree_classifier_fit_rf_categorical(X: Matrix, y: ptr = malloc((X.rows as i64) * 4) as ptr for i in 0 to X.rows { indices[i] = i } + let class_of_row: ptr = malloc((X.rows as i64) * 4) as ptr + for i in 0 to X.rows { + let mut ci: i32 = 0 + let yi: f32 = y[i] + for c in 0 to n_classes { + if yi == classes[c] { ci = c } + } + class_of_row[i] = ci + } + + # The histogram is only touched for a binned feature, and TREE_MAX_BINS + # caps a bin count, so this is the largest it can ever need to be. + let hist_len: i32 = TREE_MAX_BINS * n_classes + let scratch: RFScratch = RFScratch { + bin_card: bin_card, + bin_vals: bin_vals, + bin_of: bin_of, + bin_rows: X.rows, + hist: malloc((hist_len as i64) * 4) as ptr, + sort_values: array_new_f32(X.rows), + sort_classes: malloc((X.rows as i64) * 4) as ptr, + left_counts: malloc((n_classes as i64) * 4) as ptr, + right_counts: malloc((n_classes as i64) * 4) as ptr, + total_counts: malloc((n_classes as i64) * 4) as ptr, + present: malloc((n_classes as i64) * 4) as ptr, + row_base: malloc((X.rows as i64) * 4) as ptr, + class_idx: malloc((X.rows as i64) * 4) as ptr, + feature_mask: malloc((X.cols as i64) * 4) as ptr + } + # Categorical bookkeeping, allocated once per tree rather than per node so # the numeric path picks up no extra allocation at all. With no categorical # feature max_card is 0 and these are a 16-byte stub and one float. @@ -1249,11 +1703,22 @@ export function decision_tree_classifier_fit_rf_categorical(X: Matrix, y: ptr = malloc((ccap * 4 + ccap * (n_classes as i64)) * 4 + 16) as ptr let cat_mean: ptr = array_new_f32(max_card + 1) - let _final: i32 = _build_classifier_tree_rf(nodes, n_nodes_ptr, X, y, indices, X.rows, 0, max_depth, criterion, classes, n_classes, max_features, prng_state, cat_card, max_card, cat_iscratch, cat_mean) + let _final: i32 = _build_classifier_tree_rf(nodes, n_nodes_ptr, X, y, indices, X.rows, 0, max_depth, criterion, classes, n_classes, class_of_row, max_features, prng_state, cat_card, max_card, cat_iscratch, cat_mean, scratch) free(cat_card as ptr) free(cat_iscratch as ptr) array_free_f32(cat_mean) + free(class_of_row as ptr) + free(scratch.hist as ptr) + array_free_f32(scratch.sort_values) + free(scratch.sort_classes as ptr) + free(scratch.left_counts as ptr) + free(scratch.right_counts as ptr) + free(scratch.total_counts as ptr) + free(scratch.present as ptr) + free(scratch.row_base as ptr) + free(scratch.class_idx as ptr) + free(scratch.feature_mask as ptr) free(indices as ptr) let n_nodes: i32 = n_nodes_ptr[0] @@ -1270,6 +1735,12 @@ export function decision_tree_classifier_fit_rf_categorical(X: Matrix, y: ptr, n_classes: i32, max_depth: i32, criterion: i32, max_features: i32, prng_state: ptr, categorical: ptr) -> DecisionTreeClassifier { + return decision_tree_classifier_fit_rf_binned(X, y, n_classes, max_depth, criterion, max_features, prng_state, categorical, null, null, null) +} + export function decision_tree_classifier_predict_one(model: DecisionTreeClassifier, x: ptr) -> f32 { let mut node_idx: i32 = 0 while not model.nodes[node_idx].is_leaf { diff --git a/tests/test_additive_chi2_transform_optimization.flow b/tests/test_additive_chi2_transform_optimization.flow index 515d1ca..b0e2ea8 100644 --- a/tests/test_additive_chi2_transform_optimization.flow +++ b/tests/test_additive_chi2_transform_optimization.flow @@ -1,17 +1,12 @@ import "lib/scikit/scikit.flow" extern { - function timespec_get(ts: ptr, base: i32) -> i32 + function flow_now_ns() -> i64 } -const TIME_UTC: i32 = 1 function now_ns() -> i64 { - let ts: ptr = malloc(16) as ptr - timespec_get(ts, TIME_UTC) - let value: i64 = ts[0] * 1000000000 + ts[1] - free(ts as ptr) - return value + return flow_now_ns() } function abs_f32(x: f32) -> f32 { diff --git a/tests/test_gaussian_nb_predict_proba_benchmark.flow b/tests/test_gaussian_nb_predict_proba_benchmark.flow index 7167c63..59ecabb 100644 --- a/tests/test_gaussian_nb_predict_proba_benchmark.flow +++ b/tests/test_gaussian_nb_predict_proba_benchmark.flow @@ -1,20 +1,15 @@ import "lib/scikit/scikit.flow" extern { - function timespec_get(ts: ptr, base: i32) -> i32 + function flow_now_ns() -> i64 function log(x: f64) -> f64 function exp(x: f64) -> f64 function printf(fmt: string, ...) -> i32 } -const TIME_UTC: i32 = 1 function now_ns() -> i64 { - let ts: ptr = malloc(16) as ptr - timespec_get(ts, TIME_UTC) - let value: i64 = ts[0] * 1000000000 + ts[1] - free(ts as ptr) - return value + return flow_now_ns() } # Pre-#317 reference implementation kept only for direct A/B timing. diff --git a/tests/test_hashing_vectorizer_transform_optimization.flow b/tests/test_hashing_vectorizer_transform_optimization.flow index 7604f0f..399063f 100644 --- a/tests/test_hashing_vectorizer_transform_optimization.flow +++ b/tests/test_hashing_vectorizer_transform_optimization.flow @@ -1,17 +1,12 @@ import "lib/scikit/scikit.flow" extern { - function timespec_get(ts: ptr, base: i32) -> i32 + function flow_now_ns() -> i64 } -const TIME_UTC: i32 = 1 function now_ns() -> i64 { - let ts: ptr = malloc(16) as ptr - timespec_get(ts, TIME_UTC) - let value: i64 = ts[0] * 1000000000 + ts[1] - free(ts as ptr) - return value + return flow_now_ns() } function reference_transform(model: HashingVectorizer, tokens: ptr >, n_samples: i32, n_tokens_per_sample: ptr) -> Matrix { diff --git a/tests/test_opt_adaboostregressor_predict.flow b/tests/test_opt_adaboostregressor_predict.flow index 20a36ff..1a5b3d5 100644 --- a/tests/test_opt_adaboostregressor_predict.flow +++ b/tests/test_opt_adaboostregressor_predict.flow @@ -9,18 +9,13 @@ import "lib/scikit/scikit.flow" extern { - function timespec_get(ts: ptr, base: i32) -> i32 + function flow_now_ns() -> i64 function printf(fmt: string, ...) -> i32 } -const ABRP_TIME_UTC: i32 = 1 function abrp_now_ns() -> i64 { - let ts: ptr = malloc(16) as ptr - timespec_get(ts, ABRP_TIME_UTC) - let value: i64 = ts[0] * 1000000000 + ts[1] - free(ts as ptr) - return value + return flow_now_ns() } function abrp_abs(x: f32) -> f32 { diff --git a/tests/test_opt_baggingclassifier_fit.flow b/tests/test_opt_baggingclassifier_fit.flow index 0ac456d..4d9f42c 100644 --- a/tests/test_opt_baggingclassifier_fit.flow +++ b/tests/test_opt_baggingclassifier_fit.flow @@ -16,18 +16,13 @@ import "lib/scikit/scikit.flow" extern { - function timespec_get(ts: ptr, base: i32) -> i32 + function flow_now_ns() -> i64 function printf(fmt: string, ...) -> i32 } -const BCF_TIME_UTC: i32 = 1 function bcf_now_ns() -> i64 { - let ts: ptr = malloc(16) as ptr - timespec_get(ts, BCF_TIME_UTC) - let value: i64 = ts[0] * 1000000000 + ts[1] - free(ts as ptr) - return value + return flow_now_ns() } # Pre-#234 reference implementation, kept only for A/B parity and timing. diff --git a/tests/test_opt_baggingregressor_fit.flow b/tests/test_opt_baggingregressor_fit.flow index 9a7b407..da122f8 100644 --- a/tests/test_opt_baggingregressor_fit.flow +++ b/tests/test_opt_baggingregressor_fit.flow @@ -16,18 +16,13 @@ import "lib/scikit/scikit.flow" extern { - function timespec_get(ts: ptr, base: i32) -> i32 + function flow_now_ns() -> i64 function printf(fmt: string, ...) -> i32 } -const BRF_TIME_UTC: i32 = 1 function brf_now_ns() -> i64 { - let ts: ptr = malloc(16) as ptr - timespec_get(ts, BRF_TIME_UTC) - let value: i64 = ts[0] * 1000000000 + ts[1] - free(ts as ptr) - return value + return flow_now_ns() } function brf_abs(x: f32) -> f32 { diff --git a/tests/test_opt_baggingregressor_predict.flow b/tests/test_opt_baggingregressor_predict.flow index 45d04ce..1d6cf45 100644 --- a/tests/test_opt_baggingregressor_predict.flow +++ b/tests/test_opt_baggingregressor_predict.flow @@ -8,18 +8,13 @@ import "lib/scikit/scikit.flow" extern { - function timespec_get(ts: ptr, base: i32) -> i32 + function flow_now_ns() -> i64 function printf(fmt: string, ...) -> i32 } -const BRP_TIME_UTC: i32 = 1 function brp_now_ns() -> i64 { - let ts: ptr = malloc(16) as ptr - timespec_get(ts, BRP_TIME_UTC) - let value: i64 = ts[0] * 1000000000 + ts[1] - free(ts as ptr) - return value + return flow_now_ns() } function brp_abs(x: f32) -> f32 { diff --git a/tests/test_opt_bernoullinb_predict.flow b/tests/test_opt_bernoullinb_predict.flow index 9b1267e..260f2b8 100644 --- a/tests/test_opt_bernoullinb_predict.flow +++ b/tests/test_opt_bernoullinb_predict.flow @@ -1,20 +1,15 @@ import "lib/scikit/scikit.flow" extern { - function timespec_get(ts: ptr, base: i32) -> i32 + function flow_now_ns() -> i64 function log(x: f64) -> f64 function exp(x: f64) -> f64 function printf(fmt: string, ...) -> i32 } -const TIME_UTC: i32 = 1 function bnb_now_ns() -> i64 { - let ts: ptr = malloc(16) as ptr - timespec_get(ts, TIME_UTC) - let value: i64 = ts[0] * 1000000000 + ts[1] - free(ts as ptr) - return value + return flow_now_ns() } # Verbatim copy of the pre-optimization bernoulli_nb_predict from origin/main, diff --git a/tests/test_opt_categoricalnb_fit.flow b/tests/test_opt_categoricalnb_fit.flow index 23f6df0..cb54e21 100644 --- a/tests/test_opt_categoricalnb_fit.flow +++ b/tests/test_opt_categoricalnb_fit.flow @@ -1,19 +1,14 @@ import "lib/scikit/scikit.flow" extern { - function timespec_get(ts: ptr, base: i32) -> i32 + function flow_now_ns() -> i64 function log(x: f64) -> f64 function printf(fmt: string, ...) -> i32 } -const TIME_UTC: i32 = 1 function catfit_now_ns() -> i64 { - let ts: ptr = malloc(16) as ptr - timespec_get(ts, TIME_UTC) - let value: i64 = ts[0] * 1000000000 + ts[1] - free(ts as ptr) - return value + return flow_now_ns() } # Verbatim copy of the pre-optimization categorical_nb_fit from origin/main. diff --git a/tests/test_opt_categoricalnb_predict.flow b/tests/test_opt_categoricalnb_predict.flow index de5ae3e..eddd665 100644 --- a/tests/test_opt_categoricalnb_predict.flow +++ b/tests/test_opt_categoricalnb_predict.flow @@ -1,18 +1,13 @@ import "lib/scikit/scikit.flow" extern { - function timespec_get(ts: ptr, base: i32) -> i32 + function flow_now_ns() -> i64 function printf(fmt: string, ...) -> i32 } -const TIME_UTC: i32 = 1 function catpred_now_ns() -> i64 { - let ts: ptr = malloc(16) as ptr - timespec_get(ts, TIME_UTC) - let value: i64 = ts[0] * 1000000000 + ts[1] - free(ts as ptr) - return value + return flow_now_ns() } # Verbatim copy of the pre-optimization categorical_nb_predict from origin/main. diff --git a/tests/test_opt_cca_predict.flow b/tests/test_opt_cca_predict.flow index b40fca3..0beddd4 100644 --- a/tests/test_opt_cca_predict.flow +++ b/tests/test_opt_cca_predict.flow @@ -14,17 +14,12 @@ import "lib/scikit/cross_decomposition.flow" extern { - function timespec_get(ts: ptr, base: i32) -> i32 + function flow_now_ns() -> i64 } -const OPTCCA_TIME_UTC: i32 = 1 function optcca_now_ns() -> i64 { - let ts: ptr = malloc(16) as ptr - timespec_get(ts, OPTCCA_TIME_UTC) - let value: i64 = ts[0] * 1000000000 + ts[1] - free(ts as ptr) - return value + return flow_now_ns() } # Byte-for-byte copy of the pre-optimization cca_transform. diff --git a/tests/test_opt_columntransformer_fit.flow b/tests/test_opt_columntransformer_fit.flow index af3d158..ac9bd55 100644 --- a/tests/test_opt_columntransformer_fit.flow +++ b/tests/test_opt_columntransformer_fit.flow @@ -12,17 +12,12 @@ import "lib/scikit/scikit.flow" extern { - function timespec_get(ts: ptr, base: i32) -> i32 + function flow_now_ns() -> i64 } -const CTOPT_TIME_UTC: i32 = 1 function ctopt_now_ns() -> i64 { - let ts: ptr = malloc(16) as ptr - timespec_get(ts, CTOPT_TIME_UTC) - let value: i64 = ts[0] * 1000000000 + ts[1] - free(ts as ptr) - return value + return flow_now_ns() } function ctopt_reference_extract(X: Matrix, columns: ptr, n_columns: i32) -> Matrix { diff --git a/tests/test_opt_complementnb_fit.flow b/tests/test_opt_complementnb_fit.flow index 52c1389..1820d2b 100644 --- a/tests/test_opt_complementnb_fit.flow +++ b/tests/test_opt_complementnb_fit.flow @@ -1,18 +1,13 @@ import "lib/scikit/scikit.flow" extern { - function timespec_get(ts: ptr, base: i32) -> i32 + function flow_now_ns() -> i64 function printf(fmt: string, ...) -> i32 } -const TIME_UTC: i32 = 1 function cnb_now_ns() -> i64 { - let ts: ptr = malloc(16) as ptr - timespec_get(ts, TIME_UTC) - let value: i64 = ts[0] * 1000000000 + ts[1] - free(ts as ptr) - return value + return flow_now_ns() } function cnb_reference_fit_loop(X: Matrix, y: ptr, n_classes: i32, alpha: f32) -> ptr > { diff --git a/tests/test_opt_complementnb_predict.flow b/tests/test_opt_complementnb_predict.flow index d8edb5c..1bcb0df 100644 --- a/tests/test_opt_complementnb_predict.flow +++ b/tests/test_opt_complementnb_predict.flow @@ -1,18 +1,13 @@ import "lib/scikit/scikit.flow" extern { - function timespec_get(ts: ptr, base: i32) -> i32 + function flow_now_ns() -> i64 function printf(fmt: string, ...) -> i32 } -const TIME_UTC: i32 = 1 function cnb_now_ns() -> i64 { - let ts: ptr = malloc(16) as ptr - timespec_get(ts, TIME_UTC) - let value: i64 = ts[0] * 1000000000 + ts[1] - free(ts as ptr) - return value + return flow_now_ns() } # Verbatim copy of the pre-optimization complement_nb_predict from origin/main. diff --git a/tests/test_opt_decisiontreeclassifier_fit.flow b/tests/test_opt_decisiontreeclassifier_fit.flow index 8df4a4a..9ad2bd0 100644 --- a/tests/test_opt_decisiontreeclassifier_fit.flow +++ b/tests/test_opt_decisiontreeclassifier_fit.flow @@ -21,19 +21,14 @@ import "lib/scikit/scikit.flow" extern { - function timespec_get(ts: ptr, base: i32) -> i32 + function flow_now_ns() -> i64 function printf(fmt: string, ...) -> i32 function calloc(count: i64, size: i64) -> ptr } -const DTCF_TIME_UTC: i32 = 1 function dtcf_now_ns() -> i64 { - let ts: ptr = malloc(16) as ptr - timespec_get(ts, DTCF_TIME_UTC) - let value: i64 = ts[0] * 1000000000 + ts[1] - free(ts as ptr) - return value + return flow_now_ns() } # -------------------------------------------------------------------------- diff --git a/tests/test_opt_decisiontreeregressor_fit.flow b/tests/test_opt_decisiontreeregressor_fit.flow index 509a5c1..dd8c352 100644 --- a/tests/test_opt_decisiontreeregressor_fit.flow +++ b/tests/test_opt_decisiontreeregressor_fit.flow @@ -15,19 +15,14 @@ import "lib/scikit/scikit.flow" extern { - function timespec_get(ts: ptr, base: i32) -> i32 + function flow_now_ns() -> i64 function printf(fmt: string, ...) -> i32 function calloc(count: i64, size: i64) -> ptr } -const DTRF_TIME_UTC: i32 = 1 function dtrf_now_ns() -> i64 { - let ts: ptr = malloc(16) as ptr - timespec_get(ts, DTRF_TIME_UTC) - let value: i64 = ts[0] * 1000000000 + ts[1] - free(ts as ptr) - return value + return flow_now_ns() } function dtrf_mse(y: ptr, n: i32) -> f32 { diff --git a/tests/test_opt_featureunion_transform.flow b/tests/test_opt_featureunion_transform.flow index bee8190..74a5b9d 100644 --- a/tests/test_opt_featureunion_transform.flow +++ b/tests/test_opt_featureunion_transform.flow @@ -10,17 +10,12 @@ import "lib/scikit/scikit.flow" extern { - function timespec_get(ts: ptr, base: i32) -> i32 + function flow_now_ns() -> i64 } -const FUOPT_TIME_UTC: i32 = 1 function fuopt_now_ns() -> i64 { - let ts: ptr = malloc(16) as ptr - timespec_get(ts, FUOPT_TIME_UTC) - let value: i64 = ts[0] * 1000000000 + ts[1] - free(ts as ptr) - return value + return flow_now_ns() } function fuopt_reference_transform(fu: FeatureUnion, X: Matrix) -> Matrix { diff --git a/tests/test_opt_kneighborsclassifier_fit.flow b/tests/test_opt_kneighborsclassifier_fit.flow index 1b8d380..3019e7f 100644 --- a/tests/test_opt_kneighborsclassifier_fit.flow +++ b/tests/test_opt_kneighborsclassifier_fit.flow @@ -11,17 +11,12 @@ import "lib/scikit/neighbors.flow" extern { - function timespec_get(ts: ptr, base: i32) -> i32 + function flow_now_ns() -> i64 } -const OPTKNN_TIME_UTC: i32 = 1 function optknn_now_ns() -> i64 { - let ts: ptr = malloc(16) as ptr - timespec_get(ts, OPTKNN_TIME_UTC) - let value: i64 = ts[0] * 1000000000 + ts[1] - free(ts as ptr) - return value + return flow_now_ns() } # Byte-for-byte copy of the pre-optimization class-discovery pass, used as the diff --git a/tests/test_opt_labelpropagation_fit.flow b/tests/test_opt_labelpropagation_fit.flow index 6be2e9a..81a7ce9 100644 --- a/tests/test_opt_labelpropagation_fit.flow +++ b/tests/test_opt_labelpropagation_fit.flow @@ -15,17 +15,12 @@ import "lib/scikit/semi_supervised.flow" extern { - function timespec_get(ts: ptr, base: i32) -> i32 + function flow_now_ns() -> i64 } -const OPTLP_TIME_UTC: i32 = 1 function optlp_now_ns() -> i64 { - let ts: ptr = malloc(16) as ptr - timespec_get(ts, OPTLP_TIME_UTC) - let value: i64 = ts[0] * 1000000000 + ts[1] - free(ts as ptr) - return value + return flow_now_ns() } # Copy of the pre-optimization affinity build: full double loop, matrix_at per diff --git a/tests/test_opt_randomforestclassifier_fit.flow b/tests/test_opt_randomforestclassifier_fit.flow index 4f98080..fc1587c 100644 --- a/tests/test_opt_randomforestclassifier_fit.flow +++ b/tests/test_opt_randomforestclassifier_fit.flow @@ -13,18 +13,13 @@ import "lib/scikit/scikit.flow" extern { - function timespec_get(ts: ptr, base: i32) -> i32 + function flow_now_ns() -> i64 function printf(fmt: string, ...) -> i32 } -const RFFT_TIME_UTC: i32 = 1 function rfft_now_ns() -> i64 { - let ts: ptr = malloc(16) as ptr - timespec_get(ts, RFFT_TIME_UTC) - let value: i64 = ts[0] * 1000000000 + ts[1] - free(ts as ptr) - return value + return flow_now_ns() } function rfft_abs(x: f32) -> f32 { diff --git a/tests/test_opt_randomforestclassifier_predict.flow b/tests/test_opt_randomforestclassifier_predict.flow index 96c9752..56770c0 100644 --- a/tests/test_opt_randomforestclassifier_predict.flow +++ b/tests/test_opt_randomforestclassifier_predict.flow @@ -9,18 +9,13 @@ import "lib/scikit/scikit.flow" extern { - function timespec_get(ts: ptr, base: i32) -> i32 + function flow_now_ns() -> i64 function printf(fmt: string, ...) -> i32 } -const RFPR_TIME_UTC: i32 = 1 function rfpr_now_ns() -> i64 { - let ts: ptr = malloc(16) as ptr - timespec_get(ts, RFPR_TIME_UTC) - let value: i64 = ts[0] * 1000000000 + ts[1] - free(ts as ptr) - return value + return flow_now_ns() } # Pre-#228 reference implementation, kept only for A/B parity and timing. diff --git a/tests/test_opt_randomforestclassifier_predict_proba.flow b/tests/test_opt_randomforestclassifier_predict_proba.flow index 079af99..f726ff6 100644 --- a/tests/test_opt_randomforestclassifier_predict_proba.flow +++ b/tests/test_opt_randomforestclassifier_predict_proba.flow @@ -9,18 +9,13 @@ import "lib/scikit/scikit.flow" extern { - function timespec_get(ts: ptr, base: i32) -> i32 + function flow_now_ns() -> i64 function printf(fmt: string, ...) -> i32 } -const RFPP_TIME_UTC: i32 = 1 function rfpp_now_ns() -> i64 { - let ts: ptr = malloc(16) as ptr - timespec_get(ts, RFPP_TIME_UTC) - let value: i64 = ts[0] * 1000000000 + ts[1] - free(ts as ptr) - return value + return flow_now_ns() } function rfpp_abs(x: f32) -> f32 { diff --git a/tests/test_tree_sort_pairs.flow b/tests/test_tree_sort_pairs.flow new file mode 100644 index 0000000..2702cf4 --- /dev/null +++ b/tests/test_tree_sort_pairs.flow @@ -0,0 +1,123 @@ +# Gate for the introsort that replaced the shell sort in the tree split +# search. Checks ascending order, that the companion array stays a +# permutation, and that the value sequence is identical to the shell sort +# the classifier split scan was verified against. +import "lib/scikit/scikit.flow" + +extern { + function printf(fmt: string, ...) -> i32 +} + +function main() -> i32 { + let mut failures: i32 = 0 + let mut state: u32 = 12345 + + # Cover empty, single, already-sorted, reverse-sorted, all-equal, + # two-valued, and pseudo-random inputs across sizes that straddle the + # insertion-sort cutoff and the stack-pushing partition sizes. + for case_id in 0 to 7 { + for size_idx in 0 to 9 { + let sizes: ptr = malloc(9 * 4) as ptr + sizes[0] = 0 + sizes[1] = 1 + sizes[2] = 2 + sizes[3] = 15 + sizes[4] = 16 + sizes[5] = 17 + sizes[6] = 64 + sizes[7] = 1257 + sizes[8] = 4096 + let n: i32 = sizes[size_idx] + free(sizes as ptr) + + let v: ptr = array_new_f32(n + 1) + let c: ptr = malloc(((n + 1) as i64) * 4) as ptr + for i in 0 to n { + state = state * 1103515245 + 12345 + let r: i32 = ((state / 65536) % 1000) as i32 + if case_id == 0 { v[i] = i as f32 } + if case_id == 1 { v[i] = (n - i) as f32 } + if case_id == 2 { v[i] = 7.0 } + if case_id == 3 { v[i] = (i % 2) as f32 } + if case_id == 4 { v[i] = r as f32 } + if case_id == 5 { v[i] = (r % 5) as f32 } + if case_id == 6 { v[i] = (0.0 - (r as f32)) / 3.0 } + c[i] = i + } + + _sort_pairs_fast(v, c, n) + + # Ascending order. + for i in 1 to n { + if v[i - 1] > v[i] { + printf("FAIL order case=%d n=%d at %d\n", case_id, n, i) + failures = failures + 1 + } + } + # Companions must still be a permutation of 0..n. + let seen: ptr = malloc(((n + 1) as i64) * 4) as ptr + for i in 0 to n { seen[i] = 0 } + for i in 0 to n { + if c[i] < 0 || c[i] >= n { + printf("FAIL companion range case=%d n=%d\n", case_id, n) + failures = failures + 1 + } else { + seen[c[i]] = seen[c[i]] + 1 + } + } + for i in 0 to n { + if seen[i] != 1 { + printf("FAIL companion perm case=%d n=%d idx=%d count=%d\n", case_id, n, i, seen[i]) + failures = failures + 1 + } + } + free(seen as ptr) + array_free_f32(v) + free(c as ptr) + } + } + + # The value sequence must match the shell sort exactly, which is the + # property the classifier split scan depends on. + # Mixed signs and mixed magnitudes, which is what the radix key mapping + # has to get right: negatives invert every bit, non-negatives only the + # sign bit. Standardized features are routinely negative. + for trial in 0 to 40 { + let n: i32 = 500 + let a: ptr = array_new_f32(n) + let b: ptr = array_new_f32(n) + let ca: ptr = malloc((n as i64) * 4) as ptr + let cb: ptr = malloc((n as i64) * 4) as ptr + for i in 0 to n { + state = state * 1103515245 + 12345 + let raw: i32 = ((state / 65536) % 4001) as i32 + let mut val: f32 = ((raw - 2000) as f32) / 7.0 + if trial % 4 == 1 { val = 0.0 - (((raw % 37) as f32) / 3.0) } + if trial % 4 == 2 { val = ((raw % 37) as f32) / 3.0 } + if trial % 4 == 3 { val = ((raw % 5) as f32) - 2.0 } + a[i] = val + b[i] = val + ca[i] = i + cb[i] = i + } + _sort_pairs(a, ca, n) + _sort_pairs_fast(b, cb, n) + for i in 0 to n { + if a[i] != b[i] { + printf("FAIL sequence trial=%d idx=%d %f vs %f\n", trial, i, a[i] as f64, b[i] as f64) + failures = failures + 1 + } + } + array_free_f32(a) + array_free_f32(b) + free(ca as ptr) + free(cb as ptr) + } + + if failures == 0 { + printf("SORT OK\n") + return 0 + } + printf("SORT FAILURES: %d\n", failures) + return 1 +}