From 27dcb15ca154373bbc94555b95b9583a537453b5 Mon Sep 17 00:00:00 2001 From: Max Date: Wed, 21 Oct 2026 21:56:01 +0200 Subject: [PATCH 1/4] Add opt-in scope-profiler regions, off by default --- CHANGELOG.md | 9 +++ docs/src/content/docs/guides/profiling.md | 22 +++++++ pyproject.toml | 4 ++ src/cunumpy/__init__.py | 7 +++ src/cunumpy/_algorithms.py | 5 ++ src/cunumpy/_cuda_kernel.py | 33 ++++++----- src/cunumpy/_dispatch.py | 71 ++++++++++++----------- src/cunumpy/_linalg.py | 3 + src/cunumpy/_mirror.py | 34 ++++++----- src/cunumpy/_morton.py | 3 + src/cunumpy/_mpi.py | 41 +++++++------ src/cunumpy/_optimize.py | 4 ++ src/cunumpy/_profile_tags.py | 9 +++ src/cunumpy/_pyccel_cache.py | 5 ++ src/cunumpy/_scipy_host.py | 8 +++ tests/unit/test_opt_in_profiling.py | 37 ++++++++++++ 16 files changed, 214 insertions(+), 81 deletions(-) create mode 100644 src/cunumpy/_profile_tags.py create mode 100644 tests/unit/test_opt_in_profiling.py diff --git a/CHANGELOG.md b/CHANGELOG.md index 6b46bef..8d0fa9b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,15 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +### Added +- scope-profiler regions around the coarse operations (`Kernel` calls as + `cunumpy.kernel.`, CUDA and Pyccel compilation, `sort_by_key`, + `SegmentPlan.sum`, `compact_by_mask`, `morton_keys`, the solvers, `exchange`, + and the device copies of `mpi_buffer` and `DeviceMirror`), behind the opt-in + tag `cunumpy`: off unless the application enables it, e.g. + `sp.setup(enable_tags=["cunumpy"])` or `SCOPE_PROFILER_ENABLE_TAGS=cunumpy`. + New dependency: `scope-profiler>=0.7.5`. + ### Changed - `xp.algorithms.sort_by_key` gathers a 2-D NumPy array along its last axis (component-major markers, `axis=-1`) row by row, about 2x faster than `take` (1e6 markers: 3 rows 12.9 to 6.7 ms, diff --git a/docs/src/content/docs/guides/profiling.md b/docs/src/content/docs/guides/profiling.md index eb7ee15..001fac4 100644 --- a/docs/src/content/docs/guides/profiling.md +++ b/docs/src/content/docs/guides/profiling.md @@ -86,6 +86,28 @@ another library converting internally) show up in Nsight as `memcpy` rows on the timeline; NVTX ranges tell you which part of the step issued them. See [Moving data between host and device](/cunumpy/guides/data-movement/). +## Cunumpy's own regions in scope-profiler + +Cunumpy ships [scope-profiler](https://github.com/max-models/scope-profiler) +regions around its coarse operations: every `Kernel` call +(`cunumpy.kernel.`), CUDA and Pyccel compilation, `sort_by_key`, +`segment_sum` (`cunumpy.SegmentPlan.sum`), `compact_by_mask`, `morton_keys`, the +solvers, `exchange` and the host/device copies of `mpi_buffer` and +`DeviceMirror`. They are off by default, so an application that profiles itself +does not see them, and cost at most about 0.2 µs per call while off. Turn them on with the `cunumpy` tag: + +```python +import scope_profiler as sp + +sp.setup(enable_tags=["cunumpy"]) +``` + +or `SCOPE_PROFILER_ENABLE_TAGS=cunumpy`, or +`scope-profiler run --enable-tag cunumpy simulate.py` (`"*"` enables every +library's tag). The regions time the host side: on CuPy, a kernel region +measures the launch, not the kernel; wrap a synchronizing `timed_region` around +a loop of steps for device times. + ## A profiling checklist 1. Run a few steps with `count_transfers()` and remove every transfer from the diff --git a/pyproject.toml b/pyproject.toml index f70ac73..55943e5 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -25,6 +25,7 @@ dependencies = [ "array-api-compat", "maybempi>=0.1.2", "numpy", + "scope-profiler>=0.7.5", ] optional-dependencies.dev = [ @@ -41,6 +42,9 @@ optional-dependencies.test = [ "coverage", "pytest", "scipy" ] optional-dependencies.test-compiled = [ "cunumpy[test]", "pyccel" ] urls."Source" = "https://github.com/max-models/cunumpy" +[tool.uv.sources] +scope-profiler = { git = "https://github.com/max-models/scope-profiler", branch = "devel" } + [tool.setuptools.packages.find] where = [ "src" ] diff --git a/src/cunumpy/__init__.py b/src/cunumpy/__init__.py index f61e4ed..d8743db 100644 --- a/src/cunumpy/__init__.py +++ b/src/cunumpy/__init__.py @@ -15,6 +15,8 @@ import re as _re from importlib.metadata import PackageNotFoundError, version +import scope_profiler as _sp + from cunumpy import ( algorithms, arguments, @@ -30,6 +32,7 @@ xp, ) from cunumpy._host import evaluate_on_host, host_call, setup_on_host +from cunumpy._profile_tags import _PROFILE_TAGS from cunumpy._scipy_backend import scipy from cunumpy._staging import to_host_async from cunumpy.xp import ( @@ -54,6 +57,10 @@ use_backend, ) +# cunumpy's scope-profiler regions are off unless the application enables the +# "cunumpy" tag +_sp.register_opt_in_tag(*_PROFILE_TAGS) + try: __version__ = version("cunumpy") """The installed version of cunumpy; ``"0.0.0+unknown"`` without package metadata.""" diff --git a/src/cunumpy/_algorithms.py b/src/cunumpy/_algorithms.py index d7e1621..76355df 100644 --- a/src/cunumpy/_algorithms.py +++ b/src/cunumpy/_algorithms.py @@ -7,8 +7,10 @@ from typing import Any import array_api_compat.numpy as np +import scope_profiler as sp from cunumpy import _fake_cupy +from cunumpy._profile_tags import _PROFILE_TAGS from cunumpy.xp import assert_same_backend, get_array_backend, get_array_module @@ -202,6 +204,7 @@ def n_segments(self) -> int: """Number of output segments, fixed when the plan is created.""" return self._n_segments + @sp.profile("cunumpy.SegmentPlan.sum", tags=_PROFILE_TAGS) def sum(self, values: Any, *, out: Any = None) -> Any: """Sum the rows of `values` per segment. @@ -371,6 +374,7 @@ def _radix_argsort(keys: np.ndarray) -> np.ndarray: return order.astype(np.int64, copy=False) +@sp.profile("cunumpy.sort_by_key", tags=_PROFILE_TAGS) def sort_by_key(keys: Any, *arrays: Any, axis: int = 0) -> tuple[Any, ...]: """Sort `keys` and reorder every array the same way, in one stable argsort. @@ -460,6 +464,7 @@ def _take_along_axis(xpm: Any, array: Any, order: Any, axis: int) -> Any: return xpm.take(array, order, axis=axis) +@sp.profile("cunumpy.compact_by_mask", tags=_PROFILE_TAGS) def compact_by_mask(mask: Any, *arrays: Any, axis: int = 0) -> int: """Move the entries where `mask` is True to the front of every array, in place. diff --git a/src/cunumpy/_cuda_kernel.py b/src/cunumpy/_cuda_kernel.py index 2db6d94..3e72f02 100644 --- a/src/cunumpy/_cuda_kernel.py +++ b/src/cunumpy/_cuda_kernel.py @@ -34,7 +34,9 @@ from typing import Any, NamedTuple import numpy as np +import scope_profiler as sp +from cunumpy._profile_tags import _PROFILE_TAGS from cunumpy._transfers import _ACTIVE as _COUNTERS from cunumpy._transfers import _record_sync @@ -2387,21 +2389,22 @@ def compile(self, *, log_stream: Any = None) -> Any: "without CUNUMPY_BOUNDS_CHECK (and without debug mode, which adds " "it) for a kernel using array views on this build.", ) - if self._template_args is None: - raw = cp.RawKernel( - self._source, - self._name, - options=options, - ) - raw.compile(log_stream=log_stream) - else: - module = cp.RawModule( - code=self._source, - options=options, - name_expressions=[self.expression], - ) - module.compile(log_stream=log_stream) - raw = module.get_function(self.expression) + with sp.region("cunumpy.CudaKernel.compile", tags=_PROFILE_TAGS): + if self._template_args is None: + raw = cp.RawKernel( + self._source, + self._name, + options=options, + ) + raw.compile(log_stream=log_stream) + else: + module = cp.RawModule( + code=self._source, + options=options, + name_expressions=[self.expression], + ) + module.compile(log_stream=log_stream) + raw = module.get_function(self.expression) limits = _device_limits(device) attributes = dict(raw.attributes) static = max(0, attributes.get("shared_size_bytes", 0)) diff --git a/src/cunumpy/_dispatch.py b/src/cunumpy/_dispatch.py index 91b5708..b3bc8a2 100644 --- a/src/cunumpy/_dispatch.py +++ b/src/cunumpy/_dispatch.py @@ -30,6 +30,7 @@ import array_api_compat import numpy as np +import scope_profiler as sp from cunumpy._cuda_kernel import CudaKernel, _compile_in_threads from cunumpy._kernel import ( @@ -39,6 +40,7 @@ get_device_kernel_implementation, outputs_from_annotations, ) +from cunumpy._profile_tags import _PROFILE_TAGS from cunumpy._transfers import _ACTIVE as _COUNTERS from cunumpy._transfers import _record from cunumpy.xp import get_backend @@ -240,6 +242,7 @@ def __init__( self._host_kernel = host_kernel self._cuda_kernel = cuda_kernel self._name = name if name is not None else host_kernel.name + self._region = f"cunumpy.kernel.{self._name}" self._missing_cuda = missing_cuda self._cuda_path = None if cuda_path is None else Path(cuda_path) self._test_args_module = test_args @@ -675,41 +678,42 @@ def __call__( If the CUDA kernel runs without `n_threads`, `grid` or ``n_threads_from``. """ - if self._dispatch == "arrays": - on_device = any(_on_device(arg) for arg in args) - kernel = self._device_kernel() if on_device else self._host_kernel - else: - on_device = get_backend() == "cupy" - kernel = self.get_kernel() - if kernel is self._host_kernel: - if _COUNTERS and self._cuda_kernel is None and on_device: - _record( - "fallback", - f"Kernel {self._name!r} has no CUDA kernel: host kernel " - "called on the CuPy backend", + with sp.region(self._region, tags=_PROFILE_TAGS): + if self._dispatch == "arrays": + on_device = any(_on_device(arg) for arg in args) + kernel = self._device_kernel() if on_device else self._host_kernel + else: + on_device = get_backend() == "cupy" + kernel = self.get_kernel() + if kernel is self._host_kernel: + if _COUNTERS and self._cuda_kernel is None and on_device: + _record( + "fallback", + f"Kernel {self._name!r} has no CUDA kernel: host kernel " + "called on the CuPy backend", + ) + if ( + self._dispatch == "arrays" + and not on_device + and kernel._use_cupy is not True + ): + # only host arguments, by the choice above: nothing to convert, + # even while CuPy is the active backend + return kernel.kernel(*args) + return kernel(*args) + if n_threads is None and grid is None and kernel.n_threads_from is None: + raise ValueError( + f"{self._name}: n_threads is required to launch the CUDA kernel " + "(or pass grid, or set cuda_kernel.n_threads_from)", ) - if ( - self._dispatch == "arrays" - and not on_device - and kernel._use_cupy is not True - ): - # only host arguments, by the choice above: nothing to convert, - # even while CuPy is the active backend - return kernel.kernel(*args) - return kernel(*args) - if n_threads is None and grid is None and kernel.n_threads_from is None: - raise ValueError( - f"{self._name}: n_threads is required to launch the CUDA kernel " - "(or pass grid, or set cuda_kernel.n_threads_from)", + return kernel( + *args, + n_threads=n_threads, + grid=grid, + block=block, + shared_mem=shared_mem, + stream=stream, ) - return kernel( - *args, - n_threads=n_threads, - grid=grid, - block=block, - shared_mem=shared_mem, - stream=stream, - ) class KernelCatalog(Mapping): @@ -978,6 +982,7 @@ def check_signatures(self) -> None: + "\n ".join(problems), ) + @sp.profile("cunumpy.KernelCatalog.compile_all", tags=_PROFILE_TAGS) def compile_all(self, jobs: int | None = 1) -> list[str]: """Compile all CUDA kernels now, e.g. at setup instead of in the first step. diff --git a/src/cunumpy/_linalg.py b/src/cunumpy/_linalg.py index 41e8ffb..00af1be 100644 --- a/src/cunumpy/_linalg.py +++ b/src/cunumpy/_linalg.py @@ -5,7 +5,9 @@ from typing import Any import numpy as np +import scope_profiler as sp +from cunumpy._profile_tags import _PROFILE_TAGS from cunumpy.xp import get_array_module @@ -24,6 +26,7 @@ def _as_inexact(xpm: Any, a: Any) -> Any: return a +@sp.profile("cunumpy.solve_circulant", tags=_PROFILE_TAGS) def solve_circulant( c: Any, b: Any, diff --git a/src/cunumpy/_mirror.py b/src/cunumpy/_mirror.py index 44192f6..746ded7 100644 --- a/src/cunumpy/_mirror.py +++ b/src/cunumpy/_mirror.py @@ -12,7 +12,9 @@ from typing import Any import numpy as np +import scope_profiler as sp +from cunumpy._profile_tags import _PROFILE_TAGS from cunumpy._transfers import _ACTIVE as _COUNTERS from cunumpy._transfers import _describe, _nbytes, _record from cunumpy.xp import _cupy_backend, to_cupy @@ -160,17 +162,18 @@ def to_device(self) -> DeviceMirror: if not _cupy_backend(): return self self._check_device() - if self._device is None: - _ = self.device # allocates as a copy of the host, through to_cupy - else: - # host -> device copy into the existing device buffer - self._device.set(self._host) - if _COUNTERS: - _record( - "to_device", - f"DeviceMirror.to_device({_describe(self._host)})", - nbytes=_nbytes(self._host), - ) + with sp.region("cunumpy.DeviceMirror.to_device", tags=_PROFILE_TAGS): + if self._device is None: + _ = self.device # allocates as a copy of the host, through to_cupy + else: + # host -> device copy into the existing device buffer + self._device.set(self._host) + if _COUNTERS: + _record( + "to_device", + f"DeviceMirror.to_device({_describe(self._host)})", + nbytes=_nbytes(self._host), + ) return self def to_host(self, *, stream: Any = None, event: Any = None) -> DeviceMirror: @@ -222,10 +225,11 @@ def to_host(self, *, stream: Any = None, event: Any = None) -> DeviceMirror: if event is not None: cp.cuda.get_current_stream().wait_event(event) # device -> host copy into the existing host buffer - if stream is None: - self._device.get(out=self._host) - else: - self._device.get(out=self._host, stream=stream) + with sp.region("cunumpy.DeviceMirror.to_host", tags=_PROFILE_TAGS): + if stream is None: + self._device.get(out=self._host) + else: + self._device.get(out=self._host, stream=stream) if _COUNTERS: _record( "to_host", diff --git a/src/cunumpy/_morton.py b/src/cunumpy/_morton.py index 49406da..79ade5c 100644 --- a/src/cunumpy/_morton.py +++ b/src/cunumpy/_morton.py @@ -22,8 +22,10 @@ from typing import Any import numpy as np +import scope_profiler as sp from cunumpy._philox import _module +from cunumpy._profile_tags import _PROFILE_TAGS __all__ = [ "MAX_MORTON_LEVELS", @@ -196,6 +198,7 @@ def morton_scales(lower: Sequence[float], upper: Sequence[float], levels: int) - return float(2**levels) / (upper - lower) +@sp.profile("cunumpy.morton_keys", tags=_PROFILE_TAGS) def morton_keys( positions: Any, lower: Sequence[float], diff --git a/src/cunumpy/_mpi.py b/src/cunumpy/_mpi.py index c923ddd..30ec632 100644 --- a/src/cunumpy/_mpi.py +++ b/src/cunumpy/_mpi.py @@ -10,8 +10,10 @@ import array_api_compat import array_api_compat.numpy as np +import scope_profiler as sp from maybempi import get_mpi +from cunumpy._profile_tags import _PROFILE_TAGS from cunumpy._transfers import _ACTIVE as _COUNTERS from cunumpy._transfers import _describe, _nbytes, _record, _record_sync from cunumpy.xp import array_backend, cupy_available, to_numpy @@ -340,9 +342,10 @@ def mpi_buffer( synchronize_for_mpi(array, stream=stream, event=event) transfer_array = staging._transfer_array(array) if send: - if transfer_array is not array: - transfer_array[...] = array - transfer_array.get(out=host) + with sp.region("cunumpy.mpi_buffer.to_host", tags=_PROFILE_TAGS): + if transfer_array is not array: + transfer_array[...] = array + transfer_array.get(out=host) if _COUNTERS: _record( "to_host", @@ -351,21 +354,23 @@ def mpi_buffer( ) yield host if recv: - transfer_array.set(host) - if _COUNTERS: - _record( - "to_device", - f"mpi_buffer({_describe(array)}) staging for recv", - nbytes=_nbytes(array), - ) - # Ensure MPI's host buffer can be reused immediately on context exit. - if transfer_array is not array: - array[...] = transfer_array - if _COUNTERS: - _record_sync("mpi_buffer() staging for recv") - cp.cuda.get_current_stream().synchronize() - - + with sp.region("cunumpy.mpi_buffer.to_device", tags=_PROFILE_TAGS): + transfer_array.set(host) + if _COUNTERS: + _record( + "to_device", + f"mpi_buffer({_describe(array)}) staging for recv", + nbytes=_nbytes(array), + ) + # Ensure MPI's host buffer can be reused immediately on context exit. + if transfer_array is not array: + array[...] = transfer_array + if _COUNTERS: + _record_sync("mpi_buffer() staging for recv") + cp.cuda.get_current_stream().synchronize() + + +@sp.profile("cunumpy.exchange", tags=_PROFILE_TAGS) def exchange( comm: Any, *, diff --git a/src/cunumpy/_optimize.py b/src/cunumpy/_optimize.py index 1bde879..f638e33 100644 --- a/src/cunumpy/_optimize.py +++ b/src/cunumpy/_optimize.py @@ -6,6 +6,9 @@ from collections.abc import Callable from typing import Any, NamedTuple +import scope_profiler as sp + +from cunumpy._profile_tags import _PROFILE_TAGS from cunumpy.xp import array_backend, get_array_backend, get_array_module @@ -35,6 +38,7 @@ def _module(x0: Any) -> Any: return array_backend.xp +@sp.profile("cunumpy.newton", tags=_PROFILE_TAGS) def newton( func: Callable[..., Any], x0: Any, diff --git a/src/cunumpy/_profile_tags.py b/src/cunumpy/_profile_tags.py new file mode 100644 index 0000000..7deda28 --- /dev/null +++ b/src/cunumpy/_profile_tags.py @@ -0,0 +1,9 @@ +"""The scope-profiler tag of cunumpy's own profiling regions. + +``cunumpy/__init__.py`` registers it as opt-in, so the regions record nothing +unless the application enables the ``"cunumpy"`` tag (see +:doc:`/guides/profiling`). +""" + +#: Tags of every cunumpy region (``@sp.profile("cunumpy.", tags=_PROFILE_TAGS)``). +_PROFILE_TAGS = ("cunumpy",) diff --git a/src/cunumpy/_pyccel_cache.py b/src/cunumpy/_pyccel_cache.py index 07cec3a..4fded85 100644 --- a/src/cunumpy/_pyccel_cache.py +++ b/src/cunumpy/_pyccel_cache.py @@ -29,6 +29,10 @@ from pathlib import Path from types import ModuleType +import scope_profiler as sp + +from cunumpy._profile_tags import _PROFILE_TAGS + #: Bumped when the layout of a cached build changes (it was an ``epyccel`` #: folder before the ``pyccel compile`` command line was used). Part of the key. _LAYOUT = "pyccel-cli-1" @@ -245,6 +249,7 @@ def _build(self, module: ModuleType, language: str, staging: Path) -> Path: ) return output + @sp.profile("cunumpy.PyccelCache.compile", tags=_PROFILE_TAGS) def compile(self, module: ModuleType, *, language: str | None = None) -> ModuleType: """Return the compiled form of `module`, building it on first use. diff --git a/src/cunumpy/_scipy_host.py b/src/cunumpy/_scipy_host.py index 0a62dff..123a241 100644 --- a/src/cunumpy/_scipy_host.py +++ b/src/cunumpy/_scipy_host.py @@ -8,8 +8,11 @@ from collections.abc import Callable from typing import Any +import scope_profiler as sp + from cunumpy import xp from cunumpy._host import _to_device +from cunumpy._profile_tags import _PROFILE_TAGS def _scipy(name: str, function: str) -> Any: @@ -79,6 +82,7 @@ def constraints(self, constraints: Any) -> Any: return out +@sp.profile("cunumpy.fsolve", tags=_PROFILE_TAGS) def fsolve( func: Callable[..., Any], x0: Any, @@ -134,6 +138,7 @@ def fsolve( return bridge.result(out) +@sp.profile("cunumpy.root", tags=_PROFILE_TAGS) def root( fun: Callable[..., Any], x0: Any, @@ -190,6 +195,7 @@ def root( return bridge.result(out) +@sp.profile("cunumpy.minimize", tags=_PROFILE_TAGS) def minimize( fun: Callable[..., Any], x0: Any, @@ -266,6 +272,7 @@ def minimize( return bridge.result(out) +@sp.profile("cunumpy.quad", tags=_PROFILE_TAGS) def quad( func: Callable[..., Any], a: float, @@ -313,6 +320,7 @@ def quad( ) +@sp.profile("cunumpy.odeint", tags=_PROFILE_TAGS) def odeint( func: Callable[..., Any], y0: Any, diff --git a/tests/unit/test_opt_in_profiling.py b/tests/unit/test_opt_in_profiling.py new file mode 100644 index 0000000..e4b6c50 --- /dev/null +++ b/tests/unit/test_opt_in_profiling.py @@ -0,0 +1,37 @@ +"""cunumpy's scope-profiler regions: off by default, recorded once the +application enables the ``"cunumpy"`` tag.""" + +import numpy as np +import pytest +import scope_profiler as sp + +import cunumpy as xp + + +@pytest.fixture +def profiler(): + """A fresh scope-profiler state, reset again afterwards so that no + configuration leaks into other tests.""" + sp.reset() + yield + sp.reset() + + +def _cunumpy_regions(**setup_options): + sp.setup(output=None, **setup_options) + keys = np.asarray([2, 0, 1, 0]) + with xp.use_backend("numpy"): + xp.algorithms.sort_by_key(keys, np.arange(4.0)) + xp.kernels.Kernel(lambda a: None, name="noop")(keys) + results = sp.finalize(verbose=False, return_results=True) + return [name for name in results.region_names if name.startswith("cunumpy.")] + + +def test_regions_are_off_by_default(profiler): + assert _cunumpy_regions() == [] + + +@pytest.mark.parametrize("tags", [["cunumpy"], ["*"]]) +def test_enabled_tag_records_regions(profiler, tags): + regions = _cunumpy_regions(enable_tags=tags) + assert sorted(regions) == ["cunumpy.kernel.noop", "cunumpy.sort_by_key"] From a1cee09ab630090fad34933bb71b8d6da1828a04 Mon Sep 17 00:00:00 2001 From: Max Date: Wed, 21 Oct 2026 22:36:49 +0200 Subject: [PATCH 2/4] Merge the uv sources tables --- pyproject.toml | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 0d86ffd..f73a9f1 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -42,9 +42,6 @@ optional-dependencies.test = [ "coverage", "pytest", "scipy" ] optional-dependencies.test-compiled = [ "cunumpy[test]", "pyccel" ] urls."Source" = "https://github.com/max-models/cunumpy" -[tool.uv.sources] -scope-profiler = { git = "https://github.com/max-models/scope-profiler", branch = "devel" } - [tool.setuptools.packages.find] where = [ "src" ] @@ -55,6 +52,7 @@ cunumpy = [ "py.typed", "*.pyi", "LLM_GUIDE.md", "cuda/include/cunumpy/*.cuh" ] # and the release use PyPI only. [tool.uv.sources] maybempi = { git = "https://github.com/max-models/maybempi", branch = "devel" } +scope-profiler = { git = "https://github.com/max-models/scope-profiler", branch = "devel" } [tool.ruff] # agent worktrees and scratch scripts are local copies, not part of the project From 44c6d5dad8bc2e16a9e59632845214576c097903 Mon Sep 17 00:00:00 2001 From: Max Date: Thu, 22 Oct 2026 08:03:45 +0200 Subject: [PATCH 3/4] Install scope-profiler from devel too --- requirements-devel.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/requirements-devel.txt b/requirements-devel.txt index ab53722..84b537c 100644 --- a/requirements-devel.txt +++ b/requirements-devel.txt @@ -2,3 +2,4 @@ # them before cunumpy on every run that does not target main; main and the release use PyPI. # pip install -r requirements-devel.txt maybempi @ git+https://github.com/max-models/maybempi@devel +scope-profiler @ git+https://github.com/max-models/scope-profiler@devel From 5406263c674a751f86268717eeb34eabfa94d961 Mon Sep 17 00:00:00 2001 From: Max Date: Thu, 22 Oct 2026 08:28:09 +0200 Subject: [PATCH 4/4] Give the Pyodide test the devel branches as wheels --- .github/workflows/pyodide.yml | 10 +++++++--- tests/pyodide/run.mjs | 20 ++++++++++++++------ 2 files changed, 21 insertions(+), 9 deletions(-) diff --git a/.github/workflows/pyodide.yml b/.github/workflows/pyodide.yml index c431dee..8ba8ea2 100644 --- a/.github/workflows/pyodide.yml +++ b/.github/workflows/pyodide.yml @@ -6,8 +6,8 @@ on: pull_request: branches: [main, devel] -# PyPI only, also on devel: micropip, which installs the wheel's dependencies in Pyodide, -# cannot install from git. +# Runs that do not target main take the max-models packages from their devel branches +# (requirements-devel.txt); micropip cannot install from git, so they reach Pyodide as wheels. jobs: wheel-test: runs-on: ubuntu-latest @@ -24,6 +24,10 @@ jobs: cache-dependency-path: tests/pyodide/package-lock.json - name: Build wheel and install compiler-free test dependencies run: | + if [ "${GITHUB_BASE_REF:-$GITHUB_REF_NAME}" != main ]; then + python -m pip install -r requirements-devel.txt + python -m pip wheel --no-deps -w devel-wheels -r requirements-devel.txt + fi python -m pip install build '.[test]' python -m build --wheel - name: Run portable tests in native Python @@ -31,4 +35,4 @@ jobs: - name: Install Pyodide run: npm ci --prefix tests/pyodide - name: Run portable tests against the wheel in Pyodide - run: node tests/pyodide/run.mjs dist/cunumpy-*-py3-none-any.whl + run: node tests/pyodide/run.mjs dist/cunumpy-*-py3-none-any.whl $(ls devel-wheels/*.whl 2>/dev/null) diff --git a/tests/pyodide/run.mjs b/tests/pyodide/run.mjs index 27d9c76..dd337c8 100644 --- a/tests/pyodide/run.mjs +++ b/tests/pyodide/run.mjs @@ -11,18 +11,26 @@ process.on("uncaughtException", (error) => { process.exit(1); }); -const [wheel] = process.argv.slice(2); -if (!wheel || process.argv.length !== 3 || !wheel.endsWith("-py3-none-any.whl")) { - throw new Error("Usage: node run.mjs /path/to/cunumpy-VERSION-py3-none-any.whl"); +// Wheels after cunumpy's are dependencies installed alongside it, e.g. the devel branches of +// the max-models packages, which micropip cannot install from git. +const [wheel, ...dependencies] = process.argv.slice(2); +if (!wheel || ![wheel, ...dependencies].every((path) => path.endsWith("-py3-none-any.whl"))) { + throw new Error( + "Usage: node run.mjs /path/to/cunumpy-VERSION-py3-none-any.whl [dependency wheels...]", + ); } const pyodide = await loadPyodide(); await pyodide.loadPackage(["micropip", "pytest"]); -pyodide.FS.writeFile(`/tmp/${basename(wheel)}`, await readFile(wheel)); -pyodide.globals.set("wheel_uri", `emfs:/tmp/${basename(wheel)}`); +const uris = []; +for (const path of [...dependencies, wheel]) { + pyodide.FS.writeFile(`/tmp/${basename(path)}`, await readFile(path)); + uris.push(`emfs:/tmp/${basename(path)}`); +} +pyodide.globals.set("wheel_uris", pyodide.toPy(uris)); await pyodide.runPythonAsync(` import micropip -await micropip.install(wheel_uri) +await micropip.install(list(wheel_uris)) `); pyodide.FS.mkdirTree("/tests"); pyodide.FS.writeFile(