diff --git a/CHANGELOG.md b/CHANGELOG.md index 5cdb02e..443e62d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,10 @@ ### Added +- `sp.reset()`, for test suites: it forgets the configuration and what was recorded but keeps the + instrumentation, so a library's `@sp.profile` functions record again after the next `setup()`. + Before, the private `ProfileManager._reset()` dropped them for good, and calling one before the + next `setup()` raised `AttributeError`. - Opt-in region tags, so a library can ship its own instrumentation switched off by default: it calls `sp.register_opt_in_tag("mylib")` and tags its regions (`region`, `profile`, `define_region` and `instrument` take `tags=`). A run records them only when it enables the @@ -97,6 +101,16 @@ ### Changed +- A decorated function whose region is disabled (before `setup()`, or behind an opt-in tag that is + off) calls straight through: it skips `metadata` and no longer starts recursive tracing. +- `register_opt_in_tag()` also re-resolves the regions and decorated functions that already exist, + so a library may call it after its own imports. +- `sp.profile` keeps the decorated function's signature for type checkers, and `with sp.region(...)` + type-checks (`BaseProfileRegion` declares `__enter__`/`__exit__`). +- `import scope_profiler` no longer imports h5py or `importlib.metadata`, about half its import + time on top of NumPy. +- The docs no longer claim that disabled instrumentation costs nothing: it costs one Python call. +- The `pproc` extra takes any `maxplotlibx >= 0.1.9` instead of pinning 0.1.9. - The documentation site moved from Sphinx to Astro + Starlight with the shared [starlight-theme](https://github.com/max-models/starlight-theme): the pages are Markdown/MDX in `docs/src/content/docs/`, the tutorials Jupyter diff --git a/CMakeLists.txt b/CMakeLists.txt index aabdac1..e534694 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -1,6 +1,6 @@ cmake_minimum_required(VERSION 3.15) -project(scope-profiler VERSION 0.7.4 LANGUAGES C CXX) +project(scope-profiler VERSION 0.7.5 LANGUAGES C CXX) include(GNUInstallDirs) include(CMakePackageConfigHelpers) diff --git a/configuration.md b/configuration.md index ebcdbb9..8c60730 100644 --- a/configuration.md +++ b/configuration.md @@ -403,8 +403,9 @@ ProfileManager.setup( ## Library regions that are off by default A library can ship its own regions without them showing up in every -profile of the code that uses it. It registers an *opt-in tag* once, at -import time, and tags its regions with it: +profile of the code that uses it. It registers an *opt-in tag* once, in its +package `__init__` (before or after its own imports), and tags its regions +with it: ```python # mylib/__init__.py @@ -422,9 +423,11 @@ def fft(x): ``` `sp.instrument(module, tags=("mylib",))` tags every function it wraps. -These regions stay disabled, and a decorated function stays unwrapped, until -the run enables the tag. Untagged regions and regions with other tags are -unaffected. The application enables it without editing the library, in any +These regions stay disabled until the run enables the tag; a disabled +decorated function calls straight through, skipping `metadata` and recursive +tracing. Either way, instrumentation that is off costs one extra Python call +(roughly 0.1–0.3 µs), so keep it to operations that take microseconds or +more. Untagged regions and regions with other tags are unaffected. The application enables it without editing the library, in any of these ways (they add up): - `setup(enable_tags=["mylib"])` or `session(enable_tags=["mylib"])`, or @@ -438,6 +441,19 @@ of these ways (they add up): records when any of them is enabled. Naming an opt-in tag in `region_tags` (`--tag`) enables it too. +A test suite that profiles in more than one test resets the profiler between +them with `sp.reset()`. It forgets the configuration and what was recorded +but keeps the instrumentation, as importing the library left it, so the +library's decorated functions record again after the next `setup()`: + +```python +@pytest.fixture(autouse=True) +def _fresh_profiler(): + sp.reset() + yield + sp.reset() +``` + ## Recursive profiling of decorated entrypoints Set `recursive_profile=True` to record Python function calls made inside diff --git a/docs/src/content/docs/reference/api.mdx b/docs/src/content/docs/reference/api.mdx index cc1a7da..9fffcf8 100644 --- a/docs/src/content/docs/reference/api.mdx +++ b/docs/src/content/docs/reference/api.mdx @@ -42,6 +42,10 @@ and passes `tags=("mylib",)` to `region`, `profile`, `define_region` or `SCOPE_PROFILER_ENABLE_TAGS=mylib` or `scope-profiler run --enable-tag mylib`. The registry is process-wide and shared by every manager. +Test suites reset the profiler between tests with `sp.reset()`, which keeps +the instrumentation (decorated functions and `define_region` definitions) +so that it records again after the next `setup()`. + ## ProfileManager The main entry point for all profiling operations. Class methods provide the diff --git a/pyproject.toml b/pyproject.toml index b2250d9..62628bb 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -5,7 +5,7 @@ requires = [ "setuptools", "wheel" ] [project] name = "scope-profiler" -version = "0.7.4" +version = "0.7.5" description = "Profile code regions in python, optionally with LIKWID markers." readme = "README.md" keywords = [ "python" ] @@ -77,7 +77,7 @@ optional-dependencies.pproc = [ "ipykernel", "jupyterlab", "matplotlib", - "maxplotlibx == 0.1.9", + "maxplotlibx >= 0.1.9", "pandas", "plotly", "pyarrow", diff --git a/src/scope_profiler/__init__.py b/src/scope_profiler/__init__.py index 839bfcc..920ee4f 100644 --- a/src/scope_profiler/__init__.py +++ b/src/scope_profiler/__init__.py @@ -1,6 +1,5 @@ """scope-profiler: lightweight region-based profiling for Python and HPC applications.""" -from importlib.metadata import PackageNotFoundError, version from typing import TYPE_CHECKING, Any from scope_profiler.call_stack import ( @@ -79,21 +78,19 @@ recorded_regions = ProfileManager.recorded_regions region = ProfileManager.region register_opt_in_tag = ProfileManager.register_opt_in_tag +reset = ProfileManager.reset registered_regions = ProfileManager.registered_regions session = ProfileManager.session setup = ProfileManager.setup setup_from_env = ProfileManager.setup_from_env run = ProfileManager.run -try: - __version__ = version("scope-profiler") -except PackageNotFoundError: - __version__ = "unknown" - # Keep these imports visible to language servers without importing the optional # plotting/reporting dependencies at runtime. Dynamic module attributes are # otherwise invisible to hover, completion, and static type checkers. if TYPE_CHECKING: + __version__: str + from scope_profiler.chrome_trace_export import export_chrome_trace from scope_profiler.html_report import create_html_report from scope_profiler.inspection import collect_file_metadata, inspect_file @@ -200,6 +197,7 @@ "region_factory", "register_opt_in_tag", "registered_regions", + "reset", "run", "session", "setup", @@ -214,6 +212,15 @@ def __getattr__(name: str) -> Any: """Load optional top-level helpers only when they are first used.""" + if name == "__version__": + # Looked up on first use: importlib.metadata is about half the cost of + # ``import scope_profiler`` on top of NumPy. + from importlib.metadata import PackageNotFoundError, version + + try: + return version("scope-profiler") + except PackageNotFoundError: + return "unknown" if name in { "collect_region_statistics", "collect_roofline_points", diff --git a/src/scope_profiler/h5reader.py b/src/scope_profiler/h5reader.py index a9d40e6..734a596 100644 --- a/src/scope_profiler/h5reader.py +++ b/src/scope_profiler/h5reader.py @@ -12,7 +12,6 @@ from copy import copy from pathlib import Path -import h5py import numpy as np from scope_profiler.h5schema import HDF5SchemaError, migrate_schema, read_schema_version @@ -39,6 +38,10 @@ class CorruptProfileError(OSError): def _open_h5(file_path): """Open a profiling file, reporting an unreadable one in context.""" + # Imported here, not at module level: ``import scope_profiler`` (and so + # every library that ships regions) should not pay for h5py up front. + import h5py + try: return h5py.File(file_path, "r") except OSError as exc: diff --git a/src/scope_profiler/profile_manager.py b/src/scope_profiler/profile_manager.py index 8ee28c3..de1a4c5 100644 --- a/src/scope_profiler/profile_manager.py +++ b/src/scope_profiler/profile_manager.py @@ -13,13 +13,13 @@ import sysconfig import threading import warnings -from collections.abc import Callable +from collections.abc import Callable, Iterable from contextlib import ContextDecorator, contextmanager from contextvars import ContextVar from enum import Enum, auto from time import perf_counter_ns from types import FrameType -from typing import TYPE_CHECKING, ClassVar, NamedTuple +from typing import TYPE_CHECKING, ClassVar, NamedTuple, ParamSpec, TypeVar, overload import numpy as np @@ -63,6 +63,8 @@ _PAYLOAD_TAG = 0x5C09 _WRITE_TOKEN_TAG = 0x5C0A _UNSET = object() +_P = ParamSpec("_P") +_R = TypeVar("_R") # Tags registered with register_opt_in_tag(). Libraries register them at # import time, so this outlives every setup() and reset, and is shared by all @@ -697,13 +699,29 @@ def solve(): ... ``"*"`` enables every opt-in tag. Listing the tag in ``region_tags`` enables it too. - A disabled decorated function is left unwrapped, so instrumentation - that is off costs nothing per call. + Regions and decorated functions that already exist pick the change up, + so the call may come after the library's own imports. + + Instrumentation that is off costs one extra Python call (roughly + 0.1-0.3 µs per call), so keep it to operations that take microseconds + or more. """ for tag in tags: if not isinstance(tag, str) or not tag: raise TypeError(f"opt-in tags must be non-empty strings, got {tag!r}") - _OPT_IN_TAGS.update(tags) + new = set(tags) - _OPT_IN_TAGS + _OPT_IN_TAGS.update(new) + if new and cls._configured: + cls._rebind_tagged(new) + + @classmethod + def _rebind_tagged(cls, tags) -> None: + """Re-resolve the existing regions and decorators that carry ``tags``.""" + for name in [n for n, r in cls._regions.items() if tags.intersection(r.tags)]: + del cls._regions[name] + for name, func, _bound, region_tags in cls._decorated_functions(): + if region_tags and tags.intersection(region_tags): + cls._bind_decorated_region(name, func, _bound, region_tags) #: Original name for :meth:`region`, kept so existing instrumentation #: keeps working unchanged. Same object, not a forwarding wrapper: this @@ -934,8 +952,21 @@ def _bind_decorated_region( ) _bound[0] = region _bound[1] = region.wrap(func) + _bound[2] = isinstance(region, DisabledProfileRegion) return region + @classmethod + def _decorated_functions(cls): + """Yield ``(name, func, _bound, tags)`` for every @profile wrapper.""" + for name, entries in cls._decorators.items(): + for entry in entries: + yield (name, *entry) + + @overload + @classmethod + def profile(cls, region_name: Callable[_P, _R]) -> Callable[_P, _R]: ... + + @overload @classmethod def profile( cls, @@ -943,8 +974,18 @@ def profile( recursive: bool | None = None, metadata=None, record_outcome: bool = False, + tags: Iterable[str] | None = None, + ) -> Callable[[Callable[_P, _R]], Callable[_P, _R]]: ... + + @classmethod + def profile( + cls, + region_name=None, + recursive=None, + metadata=None, + record_outcome=False, tags=None, - ) -> Callable: + ): """ Decorator factory for profiling a function. @@ -979,7 +1020,10 @@ def profile( ``ProfileManager.setup()`` after decoration re-binds the wrapper to the new region class at zero per-call cost. This means ``@ProfileManager.profile`` can be applied at class-definition time - even when ``setup()`` is called later. + even when ``setup()`` is called later. While its region is disabled + (before ``setup()``, or behind an opt-in tag that is off) the wrapper + calls the function directly, skipping ``metadata`` and recursive + tracing. """ region_tags = None if tags is None else tuple(tags) @@ -989,7 +1033,7 @@ def decorator(func): # _bound[1] is the inner callable produced by region.wrap(func). # It is replaced (without touching the outer wrapper) whenever # set_config() is called, so there is no per-call rebind check. - _bound = [None, None] # [region, wrapped_func] + _bound = [None, func, True] # [region, wrapped_func, disabled] recursive_override = recursive def invoke(*args, **kwargs): @@ -1024,11 +1068,10 @@ def _invoke_with_metadata(values, args, kwargs): cls._bind_decorated_region(name, func, _bound, region_tags) cls._decorated_codes.add(func.__code__) - # Register so set_config() can rebind without a per-call check. - cls._decorators.setdefault(name, []).append((func, _bound, region_tags)) - @functools.wraps(func) def wrapper(*args, **kwargs): + if _bound[2]: + return _bound[1](*args, **kwargs) recursive_enabled = cls._config.recursive_profile if recursive_override is not None: recursive_enabled = recursive_override @@ -1057,6 +1100,8 @@ def wrapper(*args, **kwargs): sys.setprofile(prev_profiler) state.depth -= 1 + # Register so set_config() can rebind without a per-call check. + cls._decorators.setdefault(name, []).append((func, _bound, region_tags)) return wrapper # Support @ProfileManager.profile without parentheses @@ -2461,6 +2506,13 @@ def finalize_at_exit(): if cls._configured: cls.finalize() + if not cls._config.deactivate_file_output: + # The hook writes HDF5 at exit, and h5py must not be imported for + # the first time during interpreter shutdown (it crashes). Importing + # it now also registers its own exit hook before ours, so ours runs + # first. + import h5py # noqa: F401 + cls._auto_finalize_callback = finalize_at_exit atexit.register(finalize_at_exit) @@ -2856,9 +2908,8 @@ def set_config(cls, config: ProfilingConfig) -> None: cls._update_region_cls() # Set the proper region class # Rebind all registered decorator wrappers to the new region class. # This is the only place rebinding happens — there is no per-call check. - for name, entries in cls._decorators.items(): - for func, _bound, tags in entries: - cls._bind_decorated_region(name, func, _bound, tags) + for name, func, _bound, tags in cls._decorated_functions(): + cls._bind_decorated_region(name, func, _bound, tags) for name, (functions, tags) in cls._region_definitions.items(): cls.region(name, functions=functions, tags=tags) @@ -2922,11 +2973,31 @@ def _reset_config(cls) -> None: cls._lifecycle_state = _LifecycleState.INACTIVE @classmethod - def _reset(cls) -> None: + def reset(cls) -> None: + """Forget the configuration and everything recorded; keep the instrumentation. + + Leaves the manager as importing the instrumented code does: not + configured, every region disabled until the next :meth:`setup`. + Decorated functions and :meth:`define_region` definitions stay + registered, and decorated functions call straight through until then. + Meant for test suites that profile in more than one test:: + + @pytest.fixture(autouse=True) + def _fresh_profiler(): + sp.reset() + yield + sp.reset() + """ cls._reset_regions() cls._reset_config() - # Back to the state a fresh import leaves behind: no configuration, - # and regions disabled until setup() or get_config() says otherwise. cls._region_cls = DisabledProfileRegion + for _name, func, _bound, _tags in cls._decorated_functions(): + _bound[:] = [None, func, True] + + @classmethod + def _reset(cls) -> None: + # reset(), and forget every decorator and definition too, so that no + # test of scope-profiler itself sees another test's instrumentation. + cls.reset() cls._decorators.clear() cls._region_definitions.clear() diff --git a/src/scope_profiler/region_profiler.py b/src/scope_profiler/region_profiler.py index cef7fd1..676697e 100644 --- a/src/scope_profiler/region_profiler.py +++ b/src/scope_profiler/region_profiler.py @@ -7,6 +7,7 @@ import threading import types from time import perf_counter_ns +from typing import TYPE_CHECKING import numpy as np @@ -15,6 +16,9 @@ from scope_profiler.perf_events import PerfEventGroup, PerfEventTotals from scope_profiler.profile_config import ProfilingConfig +if TYPE_CHECKING: + from typing_extensions import Self # typing.Self needs Python 3.11 + # Parsed module ASTs, memoized by filename. Capturing a region's source only # runs once per region name (see BaseProfileRegion.set_source), but a file can # define many regions, so the parse itself is cached rather than repeated. @@ -462,6 +466,16 @@ def wrapper(*args, **kwargs): return wrapper + # Declared here, with their types, so that ``with ProfileManager.region(...)`` + # type-checks; every region class overrides both. + def __enter__(self) -> "Self": + """Start timing a ``with`` block.""" + raise NotImplementedError + + def __exit__(self, exc_type, exc_value, traceback) -> None: + """Stop timing a ``with`` block.""" + raise NotImplementedError + def append(self, start: float, end: float) -> None: """Append a start/end time pair to the buffer, growing it if needed.""" diff --git a/src/scope_profiler/tests/test_opt_in_tags.py b/src/scope_profiler/tests/test_opt_in_tags.py index b8412a4..c0fd100 100644 --- a/src/scope_profiler/tests/test_opt_in_tags.py +++ b/src/scope_profiler/tests/test_opt_in_tags.py @@ -1,5 +1,7 @@ """Opt-in region tags: library instrumentation that is off unless enabled.""" +import sys + import pytest import scope_profiler as sp @@ -195,3 +197,70 @@ def test_cli_enable_tag(tmp_path): assert set(read_h5(off).region_names) == {"user.step"} assert set(read_h5(on).region_names) == {"mylib.kernel", "user.step"} + + +def test_reset_keeps_decorated_functions_for_the_next_setup(): + sp.register_opt_in_tag("mylib") + + @sp.profile("mylib.solve", tags=("mylib",)) + def solve(): + return 1 + + sp.reset() + assert solve() == 1 # calls straight through while nothing is configured + ProfileManager.setup(output=None, enable_tags=["mylib"]) + solve() + results = ProfileManager.finalize(verbose=False, return_results=True) + + assert results["mylib.solve"].num_calls == 1 + + +def test_private_reset_forgets_decorated_functions(): + @sp.profile("solve") + def solve(): + pass + + ProfileManager._reset() + ProfileManager.setup(output=None) + + assert "solve" not in sp.registered_regions() + + +def test_disabled_decorated_function_skips_metadata_and_tracing(): + sp.register_opt_in_tag("mylib") + metadata_calls = [] + profilers = [] + + def metadata(*args, **kwargs): + metadata_calls.append(args) + return {"n": 1} + + @sp.profile("mylib.solve", tags=("mylib",), metadata=metadata, recursive=True) + def solve(): + profilers.append(sys.getprofile()) + + before = sys.getprofile() + ProfileManager.setup(output=None) + solve() + ProfileManager.finalize(verbose=False) + + assert metadata_calls == [] + assert profilers == [before] + + +def test_registering_a_tag_late_disables_existing_instrumentation(): + @sp.profile("mylib.solve", tags=("mylib",)) + def solve(): + pass + + ProfileManager.setup(output=None) + with sp.region("mylib.kernel", tags=("mylib",)): + pass + sp.register_opt_in_tag("mylib") + solve() + with sp.region("mylib.kernel", tags=("mylib",)): + pass + results = ProfileManager.finalize(verbose=False, return_results=True) + + assert "mylib.solve" not in results.region_names + assert "mylib.kernel" not in results.region_names