From 265e5f0e6be1a199e459a7cc3411b3b3af959236 Mon Sep 17 00:00:00 2001 From: Yi-111-a Date: Sat, 10 Oct 2026 03:35:10 +0800 Subject: [PATCH 1/3] Remove the VerboseModule wrapper that made `zarr.storage` unserializable `zarr.storage` was replaced at import time by `VerboseModule`, a `types.ModuleType` subclass that only existed to warn about the long-deprecated `zarr.storage.default_compressor` attribute. Serializers that dispatch on the exact type of an object miss that subclass: `cloudpickle` (used by Dask) registers a reduce for `ModuleType`, so any attempt to serialize the module raised `TypeError: cannot pickle 'VerboseModule' object`, which made `zarr.storage.ObjectStore`-backed Dask arrays impossible to send to a cluster. Drop the wrapper as suggested in #4029 and add a regression test. Setting `zarr.storage.default_compressor` no longer warns; `zarr.config` is the supported way to configure `array.v2_default_compressor`. --- changes/4029.bugfix.md | 8 ++++++++ pyproject.toml | 1 + src/zarr/storage/__init__.py | 23 ----------------------- tests/test_store/test_core.py | 14 ++++++++++++++ tests/test_v2.py | 6 ------ uv.lock | 17 ++++++++++++++++- 6 files changed, 39 insertions(+), 30 deletions(-) create mode 100644 changes/4029.bugfix.md diff --git a/changes/4029.bugfix.md b/changes/4029.bugfix.md new file mode 100644 index 0000000000..b4abda7658 --- /dev/null +++ b/changes/4029.bugfix.md @@ -0,0 +1,8 @@ +`zarr.storage` is no longer replaced by a `ModuleType` subclass at import time. +The subclass only existed to warn about the long-deprecated +`zarr.storage.default_compressor` attribute, but serializers that dispatch on +the exact type of an object (such as `cloudpickle`, which Dask uses by default) +could not serialize the module and raised +`TypeError: cannot pickle 'VerboseModule' object`. Setting +`zarr.storage.default_compressor` no longer emits a deprecation warning; use +`zarr.config` to configure `array.v2_default_compressor` instead. diff --git a/pyproject.toml b/pyproject.toml index c68450474e..98bdb0ab9c 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -109,6 +109,7 @@ homepage = "https://github.com/zarr-developers/zarr-python" # pins deliberately, e.g. via dependabot or `uv lock --upgrade`. [dependency-groups] test = [ + "cloudpickle==3.1.2", "coverage==7.16.2", "pytest==9.1.1", "pytest-asyncio==1.4.0", diff --git a/src/zarr/storage/__init__.py b/src/zarr/storage/__init__.py index f1bd1724af..1c41082ba7 100644 --- a/src/zarr/storage/__init__.py +++ b/src/zarr/storage/__init__.py @@ -1,9 +1,3 @@ -import sys -import warnings -from types import ModuleType -from typing import Any - -from zarr.errors import ZarrDeprecationWarning from zarr.storage._common import StoreLike, StorePath from zarr.storage._fsspec import FsspecStore from zarr.storage._local import LocalStore @@ -26,20 +20,3 @@ "WrapperStore", "ZipStore", ] - - -class VerboseModule(ModuleType): - def __setattr__(self, attr: str, value: Any) -> None: - if attr == "default_compressor": - warnings.warn( - "setting zarr.storage.default_compressor is deprecated, use " - "zarr.config to configure array.v2_default_compressor " - "e.g. config.set({'codecs.zstd':'numcodecs.Zstd', 'array.v2_default_compressor.numeric': 'zstd'})", - ZarrDeprecationWarning, - stacklevel=1, - ) - else: - super().__setattr__(attr, value) - - -sys.modules[__name__].__class__ = VerboseModule diff --git a/tests/test_store/test_core.py b/tests/test_store/test_core.py index daf62e3843..c928ba8417 100644 --- a/tests/test_store/test_core.py +++ b/tests/test_store/test_core.py @@ -1,8 +1,10 @@ import tempfile from collections.abc import Awaitable, Callable, Generator from pathlib import Path +from types import ModuleType from typing import Any, Literal +import cloudpickle import pytest from packaging.version import parse as parse_version @@ -434,3 +436,15 @@ def test_different_open_mode(tmp_path: Path) -> None: match="Store is not read-only but mode is 'r'. Unable to create a read-only copy of the store. Please use a read-only store or a storage class that implements .with_read_only().", ): zarr.open_array(store=zip_store, path="a", zarr_format=2, mode="r") + + +def test_storage_module_is_picklable() -> None: + # `zarr.storage` used to be swapped for a `VerboseModule` subclass of + # `types.ModuleType` in order to warn about the long-deprecated + # `zarr.storage.default_compressor` attribute. cloudpickle dispatches on the + # exact type of an object, so the subclass was not registered and + # serializing the module raised `TypeError: cannot pickle 'VerboseModule' + # object`, which broke passing `zarr.storage.ObjectStore` around with Dask. + # See issue #4029. + assert type(zarr.storage) is ModuleType + assert cloudpickle.loads(cloudpickle.dumps(zarr.storage)) is zarr.storage diff --git a/tests/test_v2.py b/tests/test_v2.py index 2e04339441..9a36aca74b 100644 --- a/tests/test_v2.py +++ b/tests/test_v2.py @@ -20,7 +20,6 @@ from zarr.core.dtype.wrapper import ZDType from zarr.core.group import Group from zarr.core.sync import sync -from zarr.errors import ZarrDeprecationWarning from zarr.storage import MemoryStore, StorePath @@ -225,11 +224,6 @@ def test_v2_non_contiguous(numpy_order: Literal["C", "F"], zarr_order: Literal[" assert (sub_arr).flags.c_contiguous -def test_default_compressor_deprecation_warning() -> None: - with pytest.warns(ZarrDeprecationWarning, match="default_compressor is deprecated"): - zarr.storage.default_compressor = "zarr.codecs.zstd.ZstdCodec()" # type: ignore[attr-defined] - - @pytest.mark.parametrize("fill_value", [None, (b"", 0, 0.0)], ids=["no_fill", "fill"]) def test_structured_dtype_roundtrip(fill_value: float | bytes, tmp_path: Path) -> None: a = np.array( diff --git a/uv.lock b/uv.lock index b14033c5b4..7bbcce0ca0 100644 --- a/uv.lock +++ b/uv.lock @@ -1,5 +1,5 @@ version = 1 -revision = 3 +revision = 5 requires-python = ">=3.12" resolution-markers = [ "python_full_version >= '3.15'", @@ -604,6 +604,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/ae/44/c1221527f6a71a01ec6fbad7fa78f1d50dfa02217385cf0fa3eec7087d59/click-8.3.3-py3-none-any.whl", hash = "sha256:a2bf429bb3033c89fa4936ffb35d5cb471e3719e1f3c8a7c3fff0b8314305613", size = 110502, upload-time = "2026-04-22T15:11:25.044Z" }, ] +[[package]] +name = "cloudpickle" +version = "3.1.2" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/27/fb/576f067976d320f5f0114a8d9fa1215425441bb35627b1993e5afd8111e5/cloudpickle-3.1.2.tar.gz", hash = "sha256:7fda9eb655c9c230dab534f1983763de5835249750e85fbcef43aaa30a9a2414", size = 22330, upload-time = "2025-11-03T09:25:26.604Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/88/39/799be3f2f0f38cc727ee3b4f1445fe6d5e4133064ec2e4115069418a5bb6/cloudpickle-3.1.2-py3-none-any.whl", hash = "sha256:9acb47f6afd73f60dc1df93bb801b472f05ff42fa6c84167d25cb206be1fbf4a", size = 22228, upload-time = "2025-11-03T09:25:25.534Z" }, +] + [[package]] name = "colorama" version = "0.4.6" @@ -3670,6 +3679,7 @@ remote = [ dev = [ { name = "astroid" }, { name = "botocore" }, + { name = "cloudpickle" }, { name = "coverage" }, { name = "fsspec" }, { name = "griffe-inherited-docstrings" }, @@ -3723,6 +3733,7 @@ release = [ ] remote-tests = [ { name = "botocore" }, + { name = "cloudpickle" }, { name = "coverage" }, { name = "fsspec" }, { name = "hypothesis" }, @@ -3743,6 +3754,7 @@ remote-tests = [ { name = "uv" }, ] test = [ + { name = "cloudpickle" }, { name = "coverage" }, { name = "hypothesis" }, { name = "numpydoc" }, @@ -3780,6 +3792,7 @@ provides-extras = ["cast-value-rs", "cli", "gpu", "optional", "remote"] dev = [ { name = "astroid", specifier = "==4.3.3" }, { name = "botocore" }, + { name = "cloudpickle", specifier = "==3.1.2" }, { name = "coverage", specifier = "==7.16.2" }, { name = "fsspec", specifier = ">=2023.10.0" }, { name = "griffe-inherited-docstrings", specifier = "==1.1.3" }, @@ -3831,6 +3844,7 @@ docs = [ release = [{ name = "towncrier", specifier = "==26.9.0" }] remote-tests = [ { name = "botocore" }, + { name = "cloudpickle", specifier = "==3.1.2" }, { name = "coverage", specifier = "==7.16.2" }, { name = "fsspec", specifier = ">=2023.10.0" }, { name = "hypothesis", specifier = "==6.168.3" }, @@ -3851,6 +3865,7 @@ remote-tests = [ { name = "uv", specifier = "==0.12.21" }, ] test = [ + { name = "cloudpickle", specifier = "==3.1.2" }, { name = "coverage", specifier = "==7.16.2" }, { name = "hypothesis", specifier = "==6.168.3" }, { name = "numpydoc", specifier = "==1.11.0" }, From 5164c415b2de513140b7d3893289890bc84ebaed Mon Sep 17 00:00:00 2001 From: Davis Vann Bennett Date: Mon, 19 Oct 2026 22:19:39 +0200 Subject: [PATCH 2/3] chore: tidy changelog fragments and test docstring for the VerboseModule removal Rename the changelog fragment from the issue number (4029) to the PR number (4505), as changes/README.md requires, and shorten it to the user-visible effect. Add a removal fragment for the dropped `zarr.storage.default_compressor` deprecation warning, as the deprecation policy in docs/contributing.md requires when a deprecated API is finally removed. Replace the comment block in test_storage_module_is_picklable with a docstring, matching the neighboring tests. Assisted-by: ClaudeCode:claude-fable-5-1 Co-Authored-By: Claude Fable 5.1 --- changes/4029.bugfix.md | 8 -------- changes/4505.bugfix.md | 1 + changes/4505.removal.md | 1 + tests/test_store/test_core.py | 13 ++++++------- 4 files changed, 8 insertions(+), 15 deletions(-) delete mode 100644 changes/4029.bugfix.md create mode 100644 changes/4505.bugfix.md create mode 100644 changes/4505.removal.md diff --git a/changes/4029.bugfix.md b/changes/4029.bugfix.md deleted file mode 100644 index b4abda7658..0000000000 --- a/changes/4029.bugfix.md +++ /dev/null @@ -1,8 +0,0 @@ -`zarr.storage` is no longer replaced by a `ModuleType` subclass at import time. -The subclass only existed to warn about the long-deprecated -`zarr.storage.default_compressor` attribute, but serializers that dispatch on -the exact type of an object (such as `cloudpickle`, which Dask uses by default) -could not serialize the module and raised -`TypeError: cannot pickle 'VerboseModule' object`. Setting -`zarr.storage.default_compressor` no longer emits a deprecation warning; use -`zarr.config` to configure `array.v2_default_compressor` instead. diff --git a/changes/4505.bugfix.md b/changes/4505.bugfix.md new file mode 100644 index 0000000000..8d50b8efb6 --- /dev/null +++ b/changes/4505.bugfix.md @@ -0,0 +1 @@ +The `zarr.storage` module can now be serialized with `cloudpickle`. Previously, code that referenced the module (for example a Dask task that used `zarr.storage.ObjectStore`) failed with `TypeError: cannot pickle 'VerboseModule' object`. diff --git a/changes/4505.removal.md b/changes/4505.removal.md new file mode 100644 index 0000000000..6549a77c95 --- /dev/null +++ b/changes/4505.removal.md @@ -0,0 +1 @@ +Setting `zarr.storage.default_compressor` no longer emits a deprecation warning. The attribute has had no effect since 3.0.0; configure `array.v2_default_compressor` through `zarr.config` instead. diff --git a/tests/test_store/test_core.py b/tests/test_store/test_core.py index c928ba8417..f0a2077a8a 100644 --- a/tests/test_store/test_core.py +++ b/tests/test_store/test_core.py @@ -439,12 +439,11 @@ def test_different_open_mode(tmp_path: Path) -> None: def test_storage_module_is_picklable() -> None: - # `zarr.storage` used to be swapped for a `VerboseModule` subclass of - # `types.ModuleType` in order to warn about the long-deprecated - # `zarr.storage.default_compressor` attribute. cloudpickle dispatches on the - # exact type of an object, so the subclass was not registered and - # serializing the module raised `TypeError: cannot pickle 'VerboseModule' - # object`, which broke passing `zarr.storage.ObjectStore` around with Dask. - # See issue #4029. + """ + `zarr.storage` is a plain `types.ModuleType` and round-trips through `cloudpickle`. + + Serializers such as `cloudpickle` dispatch on the exact type of an object, so a + `ModuleType` subclass would not be serializable. See issue #4029. + """ assert type(zarr.storage) is ModuleType assert cloudpickle.loads(cloudpickle.dumps(zarr.storage)) is zarr.storage From fbd380b26f5189c2d3839d0acc219b092c42d3e6 Mon Sep 17 00:00:00 2001 From: Davis Vann Bennett Date: Thu, 22 Oct 2026 22:25:27 +0200 Subject: [PATCH 3/3] test: drop the cloudpickle regression test for the VerboseModule removal The test only checked that `zarr.storage` is a plain `ModuleType` and that cloudpickle can pickle a plain module. The first guards against re-adding a one-off `__class__` swap; the second tests cloudpickle. Neither justified a new pinned test dependency, so the test, the `cloudpickle` pin in the `test` group, and its `uv.lock` entries go. Assisted-by: ClaudeCode:claude-opus-5-5 --- pyproject.toml | 1 - tests/test_store/test_core.py | 13 ------------- uv.lock | 17 +---------------- 3 files changed, 1 insertion(+), 30 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 98bdb0ab9c..c68450474e 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -109,7 +109,6 @@ homepage = "https://github.com/zarr-developers/zarr-python" # pins deliberately, e.g. via dependabot or `uv lock --upgrade`. [dependency-groups] test = [ - "cloudpickle==3.1.2", "coverage==7.16.2", "pytest==9.1.1", "pytest-asyncio==1.4.0", diff --git a/tests/test_store/test_core.py b/tests/test_store/test_core.py index f0a2077a8a..daf62e3843 100644 --- a/tests/test_store/test_core.py +++ b/tests/test_store/test_core.py @@ -1,10 +1,8 @@ import tempfile from collections.abc import Awaitable, Callable, Generator from pathlib import Path -from types import ModuleType from typing import Any, Literal -import cloudpickle import pytest from packaging.version import parse as parse_version @@ -436,14 +434,3 @@ def test_different_open_mode(tmp_path: Path) -> None: match="Store is not read-only but mode is 'r'. Unable to create a read-only copy of the store. Please use a read-only store or a storage class that implements .with_read_only().", ): zarr.open_array(store=zip_store, path="a", zarr_format=2, mode="r") - - -def test_storage_module_is_picklable() -> None: - """ - `zarr.storage` is a plain `types.ModuleType` and round-trips through `cloudpickle`. - - Serializers such as `cloudpickle` dispatch on the exact type of an object, so a - `ModuleType` subclass would not be serializable. See issue #4029. - """ - assert type(zarr.storage) is ModuleType - assert cloudpickle.loads(cloudpickle.dumps(zarr.storage)) is zarr.storage diff --git a/uv.lock b/uv.lock index 7bbcce0ca0..b14033c5b4 100644 --- a/uv.lock +++ b/uv.lock @@ -1,5 +1,5 @@ version = 1 -revision = 5 +revision = 3 requires-python = ">=3.12" resolution-markers = [ "python_full_version >= '3.15'", @@ -604,15 +604,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/ae/44/c1221527f6a71a01ec6fbad7fa78f1d50dfa02217385cf0fa3eec7087d59/click-8.3.3-py3-none-any.whl", hash = "sha256:a2bf429bb3033c89fa4936ffb35d5cb471e3719e1f3c8a7c3fff0b8314305613", size = 110502, upload-time = "2026-04-22T15:11:25.044Z" }, ] -[[package]] -name = "cloudpickle" -version = "3.1.2" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/27/fb/576f067976d320f5f0114a8d9fa1215425441bb35627b1993e5afd8111e5/cloudpickle-3.1.2.tar.gz", hash = "sha256:7fda9eb655c9c230dab534f1983763de5835249750e85fbcef43aaa30a9a2414", size = 22330, upload-time = "2025-11-03T09:25:26.604Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/88/39/799be3f2f0f38cc727ee3b4f1445fe6d5e4133064ec2e4115069418a5bb6/cloudpickle-3.1.2-py3-none-any.whl", hash = "sha256:9acb47f6afd73f60dc1df93bb801b472f05ff42fa6c84167d25cb206be1fbf4a", size = 22228, upload-time = "2025-11-03T09:25:25.534Z" }, -] - [[package]] name = "colorama" version = "0.4.6" @@ -3679,7 +3670,6 @@ remote = [ dev = [ { name = "astroid" }, { name = "botocore" }, - { name = "cloudpickle" }, { name = "coverage" }, { name = "fsspec" }, { name = "griffe-inherited-docstrings" }, @@ -3733,7 +3723,6 @@ release = [ ] remote-tests = [ { name = "botocore" }, - { name = "cloudpickle" }, { name = "coverage" }, { name = "fsspec" }, { name = "hypothesis" }, @@ -3754,7 +3743,6 @@ remote-tests = [ { name = "uv" }, ] test = [ - { name = "cloudpickle" }, { name = "coverage" }, { name = "hypothesis" }, { name = "numpydoc" }, @@ -3792,7 +3780,6 @@ provides-extras = ["cast-value-rs", "cli", "gpu", "optional", "remote"] dev = [ { name = "astroid", specifier = "==4.3.3" }, { name = "botocore" }, - { name = "cloudpickle", specifier = "==3.1.2" }, { name = "coverage", specifier = "==7.16.2" }, { name = "fsspec", specifier = ">=2023.10.0" }, { name = "griffe-inherited-docstrings", specifier = "==1.1.3" }, @@ -3844,7 +3831,6 @@ docs = [ release = [{ name = "towncrier", specifier = "==26.9.0" }] remote-tests = [ { name = "botocore" }, - { name = "cloudpickle", specifier = "==3.1.2" }, { name = "coverage", specifier = "==7.16.2" }, { name = "fsspec", specifier = ">=2023.10.0" }, { name = "hypothesis", specifier = "==6.168.3" }, @@ -3865,7 +3851,6 @@ remote-tests = [ { name = "uv", specifier = "==0.12.21" }, ] test = [ - { name = "cloudpickle", specifier = "==3.1.2" }, { name = "coverage", specifier = "==7.16.2" }, { name = "hypothesis", specifier = "==6.168.3" }, { name = "numpydoc", specifier = "==1.11.0" },