From 1713fd899b6c76edd64f127bc3141e4287fdd716 Mon Sep 17 00:00:00 2001 From: Jonathan Amponsah <82057176+mgalore@users.noreply.github.com> Date: Mon, 20 Jul 2026 14:08:49 +0000 Subject: [PATCH 1/4] docs: clarify Zarr v3 codec pipeline roles --- changes/4159.doc.md | 1 + docs/user-guide/arrays.md | 37 +++++++++++++++++++++++++++++++++++- src/zarr/api/asynchronous.py | 8 ++++++-- src/zarr/api/synchronous.py | 24 ++++++++++++++++------- 4 files changed, 60 insertions(+), 10 deletions(-) create mode 100644 changes/4159.doc.md diff --git a/changes/4159.doc.md b/changes/4159.doc.md new file mode 100644 index 0000000000..32738d3635 --- /dev/null +++ b/changes/4159.doc.md @@ -0,0 +1 @@ +Clarify how Zarr format 3 codec pipeline roles map to `filters`, `serializer`, and `compressors` when creating arrays. diff --git a/docs/user-guide/arrays.md b/docs/user-guide/arrays.md index a192845f9e..036c7fea24 100644 --- a/docs/user-guide/arrays.md +++ b/docs/user-guide/arrays.md @@ -195,13 +195,48 @@ arr_f = arr.with_config({"order": "F"}) print(arr_f.config) ``` +## Zarr format 3 codec pipeline + +Zarr format 3 stores a single ordered `codecs` pipeline in array metadata, but +Zarr-Python's array creation functions expose that pipeline through three +role-specific parameters: + +- `filters`: [`zarr.abc.codec.ArrayArrayCodec`][] instances. These transform chunk + arrays into chunk arrays before serialization. +- `serializer`: one [`zarr.abc.codec.ArrayBytesCodec`][] instance. This transforms + a chunk array into bytes. Every Zarr format 3 array needs exactly one + array-to-bytes codec, either supplied explicitly or chosen by default. +- `compressors`: [`zarr.abc.codec.BytesBytesCodec`][] instances. These transform + bytes into bytes after serialization. + +The `compressors` parameter is only for bytes-to-bytes codecs. If a codec is an +`ArrayBytesCodec`, pass it with `serializer`, not `compressors`. For example, the +built-in [`zarr.codecs.BytesCodec`][] can be supplied explicitly as the serializer: + +```python exec="true" session="arrays" source="above" result="ansi" +serializer = zarr.codecs.BytesCodec(endian="little") +z_explicit_serializer = zarr.create_array( + store="data/example-explicit-serializer.zarr", + shape=(100,), + chunks=(10,), + dtype="int32", + serializer=serializer, + compressors=None, +) +print(z_explicit_serializer.serializer) +print(f"Compressors: {z_explicit_serializer.compressors}") +``` + +The same rule applies to third-party Zarr format 3 codecs: if the codec is +documented as an `ArrayBytesCodec`, provide an instance as `serializer=...`. + ## Compressors A number of different compressors can be used with Zarr. Zarr includes Blosc, Zstandard and Gzip compressors. Additional compressors are available through a separate package called [NumCodecs](https://numcodecs.readthedocs.io/en/stable/) which provides various compressor libraries including LZ4, Zlib, BZ2 and LZMA. -Different compressors can be provided via the `compressors` keyword +Different bytes-to-bytes compressors can be provided via the `compressors` keyword argument accepted by all array creation functions. For example: ```python exec="true" session="arrays" source="above" result="ansi" diff --git a/src/zarr/api/asynchronous.py b/src/zarr/api/asynchronous.py index 3bdc254ea5..f4af245ece 100644 --- a/src/zarr/api/asynchronous.py +++ b/src/zarr/api/asynchronous.py @@ -1020,8 +1020,12 @@ async def create( codecs : Sequence of Codecs or dicts, optional An iterable of Codec or dict serializations of Codecs. Zarr V3 only. - The elements of `codecs` specify the transformation from array values to stored bytes. - Zarr format 3 only. Zarr format 2 arrays should use `filters` and `compressor` instead. + The elements of ``codecs`` specify the transformation from array values to stored bytes. + Zarr format 3 only. Zarr format 2 arrays should use ``filters`` and ``compressor`` instead. + In order, a Zarr format 3 pipeline contains zero or more + [`zarr.abc.codec.ArrayArrayCodec`][] filters, exactly one + [`zarr.abc.codec.ArrayBytesCodec`][] serializer, and zero or more + [`zarr.abc.codec.BytesBytesCodec`][] compressors. If no codecs are provided, default codecs will be used based on the data type of the array. For most data types, the default codecs are the tuple `(BytesCodec(), ZstdCodec())`; diff --git a/src/zarr/api/synchronous.py b/src/zarr/api/synchronous.py index 11afb4a85c..ac08d71a4e 100644 --- a/src/zarr/api/synchronous.py +++ b/src/zarr/api/synchronous.py @@ -758,8 +758,12 @@ def create( codecs : Sequence of Codecs or dicts, optional An iterable of Codec or dict serializations of Codecs. Zarr V3 only. - The elements of `codecs` specify the transformation from array values to stored bytes. - Zarr format 3 only. Zarr format 2 arrays should use `filters` and `compressor` instead. + The elements of ``codecs`` specify the transformation from array values to stored bytes. + Zarr format 3 only. Zarr format 2 arrays should use ``filters`` and ``compressor`` instead. + In order, a Zarr format 3 pipeline contains zero or more + [`zarr.abc.codec.ArrayArrayCodec`][] filters, exactly one + [`zarr.abc.codec.ArrayBytesCodec`][] serializer, and zero or more + [`zarr.abc.codec.BytesBytesCodec`][] compressors. If no codecs are provided, default codecs will be used based on the data type of the array. For most data types, the default codecs are the tuple `(BytesCodec(), ZstdCodec())`; @@ -892,9 +896,13 @@ def create_array( filters are applied (if any are specified) and the data is serialized into bytes. For Zarr format 3, a "compressor" is a codec that takes a bytestream, and - returns another bytestream. Multiple compressors may be provided for Zarr format 3. - If no `compressors` are provided, a default set of compressors will be used. - These defaults can be changed by modifying the value of `array.v3_default_compressors` + returns another bytestream. These values must be instances of + [`zarr.abc.codec.BytesBytesCodec`][], or dict representations of + [`zarr.abc.codec.BytesBytesCodec`][]. Multiple compressors may be provided + for Zarr format 3. Codecs that take an array and return bytes are serializers + and must be supplied with ``serializer`` instead. + If no ``compressors`` are provided, a default set of compressors will be used. + These defaults can be changed by modifying the value of ``array.v3_default_compressors`` in [`zarr.config`][zarr.config]. Use `None` to omit default compressors. @@ -906,8 +914,10 @@ def create_array( serializer : dict[str, JSON] | ArrayBytesCodec, optional Array-to-bytes codec to use for encoding the array data. Zarr format 3 only. Zarr format 2 arrays use implicit array-to-bytes conversion. - If no `serializer` is provided, a default serializer will be used. - These defaults can be changed by modifying the value of `array.v3_default_serializer` + Codecs that are instances of [`zarr.abc.codec.ArrayBytesCodec`][] must be + supplied here, not with ``compressors``. + If no ``serializer`` is provided, a default serializer will be used. + These defaults can be changed by modifying the value of ``array.v3_default_serializer`` in [`zarr.config`][zarr.config]. fill_value : Any, optional Fill value for the array. From a5f91c4ce5755bcf5c1d4bd603e3ef97263434d8 Mon Sep 17 00:00:00 2001 From: Jonathan Amponsah <82057176+mgalore@users.noreply.github.com> Date: Mon, 20 Jul 2026 18:07:25 +0000 Subject: [PATCH 2/4] docs: align async codec parameter docs --- src/zarr/core/array.py | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) diff --git a/src/zarr/core/array.py b/src/zarr/core/array.py index 903f7b8f26..32244be0f8 100644 --- a/src/zarr/core/array.py +++ b/src/zarr/core/array.py @@ -4643,9 +4643,13 @@ async def create_array( filters are applied (if any are specified) and the data is serialized into bytes. For Zarr format 3, a "compressor" is a codec that takes a bytestream, and - returns another bytestream. Multiple compressors may be provided for Zarr format 3. - If no `compressors` are provided, a default set of compressors will be used. - These defaults can be changed by modifying the value of `array.v3_default_compressors` + returns another bytestream. These values must be instances of + [`zarr.abc.codec.BytesBytesCodec`][], or dict representations of + [`zarr.abc.codec.BytesBytesCodec`][]. Multiple compressors may be provided + for Zarr format 3. Codecs that take an array and return bytes are serializers + and must be supplied with ``serializer`` instead. + If no ``compressors`` are provided, a default set of compressors will be used. + These defaults can be changed by modifying the value of ``array.v3_default_compressors`` in [`zarr.config`][zarr.config]. Use `None` to omit default compressors. @@ -4657,8 +4661,10 @@ async def create_array( serializer : dict[str, JSON] | ArrayBytesCodec, optional Array-to-bytes codec to use for encoding the array data. Zarr format 3 only. Zarr format 2 arrays use implicit array-to-bytes conversion. - If no `serializer` is provided, a default serializer will be used. - These defaults can be changed by modifying the value of `array.v3_default_serializer` + Codecs that are instances of [`zarr.abc.codec.ArrayBytesCodec`][] must be + supplied here, not with ``compressors``. + If no ``serializer`` is provided, a default serializer will be used. + These defaults can be changed by modifying the value of ``array.v3_default_serializer`` in [`zarr.config`][zarr.config]. fill_value : Any, optional Fill value for the array. From 7e0ec27ac6bcd9592bc3f61e745ed23fc46cc56a Mon Sep 17 00:00:00 2001 From: Davis Vann Bennett Date: Fri, 9 Oct 2026 18:24:15 +0200 Subject: [PATCH 3/4] docs: use markdown backticks and actual codec defaults Revert RST-style double backticks to single backticks in the edited docstrings, replace references to the removed array.v3_default_compressors / array.v3_default_serializer config keys with the actual Zarr format 3 defaults, and rename the changelog fragment after the PR number. Assisted-by: ClaudeCode:claude-opus-5-5 --- changes/{4159.doc.md => 4166.doc.md} | 0 src/zarr/api/asynchronous.py | 4 ++-- src/zarr/api/synchronous.py | 19 +++++++++---------- src/zarr/core/array.py | 15 +++++++-------- 4 files changed, 18 insertions(+), 20 deletions(-) rename changes/{4159.doc.md => 4166.doc.md} (100%) diff --git a/changes/4159.doc.md b/changes/4166.doc.md similarity index 100% rename from changes/4159.doc.md rename to changes/4166.doc.md diff --git a/src/zarr/api/asynchronous.py b/src/zarr/api/asynchronous.py index f4af245ece..6d5e695492 100644 --- a/src/zarr/api/asynchronous.py +++ b/src/zarr/api/asynchronous.py @@ -1020,8 +1020,8 @@ async def create( codecs : Sequence of Codecs or dicts, optional An iterable of Codec or dict serializations of Codecs. Zarr V3 only. - The elements of ``codecs`` specify the transformation from array values to stored bytes. - Zarr format 3 only. Zarr format 2 arrays should use ``filters`` and ``compressor`` instead. + The elements of `codecs` specify the transformation from array values to stored bytes. + Zarr format 3 only. Zarr format 2 arrays should use `filters` and `compressor` instead. In order, a Zarr format 3 pipeline contains zero or more [`zarr.abc.codec.ArrayArrayCodec`][] filters, exactly one [`zarr.abc.codec.ArrayBytesCodec`][] serializer, and zero or more diff --git a/src/zarr/api/synchronous.py b/src/zarr/api/synchronous.py index ac08d71a4e..370132978c 100644 --- a/src/zarr/api/synchronous.py +++ b/src/zarr/api/synchronous.py @@ -758,8 +758,8 @@ def create( codecs : Sequence of Codecs or dicts, optional An iterable of Codec or dict serializations of Codecs. Zarr V3 only. - The elements of ``codecs`` specify the transformation from array values to stored bytes. - Zarr format 3 only. Zarr format 2 arrays should use ``filters`` and ``compressor`` instead. + The elements of `codecs` specify the transformation from array values to stored bytes. + Zarr format 3 only. Zarr format 2 arrays should use `filters` and `compressor` instead. In order, a Zarr format 3 pipeline contains zero or more [`zarr.abc.codec.ArrayArrayCodec`][] filters, exactly one [`zarr.abc.codec.ArrayBytesCodec`][] serializer, and zero or more @@ -900,10 +900,9 @@ def create_array( [`zarr.abc.codec.BytesBytesCodec`][], or dict representations of [`zarr.abc.codec.BytesBytesCodec`][]. Multiple compressors may be provided for Zarr format 3. Codecs that take an array and return bytes are serializers - and must be supplied with ``serializer`` instead. - If no ``compressors`` are provided, a default set of compressors will be used. - These defaults can be changed by modifying the value of ``array.v3_default_compressors`` - in [`zarr.config`][zarr.config]. + and must be supplied with `serializer` instead. + If no `compressors` are provided, the default for Zarr format 3 is + `(ZstdCodec(),)`. Use `None` to omit default compressors. For Zarr format 2, a "compressor" can be any numcodecs codec. Only a single compressor may @@ -915,10 +914,10 @@ def create_array( Array-to-bytes codec to use for encoding the array data. Zarr format 3 only. Zarr format 2 arrays use implicit array-to-bytes conversion. Codecs that are instances of [`zarr.abc.codec.ArrayBytesCodec`][] must be - supplied here, not with ``compressors``. - If no ``serializer`` is provided, a default serializer will be used. - These defaults can be changed by modifying the value of ``array.v3_default_serializer`` - in [`zarr.config`][zarr.config]. + supplied here, not with `compressors`. + If no `serializer` is provided, a default serializer is chosen based on the data type: + `VLenUTF8Codec` for variable-length strings, `VLenBytesCodec` for variable-length + bytes, and `BytesCodec` otherwise. fill_value : Any, optional Fill value for the array. order : {"C", "F"}, optional diff --git a/src/zarr/core/array.py b/src/zarr/core/array.py index 32244be0f8..0c75a96e91 100644 --- a/src/zarr/core/array.py +++ b/src/zarr/core/array.py @@ -4647,10 +4647,9 @@ async def create_array( [`zarr.abc.codec.BytesBytesCodec`][], or dict representations of [`zarr.abc.codec.BytesBytesCodec`][]. Multiple compressors may be provided for Zarr format 3. Codecs that take an array and return bytes are serializers - and must be supplied with ``serializer`` instead. - If no ``compressors`` are provided, a default set of compressors will be used. - These defaults can be changed by modifying the value of ``array.v3_default_compressors`` - in [`zarr.config`][zarr.config]. + and must be supplied with `serializer` instead. + If no `compressors` are provided, the default for Zarr format 3 is + `(ZstdCodec(),)`. Use `None` to omit default compressors. For Zarr format 2, a "compressor" can be any numcodecs codec. Only a single compressor may @@ -4662,10 +4661,10 @@ async def create_array( Array-to-bytes codec to use for encoding the array data. Zarr format 3 only. Zarr format 2 arrays use implicit array-to-bytes conversion. Codecs that are instances of [`zarr.abc.codec.ArrayBytesCodec`][] must be - supplied here, not with ``compressors``. - If no ``serializer`` is provided, a default serializer will be used. - These defaults can be changed by modifying the value of ``array.v3_default_serializer`` - in [`zarr.config`][zarr.config]. + supplied here, not with `compressors`. + If no `serializer` is provided, a default serializer is chosen based on the data type: + `VLenUTF8Codec` for variable-length strings, `VLenBytesCodec` for variable-length + bytes, and `BytesCodec` otherwise. fill_value : Any, optional Fill value for the array. order : {"C", "F"}, optional From e1c1d5c2a5ef4566d7a77638ffb1b2614bde7811 Mon Sep 17 00:00:00 2001 From: Davis Vann Bennett Date: Fri, 9 Oct 2026 18:32:03 +0200 Subject: [PATCH 4/4] docs: mark filters/serializer/compressors as Zarr-Python aliases State that these names are not defined in the Zarr format 3 specification, and map each one to the spec's codec kind (array -> array, array -> bytes, bytes -> bytes) with links to the spec's chunk encoding section and `codecs` metadata field. Assisted-by: ClaudeCode:claude-opus-5-5 --- docs/user-guide/arrays.md | 35 ++++++++++++++++++++++++----------- src/zarr/api/asynchronous.py | 11 +++++++---- src/zarr/api/synchronous.py | 11 +++++++---- 3 files changed, 38 insertions(+), 19 deletions(-) diff --git a/docs/user-guide/arrays.md b/docs/user-guide/arrays.md index 036c7fea24..54962ecd18 100644 --- a/docs/user-guide/arrays.md +++ b/docs/user-guide/arrays.md @@ -197,17 +197,30 @@ print(arr_f.config) ## Zarr format 3 codec pipeline -Zarr format 3 stores a single ordered `codecs` pipeline in array metadata, but -Zarr-Python's array creation functions expose that pipeline through three -role-specific parameters: - -- `filters`: [`zarr.abc.codec.ArrayArrayCodec`][] instances. These transform chunk - arrays into chunk arrays before serialization. -- `serializer`: one [`zarr.abc.codec.ArrayBytesCodec`][] instance. This transforms - a chunk array into bytes. Every Zarr format 3 array needs exactly one - array-to-bytes codec, either supplied explicitly or chosen by default. -- `compressors`: [`zarr.abc.codec.BytesBytesCodec`][] instances. These transform - bytes into bytes after serialization. +Zarr format 3 stores a single ordered list of codecs in the +[`codecs`](https://zarr-specs.readthedocs.io/en/latest/v3/core/index.html#array-metadata-codecs) +field of array metadata. The +[Zarr format 3 specification](https://zarr-specs.readthedocs.io/en/latest/v3/core/index.html#chunk-encoding) +classifies each codec by the input and output of its encode transform, as +"array -> array", "array -> bytes", or "bytes -> bytes", and requires the list to +contain zero or more array -> array codecs, followed by exactly one +array -> bytes codec, followed by zero or more bytes -> bytes codecs. + +Zarr-Python's array creation functions expose that list through three +parameters: `filters`, `serializer`, and `compressors`. These names are not +defined in the Zarr format 3 specification. They are Zarr-Python aliases for +the specification's codec kinds: + +- `filters`: the array -> array codecs, given as + [`zarr.abc.codec.ArrayArrayCodec`][] instances. These transform chunk arrays + into chunk arrays before serialization. +- `serializer`: the array -> bytes codec, given as one + [`zarr.abc.codec.ArrayBytesCodec`][] instance. This transforms a chunk array + into bytes. Every Zarr format 3 array needs exactly one array -> bytes codec, + either supplied explicitly or chosen by default. +- `compressors`: the bytes -> bytes codecs, given as + [`zarr.abc.codec.BytesBytesCodec`][] instances. These transform bytes into + bytes after serialization. The `compressors` parameter is only for bytes-to-bytes codecs. If a codec is an `ArrayBytesCodec`, pass it with `serializer`, not `compressors`. For example, the diff --git a/src/zarr/api/asynchronous.py b/src/zarr/api/asynchronous.py index 6d5e695492..1e4e330130 100644 --- a/src/zarr/api/asynchronous.py +++ b/src/zarr/api/asynchronous.py @@ -1022,10 +1022,13 @@ async def create( The elements of `codecs` specify the transformation from array values to stored bytes. Zarr format 3 only. Zarr format 2 arrays should use `filters` and `compressor` instead. - In order, a Zarr format 3 pipeline contains zero or more - [`zarr.abc.codec.ArrayArrayCodec`][] filters, exactly one - [`zarr.abc.codec.ArrayBytesCodec`][] serializer, and zero or more - [`zarr.abc.codec.BytesBytesCodec`][] compressors. + The [Zarr format 3 specification](https://zarr-specs.readthedocs.io/en/latest/v3/core/index.html#chunk-encoding) + requires, in order, zero or more array -> array codecs + ([`zarr.abc.codec.ArrayArrayCodec`][]), exactly one array -> bytes codec + ([`zarr.abc.codec.ArrayBytesCodec`][]), and zero or more bytes -> bytes codecs + ([`zarr.abc.codec.BytesBytesCodec`][]). Elsewhere in Zarr-Python these are + called filters, the serializer, and compressors; those names are Zarr-Python + aliases, not terms defined in the specification. If no codecs are provided, default codecs will be used based on the data type of the array. For most data types, the default codecs are the tuple `(BytesCodec(), ZstdCodec())`; diff --git a/src/zarr/api/synchronous.py b/src/zarr/api/synchronous.py index 370132978c..2a3d83a306 100644 --- a/src/zarr/api/synchronous.py +++ b/src/zarr/api/synchronous.py @@ -760,10 +760,13 @@ def create( The elements of `codecs` specify the transformation from array values to stored bytes. Zarr format 3 only. Zarr format 2 arrays should use `filters` and `compressor` instead. - In order, a Zarr format 3 pipeline contains zero or more - [`zarr.abc.codec.ArrayArrayCodec`][] filters, exactly one - [`zarr.abc.codec.ArrayBytesCodec`][] serializer, and zero or more - [`zarr.abc.codec.BytesBytesCodec`][] compressors. + The [Zarr format 3 specification](https://zarr-specs.readthedocs.io/en/latest/v3/core/index.html#chunk-encoding) + requires, in order, zero or more array -> array codecs + ([`zarr.abc.codec.ArrayArrayCodec`][]), exactly one array -> bytes codec + ([`zarr.abc.codec.ArrayBytesCodec`][]), and zero or more bytes -> bytes codecs + ([`zarr.abc.codec.BytesBytesCodec`][]). Elsewhere in Zarr-Python these are + called filters, the serializer, and compressors; those names are Zarr-Python + aliases, not terms defined in the specification. If no codecs are provided, default codecs will be used based on the data type of the array. For most data types, the default codecs are the tuple `(BytesCodec(), ZstdCodec())`;