diff --git a/changes/4166.doc.md b/changes/4166.doc.md new file mode 100644 index 0000000000..32738d3635 --- /dev/null +++ b/changes/4166.doc.md @@ -0,0 +1 @@ +Clarify how Zarr format 3 codec pipeline roles map to `filters`, `serializer`, and `compressors` when creating arrays. diff --git a/docs/user-guide/arrays.md b/docs/user-guide/arrays.md index e8df79a102..d4a7bf6832 100644 --- a/docs/user-guide/arrays.md +++ b/docs/user-guide/arrays.md @@ -194,13 +194,61 @@ arr_f = arr.with_config({"order": "F"}) print(arr_f.config) ``` +## Zarr format 3 codec pipeline + +Zarr format 3 stores a single ordered list of codecs in the +[`codecs`](https://zarr-specs.readthedocs.io/en/latest/v3/core/index.html#array-metadata-codecs) +field of array metadata. The +[Zarr format 3 specification](https://zarr-specs.readthedocs.io/en/latest/v3/core/index.html#chunk-encoding) +classifies each codec by the input and output of its encode transform, as +"array -> array", "array -> bytes", or "bytes -> bytes", and requires the list to +contain zero or more array -> array codecs, followed by exactly one +array -> bytes codec, followed by zero or more bytes -> bytes codecs. + +Zarr-Python's array creation functions expose that list through three +parameters: `filters`, `serializer`, and `compressors`. These names are not +defined in the Zarr format 3 specification. They are Zarr-Python aliases for +the specification's codec kinds: + +- `filters`: the array -> array codecs, given as + [`zarr.abc.codec.ArrayArrayCodec`][] instances. These transform chunk arrays + into chunk arrays before serialization. +- `serializer`: the array -> bytes codec, given as one + [`zarr.abc.codec.ArrayBytesCodec`][] instance. This transforms a chunk array + into bytes. Every Zarr format 3 array needs exactly one array -> bytes codec, + either supplied explicitly or chosen by default. +- `compressors`: the bytes -> bytes codecs, given as + [`zarr.abc.codec.BytesBytesCodec`][] instances. These transform bytes into + bytes after serialization. + +The `compressors` parameter is only for bytes-to-bytes codecs. If a codec is an +`ArrayBytesCodec`, pass it with `serializer`, not `compressors`. For example, the +built-in [`zarr.codecs.BytesCodec`][] can be supplied explicitly as the serializer: + +```python exec="true" session="arrays" source="above" result="ansi" +serializer = zarr.codecs.BytesCodec(endian="little") +z_explicit_serializer = zarr.create_array( + store="data/example-explicit-serializer.zarr", + shape=(100,), + chunks=(10,), + dtype="int32", + serializer=serializer, + compressors=None, +) +print(z_explicit_serializer.serializer) +print(f"Compressors: {z_explicit_serializer.compressors}") +``` + +The same rule applies to third-party Zarr format 3 codecs: if the codec is +documented as an `ArrayBytesCodec`, provide an instance as `serializer=...`. + ## Compressors A number of different compressors can be used with Zarr. Zarr includes Blosc, Zstandard and Gzip compressors. Additional compressors are available through a separate package called [NumCodecs](https://numcodecs.readthedocs.io/en/stable/) which provides various compressor libraries including LZ4, Zlib, BZ2 and LZMA. -Different compressors can be provided via the `compressors` keyword +Different bytes-to-bytes compressors can be provided via the `compressors` keyword argument accepted by all array creation functions. For example: ```python exec="true" session="arrays" source="above" result="ansi" diff --git a/src/zarr/api/asynchronous.py b/src/zarr/api/asynchronous.py index 5725b3d13a..83181cbacf 100644 --- a/src/zarr/api/asynchronous.py +++ b/src/zarr/api/asynchronous.py @@ -1058,6 +1058,13 @@ async def create( The elements of `codecs` specify the transformation from array values to stored bytes. Zarr format 3 only. Zarr format 2 arrays should use `filters` and `compressor` instead. + The [Zarr format 3 specification](https://zarr-specs.readthedocs.io/en/latest/v3/core/index.html#chunk-encoding) + requires, in order, zero or more array -> array codecs + ([`zarr.abc.codec.ArrayArrayCodec`][]), exactly one array -> bytes codec + ([`zarr.abc.codec.ArrayBytesCodec`][]), and zero or more bytes -> bytes codecs + ([`zarr.abc.codec.BytesBytesCodec`][]). Elsewhere in Zarr-Python these are + called filters, the serializer, and compressors; those names are Zarr-Python + aliases, not terms defined in the specification. If no codecs are provided, default codecs will be used based on the data type of the array. For most data types, the default codecs are the tuple `(BytesCodec(), ZstdCodec())`; diff --git a/src/zarr/api/synchronous.py b/src/zarr/api/synchronous.py index 524f719742..e4ac86d8d9 100644 --- a/src/zarr/api/synchronous.py +++ b/src/zarr/api/synchronous.py @@ -775,6 +775,13 @@ def create( The elements of `codecs` specify the transformation from array values to stored bytes. Zarr format 3 only. Zarr format 2 arrays should use `filters` and `compressor` instead. + The [Zarr format 3 specification](https://zarr-specs.readthedocs.io/en/latest/v3/core/index.html#chunk-encoding) + requires, in order, zero or more array -> array codecs + ([`zarr.abc.codec.ArrayArrayCodec`][]), exactly one array -> bytes codec + ([`zarr.abc.codec.ArrayBytesCodec`][]), and zero or more bytes -> bytes codecs + ([`zarr.abc.codec.BytesBytesCodec`][]). Elsewhere in Zarr-Python these are + called filters, the serializer, and compressors; those names are Zarr-Python + aliases, not terms defined in the specification. If no codecs are provided, default codecs will be used based on the data type of the array. For most data types, the default codecs are the tuple `(BytesCodec(), ZstdCodec())`; @@ -928,10 +935,13 @@ def create_array( filters are applied (if any are specified) and the data is serialized into bytes. For Zarr format 3, a "compressor" is a codec that takes a bytestream, and - returns another bytestream. Multiple compressors may be provided for Zarr format 3. - If no `compressors` are provided, a default set of compressors will be used. - These defaults can be changed by modifying the value of `array.v3_default_compressors` - in [`zarr.config`][zarr.config]. + returns another bytestream. These values must be instances of + [`zarr.abc.codec.BytesBytesCodec`][], or dict representations of + [`zarr.abc.codec.BytesBytesCodec`][]. Multiple compressors may be provided + for Zarr format 3. Codecs that take an array and return bytes are serializers + and must be supplied with `serializer` instead. + If no `compressors` are provided, the default for Zarr format 3 is + `(ZstdCodec(),)`. Use `None` to omit default compressors. For Zarr format 2, a "compressor" can be any numcodecs codec. Only a single compressor may @@ -943,9 +953,11 @@ def create_array( serializer : dict[str, JSON] | ArrayBytesCodec, optional Array-to-bytes codec to use for encoding the array data. Zarr format 3 only. Zarr format 2 arrays use implicit array-to-bytes conversion. - If no `serializer` is provided, a default serializer will be used. - These defaults can be changed by modifying the value of `array.v3_default_serializer` - in [`zarr.config`][zarr.config]. + Codecs that are instances of [`zarr.abc.codec.ArrayBytesCodec`][] must be + supplied here, not with `compressors`. + If no `serializer` is provided, a default serializer is chosen based on the data type: + `VLenUTF8Codec` for variable-length strings, `VLenBytesCodec` for variable-length + bytes, and `BytesCodec` otherwise. fill_value : Any, optional Fill value for the array. order : {"C", "F"}, optional diff --git a/src/zarr/core/array.py b/src/zarr/core/array.py index 06f7338755..d392efb757 100644 --- a/src/zarr/core/array.py +++ b/src/zarr/core/array.py @@ -4802,10 +4802,13 @@ async def create_array( filters are applied (if any are specified) and the data is serialized into bytes. For Zarr format 3, a "compressor" is a codec that takes a bytestream, and - returns another bytestream. Multiple compressors may be provided for Zarr format 3. - If no `compressors` are provided, a default set of compressors will be used. - These defaults can be changed by modifying the value of `array.v3_default_compressors` - in [`zarr.config`][zarr.config]. + returns another bytestream. These values must be instances of + [`zarr.abc.codec.BytesBytesCodec`][], or dict representations of + [`zarr.abc.codec.BytesBytesCodec`][]. Multiple compressors may be provided + for Zarr format 3. Codecs that take an array and return bytes are serializers + and must be supplied with `serializer` instead. + If no `compressors` are provided, the default for Zarr format 3 is + `(ZstdCodec(),)`. Use `None` to omit default compressors. For Zarr format 2, a "compressor" can be any numcodecs codec. Only a single compressor may @@ -4817,9 +4820,11 @@ async def create_array( serializer : dict[str, JSON] | ArrayBytesCodec, optional Array-to-bytes codec to use for encoding the array data. Zarr format 3 only. Zarr format 2 arrays use implicit array-to-bytes conversion. - If no `serializer` is provided, a default serializer will be used. - These defaults can be changed by modifying the value of `array.v3_default_serializer` - in [`zarr.config`][zarr.config]. + Codecs that are instances of [`zarr.abc.codec.ArrayBytesCodec`][] must be + supplied here, not with `compressors`. + If no `serializer` is provided, a default serializer is chosen based on the data type: + `VLenUTF8Codec` for variable-length strings, `VLenBytesCodec` for variable-length + bytes, and `BytesCodec` otherwise. fill_value : Any, optional Fill value for the array. order : {"C", "F"}, optional