diff --git a/changes/4493.bugfix.md b/changes/4493.bugfix.md new file mode 100644 index 0000000000..e4c2c708c9 --- /dev/null +++ b/changes/4493.bugfix.md @@ -0,0 +1 @@ +Accept a list of byte values as a ``fill_value`` for byte data types, as permitted by the Zarr V3 spec. diff --git a/src/zarr/core/dtype/npy/bytes.py b/src/zarr/core/dtype/npy/bytes.py index daeb105819..511db575d4 100644 --- a/src/zarr/core/dtype/npy/bytes.py +++ b/src/zarr/core/dtype/npy/bytes.py @@ -18,7 +18,7 @@ check_dtype_spec_v2, v3_unstable_dtype_warning, ) -from zarr.core.dtype.npy.common import check_json_str +from zarr.core.dtype.npy.common import check_json_byte_list, check_json_str from zarr.core.dtype.wrapper import TBaseDType, ZDType from zarr.errors import DataTypeValidationError @@ -524,8 +524,10 @@ def from_json_scalar(self, data: JSON, *, zarr_format: ZarrFormat) -> np.bytes_: if check_json_str(data): return self.to_native_dtype().type(base64.standard_b64decode(data.encode("ascii"))) + if check_json_byte_list(data): + return self.to_native_dtype().type(bytes(data)) raise TypeError( - f"Invalid type: {data}. Expected a base64-encoded string." + f"Invalid type: {data}. Expected a base64-encoded string or a list of byte values." ) # pragma: no cover @property @@ -919,7 +921,11 @@ def from_json_scalar(self, data: JSON, *, zarr_format: ZarrFormat) -> np.void: """ if check_json_str(data): return self.to_native_dtype().type(base64.standard_b64decode(data)) - raise TypeError(f"Invalid type: {data}. Expected a string.") # pragma: no cover + if check_json_byte_list(data): + return self.to_native_dtype().type(bytes(data)) + raise TypeError( + f"Invalid type: {data}. Expected a base64-encoded string or a list of byte values." + ) # pragma: no cover @property def item_size(self) -> int: @@ -1200,7 +1206,11 @@ def from_json_scalar(self, data: JSON, *, zarr_format: ZarrFormat) -> bytes: if check_json_str(data): return base64.standard_b64decode(data.encode("ascii")) - raise TypeError(f"Invalid type: {data}. Expected a string.") # pragma: no cover + if check_json_byte_list(data): + return bytes(data) + raise TypeError( + f"Invalid type: {data}. Expected a base64-encoded string or a list of byte values." + ) # pragma: no cover def _check_scalar(self, data: object) -> TypeGuard[BytesLike]: """ diff --git a/src/zarr/core/dtype/npy/common.py b/src/zarr/core/dtype/npy/common.py index f413f5f678..81941554e8 100644 --- a/src/zarr/core/dtype/npy/common.py +++ b/src/zarr/core/dtype/npy/common.py @@ -564,3 +564,22 @@ def check_json_bool(data: JSON) -> TypeGuard[bool]: True if the data is a boolean, False otherwise. """ return isinstance(data, bool) + + +def check_json_byte_list(data: JSON) -> TypeGuard[list[int]]: + """ + Check if a JSON value is a list of byte values. + + Parameters + ---------- + data : JSON + The JSON value to check. + + Returns + ------- + Bool + True if the data is a list of integers in the range [0, 255], False otherwise. + """ + return isinstance(data, list) and all( + isinstance(e, int) and not isinstance(e, bool) and 0 <= e <= 255 for e in data + ) diff --git a/tests/test_dtype/test_npy/test_bytes.py b/tests/test_dtype/test_npy/test_bytes.py index 2d384d82ab..5662e6d253 100644 --- a/tests/test_dtype/test_npy/test_bytes.py +++ b/tests/test_dtype/test_npy/test_bytes.py @@ -2,6 +2,7 @@ import pytest from tests.test_dtype.test_wrapper import BaseTestZDType +from zarr.core.common import JSON from zarr.core.dtype.npy.bytes import NullTerminatedBytes, RawBytes, VariableLengthBytes from zarr.errors import UnstableSpecificationWarning @@ -171,3 +172,38 @@ def test_invalid_size(zdtype_cls: type[NullTerminatedBytes] | type[RawBytes]) -> msg = f"length must be >= 1, got {length}." with pytest.raises(ValueError, match=msg): zdtype_cls(length=length) + + +@pytest.mark.parametrize( + ("zdtype", "expected"), + [ + (NullTerminatedBytes(length=2), np.bytes_(b"ab")), + (RawBytes(length=2), np.void(b"ab")), + (VariableLengthBytes(), b"ab"), + ], +) +def test_byte_list_fill_value( + zdtype: NullTerminatedBytes | RawBytes | VariableLengthBytes, expected: object +) -> None: + """ + Test that byte data types accept a JSON list of byte values as a fill value, + per the Zarr V3 spec, and that it matches the base64 form. + """ + from_byte_list = zdtype.from_json_scalar([97, 98], zarr_format=3) + from_base64 = zdtype.from_json_scalar("YWI=", zarr_format=3) + assert from_byte_list == expected + assert from_byte_list == from_base64 + + +@pytest.mark.parametrize( + "zdtype", [NullTerminatedBytes(length=2), RawBytes(length=2), VariableLengthBytes()] +) +@pytest.mark.parametrize("data", [[256], [-1], ["a"], [97.0], [True]]) +def test_byte_list_fill_value_invalid( + zdtype: NullTerminatedBytes | RawBytes | VariableLengthBytes, data: JSON +) -> None: + """ + Test that a JSON list containing non-byte values is rejected as a fill value. + """ + with pytest.raises(TypeError, match="byte values"): + zdtype.from_json_scalar(data, zarr_format=3)