diff --git a/changes/4526.bugfix.md b/changes/4526.bugfix.md new file mode 100644 index 0000000000..6f2b513d54 --- /dev/null +++ b/changes/4526.bugfix.md @@ -0,0 +1 @@ +Array creation no longer fails when the codec chain cannot encode the array's fill value (which defaults to `dtype.default_scalar()`), e.g. a `ScaleOffset` + `CastValue` chain mapping only a subrange of the dtype. The fill value is now carried through the chain's metadata resolution unchanged if it cannot be encoded; genuine value errors for actual chunk data still surface at write time, and structurally invalid codec configurations are still rejected at creation. diff --git a/src/zarr/codecs/cast_value.py b/src/zarr/codecs/cast_value.py index 629b4e7a27..e69e362f62 100644 --- a/src/zarr/codecs/cast_value.py +++ b/src/zarr/codecs/cast_value.py @@ -371,11 +371,19 @@ def resolve_metadata(self, chunk_spec: ArraySpec) -> ArraySpec: fill = chunk_spec.fill_value fill_arr = np.array([fill], dtype=source_native) - - new_fill_arr = self._do_cast( - fill_arr, target_dtype=target_native, scalar_map=self._get_scalar_map("encode") - ) - new_fill = target_native.type(new_fill_arr[0]) + try: + new_fill_arr = self._do_cast( + fill_arr, target_dtype=target_native, scalar_map=self._get_scalar_map("encode") + ) + new_fill = target_native.type(new_fill_arr[0]) + except (ValueError, OverflowError): + # The fill value may be outside the encodable domain of this codec (e.g. a + # source value that cannot be cast to the target dtype without an explicit + # out_of_range policy). That is a property of the fill value, not of the + # codec configuration, so metadata threading (shape/dtype) must not fail + # because of it: carry the fill value forward unchanged. Genuine value + # errors for actual chunk data still surface in `_encode_sync`. + new_fill = fill return replace(chunk_spec, dtype=target_zdtype, fill_value=new_fill) diff --git a/src/zarr/codecs/scale_offset.py b/src/zarr/codecs/scale_offset.py index 2ec6015b6a..e1d00b0105 100644 --- a/src/zarr/codecs/scale_offset.py +++ b/src/zarr/codecs/scale_offset.py @@ -408,8 +408,17 @@ def resolve_metadata(self, chunk_spec: ArraySpec) -> ArraySpec: fill = np.asarray(zdtype.cast_scalar(chunk_spec.fill_value)) offset = cast("np.generic", zdtype.from_json_scalar(self.offset, zarr_format=3)) scale = cast("np.generic", zdtype.from_json_scalar(self.scale, zarr_format=3)) - new_fill = _encode(fill, offset, scale) - return replace(chunk_spec, fill_value=new_fill.reshape(()).item()) + try: + new_fill = _encode(fill, offset, scale).reshape(()).item() + except (ValueError, OverflowError): + # The fill value may lie outside the encodable domain of this codec (e.g. a + # scale/offset pair mapping only a subrange of the dtype, with the fill value + # outside that subrange). That is a property of the fill value, not of the + # codec configuration, so metadata threading (shape/dtype) must not fail + # because of it: carry the fill value forward unchanged. Genuine value + # errors for actual chunk data still surface in `_encode_sync`. + new_fill = chunk_spec.fill_value + return replace(chunk_spec, fill_value=new_fill) def _decode_sync( self, diff --git a/tests/test_codecs/test_cast_value.py b/tests/test_codecs/test_cast_value.py index c2e78770d9..22fb38172b 100644 --- a/tests/test_codecs/test_cast_value.py +++ b/tests/test_codecs/test_cast_value.py @@ -8,7 +8,7 @@ import zarr from tests.conftest import Expect, ExpectFail -from zarr.codecs import BytesCodec, TransposeCodec +from zarr.codecs import BytesCodec, ScaleOffset, TransposeCodec from zarr.codecs.cast_value import CastValue from zarr.storage import MemoryStore @@ -611,3 +611,55 @@ def make_array(filters: list[Any]) -> Any: array[:] = data assert_array_equal(array[:], data) + + +# --------------------------------------------------------------------------- +# Fill-value tolerance in resolve_metadata (issue #4511) +# --------------------------------------------------------------------------- + + +@requires_cast_value_rs +def test_creation_with_unencodable_default_fill_value() -> None: + """A ScaleOffset+CastValue chain whose valid input range excludes the default + fill value (e.g. float32 values known to lie in [100, 200], scaled to uint16) + must not fail array creation: the fill value is metadata, not chunk data, and + is never required to pass through the codec chain + (https://github.com/zarr-developers/zarr-python/issues/4511). + """ + minimum, maximum = 100.0, 200.0 + arr = zarr.create_array( + store=MemoryStore(), + shape=(4,), + dtype="float32", + chunks=(4,), + filters=[ + ScaleOffset(scale=65535 / (maximum - minimum), offset=minimum), + CastValue(data_type="uint16", rounding="nearest-even"), + ], + compressors=None, + ) + # Full-chunk writes contain no fill positions, so they encode cleanly. + data = np.array([120.0, 180.0, 100.0, 200.0], dtype="float32") + arr[:] = data + # Quantization to uint16 introduces a small rounding error. + np.testing.assert_allclose(np.asarray(arr[:]), data, atol=1e-2) + + +@requires_cast_value_rs +def test_unencodable_fill_value_does_not_mask_invalid_config() -> None: + """Tolerating an unencodable fill value must not weaken validation: a codec + configuration that is structurally incompatible with the dtype still raises + at array creation time. + """ + with pytest.raises(ValueError, match="only supports integer and floating-point"): + zarr.create_array( + store=MemoryStore(), + shape=(4,), + dtype="str", + chunks=(4,), + filters=[ + ScaleOffset(scale=1, offset=100), + CastValue(data_type="uint16", rounding="nearest-even"), + ], + compressors=None, + ) diff --git a/tests/test_codecs/test_scale_offset.py b/tests/test_codecs/test_scale_offset.py index 0100d9fe80..b81ae004cc 100644 --- a/tests/test_codecs/test_scale_offset.py +++ b/tests/test_codecs/test_scale_offset.py @@ -488,3 +488,45 @@ def test_compute_encoded_size() -> None: # The chunk_spec argument is unused; pass any sentinel assert codec.compute_encoded_size(input_byte_length=100, _chunk_spec=None) == 100 # type: ignore[arg-type] assert codec.compute_encoded_size(input_byte_length=0, _chunk_spec=None) == 0 # type: ignore[arg-type] + + +# --------------------------------------------------------------------------- +# Fill-value tolerance in resolve_metadata (issue #4511) +# --------------------------------------------------------------------------- + + +def test_creation_with_unencodable_default_fill_value() -> None: + """A scale/offset pair mapping only a subrange of the dtype makes the default + fill value (0) unencodable. Array creation must still succeed: the fill value + is metadata, not chunk data, and is never required to pass through the codec + chain (https://github.com/zarr-developers/zarr-python/issues/4511). + """ + arr = zarr.create_array( + store=MemoryStore(), + shape=(4,), + dtype="uint16", + chunks=(4,), + filters=[ScaleOffset(offset=100, scale=1)], + compressors=None, + ) + # Full-chunk writes contain no fill positions, so they encode cleanly. + # Encoded values must stay within uint16 after decode adds the offset back. + data = np.array([100, 150, 200, 65435], dtype="uint16") + arr[:] = data + np.testing.assert_array_equal(arr[:], data) + + +def test_unencodable_fill_value_does_not_mask_invalid_config() -> None: + """Tolerating an unencodable fill value must not weaken validation: a codec + configuration that is structurally incompatible with the dtype still raises + at array creation time. + """ + with pytest.raises(ValueError, match="only supports integer and floating-point"): + zarr.create_array( + store=MemoryStore(), + shape=(4,), + dtype="str", + chunks=(4,), + filters=[ScaleOffset(offset=100, scale=1)], + compressors=None, + ) diff --git a/tests/test_codecs/test_sharding.py b/tests/test_codecs/test_sharding.py index 5712c9576d..976a942406 100644 --- a/tests/test_codecs/test_sharding.py +++ b/tests/test_codecs/test_sharding.py @@ -19,6 +19,7 @@ BytesCodec, Crc32cCodec, GzipCodec, + ScaleOffset, ShardingCodec, TransposeCodec, ) @@ -1522,3 +1523,31 @@ def test_open_rejects_stored_nested_sharding_with_indivisible_inner_chunk_shape( doc["codecs"][0]["configuration"]["codecs"][0]["configuration"]["chunk_shape"] = (4, 7) with pytest.raises(ValueError, match="not divisible"): ArrayV3Metadata.from_dict(doc) + + +# --------------------------------------------------------------------------- +# Inner-codec validation tolerance for unencodable fill values (issue #4511) +# --------------------------------------------------------------------------- + + +def test_create_with_unencodable_default_fill_value_inner_codecs() -> None: + """`ShardingCodec._validate_inner_codecs` probes the inner codec chain with + `dtype.default_scalar()` as the fill value. When the chain maps only a + subrange of the dtype (e.g. a scale/offset pair for values in [100, 200]), + that probe value is unencodable — but the configuration is still valid, so + array creation must succeed + (https://github.com/zarr-developers/zarr-python/issues/4511). + """ + arr = zarr.create_array( + store=MemoryStore(), + shape=(16,), + chunks=(4,), + shards=(16,), + dtype="uint16", + filters=[ScaleOffset(offset=100, scale=1)], + ) + # Full-shard writes contain no fill positions, so they encode cleanly. + # Encoded values must stay within uint16 after decode adds the offset back. + data = np.array([100, 150, 200, 65435] * 4, dtype="uint16") + arr[:] = data + np.testing.assert_array_equal(arr[:], data)