fix: tolerate unencodable default_scalar in ShardingCodec.validate

changes/4526.bugfix.md

@@ -0,0 +1 @@
+Array creation no longer fails when the codec chain cannot encode the array's fill value (which defaults to `dtype.default_scalar()`), e.g. a `ScaleOffset` + `CastValue` chain mapping only a subrange of the dtype. The fill value is now carried through the chain's metadata resolution unchanged if it cannot be encoded; genuine value errors for actual chunk data still surface at write time, and structurally invalid codec configurations are still rejected at creation.

src/zarr/codecs/cast_value.py

@@ -371,11 +371,19 @@ def resolve_metadata(self, chunk_spec: ArraySpec) -> ArraySpec:
 
         fill = chunk_spec.fill_value
         fill_arr = np.array([fill], dtype=source_native)
-
-        new_fill_arr = self._do_cast(
-            fill_arr, target_dtype=target_native, scalar_map=self._get_scalar_map("encode")
-        )
-        new_fill = target_native.type(new_fill_arr[0])
+        try:
+            new_fill_arr = self._do_cast(
+                fill_arr, target_dtype=target_native, scalar_map=self._get_scalar_map("encode")
+            )
+            new_fill = target_native.type(new_fill_arr[0])
+        except (ValueError, OverflowError):
+            # The fill value may be outside the encodable domain of this codec (e.g. a
+            # source value that cannot be cast to the target dtype without an explicit
+            # out_of_range policy). That is a property of the fill value, not of the
+            # codec configuration, so metadata threading (shape/dtype) must not fail
+            # because of it: carry the fill value forward unchanged. Genuine value
+            # errors for actual chunk data still surface in `_encode_sync`.
+            new_fill = fill
 
         return replace(chunk_spec, dtype=target_zdtype, fill_value=new_fill)
 

src/zarr/codecs/scale_offset.py

@@ -408,8 +408,17 @@ def resolve_metadata(self, chunk_spec: ArraySpec) -> ArraySpec:
         fill = np.asarray(zdtype.cast_scalar(chunk_spec.fill_value))
         offset = cast("np.generic", zdtype.from_json_scalar(self.offset, zarr_format=3))
         scale = cast("np.generic", zdtype.from_json_scalar(self.scale, zarr_format=3))
-        new_fill = _encode(fill, offset, scale)
-        return replace(chunk_spec, fill_value=new_fill.reshape(()).item())
+        try:
+            new_fill = _encode(fill, offset, scale).reshape(()).item()
+        except (ValueError, OverflowError):
+            # The fill value may lie outside the encodable domain of this codec (e.g. a
+            # scale/offset pair mapping only a subrange of the dtype, with the fill value
+            # outside that subrange). That is a property of the fill value, not of the
+            # codec configuration, so metadata threading (shape/dtype) must not fail
+            # because of it: carry the fill value forward unchanged. Genuine value
+            # errors for actual chunk data still surface in `_encode_sync`.
+            new_fill = chunk_spec.fill_value
+        return replace(chunk_spec, fill_value=new_fill)
 
     def _decode_sync(
         self,

tests/test_codecs/test_cast_value.py

@@ -8,7 +8,7 @@
 
 import zarr
 from tests.conftest import Expect, ExpectFail
-from zarr.codecs import BytesCodec, TransposeCodec
+from zarr.codecs import BytesCodec, ScaleOffset, TransposeCodec
 from zarr.codecs.cast_value import CastValue
 from zarr.storage import MemoryStore
 
@@ -611,3 +611,55 @@ def make_array(filters: list[Any]) -> Any:
 
     array[:] = data
     assert_array_equal(array[:], data)
+
+
+# ---------------------------------------------------------------------------
+# Fill-value tolerance in resolve_metadata (issue #4511)
+# ---------------------------------------------------------------------------
+
+
+@requires_cast_value_rs
+def test_creation_with_unencodable_default_fill_value() -> None:
+    """A ScaleOffset+CastValue chain whose valid input range excludes the default
+    fill value (e.g. float32 values known to lie in [100, 200], scaled to uint16)
+    must not fail array creation: the fill value is metadata, not chunk data, and
+    is never required to pass through the codec chain
+    (https://github.com/zarr-developers/zarr-python/issues/4511).
+    """
+    minimum, maximum = 100.0, 200.0
+    arr = zarr.create_array(
+        store=MemoryStore(),
+        shape=(4,),
+        dtype="float32",
+        chunks=(4,),
+        filters=[
+            ScaleOffset(scale=65535 / (maximum - minimum), offset=minimum),
+            CastValue(data_type="uint16", rounding="nearest-even"),
+        ],
+        compressors=None,
+    )
+    # Full-chunk writes contain no fill positions, so they encode cleanly.
+    data = np.array([120.0, 180.0, 100.0, 200.0], dtype="float32")
+    arr[:] = data
+    # Quantization to uint16 introduces a small rounding error.
+    np.testing.assert_allclose(np.asarray(arr[:]), data, atol=1e-2)
+
+
+@requires_cast_value_rs
+def test_unencodable_fill_value_does_not_mask_invalid_config() -> None:
+    """Tolerating an unencodable fill value must not weaken validation: a codec
+    configuration that is structurally incompatible with the dtype still raises
+    at array creation time.
+    """
+    with pytest.raises(ValueError, match="only supports integer and floating-point"):
+        zarr.create_array(
+            store=MemoryStore(),
+            shape=(4,),
+            dtype="str",
+            chunks=(4,),
+            filters=[
+                ScaleOffset(scale=1, offset=100),
+                CastValue(data_type="uint16", rounding="nearest-even"),
+            ],
+            compressors=None,
+        )

tests/test_codecs/test_scale_offset.py

@@ -488,3 +488,45 @@ def test_compute_encoded_size() -> None:
     # The chunk_spec argument is unused; pass any sentinel
     assert codec.compute_encoded_size(input_byte_length=100, _chunk_spec=None) == 100  # type: ignore[arg-type]
     assert codec.compute_encoded_size(input_byte_length=0, _chunk_spec=None) == 0  # type: ignore[arg-type]
+
+
+# ---------------------------------------------------------------------------
+# Fill-value tolerance in resolve_metadata (issue #4511)
+# ---------------------------------------------------------------------------
+
+
+def test_creation_with_unencodable_default_fill_value() -> None:
+    """A scale/offset pair mapping only a subrange of the dtype makes the default
+    fill value (0) unencodable. Array creation must still succeed: the fill value
+    is metadata, not chunk data, and is never required to pass through the codec
+    chain (https://github.com/zarr-developers/zarr-python/issues/4511).
+    """
+    arr = zarr.create_array(
+        store=MemoryStore(),
+        shape=(4,),
+        dtype="uint16",
+        chunks=(4,),
+        filters=[ScaleOffset(offset=100, scale=1)],
+        compressors=None,
+    )
+    # Full-chunk writes contain no fill positions, so they encode cleanly.
+    # Encoded values must stay within uint16 after decode adds the offset back.
+    data = np.array([100, 150, 200, 65435], dtype="uint16")
+    arr[:] = data
+    np.testing.assert_array_equal(arr[:], data)
+
+
+def test_unencodable_fill_value_does_not_mask_invalid_config() -> None:
+    """Tolerating an unencodable fill value must not weaken validation: a codec
+    configuration that is structurally incompatible with the dtype still raises
+    at array creation time.
+    """
+    with pytest.raises(ValueError, match="only supports integer and floating-point"):
+        zarr.create_array(
+            store=MemoryStore(),
+            shape=(4,),
+            dtype="str",
+            chunks=(4,),
+            filters=[ScaleOffset(offset=100, scale=1)],
+            compressors=None,
+        )

tests/test_codecs/test_sharding.py

@@ -19,6 +19,7 @@
     BytesCodec,
     Crc32cCodec,
     GzipCodec,
+    ScaleOffset,
     ShardingCodec,
     TransposeCodec,
 )
@@ -1522,3 +1523,31 @@ def test_open_rejects_stored_nested_sharding_with_indivisible_inner_chunk_shape(
     doc["codecs"][0]["configuration"]["codecs"][0]["configuration"]["chunk_shape"] = (4, 7)
     with pytest.raises(ValueError, match="not divisible"):
         ArrayV3Metadata.from_dict(doc)
+
+
+# ---------------------------------------------------------------------------
+# Inner-codec validation tolerance for unencodable fill values (issue #4511)
+# ---------------------------------------------------------------------------
+
+
+def test_create_with_unencodable_default_fill_value_inner_codecs() -> None:
+    """`ShardingCodec._validate_inner_codecs` probes the inner codec chain with
+    `dtype.default_scalar()` as the fill value. When the chain maps only a
+    subrange of the dtype (e.g. a scale/offset pair for values in [100, 200]),
+    that probe value is unencodable — but the configuration is still valid, so
+    array creation must succeed
+    (https://github.com/zarr-developers/zarr-python/issues/4511).
+    """
+    arr = zarr.create_array(
+        store=MemoryStore(),
+        shape=(16,),
+        chunks=(4,),
+        shards=(16,),
+        dtype="uint16",
+        filters=[ScaleOffset(offset=100, scale=1)],
+    )
+    # Full-shard writes contain no fill positions, so they encode cleanly.
+    # Encoded values must stay within uint16 after decode adds the offset back.
+    data = np.array([100, 150, 200, 65435] * 4, dtype="uint16")
+    arr[:] = data
+    np.testing.assert_array_equal(arr[:], data)