Add Array.prepend and the regular_bounded chunk grid

changes/4516.feature.md

@@ -0,0 +1,4 @@
+Added ``Array.prepend``, which adds data before index 0 of an axis without moving existing chunks,
+and the ``regular_bounded`` chunk grid it switches an array to: a regular grid with an ``origin``
+and a ``lower_bound``, under which chunk grid indices can be negative. ``Array.domain`` reports the
+coordinate range of each dimension. Chunk keys with a ``-0`` component are now rejected.

docs/user-guide/arrays.md

@@ -154,6 +154,26 @@ z.append(np.vstack([a, a]), axis=1)
 print(f"Shape after second append: {z.shape}")
 ```
 
+### Prepending
+
+[`zarr.Array.prepend`][] adds data before index 0 of an axis, the way [`zarr.Array.append`][]
+adds it after the end. The existing chunks keep their keys and their bytes. Data that does not
+fill a whole chunk goes into a partly filled first chunk, like the partly filled last chunk of
+any array.
+
+The first prepend switches the chunk grid from `regular` to `regular_bounded`, which records
+the coordinate of the first element (`lower_bound`) and of the start of chunk 0 (`origin`), so
+chunks in front of the old start get negative keys such as `c/-1/0`. Indexing stays zero-based,
+and [`zarr.Array.domain`][] gives the coordinate range of each dimension. Readers that do not
+implement `regular_bounded` refuse such an array rather than misread it.
+
+```python exec="true" session="arrays" source="above" result="ansi"
+z = zarr.create_array(store='data/example-5.zarr', shape=(6, 4), dtype='int32', chunks=(4, 4), overwrite=True)
+z[:] = 1
+z.prepend(np.full((3, 4), 2, dtype='int32'))
+print(f"Shape: {z.shape}, domain: {z.domain}")
+```
+
 ## Runtime configuration
 
 Zarr arrays are parametrized with a configuration that determines certain aspects of array behavior.

src/zarr/core/array.py

@@ -138,6 +138,7 @@
 from zarr.core.metadata.v3 import (
     ChunkGridMetadata,
     RectilinearChunkGridMetadata,
+    RegularBoundedChunkGridMetadata,
     RegularChunkGridMetadata,
     create_chunk_grid_metadata,
     parse_node_type_array,
@@ -949,6 +950,20 @@ def write_chunk_sizes(self) -> tuple[tuple[int, ...], ...]:
 
         return self._chunk_grid.chunk_sizes
 
+    @property
+    def domain(self) -> tuple[tuple[int, int], ...]:
+        """The half-open coordinate range ``(start, stop)`` of each dimension.
+
+        Index ``i`` along a dimension is coordinate ``start + i``. ``start`` is 0
+        unless the array has been grown with `prepend`.
+        """
+        grid = getattr(self.metadata, "chunk_grid", None)
+        if isinstance(grid, RegularBoundedChunkGridMetadata):
+            lower_bound = grid.lower_bound
+        else:
+            lower_bound = (0,) * len(self.shape)
+        return tuple((lb, lb + n) for lb, n in zip(lower_bound, self.shape, strict=True))
+
     @property
     def shards(self) -> tuple[int, ...] | None:
         """Returns the shard shape of the Array.
@@ -1801,6 +1816,29 @@ async def append(self, data: npt.ArrayLike, axis: int = 0) -> tuple[int, ...]:
         """
         return await _append(self, data, axis)
 
+    async def prepend(self, data: npt.ArrayLike, axis: int = 0) -> tuple[int, ...]:
+        """Prepend `data` to `axis`.
+
+        Parameters
+        ----------
+        data : array-like
+            Data to be prepended.
+        axis : int
+            Axis along which to prepend.
+
+        Returns
+        -------
+        new_shape : tuple
+
+        Notes
+        -----
+        The size of all dimensions other than `axis` must match between this
+        array and `data`. Existing chunks keep their keys. The first prepend
+        switches a ``regular`` chunk grid to ``regular_bounded``, which readers
+        without that chunk grid refuse.
+        """
+        return await _prepend(self, data, axis)
+
     async def update_attributes(self, new_attributes: dict[str, JSON]) -> Self:
         """
         Asynchronously update the array's attributes.
@@ -2194,6 +2232,15 @@ def write_chunk_sizes(self) -> tuple[tuple[int, ...], ...]:
         """
         return self.async_array.write_chunk_sizes
 
+    @property
+    def domain(self) -> tuple[tuple[int, int], ...]:
+        """The half-open coordinate range ``(start, stop)`` of each dimension.
+
+        Index ``i`` along a dimension is coordinate ``start + i``. ``start`` is 0
+        unless the array has been grown with `prepend`.
+        """
+        return self.async_array.domain
+
     @property
     def shards(self) -> tuple[int, ...] | None:
         """Returns a tuple of integers describing the length of each dimension of a shard of the array.
@@ -4025,6 +4072,29 @@ def append(self, data: npt.ArrayLike, axis: int = 0) -> tuple[int, ...]:
         """
         return sync(self.async_array.append(data, axis=axis))
 
+    def prepend(self, data: npt.ArrayLike, axis: int = 0) -> tuple[int, ...]:
+        """Prepend `data` to `axis`.
+
+        Parameters
+        ----------
+        data : array-like
+            Data to be prepended.
+        axis : int
+            Axis along which to prepend.
+
+        Returns
+        -------
+        new_shape : tuple
+
+        Notes
+        -----
+        The size of all dimensions other than `axis` must match between this
+        array and `data`. Existing chunks keep their keys. The first prepend
+        switches a ``regular`` chunk grid to ``regular_bounded``, which readers
+        without that chunk grid refuse.
+        """
+        return sync(self.async_array.prepend(data, axis=axis))
+
     def update_attributes(self, new_attributes: dict[str, JSON]) -> Self:
         """
         Update the array's attributes.
@@ -5565,7 +5635,7 @@ async def _nchunks_initialized(
     grid = array._chunk_grid
     total = 0
     for key in await _shards_initialized(array):
-        spec = grid[meta.chunk_key_encoding.decode_chunk_key(key)]
+        spec = grid[meta.decode_chunk_key(key)]
         if spec is not None:
             total += product(
                 tuple(s // c for s, c in zip(spec.codec_shape, inner_chunks, strict=True))
@@ -6066,6 +6136,77 @@ async def _append(
     return new_shape
 
 
+async def _prepend(
+    array: AsyncArray[ArrayV2Metadata] | AsyncArray[ArrayV3Metadata],
+    data: npt.ArrayLike,
+    axis: int = 0,
+) -> tuple[int, ...]:
+    """
+    Prepend data to an array along the specified axis.
+
+    Parameters
+    ----------
+    array : AsyncArray
+        The array to prepend to. Must be a Zarr v3 array with a regular chunk grid.
+    data : npt.ArrayLike
+        Data to be prepended.
+    axis : int
+        Axis along which to prepend.
+
+    Returns
+    -------
+    new_shape : tuple[int, ...]
+        The new shape of the array after prepending.
+    """
+    if not hasattr(data, "shape"):
+        data = np.asanyarray(data)
+
+    metadata = array.metadata
+    if not isinstance(metadata, ArrayV3Metadata) or not isinstance(
+        metadata.chunk_grid, RegularChunkGridMetadata
+    ):
+        raise TypeError("prepend needs a Zarr v3 array with a regular chunk grid.")
+    ndim = len(metadata.shape)
+    if not -ndim <= axis < ndim:
+        raise ValueError(f"axis {axis} is out of range for a {ndim}-dimensional array.")
+    axis %= ndim
+
+    self_shape_preserved = tuple(s for i, s in enumerate(array.shape) if i != axis)
+    data_shape_preserved = tuple(s for i, s in enumerate(data.shape) if i != axis)
+    if self_shape_preserved != data_shape_preserved:
+        raise ValueError(
+            f"shape of data to prepend is not compatible with the array. "
+            f"The shape of the data is ({data_shape_preserved})"
+            f"and the shape of the array is ({self_shape_preserved})."
+            "All dimensions must match except for the dimension being "
+            "prepended."
+        )
+    extent = data.shape[axis]
+    grid = metadata.chunk_grid
+    if isinstance(grid, RegularBoundedChunkGridMetadata):
+        origin, lower_bound = grid.origin, grid.lower_bound
+    else:
+        origin = lower_bound = (0,) * ndim
+
+    new_shape = tuple(s + extent if i == axis else s for i, s in enumerate(metadata.shape))
+    new_grid = RegularBoundedChunkGridMetadata(
+        chunk_shape=grid.chunk_shape,
+        origin=origin,
+        lower_bound=tuple(lb - extent if i == axis else lb for i, lb in enumerate(lower_bound)),
+    )
+    new_metadata = replace(metadata, shape=new_shape, chunk_grid=new_grid)
+    new_chunk_grid = ChunkGrid.from_metadata(new_metadata)
+    await store_documents(array.store_path, encode_documents(array.store_path, new_metadata))
+    array._stored_document_replaced()
+    object.__setattr__(array, "metadata", new_metadata)
+    object.__setattr__(array, "_chunk_grid", new_chunk_grid)
+
+    prepend_selection = tuple(slice(None) if i != axis else slice(0, extent) for i in range(ndim))
+    await array.setitem(prepend_selection, data)
+
+    return new_shape
+
+
 async def _update_attributes(
     array: AsyncArray[ArrayV2Metadata] | AsyncArray[ArrayV3Metadata],
     new_attributes: dict[str, JSON],

src/zarr/core/chunk_grids.py

@@ -48,10 +48,12 @@
 @dataclass(frozen=True)
 class FixedDimension:
     """Uniform chunk size. Boundary chunks contain less data but are
-    encoded at full size by the codec pipeline."""
+    encoded at full size by the codec pipeline. With a non-zero ``shift`` the
+    first element sits ``shift`` positions into the first chunk's buffer."""
 
     size: int  # chunk edge length (>= 1)
     extent: int  # array dimension length (>= 0)
+    shift: int = 0  # position of array index 0 inside the first chunk, in [0, size)
     nchunks: int = field(init=False, repr=False)
     ngridcells: int = field(init=False, repr=False)
 
@@ -60,7 +62,9 @@ def __post_init__(self) -> None:
             raise ValueError(f"FixedDimension size must be >= 1, got {self.size}")
         if self.extent < 0:
             raise ValueError(f"FixedDimension extent must be >= 0, got {self.extent}")
-        n = ceildiv_int(self.extent, self.size)
+        if not 0 <= self.shift < self.size:
+            raise ValueError(f"FixedDimension shift must be in [0, {self.size}), got {self.shift}")
+        n = ceildiv_int(self.extent + self.shift, self.size) if self.extent else 0
         object.__setattr__(self, "nchunks", n)
         object.__setattr__(self, "ngridcells", n)
 
@@ -69,15 +73,19 @@ def index_to_chunk(self, idx: int) -> int:
             raise IndexError(f"Negative index {idx} is not allowed")
         if idx >= self.extent:
             raise IndexError(f"Index {idx} is out of bounds for extent {self.extent}")
-        return idx // self.size
+        return (idx + self.shift) // self.size
 
     def chunk_offset(self, chunk_ix: int) -> int:
-        """Byte-aligned start position of chunk *chunk_ix* in array coordinates.
+        """Start of the valid data region of chunk *chunk_ix* in array coordinates.
 
         Does not validate *chunk_ix* — callers must ensure it is in
         ``[0, nchunks)``. Use ``ChunkGrid.__getitem__`` for safe access.
         """
-        return chunk_ix * self.size
+        return max(0, chunk_ix * self.size - self.shift)
+
+    def buffer_offset(self, chunk_ix: int) -> int:
+        """Position of the valid data region inside the chunk's buffer."""
+        return self.shift if chunk_ix == 0 else 0
 
     def chunk_size(self, chunk_ix: int) -> int:
         """Buffer size for codec processing — always uniform.
@@ -93,7 +101,8 @@ def data_size(self, chunk_ix: int) -> int:
         Does not validate *chunk_ix* — callers must ensure it is in
         ``[0, nchunks)``. Use ``ChunkGrid.__getitem__`` for safe access.
         """
-        return max(0, min(self.size, self.extent - chunk_ix * self.size))
+        start = self.chunk_offset(chunk_ix)
+        return max(0, min((chunk_ix + 1) * self.size - self.shift, self.extent) - start)
 
     @property
     def _unique_edge_lengths(self) -> Iterable[int]:
@@ -106,7 +115,7 @@ def _unique_edge_lengths(self) -> Iterable[int]:
         return (self.size,)
 
     def indices_to_chunks(self, indices: npt.NDArray[np.intp]) -> npt.NDArray[np.intp]:
-        return indices // self.size
+        return (indices + self.shift) // self.size
 
     def with_extent(self, new_extent: int) -> FixedDimension:
         """Re-bind to *new_extent* without modifying edges.
@@ -115,15 +124,15 @@ def with_extent(self, new_extent: int) -> FixedDimension:
         are already correct. Raises on
         ``VaryingDimension`` if edges don't cover the new extent.
         """
-        return FixedDimension(size=self.size, extent=new_extent)
+        return FixedDimension(size=self.size, extent=new_extent, shift=self.shift)
 
     def resize(self, new_extent: int) -> FixedDimension:
         """Adapt for a user-initiated array resize, growing edges if needed.
 
         For ``FixedDimension`` this is identical to ``with_extent`` since
         regular grids don't store explicit edges.
         """
-        return FixedDimension(size=self.size, extent=new_extent)
+        return FixedDimension(size=self.size, extent=new_extent, shift=self.shift)
 
     @property
     def _size_repr(self) -> str:
@@ -205,6 +214,10 @@ def data_size(self, chunk_ix: int) -> int:
         offset = self.cumulative[chunk_ix - 1] if chunk_ix > 0 else 0
         return max(0, min(self.edges[chunk_ix], self.extent - offset))
 
+    def buffer_offset(self, chunk_ix: int) -> int:
+        """Position of the valid data region inside the chunk's buffer."""
+        return 0
+
     @property
     def _unique_edge_lengths(self) -> Iterable[int]:
         """Distinct chunk edge lengths for this dimension (lazily deduplicated).
@@ -271,6 +284,7 @@ def index_to_chunk(self, idx: int) -> int: ...
     def chunk_offset(self, chunk_ix: int) -> int: ...
     def chunk_size(self, chunk_ix: int) -> int: ...
     def data_size(self, chunk_ix: int) -> int: ...
+    def buffer_offset(self, chunk_ix: int) -> int: ...
     def indices_to_chunks(self, indices: npt.NDArray[np.intp]) -> npt.NDArray[np.intp]: ...
     @property
     def _unique_edge_lengths(self) -> Iterable[int]: ...
@@ -416,11 +430,27 @@ def from_metadata(cls, metadata: ArrayMetadata) -> ChunkGrid:
         For v3 metadata, dispatches on the chunk grid type.
         """
         from zarr.core.metadata import ArrayV2Metadata
-        from zarr.core.metadata.v3 import RectilinearChunkGridMetadata, RegularChunkGridMetadata
+        from zarr.core.metadata.v3 import (
+            RectilinearChunkGridMetadata,
+            RegularBoundedChunkGridMetadata,
+            RegularChunkGridMetadata,
+        )
 
         if isinstance(metadata, ArrayV2Metadata):
             return cls.from_sizes(metadata.shape, tuple(metadata.chunks))
         chunk_grid_meta = metadata.chunk_grid
+        if isinstance(chunk_grid_meta, RegularBoundedChunkGridMetadata):
+            return cls(
+                dimensions=tuple(
+                    FixedDimension(size=c, extent=e, shift=s)
+                    for c, e, s in zip(
+                        chunk_grid_meta.chunk_shape,
+                        metadata.shape,
+                        chunk_grid_meta.first_chunk_offset,
+                        strict=True,
+                    )
+                )
+            )
         if isinstance(chunk_grid_meta, RegularChunkGridMetadata):
             return cls.from_sizes(metadata.shape, tuple(chunk_grid_meta.chunk_shape))
         elif isinstance(chunk_grid_meta, RectilinearChunkGridMetadata):

src/zarr/core/chunk_key_encodings.py

@@ -23,6 +23,13 @@ def parse_separator(data: JSON) -> SeparatorLiteral:
     return cast("SeparatorLiteral", parse_field(data, Literal[".", "/"], "separator"))
 
 
+def _parse_key_component(part: str) -> int:
+    # A chunk grid index can be negative; "-0" would name the same chunk as "0".
+    if part.startswith("-0"):
+        raise ValueError(f"Invalid chunk key component: {part!r}")
+    return int(part)
+
+
 class ChunkKeyEncodingParams(TypedDict):
     name: Literal["v2", "default"]
     separator: NotRequired[SeparatorLiteral]
@@ -81,7 +88,7 @@ def decode_chunk_key(self, chunk_key: str) -> tuple[int, ...]:
         # Strip the "c<sep>" prefix (e.g. "c/" or "c.") before splitting.
         prefix = "c" + self.separator
         if chunk_key.startswith(prefix):
-            return tuple(map(int, chunk_key[len(prefix) :].split(self.separator)))
+            return tuple(map(_parse_key_component, chunk_key[len(prefix) :].split(self.separator)))
         raise ValueError(f"Invalid chunk key for default encoding: {chunk_key!r}")
 
     def encode_chunk_key(self, chunk_coords: tuple[int, ...]) -> str:
@@ -98,7 +105,7 @@ def __post_init__(self) -> None:
         object.__setattr__(self, "separator", separator_parsed)
 
     def decode_chunk_key(self, chunk_key: str) -> tuple[int, ...]:
-        return tuple(map(int, chunk_key.split(self.separator)))
+        return tuple(map(_parse_key_component, chunk_key.split(self.separator)))
 
     def encode_chunk_key(self, chunk_coords: tuple[int, ...]) -> str:
         chunk_identifier = self.separator.join(map(str, chunk_coords))