diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index a4141d7356..3f995e8b5e 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -201,6 +201,32 @@ jobs: run: | HATCH_ENV=test.py3.13-minimal just benchmark + examples: + name: Example scripts + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + fetch-depth: 0 # required for hatch version discovery when uv builds zarr + persist-credentials: false + - name: Set up Python + uses: actions/setup-python@6d4fd22b20465ccc0f7354476588e8df7c9ca0e0 # v7.1.0 + with: + python-version: '3.13' + cache: 'pip' + - name: Install uv + uses: astral-sh/setup-uv@07a047e608db84b3d8c06cd9a7c7dcc9c355c18e # v10.3.2 + - name: Install Hatch + run: python -m pip install hatch==1.16.5 + - name: Install just + uses: extractions/setup-crate@7577c1bdf2d95e6d65d532788f35ed79d4b1dda2 # v2.0.1 + with: + repo: casey/just + version: 1.58.0 + - name: Run Examples + run: | + HATCH_ENV=test.py3.13-minimal just test-examples + test-complete: name: Test complete @@ -209,7 +235,8 @@ jobs: test, test-upstream-and-min-deps, doctests, - benchmarks + benchmarks, + examples ] if: always() runs-on: ubuntu-latest diff --git a/Justfile b/Justfile index b3f187e231..df96003eef 100644 --- a/Justfile +++ b/Justfile @@ -64,6 +64,10 @@ hypothesis *args: doctest *args: hatch run doctest:pytest tests/test_docs.py -v "$@" +# Run each example script against the local checkout, in its PEP 723 environment +test-examples *args: + hatch run {{ quote(hatch_env) }}:pytest --run-examples tests/test_examples.py -v "$@" + # Run the benchmark suite benchmark *args: hatch run {{ quote(hatch_env) }}:pytest --benchmark-enable tests/benchmarks "$@" diff --git a/changes/4560.bugfix.md b/changes/4560.bugfix.md new file mode 100644 index 0000000000..3dcf01a342 --- /dev/null +++ b/changes/4560.bugfix.md @@ -0,0 +1,8 @@ +Encoding a chunk through a numcodecs filter no longer passes the +filter's flattened output to the next codec: `_encode_sync` reshapes the +flattened array to the filter's output shape (from +`resolve_metadata`), so downstream codecs (e.g. `TransposeCodec` or an `imagecodecs` serializer chained after a +`Delta` or `FixedScaleOffset` filter) receive a chunk of the expected +shape instead of a flat 1-D array that could make writing fail or +silently corrupt data. Shape-changing codecs such as `PackBits` get +their own resolved output shape. diff --git a/examples/README.md b/examples/README.md index a6b5fa2179..fa664d2c58 100644 --- a/examples/README.md +++ b/examples/README.md @@ -19,7 +19,7 @@ examples/ To add a new example: 1. Create a new subdirectory: `examples/my_example/` -2. Add your Python code: `examples/my_example/my_example.py` +2. Add your Python code: `examples/my_example/my_example.py`. Start it with a [PEP 723](https://peps.python.org/pep-0723/) header that lists its dependencies, including `zarr`. `just test-examples` runs every example against your local checkout, and CI runs it on every pull request. 3. Create documentation: `examples/my_example/README.md` 4. Create a documentation page at `docs/user-guide/examples/my_example.md`. The documentation page should simply link to the `README.md` and the source code, e.g.: diff --git a/examples/codec_pipeline_performance/codec_pipeline_performance.py b/examples/codec_pipeline_performance/codec_pipeline_performance.py index b821f3e6a7..d146c51f4a 100644 --- a/examples/codec_pipeline_performance/codec_pipeline_performance.py +++ b/examples/codec_pipeline_performance/codec_pipeline_performance.py @@ -1,5 +1,5 @@ # /// script -# requires-python = ">=3.11" +# requires-python = ">=3.12" # dependencies = [ # "zarr @ git+https://github.com/zarr-developers/zarr-python.git@main", # "numpy", diff --git a/examples/sharding_coalescing/sharding_coalescing.py b/examples/sharding_coalescing/sharding_coalescing.py index 5da62ed806..66838d2255 100644 --- a/examples/sharding_coalescing/sharding_coalescing.py +++ b/examples/sharding_coalescing/sharding_coalescing.py @@ -1,5 +1,5 @@ # /// script -# requires-python = ">=3.11" +# requires-python = ">=3.12" # dependencies = [ # "zarr @ git+https://github.com/zarr-developers/zarr-python.git@main", # "numpy", diff --git a/packages/zarr-metadata/README.md b/packages/zarr-metadata/README.md index 3c5513d1dd..12583e0c96 100644 --- a/packages/zarr-metadata/README.md +++ b/packages/zarr-metadata/README.md @@ -41,7 +41,7 @@ closely model the content of the Zarr specifications, such as: `TypedDict` with its rules, and the scope that decides which definitions read a document. `zarr_metadata.v3` and `zarr_metadata.v2` export every definition and scope of their format, such as `BYTES_CODEC` and - `CORE_AND_EXTENSIONS`. + `CORE_AND_EXTENSIONS_CONTEXT`. - **JSON type checking** ([`zarr_metadata.typed_json`](https://zarr-metadata.readthedocs.io/en/latest/api/typed_json/)): `check` checks a JSON value against any `TypedDict` and reports every problem with its location. @@ -104,7 +104,9 @@ of context-free consistency rules, e.g.: In a v3 document, validation also checks each extension point: the data type, the chunk grid, the chunk key encoding, each codec, and each storage transformer. Each one is checked by the definition that claims its name in -a scope. The scope is `CORE_AND_EXTENSIONS` unless a `context` is passed. +a scope. The scope is `CORE_AND_EXTENSIONS_CONTEXT` unless a `context` is passed. +`ZARR_PYTHON_CONTEXT` adds the data types that zarr-python writes but no +specification registers. If the definition rejects a configuration, the reader rejects it too, and a configuration key the definition does not declare is reported as `unknown_key`. If no definition in the scope claims a name, the reader @@ -232,13 +234,13 @@ each field: ```python test="true" session="readme" from zarr_metadata.model import read_array_metadata_v3 from zarr_metadata.definition import canonical_of, with_problems -from zarr_metadata.v3.definition import CORE +from zarr_metadata.v3.definition import CORE_CONTEXT reading = read_array_metadata_v3(raw) beyond_core = [ loc for loc, field in reading.fields() - if field.name is not None and CORE.claimant(field.read_as, field.name) is None + if field.name is not None and CORE_CONTEXT.claimant(field.read_as, field.name) is None ] simplest = { loc: canonical_of(field, problems) @@ -283,7 +285,7 @@ scope claims is an `UnclaimedField` too, also when it sits inside another field, such as a codec inside a shard. A model's `unclaimed_fields` lists every field nothing in scope claims, at any depth, by its location, and leaves out the members that waive `must_understand`: a reader can open -the node when it is empty. `CORE` holds +the node when it is empty. `CORE_CONTEXT` holds `CONSOLIDATED_METADATA_EXTRA_FIELD`, which reads a group's `consolidated_metadata` as the envelope the spec fixes. A group's `consolidated()` reads the documents that member lists, in the group's @@ -291,7 +293,7 @@ own scope. A v3 model consists of its document and the scope it was read in. `ZarrV3ArrayMetadata(document, context=None)` reads the document in the -scope (`CORE_AND_EXTENSIONS` when none is given) and raises +scope (`CORE_AND_EXTENSIONS_CONTEXT` when none is given) and raises `MetadataValidationError` with every problem, so an invalid model is never built. `to_json` returns the document as it was written, and `to_key_value` writes that same document to store bytes. @@ -322,7 +324,7 @@ all of them with `ZarrV3ConsolidatedMetadata`, or with `read_consolidated_metadata_v3`, which returns problems instead of raising them. A document with a problem then fails only when it is read. Pass `context=group.context` to read the documents in the group's own -scope; with no `context`, they are read in `CORE_AND_EXTENSIONS`. +scope; with no `context`, they are read in `CORE_AND_EXTENSIONS_CONTEXT`. To write consolidated metadata, build a `ZarrV3ConsolidatedMetadata` and pass it to the group's `update`. The group reads the listed models in its @@ -332,7 +334,7 @@ models as entries. Such an entry is accepted when its claims refine into the model's scope, and refused at its path otherwise. To consolidate children that were read in several scopes, consolidate them in the `union` of those scopes. Every reader takes `context=None` for the -default scope, `CORE_AND_EXTENSIONS`. +default scope, `CORE_AND_EXTENSIONS_CONTEXT`. Two models are equal when they represent the same document, even if each is spelled differently. Values the package interprets (each field, and the diff --git a/packages/zarr-metadata/changes/4575.feature.md b/packages/zarr-metadata/changes/4575.feature.md new file mode 100644 index 0000000000..bd1d87ab40 --- /dev/null +++ b/packages/zarr-metadata/changes/4575.feature.md @@ -0,0 +1 @@ +Added `zarr_metadata.testing.strategies`, Hypothesis strategies that draw valid Zarr v3 metadata fields, behind a new `testing` extra (`pip install zarr-metadata[testing]`). There is one strategy for every codec, data type, chunk grid and chunk key encoding the package defines that holds no other field. `fields` draws every envelope spelling the spec allows, or only the canonical one with `spellings="canonical"`. Codecs can be drawn for the `Chunk` they are handed, chunk grids for an array shape, and fill values for a data type. The strategies are a mapping keyed by definition, so an extension author can add strategies for their own definitions. diff --git a/packages/zarr-metadata/changes/4583.feature.md b/packages/zarr-metadata/changes/4583.feature.md new file mode 100644 index 0000000000..cbbe44f174 --- /dev/null +++ b/packages/zarr-metadata/changes/4583.feature.md @@ -0,0 +1 @@ +Add the `ZARR_PYTHON_CONTEXT` scope, which reads what zarr-python writes: `CORE_AND_EXTENSIONS_CONTEXT` plus zarr-python's own data types `null_terminated_bytes` (NumPy's `S`), `raw_bytes` (NumPy's `V`) and `variable_length_bytes`, which no specification registers. `CORE_AND_EXTENSIONS_CONTEXT` now also reads the `fixed_length_utf32` data type (NumPy's `U`) and the `vlen-utf8` and `vlen-bytes` codecs that zarr-extensions registers, so the strings zarr-python writes are checked instead of read as unclaimed. diff --git a/packages/zarr-metadata/changes/4583.removal.md b/packages/zarr-metadata/changes/4583.removal.md new file mode 100644 index 0000000000..9faa94ee31 --- /dev/null +++ b/packages/zarr-metadata/changes/4583.removal.md @@ -0,0 +1 @@ +Rename the scopes `CORE`, `CORE_AND_EXTENSIONS` and `CORE_V2` to `CORE_CONTEXT`, `CORE_AND_EXTENSIONS_CONTEXT` and `CORE_V2_CONTEXT`, so that a scope's name says it is a context. The old names, imported from `zarr_metadata.v3`, `zarr_metadata.v3.definition`, `zarr_metadata.v2` or `zarr_metadata.v2.definition`, still refer to the same scopes with a `DeprecationWarning`, and will be removed in a later release. diff --git a/packages/zarr-metadata/changes/4588.feature.md b/packages/zarr-metadata/changes/4588.feature.md new file mode 100644 index 0000000000..e066923667 --- /dev/null +++ b/packages/zarr-metadata/changes/4588.feature.md @@ -0,0 +1 @@ +`zarr_metadata.v3.data_type`, `zarr_metadata.v3.codec`, `zarr_metadata.v3.chunk_grid` and `zarr_metadata.v3.chunk_key_encoding` re-export every name their submodules export, so `from zarr_metadata.v3.data_type import BoolDataTypeName, Uint8DataTypeName` works again, as it did before 0.6.0. Each name is the same object as in its submodule, which stays its documented home. diff --git a/packages/zarr-metadata/docs/api/index.md b/packages/zarr-metadata/docs/api/index.md index 4a23c5c2fb..e95a186284 100644 --- a/packages/zarr-metadata/docs/api/index.md +++ b/packages/zarr-metadata/docs/api/index.md @@ -22,7 +22,7 @@ The package is organized to mirror the structure of the Zarr specifications: - [`zarr_metadata.v2`](v2.md) — `TypedDict` shapes for Zarr v2 documents (`.zarray`, `.zgroup`, `.zattrs`, `.zmetadata`), and `zarr_metadata.v2.definition`: the v2 kinds of metadata, the scope - `CORE_V2`, and the readers of one `dtype` or codec. Each dtype family's + `CORE_V2_CONTEXT`, and the readers of one `dtype` or codec. Each dtype family's definition is in `zarr_metadata.v2.data_type`, and each codec's in `zarr_metadata.v2.codec` - [`zarr_metadata.v3`](v3/index.md) — `TypedDict` shapes for Zarr v3 @@ -35,7 +35,7 @@ The package is organized to mirror the structure of the Zarr specifications: scope type `Context`. Its module docstring is the guide - [`zarr_metadata.v3.definition`](v3/definition.md) — the five v3 kinds of metadata (`CodecDefinition`, `DataTypeDefinition`, ...), the codec - pipeline types, and the v3 scopes `CORE` and `CORE_AND_EXTENSIONS` + pipeline types, and the v3 scopes `CORE_CONTEXT` and `CORE_AND_EXTENSIONS_CONTEXT` The document types, models, and spec vocabulary — including the store keys — are re-exported at the top level, so diff --git a/packages/zarr-metadata/docs/api/testing.md b/packages/zarr-metadata/docs/api/testing.md new file mode 100644 index 0000000000..a5d6145258 --- /dev/null +++ b/packages/zarr-metadata/docs/api/testing.md @@ -0,0 +1,8 @@ +--- +title: testing.strategies +--- + +::: zarr_metadata.testing.strategies + options: + inherited_members: false + show_if_no_docstring: false diff --git a/packages/zarr-metadata/docs/api/v3/codec.md b/packages/zarr-metadata/docs/api/v3/codec.md index cb96d2c7d5..ae5171acd0 100644 --- a/packages/zarr-metadata/docs/api/v3/codec.md +++ b/packages/zarr-metadata/docs/api/v3/codec.md @@ -22,4 +22,8 @@ title: codec ::: zarr_metadata.v3.codec.transpose +::: zarr_metadata.v3.codec.vlen_bytes + +::: zarr_metadata.v3.codec.vlen_utf8 + ::: zarr_metadata.v3.codec.zstd diff --git a/packages/zarr-metadata/docs/api/v3/data_type.md b/packages/zarr-metadata/docs/api/v3/data_type.md index 3f8444625a..2bdfe0ad2c 100644 --- a/packages/zarr-metadata/docs/api/v3/data_type.md +++ b/packages/zarr-metadata/docs/api/v3/data_type.md @@ -40,8 +40,16 @@ title: data_type ::: zarr_metadata.v3.data_type.string +::: zarr_metadata.v3.data_type.fixed_length_utf32 + ::: zarr_metadata.v3.data_type.numpy_datetime64 ::: zarr_metadata.v3.data_type.numpy_timedelta64 ::: zarr_metadata.v3.data_type.struct + +::: zarr_metadata.v3.data_type.null_terminated_bytes + +::: zarr_metadata.v3.data_type.variable_length_bytes + +::: zarr_metadata.v3.data_type.zarr_python_raw_bytes diff --git a/packages/zarr-metadata/docs/index.md b/packages/zarr-metadata/docs/index.md index b3936156d0..6b238b7017 100644 --- a/packages/zarr-metadata/docs/index.md +++ b/packages/zarr-metadata/docs/index.md @@ -39,7 +39,7 @@ closely model the content of the Zarr specifications, such as: `TypedDict` with its rules, and the scope that decides which definitions read a document. `zarr_metadata.v3` and `zarr_metadata.v2` export every definition and scope of their format, such as `BYTES_CODEC` and - `CORE_AND_EXTENSIONS`. + `CORE_AND_EXTENSIONS_CONTEXT`. - **JSON type checking** ([`zarr_metadata.typed_json`](api/typed_json.md)): `check` checks a JSON value against any `TypedDict` and reports every problem with its location. @@ -102,7 +102,9 @@ of context-free consistency rules, e.g.: In a v3 document, validation also checks each extension point: the data type, the chunk grid, the chunk key encoding, each codec, and each storage transformer. Each one is checked by the definition that claims its name in -a scope. The scope is `CORE_AND_EXTENSIONS` unless a `context` is passed. +a scope. The scope is `CORE_AND_EXTENSIONS_CONTEXT` unless a `context` is passed. +`ZARR_PYTHON_CONTEXT` adds the data types that zarr-python writes but no +specification registers. If the definition rejects a configuration, the reader rejects it too, and a configuration key the definition does not declare is reported as `unknown_key`. If no definition in the scope claims a name, the reader @@ -234,13 +236,13 @@ each field: ```python test="true" session="index" from zarr_metadata.model import read_array_metadata_v3 from zarr_metadata.definition import canonical_of, with_problems -from zarr_metadata.v3.definition import CORE +from zarr_metadata.v3.definition import CORE_CONTEXT reading = read_array_metadata_v3(raw) beyond_core = [ loc for loc, field in reading.fields() - if field.name is not None and CORE.claimant(field.read_as, field.name) is None + if field.name is not None and CORE_CONTEXT.claimant(field.read_as, field.name) is None ] simplest = { loc: canonical_of(field, problems) @@ -285,7 +287,7 @@ scope claims is an `UnclaimedField` too, also when it sits inside another field, such as a codec inside a shard. A model's `unclaimed_fields` lists every field nothing in scope claims, at any depth, by its location, and leaves out the members that waive `must_understand`: a reader can open -the node when it is empty. `CORE` holds +the node when it is empty. `CORE_CONTEXT` holds `CONSOLIDATED_METADATA_EXTRA_FIELD`, which reads a group's `consolidated_metadata` as the envelope the spec fixes. A group's `consolidated()` reads the documents that member lists, in the group's @@ -293,7 +295,7 @@ own scope. A v3 model consists of its document and the scope it was read in. `ZarrV3ArrayMetadata(document, context=None)` reads the document in the -scope (`CORE_AND_EXTENSIONS` when none is given) and raises +scope (`CORE_AND_EXTENSIONS_CONTEXT` when none is given) and raises `MetadataValidationError` with every problem, so an invalid model is never built. `to_json` returns the document as it was written, and `to_key_value` writes that same document to store bytes. @@ -324,7 +326,7 @@ all of them with `ZarrV3ConsolidatedMetadata`, or with `read_consolidated_metadata_v3`, which returns problems instead of raising them. A document with a problem then fails only when it is read. Pass `context=group.context` to read the documents in the group's own -scope; with no `context`, they are read in `CORE_AND_EXTENSIONS`. +scope; with no `context`, they are read in `CORE_AND_EXTENSIONS_CONTEXT`. To write consolidated metadata, build a `ZarrV3ConsolidatedMetadata` and pass it to the group's `update`. The group reads the listed models in its @@ -334,7 +336,7 @@ models as entries. Such an entry is accepted when its claims refine into the model's scope, and refused at its path otherwise. To consolidate children that were read in several scopes, consolidate them in the `union` of those scopes. Every reader takes `context=None` for the -default scope, `CORE_AND_EXTENSIONS`. +default scope, `CORE_AND_EXTENSIONS_CONTEXT`. Two models are equal when they represent the same document, even if each is spelled differently. Values the package interprets (each field, and the diff --git a/packages/zarr-metadata/docs/parsing-validation.md b/packages/zarr-metadata/docs/parsing-validation.md index adfe8e51c2..a4122d87ea 100644 --- a/packages/zarr-metadata/docs/parsing-validation.md +++ b/packages/zarr-metadata/docs/parsing-validation.md @@ -161,7 +161,7 @@ This procedure yields one of two values: `zarr-metadata` does not run each stage as a separate function. Instead, a single read of the document collects the problems from every stage, so a caller gets the full list of problems at once. -The **registry** from the resolve-and-validate stage is called a **scope** in this library. For Zarr V3 it is a [`ZarrV3Context`](api/v3/definition.md) object, with one slot for each kind of field (`data_types`, `codecs`, `chunk_grids`, ...), and every reader takes a `context` argument. The default scope is `CORE_AND_EXTENSIONS`, which holds the data types, codecs, chunk grids, etc. defined in the Zarr V3 spec and in [zarr-extensions](https://github.com/zarr-developers/zarr-extensions/). `CORE` holds only the definitions from the Zarr V3 spec. +The **registry** from the resolve-and-validate stage is called a **scope** in this library. For Zarr V3 it is a [`ZarrV3Context`](api/v3/definition.md) object, with one slot for each kind of field (`data_types`, `codecs`, `chunk_grids`, ...), and every reader takes a `context` argument. The default scope is `CORE_AND_EXTENSIONS_CONTEXT`, which holds the data types, codecs, chunk grids, etc. defined in the Zarr V3 spec and in [zarr-extensions](https://github.com/zarr-developers/zarr-extensions/). `CORE_CONTEXT` holds only the definitions from the Zarr V3 spec. `ZARR_PYTHON_CONTEXT` adds to `CORE_AND_EXTENSIONS_CONTEXT` the data types that zarr-python writes but no specification registers (`null_terminated_bytes` for NumPy's `S`, `raw_bytes` for NumPy's `V`, and `variable_length_bytes`), so a strict check of what zarr-python writes passes it. `read_array_metadata_v3` reads a JSON document and returns a "reading". `reading.problems` is a tuple of the problems, and `reading.metadata` is the validated model, or `None` if there are problems. Each problem has a `loc`, which is its position in the document as a tuple, a `kind`, like `missing_key` or `invalid_value`, and a human-readable `message`: @@ -193,10 +193,10 @@ To read one field on its own, such as a codec, use `resolve` from `zarr_metadata ```python test="true" session="parsing-validation" from zarr_metadata.definition import resolve -from zarr_metadata.v3.definition import CORE_AND_EXTENSIONS, CodecDefinition +from zarr_metadata.v3.definition import CORE_AND_EXTENSIONS_CONTEXT, CodecDefinition field, problems = resolve( - {"name": "gzip", "configuration": {"level": 12}}, CodecDefinition, CORE_AND_EXTENSIONS + {"name": "gzip", "configuration": {"level": 12}}, CodecDefinition, CORE_AND_EXTENSIONS_CONTEXT ) for problem in problems: print(problem.loc, problem.kind) @@ -214,17 +214,17 @@ To make an unknown data type or codec known, add its definition to the scope's s ```python test="true" session="parsing-validation" from zarr_metadata.definition import EmptyConfiguration -from zarr_metadata.v3.definition import CORE_AND_EXTENSIONS, CodecDefinition +from zarr_metadata.v3.definition import CORE_AND_EXTENSIONS_CONTEXT, CodecDefinition # The definition of the codec that was unknown: here, a codec named # `acme.codec` that takes no configuration. MY_CODEC = CodecDefinition( name="acme.codec", configuration=EmptyConfiguration, kind="bytes_bytes", size="dynamic" ) -context = CORE_AND_EXTENSIONS.update(codecs=[*CORE_AND_EXTENSIONS.codecs.values(), MY_CODEC]) +context = CORE_AND_EXTENSIONS_CONTEXT.update(codecs=[*CORE_AND_EXTENSIONS_CONTEXT.codecs.values(), MY_CODEC]) reading = read_array_metadata_v3(raw, context=context) ``` -`update` replaces each slot it is given and keeps the others. A slot can also be given as a mapping from each definition's name to it, so `CORE_AND_EXTENSIONS.update(codecs={**CORE_AND_EXTENSIONS.codecs, "gzip": MY_GZIP})` replaces the gzip definition with your own. +`update` replaces each slot it is given and keeps the others. A slot can also be given as a mapping from each definition's name to it, so `CORE_AND_EXTENSIONS_CONTEXT.update(codecs={**CORE_AND_EXTENSIONS_CONTEXT.codecs, "gzip": MY_GZIP})` replaces the gzip definition with your own. [^1]: The one exception is the `r*` data type, which is identified by a templated string with no `configuration` object. This mixes the identifier with the configuration in the same JSON value, which is not good design. So this library normalizes `r*` into the same form as all the other metadata objects: the field `"r16"` keeps its name, and is read by the `r*` definition with the configuration `{"bits": 16}`. diff --git a/packages/zarr-metadata/mkdocs.yml b/packages/zarr-metadata/mkdocs.yml index 10464e3cf0..b19c0932d2 100644 --- a/packages/zarr-metadata/mkdocs.yml +++ b/packages/zarr-metadata/mkdocs.yml @@ -28,6 +28,7 @@ nav: - ' zarr_metadata.model': api/model.md - ' zarr_metadata.problems': api/problems.md - ' zarr_metadata.pydantic': api/pydantic.md + - ' zarr_metadata.testing.strategies': api/testing.md - ' zarr_metadata.typed_json': api/typed_json.md - ' zarr_metadata.v2': api/v2.md - ' zarr_metadata.v3': diff --git a/packages/zarr-metadata/pyproject.toml b/packages/zarr-metadata/pyproject.toml index abd93bc729..25d1c7aead 100644 --- a/packages/zarr-metadata/pyproject.toml +++ b/packages/zarr-metadata/pyproject.toml @@ -43,6 +43,11 @@ dependencies = [ "typing_extensions>=4.16", ] +[project.optional-dependencies] +# `zarr_metadata.testing.strategies`. 6.85: first release with +# `characters(exclude_categories=...)`. +testing = ["hypothesis>=6.85"] + [project.urls] Homepage = "https://github.com/zarr-developers/zarr-python" Source = "https://github.com/zarr-developers/zarr-python/tree/main/packages/zarr-metadata" diff --git a/packages/zarr-metadata/src/zarr_metadata/__init__.py b/packages/zarr-metadata/src/zarr_metadata/__init__.py index bc57d90fd0..5e8b1ffcba 100644 --- a/packages/zarr-metadata/src/zarr_metadata/__init__.py +++ b/packages/zarr-metadata/src/zarr_metadata/__init__.py @@ -136,6 +136,16 @@ TransposeCodecMetadata, TransposeCodecName, ) +from zarr_metadata.v3.codec.vlen_bytes import ( + VLEN_BYTES_CODEC_NAME, + VlenBytesCodecMetadata, + VlenBytesCodecName, +) +from zarr_metadata.v3.codec.vlen_utf8 import ( + VLEN_UTF8_CODEC_NAME, + VlenUtf8CodecMetadata, + VlenUtf8CodecName, +) from zarr_metadata.v3.codec.zstd import ZSTD_CODEC_NAME, ZstdCodecMetadata, ZstdCodecName from zarr_metadata.v3.consolidated import ( ZARR_V3_CONSOLIDATED_METADATA_KEY, @@ -162,6 +172,11 @@ Complex128DataTypeName, Complex128FillValue, ) +from zarr_metadata.v3.data_type.fixed_length_utf32 import ( + FIXED_LENGTH_UTF32_DATA_TYPE_NAME, + FixedLengthUtf32DataTypeName, + FixedLengthUtf32FillValue, +) from zarr_metadata.v3.data_type.float16 import ( FLOAT16_DATA_TYPE_NAME, Float16DataTypeName, @@ -265,6 +280,7 @@ "DEFAULT_CHUNK_KEY_ENCODING_NAME", "DEFAULT_CHUNK_KEY_ENCODING_SEPARATOR", "ENDIANNESS", + "FIXED_LENGTH_UTF32_DATA_TYPE_NAME", "FLOAT16_DATA_TYPE_NAME", "FLOAT32_DATA_TYPE_NAME", "FLOAT64_DATA_TYPE_NAME", @@ -291,6 +307,8 @@ "UNSET", "V2_CHUNK_KEY_ENCODING_NAME", "V2_CHUNK_KEY_ENCODING_SEPARATOR", + "VLEN_BYTES_CODEC_NAME", + "VLEN_UTF8_CODEC_NAME", "ZARR_V2_ARRAY_DIMENSION_SEPARATOR", "ZARR_V2_ARRAY_METADATA_STORE_KEY", "ZARR_V2_ARRAY_ORDER", @@ -325,6 +343,8 @@ "DefaultChunkKeyEncodingName", "DefaultChunkKeyEncodingSeparator", "Endianness", + "FixedLengthUtf32DataTypeName", + "FixedLengthUtf32FillValue", "Float16DataTypeName", "Float16FillValue", "Float32DataTypeName", @@ -378,6 +398,10 @@ "V2ChunkKeyEncodingName", "V2ChunkKeyEncodingSeparator", "ValidationProblem", + "VlenBytesCodecMetadata", + "VlenBytesCodecName", + "VlenUtf8CodecMetadata", + "VlenUtf8CodecName", "ZarrV2ArrayDimensionSeparator", "ZarrV2ArrayMetadata", "ZarrV2ArrayMetadataJSON", diff --git a/packages/zarr-metadata/src/zarr_metadata/_renamed.py b/packages/zarr-metadata/src/zarr_metadata/_renamed.py new file mode 100644 index 0000000000..d0267a70db --- /dev/null +++ b/packages/zarr-metadata/src/zarr_metadata/_renamed.py @@ -0,0 +1,26 @@ +"""Scopes this package renamed, which keep their old names for now with a `DeprecationWarning`.""" + +import warnings +from collections.abc import Mapping + + +def renamed_scope(module: str, name: str, renames: Mapping[str, tuple[str, object]]) -> object: + """The scope that `name`, an old name in `renames`, refers to, with a `DeprecationWarning`. + + `renames` maps each old name to the scope's name now and the scope. A + module that exported the old names calls this from its `__getattr__`, + so an import of an old name keeps working. Raises `AttributeError`, as + attribute lookup does, when `name` is not an old name. + """ + if name not in renames: + raise AttributeError(f"module {module!r} has no attribute {name!r}") + new, scope = renames[name] + warnings.warn( + f"`{name}` is deprecated and will be removed; use `{new}`, the same scope", + DeprecationWarning, + stacklevel=3, + ) + return scope + + +__all__ = ["renamed_scope"] diff --git a/packages/zarr-metadata/src/zarr_metadata/definition/__init__.py b/packages/zarr-metadata/src/zarr_metadata/definition/__init__.py index 6dd9c86173..782807f4a8 100644 --- a/packages/zarr-metadata/src/zarr_metadata/definition/__init__.py +++ b/packages/zarr-metadata/src/zarr_metadata/definition/__init__.py @@ -40,7 +40,7 @@ configuration, checks the envelope of each nested field (an extra member, a `must_understand` of `false`), and then runs the rules, for one configuration: `GZIP_CODEC.read_configuration({"level": 12})`. -3. `resolve(field, CodecDefinition, CORE_AND_EXTENSIONS)` reads a whole +3. `resolve(field, CodecDefinition, CORE_AND_EXTENSIONS_CONTEXT)` reads a whole field in a scope. It checks the envelope, maps the name to a definition, checks the configuration, and reads each nested field the same way. It returns the result of the reading and every problem. The @@ -70,17 +70,17 @@ from zarr_metadata.definition import resolve from zarr_metadata.v3.codec.gzip import GZIP_CODEC - from zarr_metadata.v3.definition import CORE_AND_EXTENSIONS, CodecDefinition + from zarr_metadata.v3.definition import CORE_AND_EXTENSIONS_CONTEXT, CodecDefinition resolved, problems = resolve({"name": "gzip", "configuration": {"level": 12}}, - CodecDefinition, CORE_AND_EXTENSIONS) + CodecDefinition, CORE_AND_EXTENSIONS_CONTEXT) resolved # RefusedField(..., definition=CodecDefinition(name='gzip'), ...) problems[0].loc # ('configuration', 'level') problems[0].input # 12 dict(problems[0].ctx) # {'ge': 0, 'le': 9} resolved, problems = resolve({"name": "gzip", "configuration": {"level": 5}}, - CodecDefinition, CORE_AND_EXTENSIONS) + CodecDefinition, CORE_AND_EXTENSIONS_CONTEXT) resolved.definition is GZIP_CODEC # True resolved.configuration # {'level': 5} @@ -124,7 +124,7 @@ from zarr_metadata import ValidationProblem from zarr_metadata.definition import Nested - from zarr_metadata.v3.definition import CORE_AND_EXTENSIONS, CodecDefinition + from zarr_metadata.v3.definition import CORE_AND_EXTENSIONS_CONTEXT, CodecDefinition class AcmeLz4Configuration(TypedDict, closed=True): @@ -149,7 +149,7 @@ def acme_lz4_rules( size="dynamic", rules=acme_lz4_rules, ) - CONTEXT = CORE_AND_EXTENSIONS.update(codecs=[*CORE_AND_EXTENSIONS.codecs.values(), ACME_LZ4]) + CONTEXT = CORE_AND_EXTENSIONS_CONTEXT.update(codecs=[*CORE_AND_EXTENSIONS_CONTEXT.codecs.values(), ACME_LZ4]) The TypedDict declares how keys it does not declare are handled. With `closed=True`, such a key is a problem, as above. With `extra_items=`, diff --git a/packages/zarr-metadata/src/zarr_metadata/definition/_definition.py b/packages/zarr-metadata/src/zarr_metadata/definition/_definition.py index 5d50f8e605..0c30ffd16b 100644 --- a/packages/zarr-metadata/src/zarr_metadata/definition/_definition.py +++ b/packages/zarr-metadata/src/zarr_metadata/definition/_definition.py @@ -1374,7 +1374,7 @@ def resolve( else raises `TypeError`. """ if not _is_scope(context): - msg = f"a field is read in a scope, such as CORE_AND_EXTENSIONS, got {described(context)}" + msg = f"a field is read in a scope, such as CORE_AND_EXTENSIONS_CONTEXT, got {described(context)}" raise TypeError(msg) asked = as_kind(kind) _check_format(asked, context) diff --git a/packages/zarr-metadata/src/zarr_metadata/json_schema.py b/packages/zarr-metadata/src/zarr_metadata/json_schema.py index 9b7443a8f4..ea80ac0242 100644 --- a/packages/zarr-metadata/src/zarr_metadata/json_schema.py +++ b/packages/zarr-metadata/src/zarr_metadata/json_schema.py @@ -30,7 +30,7 @@ **A whole `zarr.json`.** `node_metadata_json_schema_v3(context=None)` writes a whole v3 `zarr.json`, array or group, as the scope reads it -(`CORE_AND_EXTENSIONS` when none is given). Its fields are written +(`CORE_AND_EXTENSIONS_CONTEXT` when none is given). Its fields are written as the scope reads them. For each definition, the schema accepts its field (its name, its configuration as its TypedDict declares it, including bounds, a `must_understand` of `true`, and its bare name when diff --git a/packages/zarr-metadata/src/zarr_metadata/model/__init__.py b/packages/zarr-metadata/src/zarr_metadata/model/__init__.py index 98776c7c5d..ab6006ca2a 100644 --- a/packages/zarr-metadata/src/zarr_metadata/model/__init__.py +++ b/packages/zarr-metadata/src/zarr_metadata/model/__init__.py @@ -9,7 +9,7 @@ `read_consolidated_metadata_v2`. A reader checks a document's JSON structure. In a v3 document it also reads each extension point (codecs, chunk grids, data types, ...) through the definition that claims its -name in a scope. The scope is `CORE_AND_EXTENSIONS` unless a `context` +name in a scope. The scope is `CORE_AND_EXTENSIONS_CONTEXT` unless a `context` is passed. A v3 reader checks the fill value against the data type the document names, the chunk grid against the shape, and the codecs as a pipeline, each codec against the chunk it receives. Each reader does one diff --git a/packages/zarr-metadata/src/zarr_metadata/model/_array_v2.py b/packages/zarr-metadata/src/zarr_metadata/model/_array_v2.py index 7768e68210..a4336ea0a5 100644 --- a/packages/zarr-metadata/src/zarr_metadata/model/_array_v2.py +++ b/packages/zarr-metadata/src/zarr_metadata/model/_array_v2.py @@ -70,7 +70,7 @@ from zarr_metadata.v2.attributes import ZARR_V2_ATTRIBUTES_STORE_KEY from zarr_metadata.v2.codec import ZarrV2CodecMetadata from zarr_metadata.v2.definition import ( - CORE_V2, + CORE_V2_CONTEXT, ZarrV2CodecDefinition, ZarrV2DataTypeDefinition, ) @@ -437,7 +437,7 @@ def in_files( def read_array_metadata_v2( value: object, *, context: ZarrV2Context | None = None ) -> ZarrV2ArrayMetadataReading: - """Read `value`, a v2 array document, in `context` (`CORE_V2` when none is given), whatever it holds. + """Read `value`, a v2 array document, in `context` (`CORE_V2_CONTEXT` when none is given), whatever it holds. The result holds everything the read finds: the dtype, the compressor and each filter as the scope read them (an `AcceptedField` read by @@ -447,7 +447,7 @@ def read_array_metadata_v2( problem. One that the scope does not claim is not a problem. `fill_value` is checked against the dtype the scope read. """ - context = scoped(context, CORE_V2) + context = scoped(context, CORE_V2_CONTEXT) reading, members = read_array_v2(value, context) if members is None: return reading @@ -510,7 +510,7 @@ class ZarrV2ArrayMetadata(Keyed): node_type: Final[Literal["array"]] = "array" # noqa: PYI064 - as above def __init__(self, document: object, context: ZarrV2Context | None = None) -> None: - context = scoped(context, CORE_V2) + context = scoped(context, CORE_V2_CONTEXT) reading, members = read_array_v2(document, context) if members is None: raise MetadataValidationError(reading.problems) @@ -731,7 +731,7 @@ def with_context(self, context: ZarrV2Context | None = None) -> ZarrV2ArrayMetad in `context`. The existing reading is reused when `context` reads every claim identically. """ - context = scoped(context, CORE_V2) + context = scoped(context, CORE_V2_CONTEXT) if disagreements(context, claims_of(self._reading.fields())).agrees: return self._of(self._document, context, self._reading, self._members) return type(self)(self._document, context=context) @@ -745,7 +745,7 @@ def refined_in(self, context: ZarrV2Context | None = None) -> ZarrV2ArrayMetadat in the document. Raises `MetadataValidationError` when a definition that `context` claims rejects what was written. """ - context = scoped(context, CORE_V2) + context = scoped(context, CORE_V2_CONTEXT) found = disagreements(context, claims_of(self._reading.fields())) if len(found.conflicts) != 0: raise ScopeConflictError(located_conflicts(self._reading.fields(), found.conflicts)) @@ -807,7 +807,9 @@ def create_default( if lengths is not None: given["chunks"] = lengths if "dtype" in given and "fill_value" not in given: - dtype, _ = resolve(given["dtype"], ZarrV2DataTypeDefinition, scoped(context, CORE_V2)) + dtype, _ = resolve( + given["dtype"], ZarrV2DataTypeDefinition, scoped(context, CORE_V2_CONTEXT) + ) if len(fill_value_problems(dtype, 0)) != 0: given["fill_value"] = None merged = {key: value for key, value in {**document, **given}.items() if value is not UNSET} @@ -836,7 +838,7 @@ def from_key_value( valid, with every problem the read finds. Raises `TypeError` for a scope of another format before any bytes are decoded. """ - context = scoped(context, CORE_V2) + context = scoped(context, CORE_V2_CONTEXT) files, problems = decoded_node_files_v2( mapping, (ZARR_V2_ARRAY_METADATA_STORE_KEY, ZARR_V2_ATTRIBUTES_STORE_KEY) ) diff --git a/packages/zarr-metadata/src/zarr_metadata/model/_array_v3.py b/packages/zarr-metadata/src/zarr_metadata/model/_array_v3.py index 53d12254bc..58fb7e30c5 100644 --- a/packages/zarr-metadata/src/zarr_metadata/model/_array_v3.py +++ b/packages/zarr-metadata/src/zarr_metadata/model/_array_v3.py @@ -62,7 +62,7 @@ # A runtime import: `get_type_hints` resolves the public signatures. from zarr_metadata.v3._context import ZarrV3Context # noqa: TC001 -from zarr_metadata.v3._core import CORE_AND_EXTENSIONS +from zarr_metadata.v3._core import CORE_AND_EXTENSIONS_CONTEXT from zarr_metadata.v3._definition import ( ArrayExtraFieldDefinition, Chunk, @@ -499,7 +499,7 @@ class ZarrV3ArrayMetadata(Keyed): node_type: Final[Literal["array"]] = "array" # noqa: PYI064 - as above def __init__(self, document: object, context: ZarrV3Context | None = None) -> None: - context = scoped(context, CORE_AND_EXTENSIONS) + context = scoped(context, CORE_AND_EXTENSIONS_CONTEXT) reading, members = read_array_v3(document, context) if members is None: raise MetadataValidationError(reading.problems) @@ -696,7 +696,7 @@ def with_context(self, context: ZarrV3Context | None = None) -> ZarrV3ArrayMetad in `context`. The existing reading is reused when `context` reads every claim identically. """ - context = scoped(context, CORE_AND_EXTENSIONS) + context = scoped(context, CORE_AND_EXTENSIONS_CONTEXT) if disagreements(context, claims_of(self._reading.fields())).agrees: return self._of(self._document, context, self._reading, self._members) return type(self)(self._document, context=context) @@ -713,7 +713,7 @@ def refined_in(self, context: ZarrV3Context | None = None) -> ZarrV3ArrayMetadat gain can expose a problem. `with_context` reads the document in any scope. """ - context = scoped(context, CORE_AND_EXTENSIONS) + context = scoped(context, CORE_AND_EXTENSIONS_CONTEXT) found = disagreements(context, claims_of(self._reading.fields())) if len(found.conflicts) != 0: raise ScopeConflictError(located_conflicts(self._reading.fields(), found.conflicts)) @@ -811,7 +811,7 @@ def from_key_value( are not JSON, or the document is not valid. Raises `TypeError` for a scope of another format before any bytes are decoded. """ - context = scoped(context, CORE_AND_EXTENSIONS) + context = scoped(context, CORE_AND_EXTENSIONS_CONTEXT) return cls(load_store_json(mapping, ZARR_V3_ARRAY_METADATA_STORE_KEY), context=context) @@ -888,7 +888,7 @@ def read_array_metadata_v3( spec's fields), iterate over the reading's `fields()`. A value that is not an object has no fields. """ - context = scoped(context, CORE_AND_EXTENSIONS) + context = scoped(context, CORE_AND_EXTENSIONS_CONTEXT) reading, members = read_array_v3(value, context) if members is None: return reading diff --git a/packages/zarr-metadata/src/zarr_metadata/model/_group_v2.py b/packages/zarr-metadata/src/zarr_metadata/model/_group_v2.py index eb290a6cb7..1a0cdc5e48 100644 --- a/packages/zarr-metadata/src/zarr_metadata/model/_group_v2.py +++ b/packages/zarr-metadata/src/zarr_metadata/model/_group_v2.py @@ -63,7 +63,7 @@ from zarr_metadata.v2.array import ZARR_V2_ARRAY_METADATA_STORE_KEY from zarr_metadata.v2.attributes import ZARR_V2_ATTRIBUTES_STORE_KEY from zarr_metadata.v2.consolidated import ZARR_V2_CONSOLIDATED_METADATA_STORE_KEY -from zarr_metadata.v2.definition import CORE_V2 +from zarr_metadata.v2.definition import CORE_V2_CONTEXT from zarr_metadata.v2.group import ZARR_V2_GROUP_METADATA_STORE_KEY, ZarrV2GroupMetadataJSON if TYPE_CHECKING: @@ -136,8 +136,8 @@ def __reduce__(self) -> str | tuple[object, ...]: def read_group_metadata_v2( value: object, *, context: ZarrV2Context | None = None ) -> ZarrV2GroupMetadataReading: - """Read `value`, a v2 group document in the merged form (`.zgroup` plus `attributes` from `.zattrs`), in `context` (`CORE_V2` when none is given), whatever it holds: every problem, and the model when there are none. A group has no fields for a scope to read; `context` is taken for consistency with every other v2 reader.""" - context = scoped(context, CORE_V2) + """Read `value`, a v2 group document in the merged form (`.zgroup` plus `attributes` from `.zattrs`), in `context` (`CORE_V2_CONTEXT` when none is given), whatever it holds: every problem, and the model when there are none. A group has no fields for a scope to read; `context` is taken for consistency with every other v2 reader.""" + context = scoped(context, CORE_V2_CONTEXT) problems = _group_problems_v2(value) if len(problems) != 0: return ZarrV2GroupMetadataReading(problems=problems) @@ -173,7 +173,7 @@ def fields(self) -> Iterator[tuple[Loc, ResolvedField[Any]]]: def read_node_metadata_v2( files: Mapping[str, object], *, context: ZarrV2Context | None = None ) -> ZarrV2NodeMetadataReading: - """Read the v2 node described by `files`, in `context` (`CORE_V2` when none is given), as the node type its files declare. + """Read the v2 node described by `files`, in `context` (`CORE_V2_CONTEXT` when none is given), as the node type its files declare. `files` maps each of the node's files to its decoded JSON: `.zarray` or `.zgroup`, and `.zattrs` when there is one. They can come from a @@ -194,7 +194,7 @@ def read_node_metadata_v2( v2 counterpart of `read_node_metadata_v3`, which reads one `zarr.json` and its `node_type`. """ - context = scoped(context, CORE_V2) + context = scoped(context, CORE_V2_CONTEXT) is_array = ZARR_V2_ARRAY_METADATA_STORE_KEY in files is_group = ZARR_V2_GROUP_METADATA_STORE_KEY in files if is_array and is_group: @@ -234,7 +234,7 @@ def node_metadata_from_key_value_v2( mapping: Mapping[StoreKey, bytes], *, context: ZarrV2Context | None = None ) -> ZarrV2NodeMetadata: """Return the model of the v2 node whose files are in `mapping`, a store's keys and bytes: each file is decoded, then read as `read_node_metadata_v2` reads it. Raises `MetadataValidationError` with every problem, including a file whose bytes are not JSON, and `TypeError` for a scope of another format before any bytes are decoded.""" - context = scoped(context, CORE_V2) + context = scoped(context, CORE_V2_CONTEXT) files, problems = decoded_node_files_v2(mapping) if len(problems) != 0: raise MetadataValidationError(problems) @@ -278,7 +278,7 @@ class ZarrV2GroupMetadata(Keyed): node_type: Final[Literal["group"]] = "group" # noqa: PYI064 - as above def __init__(self, document: object, context: ZarrV2Context | None = None) -> None: - context = scoped(context, CORE_V2) + context = scoped(context, CORE_V2_CONTEXT) problems = _group_problems_v2(document) if len(problems) != 0: raise MetadataValidationError(problems) @@ -368,7 +368,7 @@ def update(self, **members: Unpack[ZarrV2GroupMetadataUpdate]) -> ZarrV2GroupMet def with_context(self, context: ZarrV2Context | None = None) -> ZarrV2GroupMetadata: """Return this document read in `context`: the same group, holding that scope.""" - context = scoped(context, CORE_V2) + context = scoped(context, CORE_V2_CONTEXT) return self._of(self._document, context) def refined_in(self, context: ZarrV2Context | None = None) -> ZarrV2GroupMetadata: @@ -406,7 +406,7 @@ def from_key_value( valid, with every problem the read finds. Raises `TypeError` for a scope of another format before any bytes are decoded. """ - context = scoped(context, CORE_V2) + context = scoped(context, CORE_V2_CONTEXT) files, problems = decoded_node_files_v2( mapping, (ZARR_V2_GROUP_METADATA_STORE_KEY, ZARR_V2_ATTRIBUTES_STORE_KEY) ) @@ -449,8 +449,8 @@ def __reduce__(self) -> str | tuple[object, ...]: def read_consolidated_metadata_v2( value: object, *, context: ZarrV2Context | None = None ) -> ZarrV2ConsolidatedMetadataReading: - """Read `value`, a v2 `.zmetadata` document, in `context` (`CORE_V2` when none is given), whatever it holds: every problem the read finds, and the model when there are none. This is the form of `ZarrV2ConsolidatedMetadata(value, context)` that returns problems instead of raising them.""" - context = scoped(context, CORE_V2) + """Read `value`, a v2 `.zmetadata` document, in `context` (`CORE_V2_CONTEXT` when none is given), whatever it holds: every problem the read finds, and the model when there are none. This is the form of `ZarrV2ConsolidatedMetadata(value, context)` that returns problems instead of raising them.""" + context = scoped(context, CORE_V2_CONTEXT) document, nodes, files, problems = _read_consolidated_v2(value, context) if len(problems) != 0: return ZarrV2ConsolidatedMetadataReading(problems=problems) @@ -487,7 +487,7 @@ class ZarrV2ConsolidatedMetadata(Keyed): zarr_consolidated_format: Final = 1 def __init__(self, document: object, context: ZarrV2Context | None = None) -> None: - context = scoped(context, CORE_V2) + context = scoped(context, CORE_V2_CONTEXT) read, nodes, files, problems = _read_consolidated_v2(document, context) if len(problems) != 0: raise MetadataValidationError(problems) @@ -576,7 +576,7 @@ def __reduce__(self) -> tuple[type[ZarrV2ConsolidatedMetadata], tuple[object, Za def with_context(self, context: ZarrV2Context | None = None) -> ZarrV2ConsolidatedMetadata: """Return this document with every node read in `context`, whatever that changes. Raises `MetadataValidationError` when a node has a problem in `context`.""" - context = scoped(context, CORE_V2) + context = scoped(context, CORE_V2_CONTEXT) return type(self)(self._document, context=context) def refined_in(self, context: ZarrV2Context | None = None) -> ZarrV2ConsolidatedMetadata: @@ -586,7 +586,7 @@ def refined_in(self, context: ZarrV2Context | None = None) -> ZarrV2Consolidated node's entry. Raises `MetadataValidationError` when a gain exposes a problem. """ - context = scoped(context, CORE_V2) + context = scoped(context, CORE_V2_CONTEXT) conflicts: list[Conflict] = [] for path, node in self._metadata.items(): if not isinstance(node, ZarrV2ArrayMetadata): @@ -633,7 +633,7 @@ def from_key_value( are not JSON, or the document is not valid. Raises `TypeError` for a scope of another format before the bytes are decoded. """ - context = scoped(context, CORE_V2) + context = scoped(context, CORE_V2_CONTEXT) return cls( load_store_json(mapping, ZARR_V2_CONSOLIDATED_METADATA_STORE_KEY), context=context ) diff --git a/packages/zarr-metadata/src/zarr_metadata/model/_group_v3.py b/packages/zarr-metadata/src/zarr_metadata/model/_group_v3.py index 155d8e8019..a189856678 100644 --- a/packages/zarr-metadata/src/zarr_metadata/model/_group_v3.py +++ b/packages/zarr-metadata/src/zarr_metadata/model/_group_v3.py @@ -80,7 +80,7 @@ # A runtime import: `get_type_hints` resolves the public signatures. from zarr_metadata.v3._context import ZarrV3Context # noqa: TC001 -from zarr_metadata.v3._core import CORE_AND_EXTENSIONS +from zarr_metadata.v3._core import CORE_AND_EXTENSIONS_CONTEXT from zarr_metadata.v3._definition import GroupExtraFieldDefinition from zarr_metadata.v3._hierarchy import hierarchy_problems, path_faults from zarr_metadata.v3.consolidated import ( @@ -169,7 +169,7 @@ class ZarrV3GroupMetadata(Keyed): the document as written, refined, and `context` is the scope. `attributes` and `consolidated_metadata` are views of the values the read refined. Each member the spec does not define is an extra field, - in `extra_fields`, as the scope read it. `CORE` reads a + in `extra_fields`, as the scope read it. `CORE_CONTEXT` reads a `consolidated_metadata` member as the envelope the core spec fixes, and no definition reads the documents the member lists: they follow a convention of the reference implementation, not the core spec. To @@ -186,7 +186,7 @@ class ZarrV3GroupMetadata(Keyed): node_type: Final[Literal["group"]] = "group" # noqa: PYI064 - as above def __init__(self, document: object, context: ZarrV3Context | None = None) -> None: - context = scoped(context, CORE_AND_EXTENSIONS) + context = scoped(context, CORE_AND_EXTENSIONS_CONTEXT) reading, members = read_group_v3(document, context) if members is None or len(reading.problems) != 0: raise MetadataValidationError(reading.problems) @@ -281,7 +281,7 @@ def attributes(self) -> Mapping[str, JSONValue]: def extra_fields( self, ) -> Mapping[str, AcceptedField[GroupExtraFieldDefinition[Any]] | UnclaimedField]: - """Each member the spec does not define, by key, as the scope read it: an `AcceptedField` when a definition in scope claims the key (as `CORE` claims `consolidated_metadata`), and an `UnclaimedField` otherwise.""" + """Each member the spec does not define, by key, as the scope read it: an `AcceptedField` when a definition in scope claims the key (as `CORE_CONTEXT` claims `consolidated_metadata`), and an `UnclaimedField` otherwise.""" return self._shown[3] @property @@ -365,7 +365,7 @@ def update(self, **members: Unpack[ZarrV3GroupMetadataUpdate]) -> ZarrV3GroupMet def with_context(self, context: ZarrV3Context | None = None) -> ZarrV3GroupMetadata: """Return this document read in `context`, whatever that changes. Raises `MetadataValidationError` when the document has a problem in `context`. The existing reading is reused when `context` reads every claim identically.""" - context = scoped(context, CORE_AND_EXTENSIONS) + context = scoped(context, CORE_AND_EXTENSIONS_CONTEXT) if disagreements(context, claims_of(self._reading.fields())).agrees: return self._of(self._document, context, self._reading, self._members) return type(self)(self._document, context=context) @@ -382,7 +382,7 @@ def refined_in(self, context: ZarrV3Context | None = None) -> ZarrV3GroupMetadat in `context`, and `ZarrV3ConsolidatedMetadata.refined_in` checks that they read the same there. """ - context = scoped(context, CORE_AND_EXTENSIONS) + context = scoped(context, CORE_AND_EXTENSIONS_CONTEXT) found = disagreements(context, claims_of(self._reading.fields())) if len(found.conflicts) != 0: raise ScopeConflictError(located_conflicts(self._reading.fields(), found.conflicts)) @@ -409,7 +409,7 @@ def create_default( **overrides: Unpack[ZarrV3GroupMetadataUpdate], ) -> ZarrV3GroupMetadata: """Return a group with no attributes, with any document members in `overrides` applied, read in `context`. `overrides` takes what `update` takes, including `UNSET` to drop a member and a `ZarrV3ConsolidatedMetadata` as `consolidated_metadata`. Raises `MetadataValidationError` when the resulting document has a problem.""" - context = scoped(context, CORE_AND_EXTENSIONS) + context = scoped(context, CORE_AND_EXTENSIONS_CONTEXT) changes = member_changes( overrides, ZarrV3GroupMetadataUpdate, GroupExtraFieldDefinition.spec_members ) @@ -426,7 +426,7 @@ def from_json( Raises `MetadataValidationError` with every problem the read finds. A `consolidated_metadata` of `null`, which zarr-python 3.0 and 3.1 wrote, is a value the document wrote that is not an object. In a - scope that claims the member, as `CORE` does, this is a problem, + scope that claims the member, as `CORE_CONTEXT` does, this is a problem, because the spec requires an object. Use `read_repaired_node_metadata_v3` to read such a store. A member the spec does not define is kept in `extra_fields`. @@ -443,7 +443,7 @@ def from_key_value( are not JSON, or the document is not valid. Raises `TypeError` for a scope of another format before any bytes are decoded. """ - context = scoped(context, CORE_AND_EXTENSIONS) + context = scoped(context, CORE_AND_EXTENSIONS_CONTEXT) return cls(load_store_json(mapping, ZARR_V3_GROUP_METADATA_STORE_KEY), context=context) @@ -495,7 +495,7 @@ class ZarrV3ConsolidatedMetadata(Keyed): must_understand: Final = False def __init__(self, member: object, context: ZarrV3Context | None = None) -> None: - context = scoped(context, CORE_AND_EXTENSIONS) + context = scoped(context, CORE_AND_EXTENSIONS_CONTEXT) # The member is always under a group's key, even when read on its # own, so nesting depth is counted from there, as in the group. readings, members, document, problems = _read_consolidated_v3( @@ -561,7 +561,7 @@ def __reduce__(self) -> tuple[type[ZarrV3ConsolidatedMetadata], tuple[object, Za def with_context(self, context: ZarrV3Context | None = None) -> ZarrV3ConsolidatedMetadata: """Return this member with every document read in `context`, whatever that changes. Raises `MetadataValidationError` when a document has a problem in `context`. The existing readings are reused when `context` reads every claim of every document identically.""" - context = scoped(context, CORE_AND_EXTENSIONS) + context = scoped(context, CORE_AND_EXTENSIONS_CONTEXT) if all( disagreements(context, claims_of(node.reading.fields())).agrees for node in self._metadata.values() @@ -577,7 +577,7 @@ def refined_in(self, context: ZarrV3Context | None = None) -> ZarrV3Consolidated document's path. Raises `MetadataValidationError` when a gain exposes a problem. """ - context = scoped(context, CORE_AND_EXTENSIONS) + context = scoped(context, CORE_AND_EXTENSIONS_CONTEXT) conflicts = [ dataclasses.replace( conflict, loc=("metadata", path, *(() if conflict.loc is None else conflict.loc)) @@ -681,8 +681,8 @@ def __reduce__(self) -> str | tuple[object, ...]: def read_consolidated_metadata_v3( value: object, *, context: ZarrV3Context | None = None ) -> ZarrV3ConsolidatedMetadataReading: - """Read `value`, a group's `consolidated_metadata` member, in `context` (`CORE_AND_EXTENSIONS` when none is given), whatever it holds: every problem the read finds, and the model when there are none. This is the form of `ZarrV3ConsolidatedMetadata(value, context)` that returns problems instead of raising them.""" - context = scoped(context, CORE_AND_EXTENSIONS) + """Read `value`, a group's `consolidated_metadata` member, in `context` (`CORE_AND_EXTENSIONS_CONTEXT` when none is given), whatever it holds: every problem the read finds, and the model when there are none. This is the form of `ZarrV3ConsolidatedMetadata(value, context)` that returns problems instead of raising them.""" + context = scoped(context, CORE_AND_EXTENSIONS_CONTEXT) readings, members, document, problems = _read_consolidated_v3( value, context, (ZARR_V3_CONSOLIDATED_METADATA_KEY,) ) @@ -737,7 +737,7 @@ def read_node_metadata_v3( read `node_type` from unvalidated JSON, and a document in another format is reported as not v3. """ - context = scoped(context, CORE_AND_EXTENSIONS) + context = scoped(context, CORE_AND_EXTENSIONS_CONTEXT) node_type, problems = _node_type(value) if node_type == "array": return read_array_metadata_v3(value, context=context) @@ -761,7 +761,7 @@ def node_metadata_from_json_v3( `MetadataValidationError` with every problem `read_node_metadata_v3` finds, including a `node_type` that is neither array nor group. """ - context = scoped(context, CORE_AND_EXTENSIONS) + context = scoped(context, CORE_AND_EXTENSIONS_CONTEXT) reading = read_node_metadata_v3(data, context=context) if reading.metadata is None: raise MetadataValidationError(reading.problems) @@ -776,7 +776,7 @@ def node_metadata_from_key_value_v3( Raises `MetadataValidationError` when the key is missing, its bytes are not JSON, or the document is not a valid array or group. """ - context = scoped(context, CORE_AND_EXTENSIONS) + context = scoped(context, CORE_AND_EXTENSIONS_CONTEXT) # Array documents and group documents are both stored at `zarr.json`. document = load_store_json(mapping, ZARR_V3_GROUP_METADATA_STORE_KEY) return node_metadata_from_json_v3(document, context=context) @@ -853,13 +853,13 @@ def read_group_metadata_v3( reader must understand each one that does not declare `must_understand: false`; the model reports these as `must_understand_fields`. A `consolidated_metadata` member, if - present and claimed by the scope, as `CORE` claims it, is checked + present and claimed by the scope, as `CORE_CONTEXT` claims it, is checked against the envelope the core spec fixes. The documents it lists are not read; `read_consolidated_metadata_v3` reads them. This is the form of `ZarrV3GroupMetadata(value, context)` that returns problems instead of raising them. """ - context = scoped(context, CORE_AND_EXTENSIONS) + context = scoped(context, CORE_AND_EXTENSIONS_CONTEXT) reading, members = read_group_v3(value, context) if members is None or members.document is None: return reading diff --git a/packages/zarr-metadata/src/zarr_metadata/model/_repair.py b/packages/zarr-metadata/src/zarr_metadata/model/_repair.py index d16663b1fe..679a584fc8 100644 --- a/packages/zarr-metadata/src/zarr_metadata/model/_repair.py +++ b/packages/zarr-metadata/src/zarr_metadata/model/_repair.py @@ -45,9 +45,9 @@ ) from zarr_metadata.model._group_v3 import ZarrV3NodeMetadataReading, read_node_metadata_v3 from zarr_metadata.v2.array import ZARR_V2_ARRAY_METADATA_STORE_KEY -from zarr_metadata.v2.definition import CORE_V2 +from zarr_metadata.v2.definition import CORE_V2_CONTEXT from zarr_metadata.v2.group import ZARR_V2_GROUP_METADATA_STORE_KEY -from zarr_metadata.v3._core import CORE_AND_EXTENSIONS +from zarr_metadata.v3._core import CORE_AND_EXTENSIONS_CONTEXT from zarr_metadata.v3.consolidated import ZARR_V3_CONSOLIDATED_METADATA_KEY if TYPE_CHECKING: @@ -278,7 +278,7 @@ def read_repaired_node_metadata_v3( Content that no repair applies to is read as it is, and reported the same way `read_node_metadata_v3` reports it. """ - context = scoped(context, CORE_AND_EXTENSIONS) + context = scoped(context, CORE_AND_EXTENSIONS_CONTEXT) repaired, repairs = repair_node_metadata_v3(value) return ZarrV3RepairedNodeMetadataReading( read_node_metadata_v3(repaired, context=context), repairs @@ -482,7 +482,7 @@ def read_repaired_node_metadata_v2( Content that no repair applies to is read as it is, and reported the same way `read_node_metadata_v2` reports it. """ - context = scoped(context, CORE_V2) + context = scoped(context, CORE_V2_CONTEXT) repaired, repairs = repair_node_metadata_v2(files) return ZarrV2RepairedNodeMetadataReading( read_node_metadata_v2(repaired, context=context), repairs @@ -510,7 +510,7 @@ def read_repaired_consolidated_metadata_v2( read as it is, and reported the same way `read_consolidated_metadata_v2` reports it. """ - context = scoped(context, CORE_V2) + context = scoped(context, CORE_V2_CONTEXT) repaired, repairs = repair_consolidated_metadata_v2(value) return ZarrV2RepairedConsolidatedMetadataReading( read_consolidated_metadata_v2(repaired, context=context), repairs diff --git a/packages/zarr-metadata/src/zarr_metadata/pydantic.py b/packages/zarr-metadata/src/zarr_metadata/pydantic.py index f369f2beec..84ded7390e 100644 --- a/packages/zarr-metadata/src/zarr_metadata/pydantic.py +++ b/packages/zarr-metadata/src/zarr_metadata/pydantic.py @@ -20,7 +20,7 @@ `"zarr_metadata_context_v2"` item. This lets a model with fields of both formats give a scope for each format. - If no scope is given for a format, its field types read in the default - scope for that format, `CORE_AND_EXTENSIONS` or `CORE_V2`. + scope for that format, `CORE_AND_EXTENSIONS_CONTEXT` or `CORE_V2_CONTEXT`. A mapping key that starts like these keys but is not one of them raises `TypeError`, so a misspelled key is not read as no scope. To fix a field's @@ -128,8 +128,8 @@ class ArrayManifest(BaseModel): from zarr_metadata.definition._context import scoped from zarr_metadata.definition._definition import Context from zarr_metadata.definition._scope import ScopeConflictError -from zarr_metadata.v2.definition import CORE_V2, ZarrV2Context -from zarr_metadata.v3.definition import CORE_AND_EXTENSIONS, ZarrV3Context +from zarr_metadata.v2.definition import CORE_V2_CONTEXT, ZarrV2Context +from zarr_metadata.v3.definition import CORE_AND_EXTENSIONS_CONTEXT, ZarrV3Context if TYPE_CHECKING: from collections.abc import Callable, Mapping @@ -321,7 +321,7 @@ def _context_of(validation_context: object, default: _C, key: str) -> _C | None: _model.ZarrV3ArrayMetadata, _model.ZarrV3ArrayMetadata.from_json, _model.ZarrV3ArrayMetadata.refined_in, - CORE_AND_EXTENSIONS, + CORE_AND_EXTENSIONS_CONTEXT, CONTEXT_KEY, ), json_schema_input_type=_ZarrV3ArrayMetadataSchema, @@ -337,7 +337,7 @@ def _context_of(validation_context: object, default: _C, key: str) -> _C | None: _model.ZarrV2ArrayMetadata, _model.ZarrV2ArrayMetadata.from_json, _model.ZarrV2ArrayMetadata.refined_in, - CORE_V2, + CORE_V2_CONTEXT, CONTEXT_KEY_V2, ), json_schema_input_type=_ZarrV2ArrayMetadataSchema, @@ -353,7 +353,7 @@ def _context_of(validation_context: object, default: _C, key: str) -> _C | None: _model.ZarrV3GroupMetadata, _model.ZarrV3GroupMetadata.from_json, _model.ZarrV3GroupMetadata.refined_in, - CORE_AND_EXTENSIONS, + CORE_AND_EXTENSIONS_CONTEXT, CONTEXT_KEY, ), json_schema_input_type=_ZarrV3GroupMetadataSchema, @@ -369,7 +369,7 @@ def _context_of(validation_context: object, default: _C, key: str) -> _C | None: _model.ZarrV2GroupMetadata, _model.ZarrV2GroupMetadata.from_json, _model.ZarrV2GroupMetadata.refined_in, - CORE_V2, + CORE_V2_CONTEXT, CONTEXT_KEY_V2, ), json_schema_input_type=_ZarrV2GroupMetadataSchema, @@ -385,7 +385,7 @@ def _context_of(validation_context: object, default: _C, key: str) -> _C | None: _model.ZarrV3ConsolidatedMetadata, _model.ZarrV3ConsolidatedMetadata.from_json, _model.ZarrV3ConsolidatedMetadata.refined_in, - CORE_AND_EXTENSIONS, + CORE_AND_EXTENSIONS_CONTEXT, CONTEXT_KEY, ), json_schema_input_type=_ZarrV3ConsolidatedMetadataSchema, @@ -404,7 +404,7 @@ def _context_of(validation_context: object, default: _C, key: str) -> _C | None: _model.ZarrV2ConsolidatedMetadata, _model.ZarrV2ConsolidatedMetadata.from_json, _model.ZarrV2ConsolidatedMetadata.refined_in, - CORE_V2, + CORE_V2_CONTEXT, CONTEXT_KEY_V2, ), json_schema_input_type=_ZarrV2ConsolidatedMetadataSchema, @@ -419,15 +419,15 @@ def _context_of(validation_context: object, default: _C, key: str) -> _C | None: _FIELD_PARTS: Final[Mapping[type[Any], tuple[ZarrV3Context | ZarrV2Context, object]]] = ( MappingProxyType( { - _model.ZarrV3ArrayMetadata: (CORE_AND_EXTENSIONS, _ZarrV3ArrayMetadataSchema), - _model.ZarrV2ArrayMetadata: (CORE_V2, _ZarrV2ArrayMetadataSchema), - _model.ZarrV3GroupMetadata: (CORE_AND_EXTENSIONS, _ZarrV3GroupMetadataSchema), - _model.ZarrV2GroupMetadata: (CORE_V2, _ZarrV2GroupMetadataSchema), + _model.ZarrV3ArrayMetadata: (CORE_AND_EXTENSIONS_CONTEXT, _ZarrV3ArrayMetadataSchema), + _model.ZarrV2ArrayMetadata: (CORE_V2_CONTEXT, _ZarrV2ArrayMetadataSchema), + _model.ZarrV3GroupMetadata: (CORE_AND_EXTENSIONS_CONTEXT, _ZarrV3GroupMetadataSchema), + _model.ZarrV2GroupMetadata: (CORE_V2_CONTEXT, _ZarrV2GroupMetadataSchema), _model.ZarrV3ConsolidatedMetadata: ( - CORE_AND_EXTENSIONS, + CORE_AND_EXTENSIONS_CONTEXT, _ZarrV3ConsolidatedMetadataSchema, ), - _model.ZarrV2ConsolidatedMetadata: (CORE_V2, _ZarrV2ConsolidatedMetadataSchema), + _model.ZarrV2ConsolidatedMetadata: (CORE_V2_CONTEXT, _ZarrV2ConsolidatedMetadataSchema), } ) ) diff --git a/packages/zarr-metadata/src/zarr_metadata/testing/__init__.py b/packages/zarr-metadata/src/zarr_metadata/testing/__init__.py new file mode 100644 index 0000000000..271a4dfe51 --- /dev/null +++ b/packages/zarr-metadata/src/zarr_metadata/testing/__init__.py @@ -0,0 +1,6 @@ +"""Testing utilities for code that reads or writes Zarr metadata. + +`zarr_metadata.testing.strategies` holds Hypothesis strategies that draw +valid metadata. It needs the `testing` extra: +`pip install zarr-metadata[testing]`. +""" diff --git a/packages/zarr-metadata/src/zarr_metadata/testing/_chunk_grid.py b/packages/zarr-metadata/src/zarr_metadata/testing/_chunk_grid.py new file mode 100644 index 0000000000..b7528a3907 --- /dev/null +++ b/packages/zarr-metadata/src/zarr_metadata/testing/_chunk_grid.py @@ -0,0 +1,105 @@ +"""Strategies for each chunk grid this package defines. + +Over an array of a known shape, a grid is drawn to fit it: a chunk +length for each dimension, never zero ("Chunk sizes must be greater +than zero", https://github.com/zarr-developers/zarr-specs/blob/fc7dd9c9beb5a50b87f9b08b00bf50fc0048482f/docs/v3/chunk-grids/regular-grid/index.rst#L40), +and rectilinear edges that cover each dimension. Over no shape, what is +drawn is what the grid's rules accept. +""" + +from __future__ import annotations + +from typing import TYPE_CHECKING, Any, Final + +from hypothesis import strategies as st + +from zarr_metadata.testing._strategy import MAX_RANK, ChunkGridStrategy +from zarr_metadata.v3.chunk_grid.rectilinear import RECTILINEAR_CHUNK_GRID +from zarr_metadata.v3.chunk_grid.regular import REGULAR_CHUNK_GRID + +if TYPE_CHECKING: + from collections.abc import Mapping + + from zarr_metadata._common import JSONValue + from zarr_metadata.v3._definition import ChunkGridDefinition + +_MAX_LENGTH: Final = 16 +"""The longest chunk drawn along a dimension whose length is unknown.""" + +_MAX_RUNS: Final = 5 +"""The most runs of equal lengths a rectilinear dimension is drawn with.""" + +_MAX_COUNT: Final = 4 +"""The most chunks a run is drawn with.""" + + +def _length(extent: int | None) -> st.SearchStrategy[int]: + """A chunk length along a dimension of `extent`: at least 1, and up to twice the extent when it is known and not 0.""" + if extent is None or extent == 0: + return st.integers(1, _MAX_LENGTH) + return st.integers(1, 2 * extent) + + +def _regular(shape: tuple[int, ...] | None) -> st.SearchStrategy[Mapping[str, JSONValue]]: + """A chunk length for each dimension.""" + if shape is None: + lengths = st.lists(_length(None), max_size=MAX_RANK).map(tuple) + else: + lengths = st.tuples(*(_length(extent) for extent in shape)) + return lengths.map(lambda chunk_shape: {"chunk_shape": chunk_shape}) + + +def _spelled_run(draw: st.DrawFn, size: int, count: int) -> list[JSONValue]: + """`count` chunks of `size`: one `[size, count]` pair, or the size written once for each.""" + if draw(st.booleans()): + return [(size, count)] + return [size] * count + + +@st.composite +def _edges(draw: st.DrawFn, extent: int | None) -> JSONValue: + """Edge lengths along a dimension, covering `extent`, in runs spelled as pairs or written out. + + Runs are drawn apart, so two of one size can sit side by side and a + pair can count one chunk: spellings the canonical form merges. A + dimension of length zero, or of a length not known, may be covered by + no edges at all. + """ + min_runs = 0 if extent is None or extent == 0 else 1 + runs = draw( + st.lists( + st.tuples(st.integers(1, _MAX_LENGTH), st.integers(1, _MAX_COUNT)), + min_size=min_runs, + max_size=_MAX_RUNS, + ) + ) + covered = sum(size * count for size, count in runs) + if extent is not None and covered < extent: + runs.append((extent - covered, 1)) + spelled: list[JSONValue] = [] + for size, count in runs: + spelled.extend(_spelled_run(draw, size, count)) + return tuple(spelled) + + +def _rectilinear_dimension(extent: int | None) -> st.SearchStrategy[JSONValue]: + """A bare length, which repeats until it covers the dimension, or edges that cover it.""" + return _length(extent) | _edges(extent) + + +def _rectilinear(shape: tuple[int, ...] | None) -> st.SearchStrategy[Mapping[str, JSONValue]]: + """Inline chunk shapes, one entry for each dimension.""" + if shape is None: + dimensions = st.lists(_rectilinear_dimension(None), max_size=MAX_RANK).map(tuple) + else: + dimensions = st.tuples(*(_rectilinear_dimension(extent) for extent in shape)) + return dimensions.map(lambda chunk_shapes: {"kind": "inline", "chunk_shapes": chunk_shapes}) + + +CHUNK_GRID_STRATEGIES: Final[Mapping[ChunkGridDefinition[Any], ChunkGridStrategy]] = { + REGULAR_CHUNK_GRID: ChunkGridStrategy(configurations=_regular), + RECTILINEAR_CHUNK_GRID: ChunkGridStrategy(configurations=_rectilinear), +} +"""A strategy for each chunk grid.""" + +__all__ = ["CHUNK_GRID_STRATEGIES"] diff --git a/packages/zarr-metadata/src/zarr_metadata/testing/_chunk_key_encoding.py b/packages/zarr-metadata/src/zarr_metadata/testing/_chunk_key_encoding.py new file mode 100644 index 0000000000..9c525118c9 --- /dev/null +++ b/packages/zarr-metadata/src/zarr_metadata/testing/_chunk_key_encoding.py @@ -0,0 +1,40 @@ +"""Strategies for each chunk key encoding this package defines.""" + +from __future__ import annotations + +from typing import TYPE_CHECKING, Any, Final + +from hypothesis import strategies as st + +from zarr_metadata.testing._strategy import ChunkKeyEncodingStrategy +from zarr_metadata.v3.chunk_key_encoding.default import ( + DEFAULT_CHUNK_KEY_ENCODING, + DEFAULT_CHUNK_KEY_ENCODING_SEPARATOR, +) +from zarr_metadata.v3.chunk_key_encoding.v2 import ( + V2_CHUNK_KEY_ENCODING, + V2_CHUNK_KEY_ENCODING_SEPARATOR, +) + +if TYPE_CHECKING: + from collections.abc import Mapping + + from zarr_metadata.v3._definition import ChunkKeyEncodingDefinition + +CHUNK_KEY_ENCODING_STRATEGIES: Final[ + Mapping[ChunkKeyEncodingDefinition[Any], ChunkKeyEncodingStrategy] +] = { + DEFAULT_CHUNK_KEY_ENCODING: ChunkKeyEncodingStrategy( + configurations=st.fixed_dictionaries( + {}, optional={"separator": st.sampled_from(DEFAULT_CHUNK_KEY_ENCODING_SEPARATOR)} + ) + ), + V2_CHUNK_KEY_ENCODING: ChunkKeyEncodingStrategy( + configurations=st.fixed_dictionaries( + {}, optional={"separator": st.sampled_from(V2_CHUNK_KEY_ENCODING_SEPARATOR)} + ) + ), +} +"""A strategy for each chunk key encoding: a `separator`, or none, which means the encoding's default.""" + +__all__ = ["CHUNK_KEY_ENCODING_STRATEGIES"] diff --git a/packages/zarr-metadata/src/zarr_metadata/testing/_codec.py b/packages/zarr-metadata/src/zarr_metadata/testing/_codec.py new file mode 100644 index 0000000000..e966cd9087 --- /dev/null +++ b/packages/zarr-metadata/src/zarr_metadata/testing/_codec.py @@ -0,0 +1,194 @@ +"""Strategies for each codec this package defines that holds no other field. + +A codec handed an array draws what fits the chunk: a `bytes` codec an +`endian` for a type whose values take several bytes, a `transpose` an +`order` with an entry for each axis. Handed `Chunk()`, nothing is known +of the chunk, and what is drawn is what the codec's rules accept. +""" + +from __future__ import annotations + +from typing import TYPE_CHECKING, Any, Final + +from hypothesis import strategies as st + +from zarr_metadata.definition._definition import AcceptedField, EmptyConfiguration +from zarr_metadata.testing._data_type import ( + COMPLEX_WIDTHS, + FLOAT_WIDTHS, + INTEGER_RANGES, + FloatWidth, + finite_floats, + float_spellings, +) +from zarr_metadata.testing._strategy import EMPTY_CONFIGURATIONS, MAX_RANK, CodecStrategy +from zarr_metadata.v3._definition import Chunk, storage_of +from zarr_metadata.v3.codec._arithmetic import NOT_NUMBERS +from zarr_metadata.v3.codec.blosc import BLOSC_CNAME, BLOSC_CODEC, BLOSC_NO_SHUFFLE, BLOSC_SHUFFLE +from zarr_metadata.v3.codec.bytes import BYTES_CODEC, ENDIANNESS +from zarr_metadata.v3.codec.crc32c import CRC32C_CODEC +from zarr_metadata.v3.codec.gzip import GZIP_CODEC +from zarr_metadata.v3.codec.scale_offset import SCALE_OFFSET_CODEC +from zarr_metadata.v3.codec.transpose import TRANSPOSE_CODEC +from zarr_metadata.v3.codec.vlen_bytes import VLEN_BYTES_CODEC +from zarr_metadata.v3.codec.vlen_utf8 import VLEN_UTF8_CODEC +from zarr_metadata.v3.codec.zstd import ZSTD_CODEC, ZSTD_MAX_LEVEL, ZSTD_MIN_LEVEL +from zarr_metadata.v3.data_type.numpy_datetime64 import NUMPY_DATETIME64_DATA_TYPE +from zarr_metadata.v3.data_type.numpy_timedelta64 import NUMPY_TIMEDELTA64_DATA_TYPE + +if TYPE_CHECKING: + from collections.abc import Mapping + + from zarr_metadata._common import JSONValue + from zarr_metadata.v3._definition import CodecDefinition + +_BLOSC_MAX_TYPESIZE: Final = 255 +"""The widest element c-blosc takes: `BLOSC_MAX_TYPESIZE`.""" + +_BLOSC_MAX_BLOCKSIZE: Final = 2**31 - 1 +"""The largest block c-blosc takes, a signed 32-bit size; 0 lets it choose.""" + +_ENDIAN: Final = st.sampled_from(ENDIANNESS) + +_NUMPY_TIMES: Final = frozenset({NUMPY_DATETIME64_DATA_TYPE, NUMPY_TIMEDELTA64_DATA_TYPE}) +"""The numpy time types, each stored as a signed 64-bit integer.""" + +_NUMPY_TIME_RANGE: Final = (-(2**63) + 1, 2**63 - 1) +"""The signed 64-bit integers but the least, which numpy reads as `NaT`: the finite numpy times.""" + + +def _bytes(chunk: Chunk) -> st.SearchStrategy[Mapping[str, JSONValue]] | None: + """An `endian`, required for a data type whose values take several bytes; None for one whose values vary in size.""" + storage = None if chunk.data_type is None else storage_of(chunk.data_type) + if storage == "variable_length": + return None + if storage == "multi_byte": + return st.fixed_dictionaries({"endian": _ENDIAN}) + return st.fixed_dictionaries({}, optional={"endian": _ENDIAN}) + + +@st.composite +def _blosc(draw: st.DrawFn) -> Mapping[str, JSONValue]: + """A compressor, a level, a shuffle and a block size; and a `typesize`, which only `noshuffle` may leave out. + + Under `noshuffle` the spec says `typesize` "is ignored", so it is + drawn there too, a spelling that is not the simplest. + """ + shuffle = draw(st.sampled_from(BLOSC_SHUFFLE)) + typesize = st.integers(1, _BLOSC_MAX_TYPESIZE) + required: dict[str, st.SearchStrategy[JSONValue]] = { + "cname": st.sampled_from(BLOSC_CNAME), + "clevel": st.integers(0, 9), + "shuffle": st.just(shuffle), + "blocksize": st.integers(0, _BLOSC_MAX_BLOCKSIZE), + } + if shuffle == BLOSC_NO_SHUFFLE: + return draw(st.fixed_dictionaries(required, optional={"typesize": typesize})) + return draw(st.fixed_dictionaries({**required, "typesize": typesize})) + + +def _order(rank: int) -> st.SearchStrategy[Mapping[str, JSONValue]]: + return st.permutations(range(rank)).map(lambda order: {"order": tuple(order)}) + + +def _transpose(chunk: Chunk) -> st.SearchStrategy[Mapping[str, JSONValue]]: + """A permutation of the chunk's axes; of some number of axes, when the chunk does not say.""" + if chunk.rank is not None: + return _order(chunk.rank) + return st.integers(0, MAX_RANK).flatmap(_order) + + +def _finite_float_values(width: FloatWidth) -> st.SearchStrategy[JSONValue]: + """A finite float of `width` bits, as a number or the hex string of its bits.""" + return finite_floats(width).flatmap(lambda value: float_spellings(value, width)) + + +def _non_zero_float_values(width: FloatWidth) -> st.SearchStrategy[JSONValue]: + """A finite, non-zero float of `width` bits, as a number or the hex string of its bits.""" + return ( + finite_floats(width) + .filter(lambda value: value != 0) + .flatmap(lambda value: float_spellings(value, width)) + ) + + +def _scale_offset(chunk: Chunk) -> st.SearchStrategy[Mapping[str, JSONValue]] | None: + """An `offset` and a `scale`, each a finite value of the data type the codec is handed, the scale not zero; None for a data type whose values are not numbers. + + Each is a fill value of the chunk's data type + (https://github.com/zarr-developers/zarr-extensions/blob/4da7b37a84f76e660902f6d3de3eaef0e0febae6/codecs/scale_offset/README.md?plain=1#L39-L43), + a float's as a number or the hex string of its bits, a complex + number's as a pair of them. Each is drawn finite, and the scale + non-zero, so that decoding never divides by zero; a scale small + enough that decoding overflows is still drawn. A numpy time is drawn + as the signed 64-bit integer it is stored as, never `NaT`, since it + "is compatible with any codec that supports arrays of signed 64-bit + integers" + (https://github.com/zarr-developers/zarr-extensions/blob/4da7b37a84f76e660902f6d3de3eaef0e0febae6/data-types/numpy.datetime64/README.md?plain=1#L120). + A data type this package does not define, whose values it cannot + draw, takes only the configuration with neither, the identity. + Handed a chunk whose data type is unknown, the values are small + non-negative integers, which every integer and float type holds. + """ + data_type = chunk.data_type + definition = data_type.definition if isinstance(data_type, AcceptedField) else None + if definition is None: + offset: st.SearchStrategy[JSONValue] = st.integers(0, 127) + scale: st.SearchStrategy[JSONValue] = st.integers(1, 127) + elif definition.name in NOT_NUMBERS: + return None + elif definition in INTEGER_RANGES or definition in _NUMPY_TIMES: + low, high = INTEGER_RANGES.get(definition, _NUMPY_TIME_RANGE) + offset = st.integers(low, high) + scale = st.integers(low, high).filter(lambda value: value != 0) + elif definition in FLOAT_WIDTHS: + offset = _finite_float_values(FLOAT_WIDTHS[definition]) + scale = _non_zero_float_values(FLOAT_WIDTHS[definition]) + elif definition in COMPLEX_WIDTHS: + width = COMPLEX_WIDTHS[definition] + offset = st.tuples(_finite_float_values(width), _finite_float_values(width)) + # Non-zero in either part is a non-zero complex number. + scale = st.one_of( + st.tuples(_non_zero_float_values(width), _finite_float_values(width)), + st.tuples(_finite_float_values(width), _non_zero_float_values(width)), + ) + else: + return EMPTY_CONFIGURATIONS + return st.fixed_dictionaries({}, optional={"offset": offset, "scale": scale}) + + +def _vlen(definition: CodecDefinition[Any]) -> CodecStrategy: + """A codec with nothing to configure that takes only some data types: none for a chunk of another, which its own chunk rules say.""" + return CodecStrategy( + configurations=lambda chunk: ( + None + if any(definition.chunk_rules(EmptyConfiguration(), {}, chunk)) + else EMPTY_CONFIGURATIONS + ) + ) + + +def _unchunked(configurations: st.SearchStrategy[Mapping[str, JSONValue]]) -> CodecStrategy: + """A codec whose configurations do not depend on the chunk it is handed.""" + return CodecStrategy(configurations=lambda _chunk: configurations) + + +CODEC_STRATEGIES: Final[Mapping[CodecDefinition[Any], CodecStrategy]] = { + BLOSC_CODEC: _unchunked(_blosc()), + BYTES_CODEC: CodecStrategy(configurations=_bytes), + CRC32C_CODEC: _unchunked(EMPTY_CONFIGURATIONS), + GZIP_CODEC: _unchunked(st.fixed_dictionaries({"level": st.integers(0, 9)})), + SCALE_OFFSET_CODEC: CodecStrategy(configurations=_scale_offset), + TRANSPOSE_CODEC: CodecStrategy(configurations=_transpose), + VLEN_BYTES_CODEC: _vlen(VLEN_BYTES_CODEC), + VLEN_UTF8_CODEC: _vlen(VLEN_UTF8_CODEC), + ZSTD_CODEC: _unchunked( + st.fixed_dictionaries( + {"level": st.integers(ZSTD_MIN_LEVEL, ZSTD_MAX_LEVEL)}, + optional={"checksum": st.booleans()}, + ) + ), +} +"""A strategy for each codec that holds no other field.""" + +__all__ = ["CODEC_STRATEGIES"] diff --git a/packages/zarr-metadata/src/zarr_metadata/testing/_data_type.py b/packages/zarr-metadata/src/zarr_metadata/testing/_data_type.py new file mode 100644 index 0000000000..b592b2d979 --- /dev/null +++ b/packages/zarr-metadata/src/zarr_metadata/testing/_data_type.py @@ -0,0 +1,256 @@ +"""Strategies for each data type this package defines that holds no other field. + +A fill value is drawn in every spelling the data type's spec gives it: +a float's as a number, a named non-finite value or a hex string of its +bits, a `bytes` value's as an array of byte values or base64. +""" + +from __future__ import annotations + +import base64 +import struct +from typing import TYPE_CHECKING, Any, Final, Literal, cast, get_args + +from hypothesis import strategies as st + +from zarr_metadata._numpy import NUMPY_TIME_MAX_SCALE_FACTOR +from zarr_metadata.testing._strategy import EMPTY_CONFIGURATIONS, DataTypeStrategy +from zarr_metadata.v3.data_type._float import FloatSpecialFillValue +from zarr_metadata.v3.data_type._numpy_time import NUMPY_TIME_UNIT +from zarr_metadata.v3.data_type.bool import BOOL_DATA_TYPE +from zarr_metadata.v3.data_type.bytes import BYTES_DATA_TYPE +from zarr_metadata.v3.data_type.complex64 import COMPLEX64_DATA_TYPE +from zarr_metadata.v3.data_type.complex128 import COMPLEX128_DATA_TYPE +from zarr_metadata.v3.data_type.fixed_length_utf32 import ( + FIXED_LENGTH_UTF32_DATA_TYPE, + FixedLengthUtf32Configuration, +) +from zarr_metadata.v3.data_type.float16 import FLOAT16_DATA_TYPE +from zarr_metadata.v3.data_type.float32 import FLOAT32_DATA_TYPE +from zarr_metadata.v3.data_type.float64 import FLOAT64_DATA_TYPE +from zarr_metadata.v3.data_type.int8 import INT8_DATA_TYPE +from zarr_metadata.v3.data_type.int16 import INT16_DATA_TYPE +from zarr_metadata.v3.data_type.int32 import INT32_DATA_TYPE +from zarr_metadata.v3.data_type.int64 import INT64_DATA_TYPE +from zarr_metadata.v3.data_type.null_terminated_bytes import ( + NULL_TERMINATED_BYTES_DATA_TYPE, + NullTerminatedBytesConfiguration, +) +from zarr_metadata.v3.data_type.numpy_datetime64 import NUMPY_DATETIME64_DATA_TYPE +from zarr_metadata.v3.data_type.numpy_timedelta64 import NUMPY_TIMEDELTA64_DATA_TYPE +from zarr_metadata.v3.data_type.raw import RAW_BYTES_DATA_TYPE, RawBytesConfiguration +from zarr_metadata.v3.data_type.string import STRING_DATA_TYPE +from zarr_metadata.v3.data_type.uint8 import UINT8_DATA_TYPE +from zarr_metadata.v3.data_type.uint16 import UINT16_DATA_TYPE +from zarr_metadata.v3.data_type.uint32 import UINT32_DATA_TYPE +from zarr_metadata.v3.data_type.uint64 import UINT64_DATA_TYPE +from zarr_metadata.v3.data_type.variable_length_bytes import VARIABLE_LENGTH_BYTES_DATA_TYPE +from zarr_metadata.v3.data_type.zarr_python_raw_bytes import ( + ZARR_PYTHON_RAW_BYTES_DATA_TYPE, + ZarrPythonRawBytesConfiguration, +) + +if TYPE_CHECKING: + from collections.abc import Mapping + + from zarr_metadata._common import JSONValue + from zarr_metadata.v3._definition import DataTypeDefinition + +FloatWidth = Literal[16, 32, 64] + +INTEGER_RANGES: Final[Mapping[DataTypeDefinition[Any], tuple[int, int]]] = { + INT8_DATA_TYPE: (-(2**7), 2**7 - 1), + INT16_DATA_TYPE: (-(2**15), 2**15 - 1), + INT32_DATA_TYPE: (-(2**31), 2**31 - 1), + INT64_DATA_TYPE: (-(2**63), 2**63 - 1), + UINT8_DATA_TYPE: (0, 2**8 - 1), + UINT16_DATA_TYPE: (0, 2**16 - 1), + UINT32_DATA_TYPE: (0, 2**32 - 1), + UINT64_DATA_TYPE: (0, 2**64 - 1), +} +"""The values each integer data type holds.""" + +FLOAT_WIDTHS: Final[Mapping[DataTypeDefinition[Any], FloatWidth]] = { + FLOAT16_DATA_TYPE: 16, + FLOAT32_DATA_TYPE: 32, + FLOAT64_DATA_TYPE: 64, +} +"""The width in bits of each floating-point data type.""" + +COMPLEX_WIDTHS: Final[Mapping[DataTypeDefinition[Any], FloatWidth]] = { + COMPLEX64_DATA_TYPE: 32, + COMPLEX128_DATA_TYPE: 64, +} +"""The width in bits of each part of each complex data type.""" + +_MAX_RAW_BYTES: Final = 32 +"""The widest raw bits drawn, in bytes: `r256`.""" + +_MAX_BYTES: Final = 16 +"""The longest `bytes` fill value drawn.""" + + +def finite_floats(width: FloatWidth) -> st.SearchStrategy[float]: + """Finite numbers a float of `width` bits holds exactly.""" + return st.floats(width=width, allow_nan=False, allow_infinity=False) + + +_PACKED: Final[Mapping[FloatWidth, str]] = {16: ">e", 32: ">f", 64: ">d"} +"""How `struct` packs a float of each width, most significant byte first.""" + + +def _hex_spellings(digits: str) -> st.SearchStrategy[str]: + """`0x` and the digits, the letters in either case.""" + return st.sampled_from((f"0x{digits}", f"0x{digits.upper()}")) + + +def float_spellings(value: float, width: FloatWidth) -> st.SearchStrategy[JSONValue]: + """`value`, a float of `width` bits, as a number or as the hex string of its bits.""" + return st.just(value) | _hex_spellings(struct.pack(_PACKED[width], value).hex()) + + +def _float_fill_values(width: FloatWidth) -> st.SearchStrategy[JSONValue]: + """A number, an integer the type holds exactly, a named non-finite value, or the bits in hex. + + A float's fill value is any of these + (https://github.com/zarr-developers/zarr-specs/blob/fc7dd9c9beb5a50b87f9b08b00bf50fc0048482f/docs/v3/data-types/index.rst#L63-L79). + Numbers are drawn from those the type holds, so none is rounded: a + `float16` takes `1e300` by the letter of the spec, and a reader is + free to make of it what it will. + """ + # Every integer of magnitude up to 2**11 is a float16, the narrowest type. + numbers = finite_floats(width) | st.integers(-(2**11), 2**11) + named = st.sampled_from(get_args(FloatSpecialFillValue)) + bits = st.integers(0, 2**width - 1).flatmap( + lambda value: _hex_spellings(f"{value:0{width // 4}x}") + ) + return st.one_of(numbers, named, bits) + + +def _fill_values_of(values: st.SearchStrategy[JSONValue]) -> DataTypeStrategy: + """A data type with nothing to configure, whose fill values are `values`.""" + return DataTypeStrategy( + configurations=EMPTY_CONFIGURATIONS, fill_values=lambda _configuration: values + ) + + +def _byte_values(size: int) -> st.SearchStrategy[JSONValue]: + """`size` byte values, each an integer in `[0, 255]`, as a JSON array.""" + return st.lists(st.integers(0, 255), min_size=size, max_size=size).map( + lambda values: cast("JSONValue", tuple(values)) + ) + + +def _raw_bytes_fill_values(configuration: Mapping[str, JSONValue]) -> st.SearchStrategy[JSONValue]: + """One byte value for each 8 bits of the size.""" + return _byte_values(cast("RawBytesConfiguration", configuration)["bits"] // 8) + + +def _base64(value: bytes) -> JSONValue: + return base64.b64encode(value).decode("ascii") + + +def _complex_fill_values(width: FloatWidth) -> st.SearchStrategy[JSONValue]: + """A pair of float fill values of `width` bits, real then imaginary.""" + component = _float_fill_values(width) + return st.tuples(component, component).map(lambda pair: cast("JSONValue", pair)) + + +_BYTES_FILL_VALUES: Final = st.integers(0, _MAX_BYTES).flatmap(_byte_values) | st.binary( + max_size=_MAX_BYTES +).map(_base64) +"""A `bytes` fill value: byte values, or the bytes in standard-alphabet base64.""" + +_LENGTH_BYTES: Final = st.integers(1, _MAX_BYTES) +"""The `length_bytes` of zarr-python's fixed-length byte types.""" + + +def _base64_fill_values( + configuration: Mapping[str, JSONValue], +) -> st.SearchStrategy[JSONValue]: + """At most `length_bytes` bytes in standard-alphabet base64, as zarr-python's fixed-length byte types take them.""" + length_bytes = cast("ZarrPythonRawBytesConfiguration", configuration)["length_bytes"] + return st.binary(max_size=length_bytes).map(_base64) + + +_TEXT: Final = st.text(st.characters(exclude_categories=("Cs",)), max_size=16) +"""Text a JSON document can hold: no lone surrogate, which is no Unicode scalar value.""" + +_MAX_CODE_POINTS: Final = 16 +"""The most code points a drawn `fixed_length_utf32` value holds.""" + + +def _fixed_length_utf32_fill_values( + configuration: Mapping[str, JSONValue], +) -> st.SearchStrategy[JSONValue]: + """Text of at most one code point for each 4 bytes of the size, some of them trailing `U+0000` padding.""" + capacity = cast("FixedLengthUtf32Configuration", configuration)["length_bytes"] // 4 + return st.text(st.characters(exclude_categories=("Cs",)) | st.just("\x00"), max_size=capacity) + + +_NUMPY_TIME: Final = DataTypeStrategy( + configurations=st.fixed_dictionaries( + { + "unit": st.sampled_from(NUMPY_TIME_UNIT), + "scale_factor": st.integers(1, NUMPY_TIME_MAX_SCALE_FACTOR), + } + ), + fill_values=lambda _configuration: st.integers(-(2**63), 2**63 - 1) | st.just("NaT"), +) +"""Both numpy time types: a unit, and how many of it one tick is; a signed 64-bit count of ticks, or `"NaT"`.""" + +DATA_TYPE_STRATEGIES: Final[Mapping[DataTypeDefinition[Any], DataTypeStrategy]] = { + BOOL_DATA_TYPE: _fill_values_of(st.booleans()), + **{ + definition: _fill_values_of(st.integers(low, high)) + for definition, (low, high) in INTEGER_RANGES.items() + }, + **{ + definition: _fill_values_of(_float_fill_values(width)) + for definition, width in FLOAT_WIDTHS.items() + }, + **{ + definition: _fill_values_of(_complex_fill_values(width)) + for definition, width in COMPLEX_WIDTHS.items() + }, + RAW_BYTES_DATA_TYPE: DataTypeStrategy( + configurations=st.integers(1, _MAX_RAW_BYTES).map( + lambda size: RawBytesConfiguration(bits=8 * size) + ), + fill_values=_raw_bytes_fill_values, + ), + BYTES_DATA_TYPE: _fill_values_of(_BYTES_FILL_VALUES), + STRING_DATA_TYPE: _fill_values_of(_TEXT), + NULL_TERMINATED_BYTES_DATA_TYPE: DataTypeStrategy( + configurations=_LENGTH_BYTES.map( + lambda length_bytes: NullTerminatedBytesConfiguration(length_bytes=length_bytes) + ), + fill_values=_base64_fill_values, + ), + VARIABLE_LENGTH_BYTES_DATA_TYPE: _fill_values_of(st.binary(max_size=_MAX_BYTES).map(_base64)), + ZARR_PYTHON_RAW_BYTES_DATA_TYPE: DataTypeStrategy( + configurations=_LENGTH_BYTES.map( + lambda length_bytes: ZarrPythonRawBytesConfiguration(length_bytes=length_bytes) + ), + fill_values=_base64_fill_values, + ), + FIXED_LENGTH_UTF32_DATA_TYPE: DataTypeStrategy( + configurations=st.integers(1, _MAX_CODE_POINTS).map( + lambda count: FixedLengthUtf32Configuration(length_bytes=4 * count) + ), + fill_values=_fixed_length_utf32_fill_values, + ), + NUMPY_DATETIME64_DATA_TYPE: _NUMPY_TIME, + NUMPY_TIMEDELTA64_DATA_TYPE: _NUMPY_TIME, +} +"""A strategy for each data type that holds no other field.""" + +__all__ = [ + "COMPLEX_WIDTHS", + "DATA_TYPE_STRATEGIES", + "FLOAT_WIDTHS", + "INTEGER_RANGES", + "FloatWidth", + "finite_floats", + "float_spellings", +] diff --git a/packages/zarr-metadata/src/zarr_metadata/testing/_strategy.py b/packages/zarr-metadata/src/zarr_metadata/testing/_strategy.py new file mode 100644 index 0000000000..73de0f8f5f --- /dev/null +++ b/packages/zarr-metadata/src/zarr_metadata/testing/_strategy.py @@ -0,0 +1,147 @@ +"""What draws one definition's configurations: a record for each kind of definition. + +Each record holds what its kind's configurations depend on, the way +each kind of definition does: a data type's fill values depend on its +configuration, a chunk grid's on the shape of the array, a codec's on +the chunk it is handed. A chunk key encoding's depend on nothing. +""" + +from __future__ import annotations + +from abc import ABC, abstractmethod +from collections.abc import Callable, Mapping +from dataclasses import dataclass +from typing import TYPE_CHECKING, Any, ClassVar, Final, TypeAlias, cast + +from hypothesis import strategies as st + +from zarr_metadata.definition._definition import EmptyConfiguration +from zarr_metadata.v3._definition import ( + Chunk, + ChunkGridDefinition, + ChunkKeyEncodingDefinition, + CodecDefinition, + DataTypeDefinition, +) + +if TYPE_CHECKING: + from hypothesis.strategies import SearchStrategy + + from zarr_metadata._common import JSONValue + from zarr_metadata.definition._definition import Definition + +Configurations: TypeAlias = "SearchStrategy[Mapping[str, JSONValue]]" +"""Configurations of one definition, each the mapping every configuration is.""" + +NOTHING_KNOWN: Final = Chunk() +"""A chunk nothing is known of, which every codec takes.""" + +MAX_RANK: Final = 5 +"""The most axes drawn where the number of axes is not known.""" + +EMPTY_CONFIGURATIONS: Final = st.builds(EmptyConfiguration) +"""A fresh empty configuration for each draw, so a test that changes one changes no other.""" + + +class DefinitionStrategy(ABC): + """How to draw the configurations of one kind of definition.""" + + __slots__ = () + + kind: ClassVar[type[Definition[Any]]] + """The kind of definition whose configurations it draws.""" + + @abstractmethod + def unconstrained(self) -> Configurations: + """Configurations when nothing is known of what they depend on: what the definition's rules accept.""" + + +@dataclass(frozen=True, kw_only=True, slots=True) +class DataTypeStrategy(DefinitionStrategy): + """How to draw a data type's configurations, and the fill values an array of one takes.""" + + kind: ClassVar[type[Definition[Any]]] = DataTypeDefinition + + configurations: Configurations + """Configurations its definition accepts.""" + fill_values: Callable[[Mapping[str, JSONValue]], SearchStrategy[JSONValue]] + """Fill values of the data type a configuration describes.""" + + def unconstrained(self) -> Configurations: + return self.configurations + + +@dataclass(frozen=True, kw_only=True, slots=True) +class ChunkGridStrategy(DefinitionStrategy): + """How to draw a chunk grid's configurations over an array of a shape.""" + + kind: ClassVar[type[Definition[Any]]] = ChunkGridDefinition + + configurations: Callable[[tuple[int, ...] | None], Configurations] + """Configurations of grids that fit an array of the shape. + + Handed None, no shape is known, and what is drawn is what the grid's + rules accept, whatever shape it is put over. + """ + + def unconstrained(self) -> Configurations: + return self.configurations(None) + + +@dataclass(frozen=True, kw_only=True, slots=True) +class ChunkKeyEncodingStrategy(DefinitionStrategy): + """How to draw a chunk key encoding's configurations.""" + + kind: ClassVar[type[Definition[Any]]] = ChunkKeyEncodingDefinition + + configurations: Configurations + """Configurations its definition accepts.""" + + def unconstrained(self) -> Configurations: + return self.configurations + + +@dataclass(frozen=True, kw_only=True, slots=True) +class CodecStrategy(DefinitionStrategy): + """How to draw a codec's configurations, handed a chunk. + + Every codec takes `Chunk()`, a chunk nothing is known of, so a + strategy that draws nothing for it is refused when it is built. + """ + + kind: ClassVar[type[Definition[Any]]] = CodecDefinition + + configurations: Callable[[Chunk], Configurations | None] + """Configurations of the codec handed the chunk; None when no configuration takes it. + + A `bytes` codec handed a `string` chunk is None, since the spec gives + that data type a codec of its own. + """ + + def __post_init__(self) -> None: + if self.configurations(NOTHING_KNOWN) is None: + msg = "a codec strategy draws configurations for Chunk(), a chunk nothing is known of; got None" + raise ValueError(msg) + + def unconstrained(self) -> Configurations: + return cast("Configurations", self.configurations(NOTHING_KNOWN)) + + +Strategies: TypeAlias = Mapping[Any, DefinitionStrategy] +"""How to draw each definition's configurations, by the definition. + +The key is `Any`, not `Definition[Any]`: a mapping's key type is +invariant, so `{MY_CODEC: CodecStrategy(...)}`, a mapping keyed by a +`CodecDefinition`, would not be a `Mapping[Definition[Any], ...]`. +""" + +__all__ = [ + "NOTHING_KNOWN", + "ChunkGridStrategy", + "ChunkKeyEncodingStrategy", + "CodecStrategy", + "Configurations", + "DataTypeStrategy", + "DefinitionStrategy", + "Strategies", +] diff --git a/packages/zarr-metadata/src/zarr_metadata/testing/strategies.py b/packages/zarr-metadata/src/zarr_metadata/testing/strategies.py new file mode 100644 index 0000000000..8491c9c171 --- /dev/null +++ b/packages/zarr-metadata/src/zarr_metadata/testing/strategies.py @@ -0,0 +1,299 @@ +"""Hypothesis strategies for valid Zarr v3 metadata fields. + +Every strategy here draws metadata its definition accepts, and that the +spec allows: a reader, writer or converter can test itself against the +whole space of a codec's, data type's, chunk grid's or chunk key +encoding's configurations instead of the handful its authors thought to +write down. + +```python +from hypothesis import given +from zarr_metadata.testing.strategies import fields +from zarr_metadata.v3.codec.blosc import BLOSC_CODEC + +@given(fields(BLOSC_CODEC)) +def test_my_reader_reads_every_blosc_codec(field): + my_reader.read_codec(field) +``` + +What a configuration depends on, each kind's function takes: +`fill_values` a data type's configuration, `chunk_grid_configurations` +the shape of the array, `codec_configurations` the `Chunk` the codec is +handed. Left out, nothing is known of it, and what is drawn is what the +definition's rules accept. `fields` wraps configurations in the +envelopes a document writes them in. + +How each definition's configurations are drawn is a value, +`Strategies`, keyed by the definition: `DEFAULT_STRATEGIES` holds one +for each definition this package defines that holds no other field. An +extension author adds their own with +`DEFAULT_STRATEGIES | {MY_CODEC: CodecStrategy(...)}`. + +Configurations are drawn at the data level the TypedDicts declare, JSON +arrays as tuples; `json.dumps` takes them as they are. + +Requires the `testing` extra (`pip install zarr-metadata[testing]`). +""" + +from __future__ import annotations + +from functools import partial +from types import MappingProxyType +from typing import TYPE_CHECKING, Any, Final, Literal, TypeVar, cast + +from hypothesis import strategies as st + +from zarr_metadata._sentinel import UNSET +from zarr_metadata.definition._definition import Definition, as_kind, canonicalize, kind_of +from zarr_metadata.testing._chunk_grid import CHUNK_GRID_STRATEGIES +from zarr_metadata.testing._chunk_key_encoding import CHUNK_KEY_ENCODING_STRATEGIES +from zarr_metadata.testing._codec import CODEC_STRATEGIES +from zarr_metadata.testing._data_type import DATA_TYPE_STRATEGIES +from zarr_metadata.testing._strategy import ( + NOTHING_KNOWN, + ChunkGridStrategy, + ChunkKeyEncodingStrategy, + CodecStrategy, + Configurations, + DataTypeStrategy, + DefinitionStrategy, + Strategies, +) +from zarr_metadata.v3._context import ZarrV3Context +from zarr_metadata.v3._core import CORE_CONTEXT +from zarr_metadata.v3._definition import Chunk, DataTypeDefinition + +if TYPE_CHECKING: + from collections.abc import Mapping + + from hypothesis.strategies import SearchStrategy + + from zarr_metadata._common import JSONValue + from zarr_metadata.v3 import ZarrV3MetadataFieldJSON + from zarr_metadata.v3._context import ZarrV3ContextSlots + from zarr_metadata.v3._definition import ( + ChunkGridDefinition, + ChunkKeyEncodingDefinition, + CodecDefinition, + ) + +C = TypeVar("C") + +Spellings = Literal["all", "canonical"] +"""How fields are written: every spelling the spec allows, or only the simplest. + +`"all"` draws each envelope the spec gives a field -- a bare name, an +object without a configuration, an empty configuration, a +`must_understand` of `true` -- and configurations that say what a +simpler one says, as a `blosc` `typesize` under `noshuffle` does. +`"canonical"` draws each in the spelling `canonicalize` gives it, which +is what a writer that writes the fewest words writes. +""" + +DEFAULT_STRATEGIES: Final[MappingProxyType[Definition[Any], DefinitionStrategy]] = MappingProxyType( + { + **DATA_TYPE_STRATEGIES, + **CHUNK_GRID_STRATEGIES, + **CHUNK_KEY_ENCODING_STRATEGIES, + **CODEC_STRATEGIES, + } +) +"""A strategy for each definition this package defines that holds no other field.""" + + +def _strategy_for(definition: Definition[Any], strategies: Strategies) -> DefinitionStrategy: + """The strategy `strategies` holds for `definition`, which draws configurations of its kind.""" + found = strategies.get(definition) + if found is None: + msg = f"no strategy for {definition.name!r}; pass one in `strategies`" + raise LookupError(msg) + if not isinstance(definition, found.kind): + msg = ( + f"the strategy for {definition.name!r}, a {type(definition).__name__}, draws " + f"configurations of a {found.kind.__name__}: {found!r}" + ) + raise TypeError(msg) + return found + + +def data_type_configurations( + definition: DataTypeDefinition[C], *, strategies: Strategies = DEFAULT_STRATEGIES +) -> SearchStrategy[C]: + """Configurations of the data type `definition` defines.""" + found = cast("DataTypeStrategy", _strategy_for(definition, strategies)) + return cast("SearchStrategy[C]", found.configurations) + + +def fill_values( + definition: DataTypeDefinition[C], + configuration: C, + *, + strategies: Strategies = DEFAULT_STRATEGIES, +) -> SearchStrategy[JSONValue]: + """Fill values of the data type `definition` and `configuration` describe, in every spelling the spec gives them.""" + found = cast("DataTypeStrategy", _strategy_for(definition, strategies)) + return found.fill_values(cast("Mapping[str, JSONValue]", configuration)) + + +def chunk_grid_configurations( + definition: ChunkGridDefinition[C], + shape: tuple[int, ...] | None = None, + *, + strategies: Strategies = DEFAULT_STRATEGIES, +) -> SearchStrategy[C]: + """Configurations of the chunk grid `definition` defines, that fit an array of `shape`. + + With no shape, what is drawn is what the grid's rules accept. + """ + found = cast("ChunkGridStrategy", _strategy_for(definition, strategies)) + return cast("SearchStrategy[C]", found.configurations(shape)) + + +def chunk_key_encoding_configurations( + definition: ChunkKeyEncodingDefinition[C], *, strategies: Strategies = DEFAULT_STRATEGIES +) -> SearchStrategy[C]: + """Configurations of the chunk key encoding `definition` defines.""" + found = cast("ChunkKeyEncodingStrategy", _strategy_for(definition, strategies)) + return cast("SearchStrategy[C]", found.configurations) + + +def codec_configurations( + definition: CodecDefinition[C], + chunk: Chunk = NOTHING_KNOWN, + *, + strategies: Strategies = DEFAULT_STRATEGIES, +) -> SearchStrategy[C] | None: + """Configurations of the codec `definition` defines, handed `chunk`; None when none takes it. + + With no chunk, nothing is known of it, and what is drawn is what the + codec's rules accept. + """ + found = cast("CodecStrategy", _strategy_for(definition, strategies)) + return cast("SearchStrategy[C] | None", found.configurations(chunk)) + + +def _written_bare(definition: Definition[Any]) -> bool: + """Whether the spec has the field written as its bare name: a data type it defines itself. + + "If the data type is defined in this specification, then the value + must be the data type identifier provided as a string" + (https://github.com/zarr-developers/zarr-specs/blob/fc7dd9c9beb5a50b87f9b08b00bf50fc0048482f/docs/v3/core/index.rst#L509-L512). + """ + if not isinstance(definition, DataTypeDefinition): + return False + return CORE_CONTEXT.claimant(DataTypeDefinition, definition.name) is definition + + +@st.composite +def _any_spelling( + draw: st.DrawFn, definition: Definition[Any], configurations: SearchStrategy[Any] +) -> ZarrV3MetadataFieldJSON: + """A configuration of `definition` in one of the envelopes the spec allows it. + + A configuration with nothing in it is written as the bare name, an + object without one, or an object with an empty one; any object may + say `must_understand` is `true`, which is what absence means. A name + that carries the configuration, as `r16` does, and a data type the + spec defines itself, are written as the bare name. + """ + configuration = cast("Mapping[str, JSONValue]", draw(configurations)) + carried = definition._carrying_name(configuration) # pyright: ignore[reportPrivateUsage] + if carried is not None: + return carried + if _written_bare(definition): + return definition.name + empty = len(configuration) == 0 + if empty and draw(st.booleans()): + return definition.name + field: dict[str, JSONValue] = {"name": definition.name} + if not empty or draw(st.booleans()): + field["configuration"] = configuration + if draw(st.booleans()): + field["must_understand"] = True + return cast("ZarrV3MetadataFieldJSON", field) + + +def _scope_of(definition: Definition[Any]) -> ZarrV3Context: + """The scope that claims `definition` and nothing else; a `TypeError` for a definition no slot of a Zarr v3 scope holds.""" + for slot, kind in ZarrV3Context.kinds.items(): + if isinstance(definition, kind): + return ZarrV3Context(**cast("ZarrV3ContextSlots", {slot: (definition,)})) + msg = ( + f"{definition.name!r} is a {type(definition).__name__}, " + "which no slot of a Zarr v3 scope holds" + ) + raise TypeError(msg) + + +def _checked( + field: ZarrV3MetadataFieldJSON, + *, + definition: Definition[Any], + kind: type[Definition[Any]], + scope: ZarrV3Context, + spellings: Spellings, +) -> ZarrV3MetadataFieldJSON: + """`field`, or its canonical spelling, once it reads without a problem in `scope`; a `ValueError` when it does not.""" + canonical, problems = canonicalize(field, kind, scope) + if len(problems) != 0: + msg = f"{field!r}, drawn for {definition.name!r}, which is refused: {problems!r}" + raise ValueError(msg) + if spellings == "canonical": + return cast("ZarrV3MetadataFieldJSON", canonical) + return field + + +def fields( + definition: Definition[C], + configurations: SearchStrategy[C] | UNSET = UNSET, + *, + spellings: Spellings = "all", + strategies: Strategies = DEFAULT_STRATEGIES, +) -> SearchStrategy[ZarrV3MetadataFieldJSON]: + """Metadata fields of `definition`: its name and a configuration, in an envelope a document writes. + + `configurations` is what to draw the configuration from, by default + the definition's own with nothing known of what they depend on: pass + `codec_configurations(definition, chunk)` for a codec handed a chunk, + once it is known not to be None. `spellings` says whether every + spelling the spec allows is drawn, or only the simplest. + + Each field is read in a scope of its definition alone, with no chunk + known, and one its definition's rules refuse, as they refuse one + drawn from `configurations` they do not accept, raises a + `ValueError`. A codec's chunk rules are not checked: what fits the + chunk is what `codec_configurations` draws. + """ + drawn: Configurations | SearchStrategy[C] = ( + _strategy_for(definition, strategies).unconstrained() + if configurations is UNSET + else configurations + ) + checked = partial( + _checked, + definition=definition, + kind=as_kind(kind_of(definition)), + scope=_scope_of(definition), + spellings=spellings, + ) + return _any_spelling(definition, drawn).map(checked) + + +__all__ = [ + "DEFAULT_STRATEGIES", + "NOTHING_KNOWN", + "ChunkGridStrategy", + "ChunkKeyEncodingStrategy", + "CodecStrategy", + "Configurations", + "DataTypeStrategy", + "DefinitionStrategy", + "Spellings", + "Strategies", + "chunk_grid_configurations", + "chunk_key_encoding_configurations", + "codec_configurations", + "data_type_configurations", + "fields", + "fill_values", +] diff --git a/packages/zarr-metadata/src/zarr_metadata/v2/__init__.py b/packages/zarr-metadata/src/zarr_metadata/v2/__init__.py index 741ff5cfb5..af6dda6c22 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v2/__init__.py +++ b/packages/zarr-metadata/src/zarr_metadata/v2/__init__.py @@ -6,12 +6,16 @@ and `data_type`, and the v2 kinds and scope are in `definition`. This module also exports every v2 definition (`GZIP_CODEC_V2`, -`FLOAT_DATA_TYPE_V2`, ...), the scope `CORE_V2`, and the scope class +`FLOAT_DATA_TYPE_V2`, ...), the scope `CORE_V2_CONTEXT`, and the scope class `ZarrV2Context`, so that one import builds or updates a scope. Each is the same object as in its own submodule, where a definition's configuration types and constants are. """ +from typing import TYPE_CHECKING + +from zarr_metadata._renamed import renamed_scope +from zarr_metadata.v2._core import RENAMED_SCOPES_V2 from zarr_metadata.v2.codec.checksum import ( ADLER32_CODEC_V2, CRC32_CODEC_V2, @@ -59,7 +63,7 @@ DATETIME64_DATA_TYPE_V2, TIMEDELTA64_DATA_TYPE_V2, ) -from zarr_metadata.v2.definition import CORE_V2, ZarrV2Context +from zarr_metadata.v2.definition import CORE_V2_CONTEXT, ZarrV2Context __all__ = [ "ADLER32_CODEC_V2", @@ -70,7 +74,7 @@ "BYTES_DATA_TYPE_V2", "BZ2_CODEC_V2", "COMPLEX_DATA_TYPE_V2", - "CORE_V2", + "CORE_V2_CONTEXT", "CRC32C_CODEC_V2", "CRC32_CODEC_V2", "DATETIME64_DATA_TYPE_V2", @@ -98,3 +102,11 @@ "ZSTD_CODEC_V2", "ZarrV2Context", ] + + +if not TYPE_CHECKING: + # Hidden from type checkers, which would otherwise read every missing + # name of this module as a scope. + def __getattr__(name: str) -> object: + """`CORE_V2`, the scope's name before it ended in `_CONTEXT`, with a `DeprecationWarning`.""" + return renamed_scope(__name__, name, RENAMED_SCOPES_V2) diff --git a/packages/zarr-metadata/src/zarr_metadata/v2/_core.py b/packages/zarr-metadata/src/zarr_metadata/v2/_core.py new file mode 100644 index 0000000000..0daaa6c344 --- /dev/null +++ b/packages/zarr-metadata/src/zarr_metadata/v2/_core.py @@ -0,0 +1,17 @@ +"""The scope this package provides for Zarr v2.""" + +from __future__ import annotations + +from typing import Final + +from zarr_metadata.v2._context import ZarrV2Context +from zarr_metadata.v2.codec import CODECS_V2 +from zarr_metadata.v2.data_type import DATA_TYPES_V2 + +CORE_V2_CONTEXT: Final = ZarrV2Context(data_types=DATA_TYPES_V2, codecs=CODECS_V2) +"""The data types that zarr-python 2.x writes and the codecs that numcodecs 0.16 configures.""" + +RENAMED_SCOPES_V2: Final = {"CORE_V2": ("CORE_V2_CONTEXT", CORE_V2_CONTEXT)} +"""The v2 scope's name before it ended in `_CONTEXT`, with the scope's name now and the scope.""" + +__all__ = ["CORE_V2_CONTEXT", "RENAMED_SCOPES_V2"] diff --git a/packages/zarr-metadata/src/zarr_metadata/v2/definition.py b/packages/zarr-metadata/src/zarr_metadata/v2/definition.py index 8cf4990992..d0da681c58 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v2/definition.py +++ b/packages/zarr-metadata/src/zarr_metadata/v2/definition.py @@ -4,10 +4,10 @@ in `compressor` and `filters`. Zarr v2 has no extension registry, but it still has a scope: the data types that zarr-python 2.x writes (one definition per NumPy family), and the codecs that numcodecs 0.16 -configures (one per id that this package models). `CORE_V2` is that +configures (one per id that this package models). `CORE_V2_CONTEXT` is that scope. To read one field in it, call `zarr_metadata.definition.resolve` with the field's kind, as for a v3 field: -`resolve("