diff --git a/pyiceberg/catalog/rest/__init__.py b/pyiceberg/catalog/rest/__init__.py index d284b06234..6fbe44ef70 100644 --- a/pyiceberg/catalog/rest/__init__.py +++ b/pyiceberg/catalog/rest/__init__.py @@ -22,15 +22,17 @@ import time import uuid from collections import deque -from collections.abc import Mapping +from collections.abc import Callable, Mapping from enum import Enum, StrEnum from typing import ( TYPE_CHECKING, + Annotated, Any, + TypeVar, ) from urllib.parse import quote, unquote -from pydantic import ConfigDict, Field, TypeAdapter, field_validator +from pydantic import BeforeValidator, ConfigDict, Field, TypeAdapter, field_validator from requests import HTTPError, PreparedRequest, Response, Session from requests.adapters import DEFAULT_RETRIES, HTTPAdapter from tenacity import RetryCallState, retry, retry_if_exception_type, stop_after_attempt @@ -368,11 +370,23 @@ def _retry_hook(retry_state: RetryCallState) -> None: } +T = TypeVar("T") + + +def _empty_if_none(factory: Callable[[], Any]) -> BeforeValidator: + return BeforeValidator(lambda v: factory() if v is None else v) + + +# A JSON null is read as an empty list / map, like Java's `hasNonNull` +NullableList = Annotated[list[T], _empty_if_none(list)] +NullableProperties = Annotated[Properties, _empty_if_none(dict)] + + class TableResponse(IcebergBaseModel): metadata_location: str | None = Field(alias="metadata-location", default=None) metadata: TableMetadata - config: Properties = Field(default_factory=dict) - storage_credentials: list[StorageCredential] = Field(alias="storage-credentials", default_factory=list) + config: NullableProperties = Field(default_factory=dict) + storage_credentials: NullableList[StorageCredential] = Field(alias="storage-credentials", default_factory=list) labels: Labels = Field(default_factory=Labels) @@ -383,7 +397,7 @@ class LoadCredentialsResponse(IcebergBaseModel): class ViewResponse(IcebergBaseModel): metadata_location: str = Field(..., alias="metadata-location") metadata: ViewMetadata - config: Properties = Field(default_factory=dict) + config: NullableProperties = Field(default_factory=dict) labels: Labels = Field(default_factory=Labels) @@ -436,8 +450,8 @@ class RegisterViewRequest(IcebergBaseModel): class ConfigResponse(IcebergBaseModel): - defaults: Properties | None = Field(default_factory=dict) - overrides: Properties | None = Field(default_factory=dict) + defaults: NullableProperties = Field(default_factory=dict) + overrides: NullableProperties = Field(default_factory=dict) endpoints: set[Endpoint] | None = Field(default=None) idempotency_key_lifetime: str | None = Field(alias="idempotency-key-lifetime", default=None) diff --git a/tests/catalog/test_rest.py b/tests/catalog/test_rest.py index 1160d2889a..18dd3c7d6d 100644 --- a/tests/catalog/test_rest.py +++ b/tests/catalog/test_rest.py @@ -50,6 +50,7 @@ SIGV4_MAX_RETRIES_DEFAULT, SNAPSHOT_LOADING_MODE, Capability, + ConfigResponse, Endpoint, HttpMethod, RestCatalog, @@ -3838,6 +3839,31 @@ def test_load_responses_parse_labels( assert response_type.model_validate({**payload, "labels": invalid}).labels == Labels() +def test_load_responses_with_null_config_and_storage_credentials( + example_table_metadata_with_snapshot_v1_rest_json: dict[str, Any], example_view_metadata_rest_json: dict[str, Any] +) -> None: + # Like Java, an explicit null config reads as an empty config + table_response = TableResponse.model_validate( + {**example_table_metadata_with_snapshot_v1_rest_json, "config": None, "storage-credentials": None} + ) + assert table_response.config == {} + assert table_response.storage_credentials == [] + + view_response = ViewResponse.model_validate({**example_view_metadata_rest_json, "config": None}) + assert view_response.config == {} + + +def test_config_response_with_null_defaults_and_overrides() -> None: + config_response = ConfigResponse.model_validate({"defaults": None, "overrides": None}) + assert config_response.defaults == {} + assert config_response.overrides == {} + + +def test_catalog_with_null_config_defaults_and_overrides(rest_mock: Mocker) -> None: + rest_mock.get(f"{TEST_URI}v1/config", json={"defaults": None, "overrides": None}, status_code=200) + assert RestCatalog("rest", uri=TEST_URI, token=TEST_TOKEN).uri == TEST_URI + + def test_table_labels_from_rest_responses( rest_mock: Mocker, example_table_metadata_with_snapshot_v1_rest_json: dict[str, Any], table_schema_simple: Schema ) -> None: