diff --git a/pyiceberg/table/update/schema.py b/pyiceberg/table/update/schema.py index a0312745d2..72eefda841 100644 --- a/pyiceberg/table/update/schema.py +++ b/pyiceberg/table/update/schema.py @@ -50,7 +50,25 @@ UpdateTableMetadata, ) from pyiceberg.typedef import L, TableVersion -from pyiceberg.types import IcebergType, ListType, MapType, NestedField, PrimitiveType, StructType +from pyiceberg.types import ( + BinaryType, + DecimalType, + DoubleType, + FixedType, + FloatType, + IcebergType, + IntegerType, + ListType, + LongType, + MapType, + NestedField, + PrimitiveType, + StringType, + StructType, + UnknownType, + UUIDType, +) +from pyiceberg.utils.deprecated import deprecation_message if TYPE_CHECKING: import pyarrow as pa @@ -74,6 +92,34 @@ class _Move: other_field_id: int | None = None +def _is_valid_promotion(from_type: IcebergType, to_type: IcebergType) -> bool: + """Check a column type change against the spec's valid primitive type promotions. + + Do not use promote() here: it is the read-side rule and also accepts string <-> binary and fixed(16) -> uuid. + """ + if from_type == to_type: + return True + if isinstance(from_type, UnknownType): + # Unknown can be promoted to any primitive type in v3+ + return isinstance(to_type, PrimitiveType) + if isinstance(from_type, IntegerType): + return isinstance(to_type, LongType) + if isinstance(from_type, FloatType): + return isinstance(to_type, DoubleType) + if isinstance(from_type, DecimalType) and isinstance(to_type, DecimalType): + return from_type.scale == to_type.scale and from_type.precision <= to_type.precision + return False + + +def _is_deprecated_promotion(from_type: IcebergType, to_type: IcebergType) -> bool: + """Check for a non-spec type change that update_column still applies, with a deprecation warning.""" + if isinstance(from_type, StringType) and isinstance(to_type, BinaryType): + return True + if isinstance(from_type, BinaryType) and isinstance(to_type, StringType): + return True + return isinstance(from_type, FixedType) and len(from_type) == 16 and isinstance(to_type, UUIDType) + + class UpdateSchema(UpdateTableMetadata["UpdateSchema"]): _schema: Schema _last_column_id: itertools.count[int] @@ -470,11 +516,16 @@ def update_column( if not field.field_type.is_primitive: raise ValidationError(f"Cannot change column type: {field.field_type} is not a primitive") - if not self._allow_incompatible_changes and field.field_type != field_type: - try: - promote(field.field_type, field_type) - except ResolveError as e: - raise ValidationError(f"Cannot change column type: {full_name}: {field.field_type} -> {field_type}") from e + if not self._allow_incompatible_changes and not _is_valid_promotion(field.field_type, field_type): + if not _is_deprecated_promotion(field.field_type, field_type): + raise ValidationError(f"Cannot change column type: {full_name}: {field.field_type} -> {field_type}") + deprecation_message( + deprecated_in="0.13.0", + removed_in="0.14.0", + help_message=f"Column {full_name} changes from {field.field_type} to {field_type}, " + "which is not a valid type promotion. Pass allow_incompatible_changes=True to update_schema() " + "to keep this change.", + ) # if other updates for the same field exist in one transaction: if updated := self._updates.get(field.field_id): diff --git a/tests/integration/test_rest_schema.py b/tests/integration/test_rest_schema.py index 3585ac77bc..21d38071e1 100644 --- a/tests/integration/test_rest_schema.py +++ b/tests/integration/test_rest_schema.py @@ -15,6 +15,7 @@ # specific language governing permissions and limitations # under the License. # pylint:disable=redefined-outer-name +from contextlib import nullcontext from datetime import UTC, date, datetime, time from decimal import Decimal from typing import Any @@ -597,15 +598,31 @@ def test_update_types_case_insensitive(catalog: Catalog) -> None: ) -allowed_promotions = [ +# update_column still applies these with a deprecation warning +deprecated_promotions = [ (StringType(), BinaryType()), (BinaryType(), StringType()), +] + +allowed_promotions = [ + *deprecated_promotions, (IntegerType(), LongType()), (FloatType(), DoubleType()), (DecimalType(9, 2), DecimalType(18, 2)), ] +def _update_bar_type(tbl: Table, from_type: PrimitiveType, to_type: PrimitiveType) -> None: + expect_warning = ( + pytest.warns(DeprecationWarning, match=r"Deprecated in 0\.13\.0, will be removed in 0\.14\.0") + if (from_type, to_type) in deprecated_promotions + else nullcontext() + ) + with expect_warning: + with tbl.update_schema() as schema_update: + schema_update.update_column("bar", to_type) + + @pytest.mark.parametrize("from_type, to_type", allowed_promotions, ids=str) @pytest.mark.integration def test_allowed_updates(from_type: PrimitiveType, to_type: PrimitiveType, catalog: Catalog) -> None: @@ -616,8 +633,7 @@ def test_allowed_updates(from_type: PrimitiveType, to_type: PrimitiveType, catal ), ) - with tbl.update_schema() as schema_update: - schema_update.update_column("bar", to_type) + _update_bar_type(tbl, from_type, to_type) assert tbl.schema() == Schema(NestedField(field_id=1, name="bar", field_type=to_type, required=True)) @@ -663,8 +679,7 @@ def test_disallowed_updates(from_type: PrimitiveType, to_type: PrimitiveType, ca assert str(exc_info.value).startswith("Cannot change column type: bar:") else: - with tbl.update_schema() as schema_update: - schema_update.update_column("bar", to_type) + _update_bar_type(tbl, from_type, to_type) assert tbl.schema() == Schema( NestedField(field_id=1, name="bar", field_type=to_type, required=True), @@ -1591,8 +1606,9 @@ def test_add_field_to_map_key(nested_table: Table) -> None: @pytest.mark.integration def test_alter_map_key(nested_table: Table) -> None: + # The key change is not a valid promotion, so opt in to reach the map key check with pytest.raises(ValueError) as exc_info: - with nested_table.update_schema() as schema_update: + with nested_table.update_schema(allow_incompatible_changes=True) as schema_update: schema_update.update_column(("quux", "key"), BinaryType()) assert "Cannot update map keys" in str(exc_info.value) diff --git a/tests/test_schema.py b/tests/test_schema.py index 15bac3bf51..cd9d6e860e 100644 --- a/tests/test_schema.py +++ b/tests/test_schema.py @@ -15,12 +15,14 @@ # specific language governing permissions and limitations # under the License. +import warnings from textwrap import dedent from typing import Any import pyarrow as pa import pytest +from pyiceberg.catalog import Catalog from pyiceberg.exceptions import ResolveError, ValidationError from pyiceberg.schema import ( Accessor, @@ -1328,6 +1330,147 @@ def test_detect_invalid_promotion_string_to_float(table_v2: Table) -> None: _ = UpdateSchema(transaction=Transaction(table_v2), schema=current_schema).union_by_name(new_schema)._apply() +def test_update_column_rejects_invalid_promotions(catalog: Catalog) -> None: + catalog.create_namespace("default") + identifier = f"default.update_column_invalid_{catalog.name}" + tbl = catalog.create_table( + identifier, + Schema( + NestedField(field_id=1, name="name", field_type=StringType(), required=False), + NestedField(field_id=2, name="qty", field_type=IntegerType(), required=False), + NestedField(field_id=3, name="price", field_type=DecimalType(9, 2), required=False), + NestedField(field_id=4, name="key", field_type=FixedType(15), required=False), + NestedField(field_id=5, name="ratio", field_type=FloatType(), required=False), + ), + ) + metadata_location = tbl.metadata_location + + # Not valid promotions in the spec, so the update is rejected and the table is not changed + rejected = [ + ("qty", FloatType(), "int -> float"), + ("price", DecimalType(9, 3), "decimal(9, 2) -> decimal(9, 3)"), + ("price", DecimalType(5, 2), "decimal(9, 2) -> decimal(5, 2)"), + ("key", UUIDType(), "fixed[15] -> uuid"), + ("ratio", IntegerType(), "float -> int"), + ] + for name, new_type, promotion in rejected: + with pytest.raises(ValidationError) as exc_info: + tbl.update_schema().update_column(name, field_type=new_type).commit() + assert str(exc_info.value) == f"Cannot change column type: {name}: {promotion}" + + assert catalog.load_table(identifier).metadata_location == metadata_location + + # Valid promotions in the spec, and a type that does not change + for name, new_type in [("name", StringType()), ("qty", LongType()), ("price", DecimalType(18, 2)), ("ratio", DoubleType())]: + tbl.update_schema().update_column(name, field_type=new_type).commit() + + assert catalog.load_table(identifier).schema() == Schema( + NestedField(field_id=1, name="name", field_type=StringType(), required=False), + NestedField(field_id=2, name="qty", field_type=LongType(), required=False), + NestedField(field_id=3, name="price", field_type=DecimalType(18, 2), required=False), + NestedField(field_id=4, name="key", field_type=FixedType(15), required=False), + NestedField(field_id=5, name="ratio", field_type=DoubleType(), required=False), + ) + + +def test_update_column_warns_for_non_spec_promotions(catalog: Catalog) -> None: + catalog.create_namespace("default") + identifier = f"default.update_column_deprecated_{catalog.name}" + tbl = catalog.create_table( + identifier, + Schema( + NestedField(field_id=1, name="name", field_type=StringType(), required=False), + NestedField(field_id=2, name="blob", field_type=BinaryType(), required=False), + NestedField(field_id=3, name="key", field_type=FixedType(16), required=False), + ), + ) + + # Not in the spec, but still applied with a deprecation warning + for name, old_type, new_type in [ + ("name", StringType(), BinaryType()), + ("blob", BinaryType(), StringType()), + ("key", FixedType(16), UUIDType()), + ]: + with pytest.warns(DeprecationWarning) as record: + tbl.update_schema().update_column(name, field_type=new_type).commit() + assert [str(w.message) for w in record if issubclass(w.category, DeprecationWarning)] == [ + f"Deprecated in 0.13.0, will be removed in 0.14.0. Column {name} changes from {old_type} to {new_type}, " + "which is not a valid type promotion. Pass allow_incompatible_changes=True to update_schema() to keep this change." + ] + + assert catalog.load_table(identifier).schema() == Schema( + NestedField(field_id=1, name="name", field_type=BinaryType(), required=False), + NestedField(field_id=2, name="blob", field_type=StringType(), required=False), + NestedField(field_id=3, name="key", field_type=UUIDType(), required=False), + ) + + +def test_update_column_no_warning_with_allow_incompatible_changes(catalog: Catalog) -> None: + catalog.create_namespace("default") + identifier = f"default.update_column_opt_in_{catalog.name}" + catalog.create_table( + identifier, + Schema( + NestedField(field_id=1, name="name", field_type=StringType(), required=False), + NestedField(field_id=2, name="blob", field_type=BinaryType(), required=False), + NestedField(field_id=3, name="key", field_type=FixedType(16), required=False), + ), + ) + + # The explicit opt-in applies the same changes without a warning + for name, new_type in [("name", BinaryType()), ("blob", StringType()), ("key", UUIDType())]: + with warnings.catch_warnings(record=True) as record: + warnings.simplefilter("always") + catalog.load_table(identifier).update_schema(allow_incompatible_changes=True).update_column( + name, field_type=new_type + ).commit() + assert not [w for w in record if issubclass(w.category, DeprecationWarning)] + + assert catalog.load_table(identifier).schema() == Schema( + NestedField(field_id=1, name="name", field_type=BinaryType(), required=False), + NestedField(field_id=2, name="blob", field_type=StringType(), required=False), + NestedField(field_id=3, name="key", field_type=UUIDType(), required=False), + ) + + +def test_update_column_promotes_unknown_to_primitive_only(table_v3: Table) -> None: + current_schema = Schema(NestedField(field_id=1, name="aCol", field_type=UnknownType(), required=False)) + + applied = ( + UpdateSchema(transaction=Transaction(table_v3), schema=current_schema) + .update_column("aCol", field_type=StringType()) + ._apply() + ) + assert applied.find_field("aCol").field_type == StringType() + + with pytest.raises(ValidationError) as exc_info: + UpdateSchema(transaction=Transaction(table_v3), schema=current_schema).update_column( + "aCol", field_type=ListType(element_id=2, element_type=StringType(), element_required=False) + ) + assert str(exc_info.value) == "Cannot change column type: aCol: unknown -> list" + + +def test_union_by_name_fixed_uuid_and_string_binary(table_v2: Table) -> None: + fixed_list = ListType(element_id=2, element_type=FixedType(16), element_required=False) + uuid_list = ListType(element_id=2, element_type=UUIDType(), element_required=False) + # An incoming UUID over an existing fixed(16) field warns, also for a list element. The other + # pairs keep the existing type without a warning. + for current_type, new_type, applied_type, warns in [ + (FixedType(16), UUIDType(), UUIDType(), True), + (UUIDType(), FixedType(16), UUIDType(), False), + (StringType(), BinaryType(), StringType(), False), + (BinaryType(), StringType(), BinaryType(), False), + (fixed_list, uuid_list, uuid_list, True), + ]: + current_schema = Schema(NestedField(field_id=1, name="aCol", field_type=current_type, required=False)) + new_schema = Schema(NestedField(field_id=1, name="aCol", field_type=new_type, required=False)) + with warnings.catch_warnings(record=True) as record: + warnings.simplefilter("always") + applied = UpdateSchema(transaction=Transaction(table_v2), schema=current_schema).union_by_name(new_schema)._apply() + assert bool([w for w in record if issubclass(w.category, DeprecationWarning)]) == warns + assert applied.find_field("aCol").field_type == applied_type + + # decimal(P,S) Fixed-point decimal; precision P, scale S -> Scale is fixed [1], # precision must be 38 or less def test_type_promote_decimal_to_fixed_scale_with_wider_precision(table_v2: Table) -> None: