From 7b2ad7f793eaf873ae6da555dc9c6d6105fac1e0 Mon Sep 17 00:00:00 2001 From: rahulsmahadev Date: Mon, 16 Nov 2026 18:22:54 +0000 Subject: [PATCH 1/2] feat: write gzip metadata files for write.metadata.compression-codec Metastore catalogs wrote table metadata as .metadata.json even when write.metadata.compression-codec was gzip. Name new metadata files .gz.metadata.json for gzip, as Java does. None stays the default, and other values raise only when metadata is written, so no-op commits on existing tables keep working. Co-authored-by: Isaac --- mkdocs/docs/configuration.md | 1 + pyiceberg/catalog/__init__.py | 9 ++- pyiceberg/table/__init__.py | 3 + pyiceberg/table/locations.py | 11 ++- tests/catalog/test_catalog_behaviors.py | 100 ++++++++++++++++++++++++ tests/table/test_locations.py | 26 ++++++ 6 files changed, 145 insertions(+), 5 deletions(-) diff --git a/mkdocs/docs/configuration.md b/mkdocs/docs/configuration.md index 14cb9c7467..8a9f644092 100644 --- a/mkdocs/docs/configuration.md +++ b/mkdocs/docs/configuration.md @@ -93,6 +93,7 @@ Iceberg tables support table properties to configure table behavior. | `write.summary.partition-limit` | Integer | 0 | Maximum number of changed partitions for which per-partition summaries are included in the snapshot summary. The default of 0 disables them | | `write.metadata.previous-versions-max` | Integer | 100 | The max number of previous version metadata files to keep before deleting after commit. | | `write.metadata.delete-after-commit.enabled` | Boolean | False | Whether to automatically delete old *tracked* metadata files after each table commit. It will retain a number of the most recent metadata files, which can be set using property `write.metadata.previous-versions-max`. | +| `write.metadata.compression-codec` | `{none,gzip}` | none | Compression for the table metadata files that PyIceberg writes. `gzip` names new metadata files `*.gz.metadata.json` and gzips them. REST catalogs are unaffected, since the server chooses the metadata file location. | | `write.object-storage.enabled` | Boolean | False | Enables the [`ObjectStoreLocationProvider`](configuration.md#object-store-location-provider) that adds a hash component to file paths. | | `write.object-storage.partitioned-paths` | Boolean | True | Controls whether [partition values are included in file paths](configuration.md#partition-exclusion) when object storage is enabled | | `write.py-location-provider.impl` | String of form `module.ClassName` | null | Optional, [custom `LocationProvider`](configuration.md#loading-a-custom-location-provider) implementation | diff --git a/pyiceberg/catalog/__init__.py b/pyiceberg/catalog/__init__.py index ce0902f338..1c063cb526 100644 --- a/pyiceberg/catalog/__init__.py +++ b/pyiceberg/catalog/__init__.py @@ -1117,9 +1117,12 @@ def _update_and_stage_table( metadata_location=current_table.metadata_location if current_table else None, ) - new_metadata_version = self._parse_metadata_version(current_table.metadata_location) + 1 if current_table else 0 - provider = load_location_provider(updated_metadata.location, updated_metadata.properties) - new_metadata_location = provider.new_table_metadata_file_location(new_metadata_version) + if current_table and updated_metadata == current_table.metadata: + new_metadata_location = current_table.metadata_location + else: + new_metadata_version = self._parse_metadata_version(current_table.metadata_location) + 1 if current_table else 0 + provider = load_location_provider(updated_metadata.location, updated_metadata.properties) + new_metadata_location = provider.new_table_metadata_file_location(new_metadata_version) return StagedTable( identifier=table_identifier, diff --git a/pyiceberg/table/__init__.py b/pyiceberg/table/__init__.py index 9ef7519865..f056e5ae6b 100644 --- a/pyiceberg/table/__init__.py +++ b/pyiceberg/table/__init__.py @@ -227,6 +227,9 @@ class TableProperties: MANIFEST_MERGE_ENABLED = "commit.manifest-merge.enabled" MANIFEST_MERGE_ENABLED_DEFAULT = False + METADATA_COMPRESSION = "write.metadata.compression-codec" + METADATA_COMPRESSION_DEFAULT = "none" + METADATA_PREVIOUS_VERSIONS_MAX = "write.metadata.previous-versions-max" METADATA_PREVIOUS_VERSIONS_MAX_DEFAULT = 100 diff --git a/pyiceberg/table/locations.py b/pyiceberg/table/locations.py index 771c6b5a0f..5fac1aa62c 100644 --- a/pyiceberg/table/locations.py +++ b/pyiceberg/table/locations.py @@ -80,12 +80,19 @@ def new_table_metadata_file_location(self, new_version: int = 0) -> str: str: fully-qualified URI for the new table metadata file. Raises: - ValueError: If the version is negative. + ValueError: If the version is negative, or the metadata compression codec is not supported. """ + from pyiceberg.table import TableProperties + if new_version < 0: raise ValueError(f"Table metadata version: `{new_version}` must be a non-negative integer") - file_name = f"{new_version:05d}-{uuid.uuid4()}.metadata.json" + codec = self.table_properties.get(TableProperties.METADATA_COMPRESSION, TableProperties.METADATA_COMPRESSION_DEFAULT) + extension = {"none": "", "gzip": ".gz"}.get(codec.lower()) + if extension is None: + raise ValueError(f"Invalid {TableProperties.METADATA_COMPRESSION}: {codec}, expected none or gzip") + + file_name = f"{new_version:05d}-{uuid.uuid4()}{extension}.metadata.json" return self.new_metadata_location(file_name) def new_metadata_location(self, metadata_file_name: str) -> str: diff --git a/tests/catalog/test_catalog_behaviors.py b/tests/catalog/test_catalog_behaviors.py index cd817512b4..7f99dcacab 100644 --- a/tests/catalog/test_catalog_behaviors.py +++ b/tests/catalog/test_catalog_behaviors.py @@ -834,6 +834,106 @@ def test_table_metadata_writes_reflect_latest_path( assert table.location_provider().new_metadata_location("metadata.json") == f"{new_metadata_path}/metadata.json" +def test_table_writes_gzip_metadata_when_compression_codec_is_set( + catalog: Catalog, + test_table_identifier: Identifier, + table_schema_simple: Schema, +) -> None: + namespace = Catalog.namespace_from(test_table_identifier) + catalog.create_namespace(namespace) + table = catalog.create_table( + identifier=test_table_identifier, + schema=table_schema_simple, + properties={TableProperties.METADATA_COMPRESSION: "gzip"}, + ) + assert table.metadata_location.endswith(".gz.metadata.json") + with table.io.new_input(table.metadata_location).open() as stream: + assert stream.read(2) == b"\x1f\x8b" # gzip magic number + + df = pa.Table.from_pydict( + {"foo": ["a"], "bar": [1], "baz": [True]}, + schema=schema_to_pyarrow(table_schema_simple), + ) + table.append(df) + + # a later commit keeps the codec, and the version is parsed from the gzip file name + assert table.metadata_location.endswith(".gz.metadata.json") + assert table.metadata_location.split("/")[-1].startswith("00001-") + loaded_table = catalog.load_table(test_table_identifier) + assert loaded_table.metadata_location == table.metadata_location + assert loaded_table.metadata == table.metadata + + +def test_table_metadata_compression_codec_default_and_set_on_commit( + catalog: Catalog, + test_table_identifier: Identifier, + table_schema_simple: Schema, +) -> None: + namespace = Catalog.namespace_from(test_table_identifier) + catalog.create_namespace(namespace) + table = catalog.create_table(identifier=test_table_identifier, schema=table_schema_simple) + assert table.metadata_location.endswith(".metadata.json") + assert not table.metadata_location.endswith(".gz.metadata.json") + + # the codec set in a commit applies to the metadata file that the commit writes + table.transaction().set_properties({TableProperties.METADATA_COMPRESSION: "gzip"}).commit_transaction() + assert table.metadata_location.endswith(".gz.metadata.json") + assert catalog.load_table(test_table_identifier).metadata_location == table.metadata_location + + +def test_table_metadata_invalid_compression_codec_raises( + catalog: Catalog, + test_table_identifier: Identifier, + table_schema_simple: Schema, +) -> None: + namespace = Catalog.namespace_from(test_table_identifier) + catalog.create_namespace(namespace) + with pytest.raises(ValueError, match="snappy"): + catalog.create_table( + identifier=test_table_identifier, + schema=table_schema_simple, + properties={TableProperties.METADATA_COMPRESSION: "snappy"}, + ) + assert not catalog.table_exists(test_table_identifier) + + table = catalog.create_table(identifier=test_table_identifier, schema=table_schema_simple) + with pytest.raises(ValueError, match="snappy"): + table.transaction().set_properties({TableProperties.METADATA_COMPRESSION: "snappy"}).commit_transaction() + assert catalog.load_table(test_table_identifier).metadata_location == table.metadata_location + + +def test_table_metadata_legacy_invalid_compression_codec_on_commit( + catalog: Catalog, + test_table_identifier: Identifier, + metadata_location: str, + tmp_path: Path, +) -> None: + # metadata registered with a codec this version rejects: commits without changes still work + # the table location is local, so the commit that removes the codec can write its metadata file + metadata = json.loads(Path(metadata_location).read_text()) + metadata["location"] = f"file://{tmp_path}/table" + metadata.setdefault("properties", {})[TableProperties.METADATA_COMPRESSION] = "snappy" + legacy_location = str(tmp_path / "table" / "metadata" / "00000-6c97e413-d51b-4538-ac70-12fe2a85cb83.metadata.json") + Path(legacy_location).parent.mkdir(parents=True) + Path(legacy_location).write_text(json.dumps(metadata)) + namespace = Catalog.namespace_from(test_table_identifier) + catalog.create_namespace(namespace) + table = catalog.register_table(test_table_identifier, legacy_location) + + table.transaction().set_properties({}).commit_transaction() + assert table.metadata_location == legacy_location + + # a change writes new metadata, so the unsupported codec raises + with pytest.raises(ValueError, match="snappy"): + table.transaction().set_properties({"owner": "test"}).commit_transaction() + assert catalog.load_table(test_table_identifier).metadata_location == legacy_location + + # removing the property writes new metadata with the default codec + table.transaction().remove_properties(TableProperties.METADATA_COMPRESSION).commit_transaction() + assert table.metadata_location != legacy_location + assert table.metadata_location.endswith(".metadata.json") + + @pytest.mark.parametrize("format_version", [1, 2]) def test_write_and_evolve(catalog: Catalog, format_version: int) -> None: identifier = f"default.arrow_write_data_and_evolve_schema_v{format_version}" diff --git a/tests/table/test_locations.py b/tests/table/test_locations.py index 6dd7520f3c..9dd465ad50 100644 --- a/tests/table/test_locations.py +++ b/tests/table/test_locations.py @@ -182,6 +182,32 @@ def test_location_provider_metadata_location_with_custom_path() -> None: assert provider.new_metadata_location("metadata.json") == "s3://table-location/custom/path/metadata.json" +def test_new_table_metadata_file_location_compression_codec() -> None: + for codec, extension in [("none", ""), ("NONE", ""), ("gzip", ".gz"), ("GZIP", ".gz")]: + provider = load_location_provider( + table_location="table_location", + table_properties={TableProperties.METADATA_COMPRESSION: codec}, + ) + + location = provider.new_table_metadata_file_location(1) + + assert location.startswith("table_location/metadata/00001-") + assert location.endswith(f"{extension}.metadata.json") + # what remains after the version and the codec extension is exactly the uuid + file_uuid = location.removeprefix("table_location/metadata/00001-").removesuffix(f"{extension}.metadata.json") + assert len(file_uuid) == 36 + + +def test_new_table_metadata_file_location_invalid_compression_codec() -> None: + provider = load_location_provider( + table_location="table_location", + table_properties={TableProperties.METADATA_COMPRESSION: "snappy"}, + ) + + with pytest.raises(ValueError, match="snappy"): + provider.new_table_metadata_file_location() + + def test_metadata_location_with_trailing_slash() -> None: provider = load_location_provider( table_location="table_location", From f2e632dcc36f10c27759b1edcb870a7fb27d6da6 Mon Sep 17 00:00:00 2001 From: rahulsmahadev Date: Tue, 17 Nov 2026 13:29:46 +0000 Subject: [PATCH 2/2] Define the metadata codec suffixes next to the gzip reader Review feedback: keep the codec names and the .gz suffix in serializers.py with the reader's GZIP constant, build the error message from the mapping, and leave REST out of the property docs. Co-authored-by: Isaac --- mkdocs/docs/configuration.md | 2 +- pyiceberg/serializers.py | 4 ++++ pyiceberg/table/locations.py | 7 +++++-- tests/table/test_locations.py | 2 +- 4 files changed, 11 insertions(+), 4 deletions(-) diff --git a/mkdocs/docs/configuration.md b/mkdocs/docs/configuration.md index 8a9f644092..c2f3ef5870 100644 --- a/mkdocs/docs/configuration.md +++ b/mkdocs/docs/configuration.md @@ -93,7 +93,7 @@ Iceberg tables support table properties to configure table behavior. | `write.summary.partition-limit` | Integer | 0 | Maximum number of changed partitions for which per-partition summaries are included in the snapshot summary. The default of 0 disables them | | `write.metadata.previous-versions-max` | Integer | 100 | The max number of previous version metadata files to keep before deleting after commit. | | `write.metadata.delete-after-commit.enabled` | Boolean | False | Whether to automatically delete old *tracked* metadata files after each table commit. It will retain a number of the most recent metadata files, which can be set using property `write.metadata.previous-versions-max`. | -| `write.metadata.compression-codec` | `{none,gzip}` | none | Compression for the table metadata files that PyIceberg writes. `gzip` names new metadata files `*.gz.metadata.json` and gzips them. REST catalogs are unaffected, since the server chooses the metadata file location. | +| `write.metadata.compression-codec` | `{none,gzip}` | none | Compression for the table metadata files that PyIceberg writes. `gzip` names new metadata files `*.gz.metadata.json` and gzips them. | | `write.object-storage.enabled` | Boolean | False | Enables the [`ObjectStoreLocationProvider`](configuration.md#object-store-location-provider) that adds a hash component to file paths. | | `write.object-storage.partitioned-paths` | Boolean | True | Controls whether [partition values are included in file paths](configuration.md#partition-exclusion) when object storage is enabled | | `write.py-location-provider.impl` | String of form `module.ClassName` | null | Optional, [custom `LocationProvider`](configuration.md#loading-a-custom-location-provider) implementation | diff --git a/pyiceberg/serializers.py b/pyiceberg/serializers.py index 726e6b5f62..1577b69ca2 100644 --- a/pyiceberg/serializers.py +++ b/pyiceberg/serializers.py @@ -27,6 +27,10 @@ from pyiceberg.utils.config import Config GZIP = "gzip" +NONE = "none" + +# Suffix before ".metadata.json" for each write.metadata.compression-codec value +METADATA_FILE_EXTENSIONS = {NONE: "", GZIP: ".gz"} class Compressor(ABC): diff --git a/pyiceberg/table/locations.py b/pyiceberg/table/locations.py index 5fac1aa62c..ff83d7e82a 100644 --- a/pyiceberg/table/locations.py +++ b/pyiceberg/table/locations.py @@ -82,15 +82,18 @@ def new_table_metadata_file_location(self, new_version: int = 0) -> str: Raises: ValueError: If the version is negative, or the metadata compression codec is not supported. """ + from pyiceberg.serializers import METADATA_FILE_EXTENSIONS from pyiceberg.table import TableProperties if new_version < 0: raise ValueError(f"Table metadata version: `{new_version}` must be a non-negative integer") codec = self.table_properties.get(TableProperties.METADATA_COMPRESSION, TableProperties.METADATA_COMPRESSION_DEFAULT) - extension = {"none": "", "gzip": ".gz"}.get(codec.lower()) + extension = METADATA_FILE_EXTENSIONS.get(codec.lower()) if extension is None: - raise ValueError(f"Invalid {TableProperties.METADATA_COMPRESSION}: {codec}, expected none or gzip") + raise ValueError( + f"Invalid {TableProperties.METADATA_COMPRESSION}: {codec}, expected one of {', '.join(METADATA_FILE_EXTENSIONS)}" + ) file_name = f"{new_version:05d}-{uuid.uuid4()}{extension}.metadata.json" return self.new_metadata_location(file_name) diff --git a/tests/table/test_locations.py b/tests/table/test_locations.py index 9dd465ad50..da86a17352 100644 --- a/tests/table/test_locations.py +++ b/tests/table/test_locations.py @@ -204,7 +204,7 @@ def test_new_table_metadata_file_location_invalid_compression_codec() -> None: table_properties={TableProperties.METADATA_COMPRESSION: "snappy"}, ) - with pytest.raises(ValueError, match="snappy"): + with pytest.raises(ValueError, match=r"Invalid write\.metadata\.compression-codec: snappy, expected one of none, gzip"): provider.new_table_metadata_file_location()