From adb265f681de962434593471a153ade94a6baaf1 Mon Sep 17 00:00:00 2001 From: Mikhail Koviazin Date: Fri, 21 Aug 2026 09:18:11 +0200 Subject: [PATCH 1/5] Cherry-pick of https://github.com/Altinity/ClickHouse/pull/2183 with unresolved conflict markers (resolution in next commit) --- Original cherry-pick message follows: Merge pull request #2183 from Altinity/feature/antalya-26.6/iceberg-puffin-deletion-vectors-read-2 Iceberg deletion vector support (attempt #2) # Conflicts: # docs/en/sql-reference/table-functions/icebergCluster.md # docs/reference/statements/system.mdx # src/Access/Common/AccessType.h # src/AggregateFunctions/AggregateFunctionGroupBitmapData.h # src/Common/ProfileEvents.cpp # src/Core/Settings.cpp # src/Interpreters/InterpreterSystemQuery.cpp # src/Parsers/ASTSystemQuery.cpp # src/Parsers/ASTSystemQuery.h # src/Parsers/ParserSystemQuery.cpp # src/Processors/Formats/Impl/Parquet/Reader.cpp # src/Storages/ObjectStorage/DataLakes/Common/AvroForIcebergDeserializer.cpp # src/Storages/ObjectStorage/DataLakes/Iceberg/IcebergIterator.cpp # src/Storages/ObjectStorage/DataLakes/Iceberg/IcebergIterator.h # src/Storages/ObjectStorage/DataLakes/Iceberg/IcebergMetadata.cpp # src/Storages/ObjectStorage/DataLakes/Iceberg/ManifestFileIterator.cpp # src/Storages/ObjectStorage/DataLakes/Iceberg/Mutations.cpp # src/Storages/ObjectStorage/DataLakes/Iceberg/Snapshot.h # src/Storages/ObjectStorage/IObjectIterator.h # src/Storages/ObjectStorage/StorageObjectStorageSource.cpp # src/Storages/ObjectStorage/tests/gtest_rendezvous_hashing.cpp # tests/queries/0_stateless/01271_show_privileges.reference # tests/queries/0_stateless/04117_parser_system_query_variants.reference # tests/queries/0_stateless/04117_parser_system_query_variants.sql --- .../table-functions/icebergCluster.md | 136 +++ .../table-engines/integrations/iceberg.mdx | 14 +- docs/reference/formats/Puffin/Puffin.mdx | 10 +- .../formats/Puffin/PuffinMetadata.mdx | 8 +- .../functions/table-functions/iceberg.mdx | 23 +- docs/reference/statements/system.mdx | 6 + programs/local/LocalServer.cpp | 15 + programs/server/Server.cpp | 15 + src/Access/Common/AccessType.h | 4 + .../AggregateFunctionGroupBitmapData.h | 144 +++ .../BuzzHouse/Generator/SessionSettings.cpp | 1 + src/Common/CacheBase.h | 61 +- src/Common/CurrentMetrics.cpp | 2 + src/Common/ProfileEvents.cpp | 8 + src/Common/tests/gtest_lru_cache.cpp | 67 ++ src/Core/Defines.h | 4 + src/Core/ServerSettings.cpp | 9 + src/Core/Settings.cpp | 10 + src/Core/SettingsChangesHistory.cpp | 3 + src/Formats/FormatFilterInfo.cpp | 4 +- src/Formats/FormatFilterInfo.h | 1 + src/Interpreters/ClusterFunctionReadTask.cpp | 53 + src/Interpreters/Context.cpp | 48 + src/Interpreters/Context.h | 6 + src/Interpreters/InterpreterSystemQuery.cpp | 10 + .../gtest_cluster_function_read_task.cpp | 314 +++++ src/Parsers/ASTSystemQuery.cpp | 5 + src/Parsers/ASTSystemQuery.h | 4 + src/Parsers/ParserSystemQuery.cpp | 5 + .../Formats/Impl/Parquet/Reader.cpp | 50 +- src/Processors/Formats/Impl/Parquet/Reader.h | 5 + .../Impl/ParquetV3BlockInputFormat.cpp | 68 +- .../Formats/Impl/ParquetV3BlockInputFormat.h | 11 +- .../Formats/Impl/PuffinBlockInputFormat.cpp | 393 ++++--- .../Formats/Impl/PuffinBlockInputFormat.h | 20 + ...gtest_parquet_row_group_global_offsets.cpp | 85 ++ ...est_parquet_v3_need_only_count_buckets.cpp | 191 ++++ .../Common/AvroForIcebergDeserializer.cpp | 45 +- .../DataLakes/DataLakeObjectMetadata.h | 9 + .../DataLakes/DeletionVectorTransform.cpp | 77 ++ .../DataLakes/Iceberg/Constant.h | 2 + .../Iceberg/IcebergDataObjectInfo.cpp | 37 +- .../DataLakes/Iceberg/IcebergDataObjectInfo.h | 12 + .../Iceberg/IcebergDeletionVector.cpp | 395 +++++++ .../DataLakes/Iceberg/IcebergDeletionVector.h | 38 + .../DataLakes/Iceberg/IcebergIterator.cpp | 162 ++- .../DataLakes/Iceberg/IcebergIterator.h | 5 + .../DataLakes/Iceberg/IcebergMetadata.cpp | 71 +- .../DataLakes/Iceberg/ManifestFile.cpp | 65 ++ .../DataLakes/Iceberg/ManifestFile.h | 37 +- .../Iceberg/ManifestFileIterator.cpp | 50 +- .../DataLakes/Iceberg/ManifestFileIterator.h | 4 +- .../DataLakes/Iceberg/Mutations.cpp | 21 +- .../DataLakes/Iceberg/Snapshot.h | 23 +- .../tests/gtest_iceberg_count_shortcuts.cpp | 249 ++++ .../gtest_iceberg_data_object_info_clone.cpp | 64 ++ ...test_iceberg_deletion_vector_positions.cpp | 90 ++ ...test_iceberg_parquet_row_deletes_guard.cpp | 47 + .../DataLakes/PuffinDeletionVectorReader.cpp | 297 +++++ .../DataLakes/PuffinDeletionVectorReader.h | 80 ++ .../DataLakes/PuffinFilesCache.cpp | 338 ++++++ .../DataLakes/PuffinFilesCache.h | 341 ++++++ ...deletion_vector_before_equality_filter.cpp | 86 ++ .../gtest_deletion_vector_need_only_count.cpp | 127 +++ .../gtest_has_non_empty_excluded_rows.cpp | 28 + .../gtest_puffin_deletion_vector_bounds.cpp | 63 + ...est_puffin_deletion_vector_cardinality.cpp | 96 ++ .../gtest_puffin_deletion_vector_envelope.cpp | 124 ++ ...est_puffin_deletion_vector_footer_bind.cpp | 258 +++++ .../gtest_puffin_dv_referenced_data_file.cpp | 71 ++ .../tests/gtest_puffin_files_cache_clone.cpp | 92 ++ .../tests/gtest_puffin_files_cache_key.cpp | 199 ++++ .../gtest_puffin_files_cache_metrics.cpp | 203 ++++ .../tests/gtest_puffin_files_cache_weight.cpp | 157 +++ .../tests/gtest_puffin_footer_cache.cpp | 295 +++++ ...gtest_puffin_non_seekable_buffer_limit.cpp | 53 + src/Storages/ObjectStorage/IObjectIterator.h | 6 + .../StorageObjectStorageSource.cpp | 60 +- .../tests/gtest_rendezvous_hashing.cpp | 5 + .../test_deletion_vectors.py | 1015 +++++++++++++++++ .../01271_show_privileges.reference | 4 + .../04077_puffin_happy_path.reference | 6 + .../0_stateless/04077_puffin_happy_path.sh | 1 + ...117_parser_system_query_variants.reference | 5 + .../04117_parser_system_query_variants.sql | 5 + .../04261_iceberg_deletion_vector.reference | 197 ++++ .../04261_iceberg_deletion_vector.sh | 16 + ..._iceberg_deletion_vector_complex.reference | 172 +++ .../04262_iceberg_deletion_vector_complex.sh | 32 + ...04263_iceberg_puffin_files_cache.reference | 19 + .../04263_iceberg_puffin_files_cache.sh | 65 ++ ...ceberg_v3_row_lineage_reserved_field_id.sh | 8 +- .../04549_puffin_allow_seeks_off.sh | 2 + ...4613_puffin_metadata_type_errors.reference | 4 + .../04613_puffin_metadata_type_errors.sh | 6 + ...ount_from_files_cache_row_policy.reference | 2 + ...04656_count_from_files_cache_row_policy.sh | 30 + ...71_iceberg_v3_mutations_rejected.reference | 3 + .../04671_iceberg_v3_mutations_rejected.sh | 41 + ...2_iceberg_dv_count_metadata_only.reference | 4 + .../04672_iceberg_dv_count_metadata_only.sh | 79 ++ .../data_minio/dv_puffin_warehouse/README.md | 35 + ...9df-a585-047546b341d9-00001-deletes.puffin | Bin 0 -> 2141 bytes ...ced-935b-d2b8ef79eb76-00001-deletes.puffin | Bin 0 -> 1346 bytes ...d07-9633-95670a571224-00001-deletes.puffin | Bin 0 -> 539 bytes ...3b9-9112-c569f91929b9-00001-deletes.puffin | Bin 0 -> 1337 bytes ...94e-b336-f4d3301291b4-00001-deletes.puffin | Bin 0 -> 1351 bytes ...429-4dc8-9457-9b0d5a6a6e9b-0-00005.parquet | Bin 0 -> 779 bytes ...0de-4110-9928-e74531d8b125-0-00005.parquet | Bin 0 -> 1003 bytes ...-de4e-4d07-9633-95670a571224-00001.parquet | Bin 0 -> 969 bytes ...f4f-47e6-ad97-00c53284ff87-0-00001.parquet | Bin 0 -> 1173 bytes ...256-45aa-9ed1-48200445069f-0-00005.parquet | Bin 0 -> 759 bytes ...429-4dc8-9457-9b0d5a6a6e9b-0-00001.parquet | Bin 0 -> 766 bytes ...0de-4110-9928-e74531d8b125-0-00001.parquet | Bin 0 -> 1027 bytes ...ebb-4cfc-a3b7-3f2a1034a6dc-0-00001.parquet | Bin 0 -> 1192 bytes ...256-45aa-9ed1-48200445069f-0-00001.parquet | Bin 0 -> 784 bytes ...429-4dc8-9457-9b0d5a6a6e9b-0-00004.parquet | Bin 0 -> 791 bytes ...0de-4110-9928-e74531d8b125-0-00004.parquet | Bin 0 -> 1043 bytes ...8a4-482a-9a88-e08ad8bdd2e6-0-00001.parquet | Bin 0 -> 1197 bytes ...256-45aa-9ed1-48200445069f-0-00004.parquet | Bin 0 -> 806 bytes ...429-4dc8-9457-9b0d5a6a6e9b-0-00002.parquet | Bin 0 -> 800 bytes ...0de-4110-9928-e74531d8b125-0-00002.parquet | Bin 0 -> 1002 bytes ...583-48b6-983a-c2993f261fec-0-00001.parquet | Bin 0 -> 1148 bytes ...256-45aa-9ed1-48200445069f-0-00002.parquet | Bin 0 -> 772 bytes ...429-4dc8-9457-9b0d5a6a6e9b-0-00003.parquet | Bin 0 -> 800 bytes ...0de-4110-9928-e74531d8b125-0-00003.parquet | Bin 0 -> 1028 bytes ...8dd-4d8a-a31a-6db198e852d4-0-00001.parquet | Bin 0 -> 1176 bytes ...256-45aa-9ed1-48200445069f-0-00003.parquet | Bin 0 -> 766 bytes ...03abfe-66a8-43ab-a765-a9e8072b59ed-m0.avro | Bin 0 -> 8221 bytes ...ca5b2b-d3f2-44b4-8494-f0aadd4e0b34-m0.avro | Bin 0 -> 8126 bytes ...1000ac-b08e-430f-9c79-e40d5e4aa42d-m0.avro | Bin 0 -> 8145 bytes ...1000ac-b08e-430f-9c79-e40d5e4aa42d-m1.avro | Bin 0 -> 8132 bytes ...ec1ceb-2dbe-4ab7-ab4e-0d18bed2d2da-m0.avro | Bin 0 -> 8230 bytes ...caec64-2cad-46f5-a966-dd75e3635abc-m0.avro | Bin 0 -> 8135 bytes ...caec64-2cad-46f5-a966-dd75e3635abc-m1.avro | Bin 0 -> 8141 bytes ...66c606-c296-4fe4-a5f8-d8be592f6b96-m0.avro | Bin 0 -> 8142 bytes ...66c606-c296-4fe4-a5f8-d8be592f6b96-m1.avro | Bin 0 -> 8318 bytes ...66c606-c296-4fe4-a5f8-d8be592f6b96-m2.avro | Bin 0 -> 8298 bytes ...66c606-c296-4fe4-a5f8-d8be592f6b96-m3.avro | Bin 0 -> 8297 bytes ...66c606-c296-4fe4-a5f8-d8be592f6b96-m4.avro | Bin 0 -> 8436 bytes ...19b33f-2be5-4073-820c-82cb1c7d76cb-m0.avro | Bin 0 -> 8179 bytes ...8e784a-6b9c-4894-827d-c56978599ecd-m0.avro | Bin 0 -> 8301 bytes ...1c4572-6006-4649-85b1-47e82e73a396-m0.avro | Bin 0 -> 8157 bytes ...-c21c4572-6006-4649-85b1-47e82e73a396.avro | Bin 0 -> 5015 bytes ...-b68e784a-6b9c-4894-827d-c56978599ecd.avro | Bin 0 -> 5072 bytes ...-23ca5b2b-d3f2-44b4-8494-f0aadd4e0b34.avro | Bin 0 -> 4885 bytes ...-0c03abfe-66a8-43ab-a765-a9e8072b59ed.avro | Bin 0 -> 4966 bytes ...-6a19b33f-2be5-4073-820c-82cb1c7d76cb.avro | Bin 0 -> 5124 bytes ...-321000ac-b08e-430f-9c79-e40d5e4aa42d.avro | Bin 0 -> 4908 bytes ...-39ec1ceb-2dbe-4ab7-ab4e-0d18bed2d2da.avro | Bin 0 -> 4800 bytes ...-5b66c606-c296-4fe4-a5f8-d8be592f6b96.avro | Bin 0 -> 5139 bytes ...-41caec64-2cad-46f5-a966-dd75e3635abc.avro | Bin 0 -> 5192 bytes .../metadata/v1.metadata.json | 1 + .../metadata/v10.metadata.json | 1 + .../metadata/v11.metadata.json | 1 + .../metadata/v2.metadata.json | 1 + .../metadata/v3.metadata.json | 1 + .../metadata/v4.metadata.json | 1 + .../metadata/v5.metadata.json | 1 + .../metadata/v6.metadata.json | 1 + .../metadata/v7.metadata.json | 1 + .../metadata/v8.metadata.json | 1 + .../metadata/v9.metadata.json | 1 + .../metadata/version-hint.text | 1 + ...697-43f9-b2ea-9894139e9373-0-00001.parquet | Bin 0 -> 734 bytes ...65e-b4d7-6366f5ebda40-00001-deletes.puffin | Bin 0 -> 532 bytes ...8edb12-fdf0-4ce1-b6ab-c9c2a05d6d57-m0.avro | Bin 0 -> 7802 bytes ...4940c4-6b84-4560-85a3-35e17802d85d-m0.avro | Bin 0 -> 7785 bytes ...-c34940c4-6b84-4560-85a3-35e17802d85d.avro | Bin 0 -> 4789 bytes ...-0e8edb12-fdf0-4ce1-b6ab-c9c2a05d6d57.avro | Bin 0 -> 4870 bytes .../metadata/v1.metadata.json | 1 + .../metadata/v2.metadata.json | 1 + .../metadata/v3.metadata.json | 1 + .../metadata/version-hint.text | 1 + .../data_minio/generate_iceberg_dv_fixture.py | 160 +++ .../dv_sequence_number_not_minus_one.puffin | Bin 0 -> 297 bytes .../dv_snapshot_id_not_minus_one.puffin | Bin 0 -> 297 bytes .../data_puffin/generate_puffin_fixtures.py | 77 +- .../data_puffin/mixed_blob_types.puffin | Bin 0 -> 444 bytes 179 files changed, 7979 insertions(+), 320 deletions(-) create mode 100644 docs/en/sql-reference/table-functions/icebergCluster.md create mode 100644 src/Interpreters/tests/gtest_cluster_function_read_task.cpp create mode 100644 src/Processors/tests/gtest_parquet_row_group_global_offsets.cpp create mode 100644 src/Processors/tests/gtest_parquet_v3_need_only_count_buckets.cpp create mode 100644 src/Storages/ObjectStorage/DataLakes/Iceberg/IcebergDeletionVector.cpp create mode 100644 src/Storages/ObjectStorage/DataLakes/Iceberg/IcebergDeletionVector.h create mode 100644 src/Storages/ObjectStorage/DataLakes/Iceberg/tests/gtest_iceberg_count_shortcuts.cpp create mode 100644 src/Storages/ObjectStorage/DataLakes/Iceberg/tests/gtest_iceberg_data_object_info_clone.cpp create mode 100644 src/Storages/ObjectStorage/DataLakes/Iceberg/tests/gtest_iceberg_deletion_vector_positions.cpp create mode 100644 src/Storages/ObjectStorage/DataLakes/Iceberg/tests/gtest_iceberg_parquet_row_deletes_guard.cpp create mode 100644 src/Storages/ObjectStorage/DataLakes/PuffinDeletionVectorReader.cpp create mode 100644 src/Storages/ObjectStorage/DataLakes/PuffinDeletionVectorReader.h create mode 100644 src/Storages/ObjectStorage/DataLakes/PuffinFilesCache.cpp create mode 100644 src/Storages/ObjectStorage/DataLakes/PuffinFilesCache.h create mode 100644 src/Storages/ObjectStorage/DataLakes/tests/gtest_deletion_vector_before_equality_filter.cpp create mode 100644 src/Storages/ObjectStorage/DataLakes/tests/gtest_deletion_vector_need_only_count.cpp create mode 100644 src/Storages/ObjectStorage/DataLakes/tests/gtest_has_non_empty_excluded_rows.cpp create mode 100644 src/Storages/ObjectStorage/DataLakes/tests/gtest_puffin_deletion_vector_bounds.cpp create mode 100644 src/Storages/ObjectStorage/DataLakes/tests/gtest_puffin_deletion_vector_cardinality.cpp create mode 100644 src/Storages/ObjectStorage/DataLakes/tests/gtest_puffin_deletion_vector_envelope.cpp create mode 100644 src/Storages/ObjectStorage/DataLakes/tests/gtest_puffin_deletion_vector_footer_bind.cpp create mode 100644 src/Storages/ObjectStorage/DataLakes/tests/gtest_puffin_dv_referenced_data_file.cpp create mode 100644 src/Storages/ObjectStorage/DataLakes/tests/gtest_puffin_files_cache_clone.cpp create mode 100644 src/Storages/ObjectStorage/DataLakes/tests/gtest_puffin_files_cache_key.cpp create mode 100644 src/Storages/ObjectStorage/DataLakes/tests/gtest_puffin_files_cache_metrics.cpp create mode 100644 src/Storages/ObjectStorage/DataLakes/tests/gtest_puffin_files_cache_weight.cpp create mode 100644 src/Storages/ObjectStorage/DataLakes/tests/gtest_puffin_footer_cache.cpp create mode 100644 src/Storages/ObjectStorage/DataLakes/tests/gtest_puffin_non_seekable_buffer_limit.cpp create mode 100644 tests/integration/test_storage_iceberg_with_spark/test_deletion_vectors.py create mode 100644 tests/queries/0_stateless/04261_iceberg_deletion_vector.reference create mode 100755 tests/queries/0_stateless/04261_iceberg_deletion_vector.sh create mode 100644 tests/queries/0_stateless/04262_iceberg_deletion_vector_complex.reference create mode 100755 tests/queries/0_stateless/04262_iceberg_deletion_vector_complex.sh create mode 100644 tests/queries/0_stateless/04263_iceberg_puffin_files_cache.reference create mode 100755 tests/queries/0_stateless/04263_iceberg_puffin_files_cache.sh create mode 100644 tests/queries/0_stateless/04656_count_from_files_cache_row_policy.reference create mode 100755 tests/queries/0_stateless/04656_count_from_files_cache_row_policy.sh create mode 100644 tests/queries/0_stateless/04671_iceberg_v3_mutations_rejected.reference create mode 100755 tests/queries/0_stateless/04671_iceberg_v3_mutations_rejected.sh create mode 100644 tests/queries/0_stateless/04672_iceberg_dv_count_metadata_only.reference create mode 100755 tests/queries/0_stateless/04672_iceberg_dv_count_metadata_only.sh create mode 100644 tests/queries/0_stateless/data_minio/dv_puffin_warehouse/README.md create mode 100644 tests/queries/0_stateless/data_minio/dv_puffin_warehouse/default/dv_puffin_complex/data/00000-14-ba106366-1379-49df-a585-047546b341d9-00001-deletes.puffin create mode 100644 tests/queries/0_stateless/data_minio/dv_puffin_warehouse/default/dv_puffin_complex/data/00000-18-0da8af70-7e2e-4ced-935b-d2b8ef79eb76-00001-deletes.puffin create mode 100644 tests/queries/0_stateless/data_minio/dv_puffin_warehouse/default/dv_puffin_complex/data/00000-20-04b90311-de4e-4d07-9633-95670a571224-00001-deletes.puffin create mode 100644 tests/queries/0_stateless/data_minio/dv_puffin_warehouse/default/dv_puffin_complex/data/00000-6-74340066-9c57-43b9-9112-c569f91929b9-00001-deletes.puffin create mode 100644 tests/queries/0_stateless/data_minio/dv_puffin_warehouse/default/dv_puffin_complex/data/00000-9-dcc19f77-3570-494e-b336-f4d3301291b4-00001-deletes.puffin create mode 100644 tests/queries/0_stateless/data_minio/dv_puffin_warehouse/default/dv_puffin_complex/data/id_bucket=0/00000-11-499dd379-3429-4dc8-9457-9b0d5a6a6e9b-0-00005.parquet create mode 100644 tests/queries/0_stateless/data_minio/dv_puffin_warehouse/default/dv_puffin_complex/data/id_bucket=0/00000-16-03e76970-c0de-4110-9928-e74531d8b125-0-00005.parquet create mode 100644 tests/queries/0_stateless/data_minio/dv_puffin_warehouse/default/dv_puffin_complex/data/id_bucket=0/00000-20-04b90311-de4e-4d07-9633-95670a571224-00001.parquet create mode 100644 tests/queries/0_stateless/data_minio/dv_puffin_warehouse/default/dv_puffin_complex/data/id_bucket=0/00000-25-ee6fe6d5-2f4f-47e6-ad97-00c53284ff87-0-00001.parquet create mode 100644 tests/queries/0_stateless/data_minio/dv_puffin_warehouse/default/dv_puffin_complex/data/id_bucket=0/00000-4-fe8ff9da-4256-45aa-9ed1-48200445069f-0-00005.parquet create mode 100644 tests/queries/0_stateless/data_minio/dv_puffin_warehouse/default/dv_puffin_complex/data/id_bucket=1/00000-11-499dd379-3429-4dc8-9457-9b0d5a6a6e9b-0-00001.parquet create mode 100644 tests/queries/0_stateless/data_minio/dv_puffin_warehouse/default/dv_puffin_complex/data/id_bucket=1/00000-16-03e76970-c0de-4110-9928-e74531d8b125-0-00001.parquet create mode 100644 tests/queries/0_stateless/data_minio/dv_puffin_warehouse/default/dv_puffin_complex/data/id_bucket=1/00000-24-0b9f04c6-aebb-4cfc-a3b7-3f2a1034a6dc-0-00001.parquet create mode 100644 tests/queries/0_stateless/data_minio/dv_puffin_warehouse/default/dv_puffin_complex/data/id_bucket=1/00000-4-fe8ff9da-4256-45aa-9ed1-48200445069f-0-00001.parquet create mode 100644 tests/queries/0_stateless/data_minio/dv_puffin_warehouse/default/dv_puffin_complex/data/id_bucket=2/00000-11-499dd379-3429-4dc8-9457-9b0d5a6a6e9b-0-00004.parquet create mode 100644 tests/queries/0_stateless/data_minio/dv_puffin_warehouse/default/dv_puffin_complex/data/id_bucket=2/00000-16-03e76970-c0de-4110-9928-e74531d8b125-0-00004.parquet create mode 100644 tests/queries/0_stateless/data_minio/dv_puffin_warehouse/default/dv_puffin_complex/data/id_bucket=2/00000-23-292ac3e8-88a4-482a-9a88-e08ad8bdd2e6-0-00001.parquet create mode 100644 tests/queries/0_stateless/data_minio/dv_puffin_warehouse/default/dv_puffin_complex/data/id_bucket=2/00000-4-fe8ff9da-4256-45aa-9ed1-48200445069f-0-00004.parquet create mode 100644 tests/queries/0_stateless/data_minio/dv_puffin_warehouse/default/dv_puffin_complex/data/id_bucket=3/00000-11-499dd379-3429-4dc8-9457-9b0d5a6a6e9b-0-00002.parquet create mode 100644 tests/queries/0_stateless/data_minio/dv_puffin_warehouse/default/dv_puffin_complex/data/id_bucket=3/00000-16-03e76970-c0de-4110-9928-e74531d8b125-0-00002.parquet create mode 100644 tests/queries/0_stateless/data_minio/dv_puffin_warehouse/default/dv_puffin_complex/data/id_bucket=3/00000-21-55307a4a-f583-48b6-983a-c2993f261fec-0-00001.parquet create mode 100644 tests/queries/0_stateless/data_minio/dv_puffin_warehouse/default/dv_puffin_complex/data/id_bucket=3/00000-4-fe8ff9da-4256-45aa-9ed1-48200445069f-0-00002.parquet create mode 100644 tests/queries/0_stateless/data_minio/dv_puffin_warehouse/default/dv_puffin_complex/data/id_bucket=4/00000-11-499dd379-3429-4dc8-9457-9b0d5a6a6e9b-0-00003.parquet create mode 100644 tests/queries/0_stateless/data_minio/dv_puffin_warehouse/default/dv_puffin_complex/data/id_bucket=4/00000-16-03e76970-c0de-4110-9928-e74531d8b125-0-00003.parquet create mode 100644 tests/queries/0_stateless/data_minio/dv_puffin_warehouse/default/dv_puffin_complex/data/id_bucket=4/00000-22-cbc4c1d2-88dd-4d8a-a31a-6db198e852d4-0-00001.parquet create mode 100644 tests/queries/0_stateless/data_minio/dv_puffin_warehouse/default/dv_puffin_complex/data/id_bucket=4/00000-4-fe8ff9da-4256-45aa-9ed1-48200445069f-0-00003.parquet create mode 100644 tests/queries/0_stateless/data_minio/dv_puffin_warehouse/default/dv_puffin_complex/metadata/0c03abfe-66a8-43ab-a765-a9e8072b59ed-m0.avro create mode 100644 tests/queries/0_stateless/data_minio/dv_puffin_warehouse/default/dv_puffin_complex/metadata/23ca5b2b-d3f2-44b4-8494-f0aadd4e0b34-m0.avro create mode 100644 tests/queries/0_stateless/data_minio/dv_puffin_warehouse/default/dv_puffin_complex/metadata/321000ac-b08e-430f-9c79-e40d5e4aa42d-m0.avro create mode 100644 tests/queries/0_stateless/data_minio/dv_puffin_warehouse/default/dv_puffin_complex/metadata/321000ac-b08e-430f-9c79-e40d5e4aa42d-m1.avro create mode 100644 tests/queries/0_stateless/data_minio/dv_puffin_warehouse/default/dv_puffin_complex/metadata/39ec1ceb-2dbe-4ab7-ab4e-0d18bed2d2da-m0.avro create mode 100644 tests/queries/0_stateless/data_minio/dv_puffin_warehouse/default/dv_puffin_complex/metadata/41caec64-2cad-46f5-a966-dd75e3635abc-m0.avro create mode 100644 tests/queries/0_stateless/data_minio/dv_puffin_warehouse/default/dv_puffin_complex/metadata/41caec64-2cad-46f5-a966-dd75e3635abc-m1.avro create mode 100644 tests/queries/0_stateless/data_minio/dv_puffin_warehouse/default/dv_puffin_complex/metadata/5b66c606-c296-4fe4-a5f8-d8be592f6b96-m0.avro create mode 100644 tests/queries/0_stateless/data_minio/dv_puffin_warehouse/default/dv_puffin_complex/metadata/5b66c606-c296-4fe4-a5f8-d8be592f6b96-m1.avro create mode 100644 tests/queries/0_stateless/data_minio/dv_puffin_warehouse/default/dv_puffin_complex/metadata/5b66c606-c296-4fe4-a5f8-d8be592f6b96-m2.avro create mode 100644 tests/queries/0_stateless/data_minio/dv_puffin_warehouse/default/dv_puffin_complex/metadata/5b66c606-c296-4fe4-a5f8-d8be592f6b96-m3.avro create mode 100644 tests/queries/0_stateless/data_minio/dv_puffin_warehouse/default/dv_puffin_complex/metadata/5b66c606-c296-4fe4-a5f8-d8be592f6b96-m4.avro create mode 100644 tests/queries/0_stateless/data_minio/dv_puffin_warehouse/default/dv_puffin_complex/metadata/6a19b33f-2be5-4073-820c-82cb1c7d76cb-m0.avro create mode 100644 tests/queries/0_stateless/data_minio/dv_puffin_warehouse/default/dv_puffin_complex/metadata/b68e784a-6b9c-4894-827d-c56978599ecd-m0.avro create mode 100644 tests/queries/0_stateless/data_minio/dv_puffin_warehouse/default/dv_puffin_complex/metadata/c21c4572-6006-4649-85b1-47e82e73a396-m0.avro create mode 100644 tests/queries/0_stateless/data_minio/dv_puffin_warehouse/default/dv_puffin_complex/metadata/snap-212327459647764229-1-c21c4572-6006-4649-85b1-47e82e73a396.avro create mode 100644 tests/queries/0_stateless/data_minio/dv_puffin_warehouse/default/dv_puffin_complex/metadata/snap-2216958009875447676-1-b68e784a-6b9c-4894-827d-c56978599ecd.avro create mode 100644 tests/queries/0_stateless/data_minio/dv_puffin_warehouse/default/dv_puffin_complex/metadata/snap-2245381059480343696-1-23ca5b2b-d3f2-44b4-8494-f0aadd4e0b34.avro create mode 100644 tests/queries/0_stateless/data_minio/dv_puffin_warehouse/default/dv_puffin_complex/metadata/snap-243132775900853210-1-0c03abfe-66a8-43ab-a765-a9e8072b59ed.avro create mode 100644 tests/queries/0_stateless/data_minio/dv_puffin_warehouse/default/dv_puffin_complex/metadata/snap-399037466005519109-1-6a19b33f-2be5-4073-820c-82cb1c7d76cb.avro create mode 100644 tests/queries/0_stateless/data_minio/dv_puffin_warehouse/default/dv_puffin_complex/metadata/snap-7131460395349610904-1-321000ac-b08e-430f-9c79-e40d5e4aa42d.avro create mode 100644 tests/queries/0_stateless/data_minio/dv_puffin_warehouse/default/dv_puffin_complex/metadata/snap-8352556540874311077-1-39ec1ceb-2dbe-4ab7-ab4e-0d18bed2d2da.avro create mode 100644 tests/queries/0_stateless/data_minio/dv_puffin_warehouse/default/dv_puffin_complex/metadata/snap-8541702041949031574-1-5b66c606-c296-4fe4-a5f8-d8be592f6b96.avro create mode 100644 tests/queries/0_stateless/data_minio/dv_puffin_warehouse/default/dv_puffin_complex/metadata/snap-9221420601809522049-1-41caec64-2cad-46f5-a966-dd75e3635abc.avro create mode 100644 tests/queries/0_stateless/data_minio/dv_puffin_warehouse/default/dv_puffin_complex/metadata/v1.metadata.json create mode 100644 tests/queries/0_stateless/data_minio/dv_puffin_warehouse/default/dv_puffin_complex/metadata/v10.metadata.json create mode 100644 tests/queries/0_stateless/data_minio/dv_puffin_warehouse/default/dv_puffin_complex/metadata/v11.metadata.json create mode 100644 tests/queries/0_stateless/data_minio/dv_puffin_warehouse/default/dv_puffin_complex/metadata/v2.metadata.json create mode 100644 tests/queries/0_stateless/data_minio/dv_puffin_warehouse/default/dv_puffin_complex/metadata/v3.metadata.json create mode 100644 tests/queries/0_stateless/data_minio/dv_puffin_warehouse/default/dv_puffin_complex/metadata/v4.metadata.json create mode 100644 tests/queries/0_stateless/data_minio/dv_puffin_warehouse/default/dv_puffin_complex/metadata/v5.metadata.json create mode 100644 tests/queries/0_stateless/data_minio/dv_puffin_warehouse/default/dv_puffin_complex/metadata/v6.metadata.json create mode 100644 tests/queries/0_stateless/data_minio/dv_puffin_warehouse/default/dv_puffin_complex/metadata/v7.metadata.json create mode 100644 tests/queries/0_stateless/data_minio/dv_puffin_warehouse/default/dv_puffin_complex/metadata/v8.metadata.json create mode 100644 tests/queries/0_stateless/data_minio/dv_puffin_warehouse/default/dv_puffin_complex/metadata/v9.metadata.json create mode 100644 tests/queries/0_stateless/data_minio/dv_puffin_warehouse/default/dv_puffin_complex/metadata/version-hint.text create mode 100644 tests/queries/0_stateless/data_minio/dv_puffin_warehouse/default/dv_puffin_source/data/00000-0-e150e3cb-5697-43f9-b2ea-9894139e9373-0-00001.parquet create mode 100644 tests/queries/0_stateless/data_minio/dv_puffin_warehouse/default/dv_puffin_source/data/00000-2-ee31c25d-1223-465e-b4d7-6366f5ebda40-00001-deletes.puffin create mode 100644 tests/queries/0_stateless/data_minio/dv_puffin_warehouse/default/dv_puffin_source/metadata/0e8edb12-fdf0-4ce1-b6ab-c9c2a05d6d57-m0.avro create mode 100644 tests/queries/0_stateless/data_minio/dv_puffin_warehouse/default/dv_puffin_source/metadata/c34940c4-6b84-4560-85a3-35e17802d85d-m0.avro create mode 100644 tests/queries/0_stateless/data_minio/dv_puffin_warehouse/default/dv_puffin_source/metadata/snap-1455293623190430422-1-c34940c4-6b84-4560-85a3-35e17802d85d.avro create mode 100644 tests/queries/0_stateless/data_minio/dv_puffin_warehouse/default/dv_puffin_source/metadata/snap-8278841256334127309-1-0e8edb12-fdf0-4ce1-b6ab-c9c2a05d6d57.avro create mode 100644 tests/queries/0_stateless/data_minio/dv_puffin_warehouse/default/dv_puffin_source/metadata/v1.metadata.json create mode 100644 tests/queries/0_stateless/data_minio/dv_puffin_warehouse/default/dv_puffin_source/metadata/v2.metadata.json create mode 100644 tests/queries/0_stateless/data_minio/dv_puffin_warehouse/default/dv_puffin_source/metadata/v3.metadata.json create mode 100644 tests/queries/0_stateless/data_minio/dv_puffin_warehouse/default/dv_puffin_source/metadata/version-hint.text create mode 100644 tests/queries/0_stateless/data_minio/generate_iceberg_dv_fixture.py create mode 100644 tests/queries/0_stateless/data_puffin/dv_sequence_number_not_minus_one.puffin create mode 100644 tests/queries/0_stateless/data_puffin/dv_snapshot_id_not_minus_one.puffin create mode 100644 tests/queries/0_stateless/data_puffin/mixed_blob_types.puffin diff --git a/docs/en/sql-reference/table-functions/icebergCluster.md b/docs/en/sql-reference/table-functions/icebergCluster.md new file mode 100644 index 000000000000..312e5b30ec0b --- /dev/null +++ b/docs/en/sql-reference/table-functions/icebergCluster.md @@ -0,0 +1,136 @@ +--- +description: 'An extension to the iceberg table function which allows processing files + from Apache Iceberg in parallel from many nodes in a specified cluster.' +sidebar_label: 'icebergCluster' +sidebar_position: 91 +slug: /sql-reference/table-functions/icebergCluster +title: 'icebergCluster' +doc_type: 'reference' +--- + +This is an extension to the [iceberg](/sql-reference/table-functions/iceberg.md) table function. + +Allows processing files from Apache [Iceberg](https://iceberg.apache.org/) in parallel from many nodes in a specified cluster. On initiator it creates a connection to all nodes in the cluster and dispatches each file dynamically. On the worker node it asks the initiator about the next task to process and processes it. This is repeated until all tasks are finished. + +## Deletion vectors on cluster reads {#deletion-vectors-cluster} + +Iceberg v3 [deletion vectors](https://iceberg.apache.org/spec/#deletion-vectors) are loaded on the **initiator** while it distributes tasks: for each data file the initiator reads the Puffin blob, validates it, materializes deleted row positions, and attaches the bitmap to the task sent to workers. Workers apply that bitmap when reading Parquet; they do not fetch or parse the Puffin file again for that path. See [Processing of tables with deleted rows](/sql-reference/table-functions/iceberg.md#deleted-rows) for format limits and caching. + +On wide tables with many deletion vectors, initiator-side decode and per-task bitmap serialization can become a bottleneck even when Parquet reads are well parallelized across the cluster. + +## Syntax {#syntax} + +```sql +icebergS3Cluster(cluster_name, url [, NOSIGN | access_key_id, secret_access_key, [session_token]] [,format] [,compression_method] [,extra_credentials]) +icebergS3Cluster(cluster_name, named_collection[, option=value [,..]]) + +icebergAzureCluster(cluster_name, connection_string|storage_account_url, container_name, blobpath, [,account_name], [,account_key] [,format] [,compression_method]) +icebergAzureCluster(cluster_name, named_collection[, option=value [,..]]) + +icebergHDFSCluster(cluster_name, path_to_table, [,format] [,compression_method]) +icebergHDFSCluster(cluster_name, named_collection[, option=value [,..]]) +``` + +## Arguments {#arguments} + +- `cluster_name` — Name of a cluster that is used to build a set of addresses and connection parameters to remote and local servers. +- Description of all other arguments coincides with description of arguments in equivalent [iceberg](/sql-reference/table-functions/iceberg.md) table function. +- An optional `extra_credentials` parameter can be used to pass a `role_arn` for role-based access in ClickHouse Cloud. See [Secure S3](/cloud/data-sources/secure-s3) for configuration steps. + +**Returned value** + +A table with the specified structure for reading data from cluster in the specified Iceberg table. + +**Examples** + +```sql +SELECT * FROM icebergS3Cluster('cluster_simple', 'http://test.s3.amazonaws.com/clickhouse-bucket/test_table', 'test', 'test') +``` + +## Virtual Columns {#virtual-columns} + +- `_path` — Path to the file. Type: `LowCardinality(String)`. +- `_file` — Name of the file. Type: `LowCardinality(String)`. +- `_size` — Size of the file in bytes. Type: `Nullable(UInt64)`. If the file size is unknown, the value is `NULL`. +- `_time` — Last modified time of the file. Type: `Nullable(DateTime)`. If the time is unknown, the value is `NULL`. +- `_etag` — The etag of the file. Type: `LowCardinality(String)`. If the etag is unknown, the value is `NULL`. + +## Altinity Antalya branch + +### `icebergLocalCluster` table function + +Only in the Altinity Antalya branch, `icebergLocalCluster` designed to make distributed cluster queries when Iceberg data is stored on shared network storage mounted with a local path. The path must be identical on all replicas. + +```sql +icebergLocalCluster(cluster_name, path_to_table, [,format] [,compression_method]) +``` + +### Specify storage type in function arguments + +Only in the Altinity Antalya branch, the `icebergCluster` table function supports all storage backends. The storage backend can be specified using the named argument `storage_type`. Valid values include `s3`, `azure`, `hdfs`, and `local`. + +```sql +icebergCluster(storage_type='s3', cluster_name, url [, NOSIGN | access_key_id, secret_access_key, [session_token]] [,format] [,compression_method]) + +icebergCluster(storage_type='azure', cluster_name, connection_string|storage_account_url, container_name, blobpath, [,account_name], [,account_key] [,format] [,compression_method]) + +icebergCluster(storage_type='hdfs', cluster_name, path_to_table, [,format] [,compression_method]) + +icebergCluster(storage_type='local', cluster_name, path_to_table, [,format] [,compression_method]) +``` + +### Specify storage type in a named collection + +Only in the Altinity Antalya branch, `storage_type` can be part of a named collection. + +```xml + + + + http://test.s3.amazonaws.com/clickhouse-bucket/ + test + test + auto + auto + s3 + + + +``` + +```sql +icebergCluster(iceberg_conf[, option=value [,..]]) +``` + +The default value for `storage_type` is `s3`. + +### `object_storage_cluster` setting. + +Only in the Altinity Antalya branch, an alternative syntax for `icebergCluster` table function is available. This allows the `iceberg` function to be used with the non-empty `object_storage_cluster` setting, specifying a cluster name. This enables distributed queries over Iceberg table across a ClickHouse cluster. + +```sql +icebergS3(url [, NOSIGN | access_key_id, secret_access_key, [session_token]] [,format] [,compression_method]) SETTINGS object_storage_cluster='cluster_name' + +icebergAzure(connection_string|storage_account_url, container_name, blobpath, [,account_name], [,account_key] [,format] [,compression_method]) SETTINGS object_storage_cluster='cluster_name' + +icebergHDFS(path_to_table, [,format] [,compression_method]) SETTINGS object_storage_cluster='cluster_name' + +icebergLocal(path_to_table, [,format] [,compression_method]) SETTINGS object_storage_cluster='cluster_name' + +icebergS3(option=value [,..]) SETTINGS object_storage_cluster='cluster_name' + +iceberg(storage_type='s3', url [, NOSIGN | access_key_id, secret_access_key, [session_token]] [,format] [,compression_method]) SETTINGS object_storage_cluster='cluster_name' + +iceberg(storage_type='azure', connection_string|storage_account_url, container_name, blobpath, [,account_name], [,account_key] [,format] [,compression_method]) SETTINGS object_storage_cluster='cluster_name' + +iceberg(storage_type='hdfs', path_to_table, [,format] [,compression_method]) SETTINGS object_storage_cluster='cluster_name' + +iceberg(storage_type='local', path_to_table, [,format] [,compression_method]) SETTINGS object_storage_cluster='cluster_name' + +iceberg(iceberg_conf[, option=value [,..]]) SETTINGS object_storage_cluster='cluster_name' +``` + +**See Also** + +- [Iceberg engine](/engines/table-engines/integrations/iceberg.md) +- [Iceberg table function](/sql-reference/table-functions/iceberg.md) diff --git a/docs/reference/engines/table-engines/integrations/iceberg.mdx b/docs/reference/engines/table-engines/integrations/iceberg.mdx index af8c9a84bb5d..5472415d2a99 100644 --- a/docs/reference/engines/table-engines/integrations/iceberg.mdx +++ b/docs/reference/engines/table-engines/integrations/iceberg.mdx @@ -149,9 +149,19 @@ ClickHouse supports reading Iceberg tables that use the following deletion metho - [Position deletes](https://iceberg.apache.org/spec/#position-delete-files) - [Equality deletes](https://iceberg.apache.org/spec/#equality-delete-files) (supported from version 25.8+) +- [Deletion vectors](https://iceberg.apache.org/spec/#deletion-vectors) stored in Puffin files (Iceberg v3, read-only) -The following deletion method is **not supported**: -- [Deletion vectors](https://iceberg.apache.org/spec/#deletion-vectors) (introduced in v3) +The following limitations apply to deletion vectors: + +- Only `deletion-vector-v1` Puffin blobs are supported +- Data files must be in Parquet format +- Column-scoped deletion vectors (user column ids in puffin `fields`) are not supported. Writers may set `fields` to `[]` or to the Iceberg reserved `_pos` id (`2147483645`) for file-scoped deletion vectors. +- Writing deletion vectors is not supported +- `DELETE` / `UPDATE` mutations on Iceberg format version 3+ tables are rejected (writers must not add position-delete files) + +Parsed deletion vectors can be cached in memory when `use_puffin_files_cache` is enabled and the puffin file has a non-empty `etag`. Empty deletion vectors are cached as well, so repeated reads do not re-fetch the puffin file. Parsed footers for coalesced multi-DV Puffin files are memoized with that cache (same identity: storage, path, `etag`) so slices share one footer parse; the memo shares `puffin_files_cache_size` / max-entry limits and is dropped when the cache is disabled (`puffin_files_cache_size=0`) or cleared. The cache can be cleared with `SYSTEM DROP PUFFIN FILES CACHE`. + +For [`icebergCluster`](/sql-reference/table-functions/icebergCluster.md) (and `object_storage_cluster`), the initiator loads and materializes each data file's deletion vector while distributing tasks, then sends the resulting row bitmap to workers with the task. Workers apply the bitmap; they do not re-read the Puffin blob for that path. On wide v3 tables this can make the initiator a serialization point for deletion-vector I/O and decode. ### Basic usage {#basic-usage} ```sql diff --git a/docs/reference/formats/Puffin/Puffin.mdx b/docs/reference/formats/Puffin/Puffin.mdx index 96f321ad4d0a..81b5764ad215 100644 --- a/docs/reference/formats/Puffin/Puffin.mdx +++ b/docs/reference/formats/Puffin/Puffin.mdx @@ -1,6 +1,10 @@ --- description: 'Documentation for the Puffin format' +input_format: true +output_format: false keywords: ['Puffin'] +sidebar_label: 'Puffin' +sidebar_position: 1 slug: /interfaces/formats/Puffin title: 'Puffin' doc_type: 'reference' @@ -10,16 +14,16 @@ doc_type: 'reference' Input format for reading [Apache Iceberg Puffin](https://iceberg.apache.org/puffin-spec/) files. -The format exposes deleted row positions from `deletion-vector-v1` blobs. It is the only supported blob type: a file containing any other blob type (for example `apache-datasketches-theta-v1`) is rejected. +The format exposes deleted row positions from `deletion-vector-v1` blobs. Other blob types (for example `apache-datasketches-theta-v1`) are skipped. If a puffin file contains multiple `deletion-vector-v1` blobs, the format outputs one row per such blob. Fixed output columns: - `referenced_data_file` (`String`) - location of the data file the deletion vector applies to (`referenced-data-file` blob property) - `deleted_rows` (`Array(UInt64)`) - 64-bit row positions deleted according to the deletion vector roaring bitmap -Deletion vectors whose declared `cardinality` exceeds an absolute materialization ceiling are rejected when `deleted_rows` is requested. Footer `deletion-vector-v1` properties (including that `cardinality` parses as an unsigned integer) are always validated. Selecting only `referenced_data_file` skips on-disk payload I/O and therefore also skips envelope, CRC, roaring deserialize, and the materialization ceiling — intentionally, so a path-only projection does not read up to the blob-size cap. +Deletion vectors whose declared `cardinality` exceeds an absolute materialization ceiling are rejected when `deleted_rows` is requested, **before** envelope peek or full blob allocation (same fail-closed order as the Iceberg deletion-vector reader). Footer `deletion-vector-v1` properties are always validated: `cardinality` must parse as an unsigned integer, `snapshot-id` / `sequence-number` must be `-1`, and `fields` must be either empty or the singleton Iceberg reserved `_pos` id (`2147483645`) that Spark writes for file-scoped DVs — other `fields` lists (column-scoped DVs) are rejected. Selecting only `referenced_data_file` skips on-disk payload I/O and therefore also skips envelope, CRC, roaring deserialize, and the materialization ceiling — intentionally, so a path-only projection does not read up to the blob-size cap. -On-disk `deletion-vector-v1` blob length is bounded by an absolute ceiling (aligned with Iceberg's 2 GiB content-size check). When `deleted_rows` is requested, the reader peeks the envelope header (combined length and magic) before allocating the full payload; CRC is verified after the bounded read. +On-disk `deletion-vector-v1` blob length is bounded by an absolute ceiling (aligned with Iceberg's 2 GiB content-size check). When `deleted_rows` is requested and cardinality is within the materialization ceiling, the reader peeks the envelope header (combined length and magic) before allocating the full payload; CRC is verified after the bounded read. LZ4-compressed and uncompressed puffin footers are supported. Footer payload size (and declared LZ4 content size) is bounded by a compression ratio where applicable and an absolute ceiling; oversized footers are rejected before allocation. diff --git a/docs/reference/formats/Puffin/PuffinMetadata.mdx b/docs/reference/formats/Puffin/PuffinMetadata.mdx index cace329a9705..7c45ac36f764 100644 --- a/docs/reference/formats/Puffin/PuffinMetadata.mdx +++ b/docs/reference/formats/Puffin/PuffinMetadata.mdx @@ -1,6 +1,10 @@ --- description: 'Documentation for the PuffinMetadata format' +input_format: true +output_format: false keywords: ['PuffinMetadata'] +sidebar_label: 'PuffinMetadata' +sidebar_position: 2 slug: /interfaces/formats/PuffinMetadata title: 'PuffinMetadata' doc_type: 'reference' @@ -9,9 +13,7 @@ doc_type: 'reference' ## Description {#description} Special input format for reading [Apache Iceberg Puffin](https://iceberg.apache.org/puffin-spec/) file footer metadata. -It outputs one row per blob entry from the footer `BlobMetadata` list. - -`deletion-vector-v1` is the only supported blob type: a file containing any other blob type (for example `apache-datasketches-theta-v1`) is rejected. +It outputs one row per blob entry from the footer `BlobMetadata` list, including non-deletion-vector types (for example `apache-datasketches-theta-v1`). Full deletion-vector property validation applies only to `deletion-vector-v1` entries. Fixed output columns: - `blob_type` (`String`) - blob type, for example `deletion-vector-v1` diff --git a/docs/reference/functions/table-functions/iceberg.mdx b/docs/reference/functions/table-functions/iceberg.mdx index 211a6d285c6a..07fef0d65761 100644 --- a/docs/reference/functions/table-functions/iceberg.mdx +++ b/docs/reference/functions/table-functions/iceberg.mdx @@ -122,11 +122,23 @@ ClickHouse supports time travel for Iceberg tables, allowing you to query histor ## Processing of tables with deleted rows {#deleted-rows} -Currently, only Iceberg tables with [position deletes](https://iceberg.apache.org/spec/#position-delete-files) are supported. +ClickHouse supports reading Iceberg tables that use the following deletion methods: -The following deletion methods are **not supported**: -- [Equality deletes](https://iceberg.apache.org/spec/#equality-delete-files) -- [Deletion vectors](https://iceberg.apache.org/spec/#deletion-vectors) (introduced in v3) +- [Position deletes](https://iceberg.apache.org/spec/#position-delete-files) +- [Equality deletes](https://iceberg.apache.org/spec/#equality-delete-files) (supported from version 25.8+) +- [Deletion vectors](https://iceberg.apache.org/spec/#deletion-vectors) stored in Puffin files (Iceberg v3, read-only) + +The following limitations apply to deletion vectors: + +- Only `deletion-vector-v1` Puffin blobs are supported +- Data files must be in Parquet format +- Column-scoped deletion vectors (user column ids in puffin `fields`) are not supported. Writers may set `fields` to `[]` or to the Iceberg reserved `_pos` id (`2147483645`) for file-scoped deletion vectors. +- Writing deletion vectors is not supported +- `DELETE` / `UPDATE` mutations on Iceberg format version 3+ tables are rejected (writers must not add position-delete files) + +Parsed deletion vectors can be cached in memory when `use_puffin_files_cache` is enabled and the puffin file has a non-empty `etag`. Empty deletion vectors are cached as well, so repeated reads do not re-fetch the puffin file. Parsed footers for coalesced multi-DV Puffin files are memoized with that cache (same identity: storage, path, `etag`) so slices share one footer parse; the memo shares `puffin_files_cache_size` / max-entry limits and is dropped when the cache is disabled (`puffin_files_cache_size=0`) or cleared. The cache can be cleared with `SYSTEM DROP PUFFIN FILES CACHE`. + +For [`icebergCluster`](/sql-reference/table-functions/icebergCluster.md) (and `object_storage_cluster`), the initiator loads and materializes each data file's deletion vector while distributing tasks, then sends the resulting row bitmap to workers with the task. Workers apply the bitmap; they do not re-read the Puffin blob for that path. On wide v3 tables this can make the initiator a serialization point for deletion-vector I/O and decode. ### Basic usage {#basic-usage} @@ -386,8 +398,9 @@ y: 993 ### DELETE {#iceberg-writes-delete} -Deleting extra rows in the merge-on-read format is also supported in ClickHouse. +Deleting extra rows in the merge-on-read format is also supported in ClickHouse for Iceberg format version 2. This query will create a new snapshot with position delete files. +Mutations on format version 3+ tables are rejected until ClickHouse can write deletion vectors (Iceberg v3 writers must not add new position-delete files). ### Example {#example-iceberg-writes-delete} diff --git a/docs/reference/statements/system.mdx b/docs/reference/statements/system.mdx index ce4d37411604..3a99b8b5b007 100644 --- a/docs/reference/statements/system.mdx +++ b/docs/reference/statements/system.mdx @@ -134,12 +134,18 @@ Clears the per-URL Confluent Schema Registry caches used by the `AvroConfluent` Clears the parquet metadata cache. +<<<<<<< HEAD:docs/reference/statements/system.mdx ## SYSTEM CLEAR|DROP PAIMON METADATA CACHE {#drop-paimon-metadata-cache} Clears the in-memory cache of parsed Paimon metadata files (manifest lists and manifests). ## SYSTEM CLEAR|DROP POINT IN POLYGON CACHE {#drop-point-in-polygon-cache} Clears the cache of preprocessed constant polygons used by the function [`pointInPolygon`](/reference/functions/regular-functions/geo/coordinates#pointinpolygon). The configured size limit (the `point_in_polygon_cache_size` server setting) is left unchanged, so the cache keeps accepting entries afterwards. To disable the cache instead, set `point_in_polygon_cache_size` to `0`. +======= +## SYSTEM DROP PUFFIN FILES CACHE {#drop-puffin-files-cache} + +Clears the Puffin files cache used for parsed Iceberg puffin file content such as deletion vectors. +>>>>>>> 4b7cecaa3cf (Merge pull request #2183 from Altinity/feature/antalya-26.6/iceberg-puffin-deletion-vectors-read-2):docs/en/sql-reference/statements/system.md ## SYSTEM CLEAR|DROP TEXT INDEX CACHES {#drop-text-index-caches} diff --git a/programs/local/LocalServer.cpp b/programs/local/LocalServer.cpp index ca93459ba7c5..822087044f05 100644 --- a/programs/local/LocalServer.cpp +++ b/programs/local/LocalServer.cpp @@ -183,6 +183,10 @@ namespace ServerSetting extern const ServerSettingsUInt64 parquet_metadata_cache_size; extern const ServerSettingsUInt64 parquet_metadata_cache_max_entries; extern const ServerSettingsDouble parquet_metadata_cache_size_ratio; + extern const ServerSettingsString puffin_files_cache_policy; + extern const ServerSettingsUInt64 puffin_files_cache_size; + extern const ServerSettingsUInt64 puffin_files_cache_max_entries; + extern const ServerSettingsDouble puffin_files_cache_size_ratio; extern const ServerSettingsUInt64 max_active_parts_loading_thread_pool_size; extern const ServerSettingsUInt64 max_io_thread_pool_free_size; extern const ServerSettingsUInt64 max_io_thread_pool_size; @@ -1635,6 +1639,17 @@ void LocalServer::processConfig() global_context->setParquetMetadataCache(parquet_metadata_cache_policy, parquet_metadata_cache_size, parquet_metadata_cache_max_entries, parquet_metadata_cache_size_ratio); #endif + String puffin_files_cache_policy = server_settings[ServerSetting::puffin_files_cache_policy]; + size_t puffin_files_cache_size = server_settings[ServerSetting::puffin_files_cache_size]; + size_t puffin_files_cache_max_entries = server_settings[ServerSetting::puffin_files_cache_max_entries]; + double puffin_files_cache_size_ratio = server_settings[ServerSetting::puffin_files_cache_size_ratio]; + if (puffin_files_cache_size > max_cache_size) + { + puffin_files_cache_size = max_cache_size; + LOG_INFO(log, "Lowered Puffin files cache size to {} because the system has limited RAM", formatReadableSizeWithBinarySuffix(puffin_files_cache_size)); + } + global_context->setPuffinFilesCache(puffin_files_cache_policy, puffin_files_cache_size, puffin_files_cache_max_entries, puffin_files_cache_size_ratio); + Names allowed_disks_table_engines; splitInto<','>(allowed_disks_table_engines, server_settings[ServerSetting::allowed_disks_for_table_engines].value); global_context->setAllowedDisksForTableEngines(std::unordered_set(allowed_disks_table_engines.begin(), allowed_disks_table_engines.end())); diff --git a/programs/server/Server.cpp b/programs/server/Server.cpp index bbe9c05f88e8..e9becbea0a59 100644 --- a/programs/server/Server.cpp +++ b/programs/server/Server.cpp @@ -302,6 +302,10 @@ namespace ServerSetting extern const ServerSettingsUInt64 parquet_metadata_cache_size; extern const ServerSettingsUInt64 parquet_metadata_cache_max_entries; extern const ServerSettingsDouble parquet_metadata_cache_size_ratio; + extern const ServerSettingsString puffin_files_cache_policy; + extern const ServerSettingsUInt64 puffin_files_cache_size; + extern const ServerSettingsUInt64 puffin_files_cache_max_entries; + extern const ServerSettingsDouble puffin_files_cache_size_ratio; extern const ServerSettingsUInt64 io_thread_pool_queue_size; extern const ServerSettingsBool jemalloc_enable_global_profiler; extern const ServerSettingsBool jemalloc_collect_global_profile_samples_in_trace_log; @@ -2508,6 +2512,16 @@ try } global_context->setParquetMetadataCache(parquet_metadata_cache_policy, parquet_metadata_cache_size, parquet_metadata_cache_max_entries, parquet_metadata_cache_size_ratio); #endif + String puffin_files_cache_policy = server_settings[ServerSetting::puffin_files_cache_policy]; + size_t puffin_files_cache_size = server_settings[ServerSetting::puffin_files_cache_size]; + size_t puffin_files_cache_max_entries = server_settings[ServerSetting::puffin_files_cache_max_entries]; + double puffin_files_cache_size_ratio = server_settings[ServerSetting::puffin_files_cache_size_ratio]; + if (puffin_files_cache_size > max_cache_size) + { + puffin_files_cache_size = max_cache_size; + LOG_INFO(log, "Lowered Puffin files cache size to {} because the system has limited RAM", formatReadableSizeWithBinarySuffix(puffin_files_cache_size)); + } + global_context->setPuffinFilesCache(puffin_files_cache_policy, puffin_files_cache_size, puffin_files_cache_max_entries, puffin_files_cache_size_ratio); Names allowed_disks_table_engines; splitInto<','>(allowed_disks_table_engines, server_settings[ServerSetting::allowed_disks_for_table_engines].value); @@ -3015,6 +3029,7 @@ try #if USE_PARQUET global_context->updateParquetMetadataCacheConfiguration(config(), max_cache_size_in_bytes); #endif + global_context->updatePuffinFilesCacheConfiguration(config(), max_cache_size_in_bytes); } #if USE_SSL diff --git a/src/Access/Common/AccessType.h b/src/Access/Common/AccessType.h index 9314d0dcddab..94abce275135 100644 --- a/src/Access/Common/AccessType.h +++ b/src/Access/Common/AccessType.h @@ -329,7 +329,11 @@ enum class AccessType : uint8_t M(SYSTEM_DROP_PAIMON_METADATA_CACHE, "SYSTEM CLEAR PAIMON_METADATA_CACHE, SYSTEM DROP PAIMON_METADATA_CACHE", GLOBAL, SYSTEM_DROP_CACHE) \ M(SYSTEM_DROP_AVRO_SCHEMA_CACHE, "SYSTEM CLEAR AVRO SCHEMA CACHE, SYSTEM DROP AVRO SCHEMA CACHE, DROP AVRO SCHEMA CACHE", GLOBAL, SYSTEM_DROP_CACHE) \ M(SYSTEM_DROP_PARQUET_METADATA_CACHE, "SYSTEM DROP PARQUET_METADATA_CACHE", GLOBAL, SYSTEM_DROP_CACHE) \ +<<<<<<< HEAD M(SYSTEM_DROP_POINT_IN_POLYGON_CACHE, "SYSTEM CLEAR POINT IN POLYGON CACHE, SYSTEM DROP POINT IN POLYGON CACHE, DROP POINT IN POLYGON CACHE", GLOBAL, SYSTEM_DROP_CACHE) \ +======= + M(SYSTEM_DROP_PUFFIN_FILES_CACHE, "SYSTEM DROP PUFFIN_FILES_CACHE", GLOBAL, SYSTEM_DROP_CACHE) \ +>>>>>>> 4b7cecaa3cf (Merge pull request #2183 from Altinity/feature/antalya-26.6/iceberg-puffin-deletion-vectors-read-2) M(SYSTEM_PREWARM_PRIMARY_INDEX_CACHE, "SYSTEM PREWARM PRIMARY INDEX, PREWARM PRIMARY INDEX CACHE, PREWARM PRIMARY INDEX", GLOBAL, SYSTEM_DROP_CACHE) \ M(SYSTEM_DROP_PRIMARY_INDEX_CACHE, "SYSTEM CLEAR PRIMARY INDEX CACHE, SYSTEM DROP PRIMARY INDEX, DROP PRIMARY INDEX CACHE, DROP PRIMARY INDEX", GLOBAL, SYSTEM_DROP_CACHE) \ M(SYSTEM_DROP_UNCOMPRESSED_CACHE, "SYSTEM CLEAR UNCOMPRESSED CACHE, SYSTEM DROP UNCOMPRESSED, DROP UNCOMPRESSED CACHE, DROP UNCOMPRESSED", GLOBAL, SYSTEM_DROP_CACHE) \ diff --git a/src/AggregateFunctions/AggregateFunctionGroupBitmapData.h b/src/AggregateFunctions/AggregateFunctionGroupBitmapData.h index eb852d24eb9f..805cd32392f5 100644 --- a/src/AggregateFunctions/AggregateFunctionGroupBitmapData.h +++ b/src/AggregateFunctions/AggregateFunctionGroupBitmapData.h @@ -35,6 +35,72 @@ enum BitmapKind Bitmap = 1 }; +/// Approximate heap footprint of a 32-bit CRoaring bitmap (index capacity + container capacities). +/// Unlike `Roaring::getSizeInBytes()`, array/run containers use allocated capacity, not cardinality. +inline UInt64 estimateRoaring32AllocatedBytes(const roaring::Roaring & bitmap) +{ + using namespace roaring::internal; + const roaring::roaring_array_t * ra = &bitmap.roaring.high_low_container; + UInt64 bytes = sizeof(roaring::Roaring); + if (ra->allocation_size > 0) + { + bytes += static_cast(ra->allocation_size) + * (sizeof(uint16_t) + sizeof(uint8_t) + sizeof(container_t *)); + } + for (int32_t i = 0; i < ra->size; ++i) + { + uint8_t typecode = ra->typecodes[i]; + const container_t * c = container_unwrap_shared(ra->containers[i], &typecode); + switch (typecode) + { + case ARRAY_CONTAINER_TYPE: + { + const array_container_t * ac = const_CAST_array(c); + bytes += sizeof(array_container_t) + static_cast(ac->capacity) * sizeof(uint16_t); + break; + } + case BITSET_CONTAINER_TYPE: + { + bytes += sizeof(bitset_container_t) + BITSET_CONTAINER_SIZE_IN_WORDS * sizeof(uint64_t); + break; + } + case RUN_CONTAINER_TYPE: + { + const run_container_t * rc = const_CAST_run(c); + bytes += sizeof(run_container_t) + static_cast(rc->capacity) * sizeof(rle16_t); + break; + } + default: + break; + } + } + return bytes; +} + +/// Count distinct high-32 keys in a Roaring64Map. Fast-path when min/max share one key. +inline UInt64 countRoaring64MapHighKeys(const roaring::Roaring64Map & bitmap) +{ + if (bitmap.isEmpty()) + return 0; + + const UInt64 min_value = bitmap.minimum(); + const UInt64 max_value = bitmap.maximum(); + if ((min_value >> 32) == (max_value >> 32)) + return 1; + + UInt64 keys = 0; + UInt64 prev_high = ~UInt64{0}; + for (auto it = bitmap.begin(); it != bitmap.end(); ++it) + { + const UInt64 high = static_cast(*it) >> 32; + if (high != prev_high) + { + ++keys; + prev_high = high; + } + } + return keys; +} /** * For a small number of values - an array of fixed size "on the stack". @@ -94,6 +160,34 @@ class RoaringBitmapWithSmallSet : private boost::noncopyable return roaring_bitmap->cardinality(); } + UInt64 getAllocatedBytes() const + { + if (isSmall()) + return sizeof(small); + + /// Prefer a heap estimate over `getSizeInBytes()` (serialization size). Include a small + /// allowance for the shared_ptr control block that owns `roaring_bitmap`. + constexpr UInt64 SHARED_PTR_CONTROL_BLOCK = 32; + + if constexpr (sizeof(T) < 8) + { + return estimateRoaring32AllocatedBytes(*roaring_bitmap) + SHARED_PTR_CONTROL_BLOCK; + } + else + { + /// Roaring64Map keeps roarings private; approximate as native serialization size plus + /// per-high-key map/Roaring/container overhead (serialization undercounts capacity and + /// std::map nodes — important for sparse high keys). + constexpr UInt64 PER_HIGH_KEY_OVERHEAD = + 4 * sizeof(void *) + sizeof(UInt32) + sizeof(roaring::Roaring) + 64; + + const UInt64 serialized = roaring_bitmap->getSizeInBytes(/*portable=*/false); + const UInt64 high_keys = countRoaring64MapHighKeys(*roaring_bitmap); + return serialized + high_keys * PER_HIGH_KEY_OVERHEAD + sizeof(roaring::Roaring64Map) + + SHARED_PTR_CONTROL_BLOCK; + } + } + void merge(const RoaringBitmapWithSmallSet & r1) { if (r1.isLarge()) @@ -532,7 +626,57 @@ class RoaringBitmapWithSmallSet : private boost::noncopyable } /** +<<<<<<< HEAD * Return new set of the smallest `limit` values (as UnsignedT) which are no less than `range_start`. +======= + * Count set bits in `[range_start, range_end)` without allocating a result bitmap. + * Used by need-only-count DV filtering to avoid an O(N) dense Filter over file rows. + * Implemented via roaring `rank` so repeated per-row-group queries stay O(containers), + * not O(row_groups × cardinality). + */ + UInt64 rb_range_cardinality(UInt64 range_start, UInt64 range_end) const /// NOLINT + { + if (range_start >= range_end) + return 0; + + if (isSmall()) + { + UInt64 count = 0; + for (const auto & x : small) + { + const UInt64 val = static_cast(x.getValue()); + if (val >= range_start && val < range_end) + ++count; + } + return count; + } + + /// |bitmap ∩ [start, end)| = rank(end - 1) - rank(start - 1). Same formula as DeleteBitmap. + if constexpr (sizeof(T) < 8) + { + constexpr UInt64 max_row = std::numeric_limits::max(); + if (range_start > max_row) + return 0; + const UInt64 hi_inclusive = std::min(range_end - 1, max_row); + if (hi_inclusive < range_start) + return 0; + const UInt64 upper = roaring_bitmap->rank(static_cast(hi_inclusive)); + const UInt64 lower = (range_start == 0) ? 0 : roaring_bitmap->rank(static_cast(range_start - 1)); + return upper - lower; + } + else + { + const UInt64 hi_inclusive = range_end - 1; + const UInt64 upper = roaring_bitmap->rank(hi_inclusive); + const UInt64 lower = (range_start == 0) ? 0 : roaring_bitmap->rank(range_start - 1); + return upper - lower; + } + } + + /** + * Return new set of the smallest `limit` values in set which is no less than `range_start`. + * It's used in subset and currently only support UInt32 +>>>>>>> 4b7cecaa3cf (Merge pull request #2183 from Altinity/feature/antalya-26.6/iceberg-puffin-deletion-vectors-read-2) */ UInt64 rb_limit(UInt64 range_start, UInt64 limit, RoaringBitmapWithSmallSet & r1) const /// NOLINT { diff --git a/src/Client/BuzzHouse/Generator/SessionSettings.cpp b/src/Client/BuzzHouse/Generator/SessionSettings.cpp index 51211ceebf79..7d29a36c07e6 100644 --- a/src/Client/BuzzHouse/Generator/SessionSettings.cpp +++ b/src/Client/BuzzHouse/Generator/SessionSettings.cpp @@ -1843,6 +1843,7 @@ static std::unordered_map serverSettings2 = { {"use_page_cache_for_local_disks", trueOrFalseSetting}, {"use_page_cache_for_object_storage", trueOrFalseSetting}, {"use_parquet_metadata_cache", trueOrFalseSetting}, + {"use_puffin_files_cache", trueOrFalseSetting}, {"use_query_cache", trueOrFalseSetting}, {"use_roaring_bitmap_iceberg_positional_deletes", trueOrFalseSetting}, {"use_skip_indexes_if_final_exact_mode", CHSetting(trueOrFalse, {"0", "1"}, true)}, diff --git a/src/Common/CacheBase.h b/src/Common/CacheBase.h index 3ca63aa79488..29d06f6382bc 100644 --- a/src/Common/CacheBase.h +++ b/src/Common/CacheBase.h @@ -10,6 +10,7 @@ #include #include +#include #include #include #include @@ -23,6 +24,19 @@ namespace ErrorCodes extern const int BAD_ARGUMENTS; } +/// Outcome of `CacheBase::getOrSetWithOutcome`, decided under the cache lock together with the +/// returned value so callers can classify hits/misses without a follow-up `contains()` race. +enum class CacheGetOrSetOutcome : uint8_t +{ + /// Value was resident when returned (this caller did not run `load_func`). + Hit, + /// This caller ran `load_func` and inserted the value into the cache. + MissInserted, + /// Value was produced (by this caller or a stampede peer) but is not resident — e.g. a concurrent + /// `clear()` discarded the insert token. The returned `MappedPtr` is still valid for the caller. + MissNotResident, +}; + /// Thread-safe cache that evicts entries using special cache policy /// (default policy evicts entries which are not used for a long time). /// WeightFunction is a functor that takes Mapped as a parameter and returns "weight" (approximate size) @@ -156,9 +170,9 @@ class CacheBase /// Exceptions occurring in load_func will be propagated to the caller. Another thread from the /// set of concurrent threads will then try to call its load_func etc. /// - /// Returns std::pair of the cached value and a bool indicating whether the value was produced during this call. + /// Returns the value together with an outcome that is atomic with residency (see CacheGetOrSetOutcome). template - std::pair getOrSet(const Key & key, LoadFunc && load_func) + std::pair getOrSetWithOutcome(const Key & key, LoadFunc && load_func) { InsertTokenHolder token_holder; { @@ -167,7 +181,7 @@ class CacheBase if (val) { ++hits; - return std::make_pair(val, false); + return std::make_pair(val, CacheGetOrSetOutcome::Hit); } auto & token = insert_tokens[key]; @@ -186,8 +200,18 @@ class CacheBase if (token->value) { /// Another thread already produced the value while we waited for token->mutex. - ++hits; - return std::make_pair(token->value, false); + /// If a concurrent clear() discarded that insert, the value is not resident — + /// count a miss (same class of outcome as the producer when insertion is skipped). + { + std::lock_guard cache_lock(mutex); + if (auto cached = cache_policy->get(key)) + { + ++hits; + return std::make_pair(std::move(cached), CacheGetOrSetOutcome::Hit); + } + } + ++misses; + return std::make_pair(token->value, CacheGetOrSetOutcome::MissNotResident); } ++misses; @@ -197,18 +221,28 @@ class CacheBase /// Insert the new value only if the token is still in present in insert_tokens. /// (The token may be absent because of a concurrent clear() call). - bool result = false; auto token_it = insert_tokens.find(key); if (token_it != insert_tokens.end() && token_it->second.get() == token) { cache_policy->set(key, token->value); - result = true; + if (!token->cleaned_up) + token_holder.cleanup(token_lock, cache_lock); + return std::make_pair(token->value, CacheGetOrSetOutcome::MissInserted); } if (!token->cleaned_up) token_holder.cleanup(token_lock, cache_lock); - return std::make_pair(token->value, result); + return std::make_pair(token->value, CacheGetOrSetOutcome::MissNotResident); + } + + /// Same as getOrSetWithOutcome, but the bool is true only when this call inserted the value + /// (`CacheGetOrSetOutcome::MissInserted`). Prefer getOrSetWithOutcome when classifying hits/misses. + template + std::pair getOrSet(const Key & key, LoadFunc && load_func) + { + auto [value, outcome] = getOrSetWithOutcome(key, std::forward(load_func)); + return std::make_pair(std::move(value), outcome == CacheGetOrSetOutcome::MissInserted); } void getStats(size_t & out_hits, size_t & out_misses) const @@ -218,6 +252,17 @@ class CacheBase out_misses = misses; } + /// Number of concurrent getOrSet callers holding the insert token for `key`, or 0 if none. + /// Useful for tests that wait until a second thread has joined an in-flight load. + size_t getInsertTokenRefcount(const Key & key) const + { + std::lock_guard lock(mutex); + auto it = insert_tokens.find(key); + if (it == insert_tokens.end()) + return 0; + return it->second->refcount; + } + std::vector dump() const { std::lock_guard lock(mutex); diff --git a/src/Common/CurrentMetrics.cpp b/src/Common/CurrentMetrics.cpp index 10e6f69d5ce5..bb9bd52776cf 100644 --- a/src/Common/CurrentMetrics.cpp +++ b/src/Common/CurrentMetrics.cpp @@ -370,6 +370,8 @@ M(PaimonMetadataFilesCacheFiles, "Number of cached files in the Paimon metadata cache") \ M(ParquetMetadataCacheBytes, "Size of the Parquet metadata cache in bytes") \ M(ParquetMetadataCacheFiles, "Number of cached files in the Parquet metadata cache") \ + M(PuffinFilesCacheBytes, "Size of the Puffin files cache in bytes (deletion vectors plus memoized footers)") \ + M(PuffinFilesCacheFiles, "Number of cached entries in the Puffin files cache (deletion vectors plus memoized footers)") \ M(AvroSchemaCacheBytes, "Size of the Avro schema cache in bytes") \ M(AvroSchemaCacheCells, "Number of cached Avro schemas, including both registered and fetched schemas.") \ M(AvroSchemaRegistryCacheBytes, "Size of the Avro schema registry cache in bytes") \ diff --git a/src/Common/ProfileEvents.cpp b/src/Common/ProfileEvents.cpp index b7759df73fcb..6cf52db6b2a5 100644 --- a/src/Common/ProfileEvents.cpp +++ b/src/Common/ProfileEvents.cpp @@ -120,10 +120,18 @@ M(IcebergMetadataReturnedObjectInfos, "Total number of returned object infos from iceberg iterator.", ValueType::Number) \ M(IcebergMinMaxNonPrunedDeleteFiles, "Total number of accepted data files-position delete file pairs by minmax analysis from pairs suitable by partitioning and sequence number.", ValueType::Number) \ M(IcebergMinMaxPrunedDeleteFiles, "Total number of accepted data files-position delete file pairs by minmax analysis from pairs suitable by partitioning and sequence number.", ValueType::Number) \ +<<<<<<< HEAD M(OneLakeAccessTokenRequests, "Number of access token requests issued by the OneLake catalog via the Entra ID refresh token grant.", ValueType::Number) \ M(OneLakeAccessTokenRequestFailures, "Number of failed access token requests issued by the OneLake catalog via the Entra ID refresh token grant (including expired or revoked refresh tokens).", ValueType::Number) \ M(OneLakeAccessTokenRequestMicroseconds, "Total time spent requesting access tokens via the Entra ID refresh token grant in the OneLake catalog.", ValueType::Microseconds) \ M(OneLakeAccessTokenExpirations, "Number of times the cached OneLake access token was found expired and renewed transparently.", ValueType::Number) \ +======= + M(PuffinFilesRead, "Number of Puffin files read (footer or deletion vector blob).", ValueType::Number) \ + M(PuffinFileReadMicroseconds, "Total time spent reading Puffin files.", ValueType::Microseconds) \ + M(PuffinFilesCacheHits, "Number of times parsed Puffin file content has been found in the cache.", ValueType::Number) \ + M(PuffinFilesCacheMisses, "Number of times parsed Puffin file content has not been found in the cache and had to be read from disk.", ValueType::Number) \ + M(PuffinFilesCacheWeightLost, "Approximate number of bytes evicted from the Puffin files cache.", ValueType::Number) \ +>>>>>>> 4b7cecaa3cf (Merge pull request #2183 from Altinity/feature/antalya-26.6/iceberg-puffin-deletion-vectors-read-2) M(VectorSimilarityIndexCacheHits, "Number of times an index granule has been found in the vector index cache.", ValueType::Number) \ M(VectorSimilarityIndexCacheMisses, "Number of times an index granule has not been found in the vector index cache and had to be read from disk.", ValueType::Number) \ M(VectorSimilarityIndexCacheWeightLost, "Approximate number of bytes evicted from the vector index cache.", ValueType::Number) \ diff --git a/src/Common/tests/gtest_lru_cache.cpp b/src/Common/tests/gtest_lru_cache.cpp index ed9c3c34c45f..ff6ddf4df52c 100644 --- a/src/Common/tests/gtest_lru_cache.cpp +++ b/src/Common/tests/gtest_lru_cache.cpp @@ -105,6 +105,73 @@ TEST(LRUCache, getOrSet) ASSERT_TRUE(*value == 10); } +TEST(LRUCache, getOrSetWithOutcomeHitAndMissInserted) +{ + using SimpleCacheBase = DB::CacheBase; + SimpleCacheBase cache("LRU", CurrentMetrics::end(), CurrentMetrics::end(), /*max_size_in_bytes*/ 10, /*max_count*/ 10, /*size_ratio*/ 0.5); + + size_t loads = 0; + auto load = [&]() + { + ++loads; + return std::make_shared(42); + }; + + { + auto [value, outcome] = cache.getOrSetWithOutcome(1, load); + ASSERT_NE(value, nullptr); + EXPECT_EQ(*value, 42); + EXPECT_EQ(outcome, DB::CacheGetOrSetOutcome::MissInserted); + EXPECT_EQ(loads, 1u); + } + { + auto [value, outcome] = cache.getOrSetWithOutcome(1, load); + ASSERT_NE(value, nullptr); + EXPECT_EQ(*value, 42); + EXPECT_EQ(outcome, DB::CacheGetOrSetOutcome::Hit); + EXPECT_EQ(loads, 1u); + } +} + +TEST(LRUCache, getOrSetWithOutcomeClearDuringLoadIsMissNotResident) +{ + using SimpleCacheBase = DB::CacheBase; + SimpleCacheBase cache("LRU", CurrentMetrics::end(), CurrentMetrics::end(), /*max_size_in_bytes*/ 10, /*max_count*/ 10, /*size_ratio*/ 0.5); + + auto [value, outcome] = cache.getOrSetWithOutcome( + 1, + [&]() + { + cache.clear(); + return std::make_shared(7); + }); + + ASSERT_NE(value, nullptr); + EXPECT_EQ(*value, 7); + EXPECT_EQ(outcome, DB::CacheGetOrSetOutcome::MissNotResident); + EXPECT_FALSE(cache.contains(1)); +} + +TEST(LRUCache, getOrSetWithOutcomeHitRemainsHitIfClearedAfterReturn) +{ + using SimpleCacheBase = DB::CacheBase; + SimpleCacheBase cache("LRU", CurrentMetrics::end(), CurrentMetrics::end(), /*max_size_in_bytes*/ 10, /*max_count*/ 10, /*size_ratio*/ 0.5); + + cache.getOrSetWithOutcome(1, []() { return std::make_shared(1); }); + + auto [value, outcome] = cache.getOrSetWithOutcome(1, []() { return std::make_shared(2); }); + EXPECT_EQ(outcome, DB::CacheGetOrSetOutcome::Hit); + ASSERT_NE(value, nullptr); + EXPECT_EQ(*value, 1); + + /// A follow-up contains() after clear would return false; callers must use `outcome`, not a + /// second residency check, when classifying hits (PuffinFilesCache used to race here). + cache.clear(); + EXPECT_FALSE(cache.contains(1)); + EXPECT_EQ(outcome, DB::CacheGetOrSetOutcome::Hit); + EXPECT_EQ(*value, 1); +} + TEST(LRUCache, noOnRemoveEntryCallback) { diff --git a/src/Core/Defines.h b/src/Core/Defines.h index 68c10a22c5e3..71aabf5549de 100644 --- a/src/Core/Defines.h +++ b/src/Core/Defines.h @@ -139,6 +139,10 @@ static constexpr auto DEFAULT_PARQUET_METADATA_CACHE_POLICY = "SLRU"; static constexpr auto DEFAULT_PARQUET_METADATA_CACHE_MAX_SIZE = 512_MiB; static constexpr auto DEFAULT_PARQUET_METADATA_CACHE_SIZE_RATIO = 0.5; static constexpr auto DEFAULT_PARQUET_METADATA_CACHE_MAX_ENTRIES = 5000; +static constexpr auto DEFAULT_PUFFIN_FILES_CACHE_POLICY = "SLRU"; +static constexpr auto DEFAULT_PUFFIN_FILES_CACHE_MAX_SIZE = 512_MiB; +static constexpr auto DEFAULT_PUFFIN_FILES_CACHE_SIZE_RATIO = 0.5; +static constexpr auto DEFAULT_PUFFIN_FILES_CACHE_MAX_ENTRIES = 5000; static constexpr auto DEFAULT_QUERY_CONDITION_CACHE_POLICY = "SLRU"; static constexpr auto DEFAULT_QUERY_CONDITION_CACHE_MAX_SIZE = 100_MiB; static constexpr auto DEFAULT_QUERY_CONDITION_CACHE_SIZE_RATIO = 0.5l; diff --git a/src/Core/ServerSettings.cpp b/src/Core/ServerSettings.cpp index 7d8fc679f59c..cb002969e31e 100644 --- a/src/Core/ServerSettings.cpp +++ b/src/Core/ServerSettings.cpp @@ -29,6 +29,7 @@ #if USE_PARQUET # include #endif +#include #include #if ENABLE_DISTRIBUTED_CACHE # include @@ -622,6 +623,10 @@ This setting can be modified at runtime and will take effect immediately. DECLARE(UInt64, parquet_metadata_cache_size, DEFAULT_PARQUET_METADATA_CACHE_MAX_SIZE, "Maximum size of parquet metadata cache in bytes. Zero means disabled.", 0) \ DECLARE(UInt64, parquet_metadata_cache_max_entries, DEFAULT_PARQUET_METADATA_CACHE_MAX_ENTRIES, "Maximum size of parquet metadata files cache in entries. Zero means disabled.", 0) \ DECLARE(Double, parquet_metadata_cache_size_ratio, DEFAULT_PARQUET_METADATA_CACHE_SIZE_RATIO, "The size of the protected queue (in case of SLRU policy) in the parquet metadata cache relative to the cache's total size.", 0) \ + DECLARE(String, puffin_files_cache_policy, DEFAULT_PUFFIN_FILES_CACHE_POLICY, "Puffin files cache policy name (SLRU or LRU).", 0) \ + DECLARE(UInt64, puffin_files_cache_size, DEFAULT_PUFFIN_FILES_CACHE_MAX_SIZE, "Maximum size of Puffin files cache in bytes. Zero means disabled.", 0) \ + DECLARE(UInt64, puffin_files_cache_max_entries, DEFAULT_PUFFIN_FILES_CACHE_MAX_ENTRIES, "Maximum number of entries in the Puffin files cache. Zero means unlimited.", 0) \ + DECLARE(Double, puffin_files_cache_size_ratio, DEFAULT_PUFFIN_FILES_CACHE_SIZE_RATIO, "The size of the protected queue (in case of SLRU policy) in the Puffin files cache relative to the cache's total size.", 0) \ DECLARE(String, allowed_disks_for_table_engines, "", "List of disks allowed for use with Iceberg", 0) \ DECLARE(String, vector_similarity_index_cache_policy, DEFAULT_VECTOR_SIMILARITY_INDEX_CACHE_POLICY, "Vector similarity index cache policy name.", 0) \ DECLARE(UInt64, vector_similarity_index_cache_size, DEFAULT_VECTOR_SIMILARITY_INDEX_CACHE_MAX_SIZE, R"(Size of cache for vector similarity indexes. Zero means disabled. @@ -3683,6 +3688,10 @@ ChangeableSettingsMap collectChangeableServerSettings(ContextPtr context) {"parquet_metadata_cache_size", {std::to_string(context->getParquetMetadataCache()->maxSizeInBytes()), ChangeableWithoutRestart::Yes}}); #endif + if (context->getPuffinFilesCache()) + changeable_settings.insert( + {"puffin_files_cache_size", + {std::to_string(context->getPuffinFilesCache()->maxSizeInBytes()), ChangeableWithoutRestart::Yes}}); /// `keeper_hosts` is not a regular config setting; it is derived from the `` config and follows /// it on config reload, so the live value diverges from the empty default stored in `ServerSettings`. diff --git a/src/Core/Settings.cpp b/src/Core/Settings.cpp index d0fa079ac340..f51912d4d104 100644 --- a/src/Core/Settings.cpp +++ b/src/Core/Settings.cpp @@ -6030,6 +6030,7 @@ Minimum time of delay between 2 background compaction operations. DECLARE(Seconds, iceberg_compaction_data_cleanup, 60 * 60 * 3, R"( The time after which the data will be deleted. )", 0) \ +<<<<<<< HEAD DECLARE(UInt64, iceberg_compaction_commit_batch_size, 100, R"( Number of merged data files that background Iceberg compaction accumulates before publishing them in a new snapshot. @@ -6037,6 +6038,15 @@ Compaction results are published in any case once there are no candidates left t how long already merged files stay unpublished while compaction keeps finding new work. `0` restores the old behaviour of publishing only when compaction runs out of candidates - a state that is never reached while the table keeps receiving new data files, so every merged output is then written to object storage and never referenced by a snapshot. +======= + DECLARE(Bool, use_puffin_files_cache, true, R"( +If turned on, Iceberg reads may utilize the Puffin files cache for parsed puffin file content such as deletion vectors. + +Possible values: + +- 0 - Disabled +- 1 - Enabled +>>>>>>> 4b7cecaa3cf (Merge pull request #2183 from Altinity/feature/antalya-26.6/iceberg-puffin-deletion-vectors-read-2) )", 0) \ DECLARE(Bool, use_query_cache, false, R"( If turned on, `SELECT` queries may utilize the [query cache](/concepts/features/performance/caches/query-cache). Parameters [enable_reads_from_query_cache](#enable_reads_from_query_cache) diff --git a/src/Core/SettingsChangesHistory.cpp b/src/Core/SettingsChangesHistory.cpp index b9acda40b208..24a230c7a01d 100644 --- a/src/Core/SettingsChangesHistory.cpp +++ b/src/Core/SettingsChangesHistory.cpp @@ -215,6 +215,9 @@ const VersionToSettingsChangesMap & getSettingsChangesHistory() {"show_remote_databases_in_system_tables", true, true, "New setting to control whether `MySQL` and `PostgreSQL` databases are shown in `system.tables`, `system.columns` and `system.completions`."}, {"use_constant_folding_in_index_analysis", false, false, "New setting to fold partition-level constants into the filter predicate per part during MergeTree index analysis, improving pruning for filters whose branches depend on partition values."}, {"join_runtime_filter_size_from_hash_table_stats", false, true, "Use hash table size statistics collected from previous executions to size the JOIN runtime filter. When disabled, fall back to the fixed `join_runtime_bloom_filter_bytes`."}, + addSettingsChanges(settings_changes_history, "26.6.2.20001.altinityantalya", + { + {"use_puffin_files_cache", false, true, "Enables cache of parsed Puffin file content such as deletion vectors."}, }); addSettingsChanges(settings_changes_history, "26.6", diff --git a/src/Formats/FormatFilterInfo.cpp b/src/Formats/FormatFilterInfo.cpp index b585592a77d9..cee9aa77ad60 100644 --- a/src/Formats/FormatFilterInfo.cpp +++ b/src/Formats/FormatFilterInfo.cpp @@ -83,7 +83,9 @@ FormatFilterInfo::FormatFilterInfo() = default; bool FormatFilterInfo::hasFilter() const { - return filter_actions_dag != nullptr; + /// Any of these can reduce the number of rows emitted by the reader pipeline. + /// Count-from-files cache must not be populated when they are present. + return filter_actions_dag != nullptr || row_level_filter != nullptr || prewhere_info != nullptr; } namespace diff --git a/src/Formats/FormatFilterInfo.h b/src/Formats/FormatFilterInfo.h index 620a0b0253aa..32a82bac459a 100644 --- a/src/Formats/FormatFilterInfo.h +++ b/src/Formats/FormatFilterInfo.h @@ -122,6 +122,7 @@ struct FormatFilterInfo std::exception_ptr init_exception; public: + /// True if WHERE / PREWHERE / row-policy filters may change the emitted row count. bool hasFilter() const; /// Creates `key_condition` and `additional_columns` with std::call_once semantics. diff --git a/src/Interpreters/ClusterFunctionReadTask.cpp b/src/Interpreters/ClusterFunctionReadTask.cpp index 71a4a6f956ed..f8d57daccaf2 100644 --- a/src/Interpreters/ClusterFunctionReadTask.cpp +++ b/src/Interpreters/ClusterFunctionReadTask.cpp @@ -84,6 +84,59 @@ void ClusterFunctionReadTaskResponse::serialize(WriteBuffer & out, size_t worker { auto protocol_version = std::min(static_cast(worker_protocol_version), static_cast(DBMS_CLUSTER_PROCESSING_PROTOCOL_VERSION)); + + /// Fail closed: protocol < 2 omits `schema_transform`, so workers would skip data-lake schema + /// evolution and return wrong columns / values. + if (protocol_version < DBMS_CLUSTER_PROCESSING_PROTOCOL_VERSION_WITH_DATA_LAKE_METADATA + && data_lake_metadata.schema_transform + && !data_lake_metadata.schema_transform->getInputs().empty()) + { + throw Exception( + ErrorCodes::UNKNOWN_PROTOCOL, + "Worker protocol version {} cannot carry `schema_transform`, which is required for " + "distributed data-lake reads with schema evolution (minimum protocol version: {})", + protocol_version, + DBMS_CLUSTER_PROCESSING_PROTOCOL_VERSION_WITH_DATA_LAKE_METADATA); + } + + /// Fail closed: downgrading would omit deletion / selection vectors and return deleted rows. + if (protocol_version < DBMS_CLUSTER_PROCESSING_PROTOCOL_VERSION_WITH_EXCLUDED_ROWS + && hasNonEmptyExcludedRows(data_lake_metadata)) + { + throw Exception( + ErrorCodes::UNKNOWN_PROTOCOL, + "Worker protocol version {} cannot carry `excluded_rows`, which is required for distributed " + "reads with deletion vectors / selection vectors (minimum protocol version: {})", + protocol_version, + DBMS_CLUSTER_PROCESSING_PROTOCOL_VERSION_WITH_EXCLUDED_ROWS); + } + + /// Fail closed: protocol < 3 omits `iceberg_info`, so workers rebuild a plain `ObjectInfo` + /// and lose Iceberg schema IDs / file format / delete transforms. + if (protocol_version < DBMS_CLUSTER_PROCESSING_PROTOCOL_VERSION_WITH_ICEBERG_METADATA + && iceberg_info.has_value()) + { + throw Exception( + ErrorCodes::UNKNOWN_PROTOCOL, + "Worker protocol version {} cannot carry `iceberg_info` " + "(minimum protocol version: {})", + protocol_version, + DBMS_CLUSTER_PROCESSING_PROTOCOL_VERSION_WITH_ICEBERG_METADATA); + } + + /// Fail closed: protocol < 4 omits `file_bucket_info`, so each bucket task becomes a full-file + /// read and bucket-split cluster queries return duplicated rows. + if (protocol_version < DBMS_CLUSTER_PROCESSING_PROTOCOL_VERSION_WITH_FILE_BUCKETS_INFO + && file_bucket_info) + { + throw Exception( + ErrorCodes::UNKNOWN_PROTOCOL, + "Worker protocol version {} cannot carry `file_bucket_info`, which is required for " + "distributed bucket-split reads (minimum protocol version: {})", + protocol_version, + DBMS_CLUSTER_PROCESSING_PROTOCOL_VERSION_WITH_FILE_BUCKETS_INFO); + } + writeVarUInt(protocol_version, out); writeStringBinary(path, out); diff --git a/src/Interpreters/Context.cpp b/src/Interpreters/Context.cpp index e08f8f77e246..19cd2977eb87 100644 --- a/src/Interpreters/Context.cpp +++ b/src/Interpreters/Context.cpp @@ -58,6 +58,7 @@ #include #include #include +#include #include #include #include @@ -638,6 +639,7 @@ struct ContextSharedPart : boost::noncopyable #if USE_PARQUET mutable ParquetMetadataCachePtr parquet_metadata_cache TSA_GUARDED_BY(mutex); /// Cache of deserialized parquet metadata files. #endif + mutable PuffinFilesCachePtr puffin_files_cache TSA_GUARDED_BY(mutex); /// Cache of parsed puffin file content. AsynchronousMetrics * asynchronous_metrics TSA_GUARDED_BY(mutex) = nullptr; /// Points to asynchronous metrics mutable PageCachePtr page_cache TSA_GUARDED_BY(mutex); /// Userspace page cache. ProcessList process_list; /// Executing queries at the moment. @@ -5223,6 +5225,52 @@ void Context::clearParquetMetadataCache() const } #endif +void Context::setPuffinFilesCache(const String & cache_policy, size_t max_size_in_bytes, size_t max_entries, double size_ratio) +{ + std::lock_guard lock(shared->mutex); + + if (shared->puffin_files_cache) + throw Exception(ErrorCodes::LOGICAL_ERROR, "Puffin files cache has been already created."); + + shared->puffin_files_cache = std::make_shared(cache_policy, max_size_in_bytes, max_entries, size_ratio); +} + +void Context::updatePuffinFilesCacheConfiguration(const Poco::Util::AbstractConfiguration & config, size_t max_cache_size) +{ + std::lock_guard lock(shared->mutex); + + if (!shared->puffin_files_cache) + throw Exception(ErrorCodes::LOGICAL_ERROR, "Puffin files cache was not created yet."); + + size_t size = config.getUInt64("puffin_files_cache_size", DEFAULT_PUFFIN_FILES_CACHE_MAX_SIZE); + size_t max_entries = config.getUInt64("puffin_files_cache_max_entries", DEFAULT_PUFFIN_FILES_CACHE_MAX_ENTRIES); + if (size > max_cache_size) + { + size = max_cache_size; + LOG_DEBUG(shared->log, "Lowered Puffin files cache size to {} because the system has limited RAM", formatReadableSizeWithBinarySuffix(size)); + } + shared->puffin_files_cache->setMaxSizeInBytes(size); + shared->puffin_files_cache->setMaxCount(max_entries); +} + +std::shared_ptr Context::getPuffinFilesCache() const +{ + SharedLockGuard lock(shared->mutex); + + if (!shared->puffin_files_cache) + throw Exception(ErrorCodes::LOGICAL_ERROR, "Puffin files cache was not created yet."); + return shared->puffin_files_cache; +} + +void Context::clearPuffinFilesCache() const +{ + auto cache = getPuffinFilesCache(); + + /// Clear the cache without holding context mutex to avoid blocking context for a long time + if (cache) + cache->clear(); +} + void Context::setQueryConditionCache(const String & cache_policy, size_t max_size_in_bytes, double size_ratio) { std::lock_guard lock(shared->mutex); diff --git a/src/Interpreters/Context.h b/src/Interpreters/Context.h index ba6222079ecc..ff07412d5964 100644 --- a/src/Interpreters/Context.h +++ b/src/Interpreters/Context.h @@ -111,6 +111,7 @@ class UncompressedCache; class IcebergMetadataFilesCache; class PaimonMetadataFilesCache; class ParquetMetadataCache; +class PuffinFilesCache; class VectorSimilarityIndexCache; class TextIndexTokensCache; class TextIndexHeaderCache; @@ -1625,6 +1626,11 @@ class Context: public ContextData, public std::enable_shared_from_this void clearParquetMetadataCache() const; #endif + void setPuffinFilesCache(const String & cache_policy, size_t max_size_in_bytes, size_t max_entries, double size_ratio); + void updatePuffinFilesCacheConfiguration(const Poco::Util::AbstractConfiguration & config, size_t max_cache_size); + std::shared_ptr getPuffinFilesCache() const; + void clearPuffinFilesCache() const; + void setAllowedDisksForTableEngines(std::unordered_set && allowed_disks_) { allowed_disks = std::move(allowed_disks_); } const std::unordered_set & getAllowedDisksForTableEngines() const { return allowed_disks; } diff --git a/src/Interpreters/InterpreterSystemQuery.cpp b/src/Interpreters/InterpreterSystemQuery.cpp index 9efabe1c7877..8cac9a6d7f7a 100644 --- a/src/Interpreters/InterpreterSystemQuery.cpp +++ b/src/Interpreters/InterpreterSystemQuery.cpp @@ -504,9 +504,15 @@ BlockIO InterpreterSystemQuery::execute() #else throw Exception(ErrorCodes::SUPPORT_IS_DISABLED, "The server was compiled without the support for Parquet"); #endif +<<<<<<< HEAD case Type::CLEAR_POINT_IN_POLYGON_CACHE: getContext()->checkAccess(AccessType::SYSTEM_DROP_POINT_IN_POLYGON_CACHE); clearPointInPolygonCache(); +======= + case Type::CLEAR_PUFFIN_FILES_CACHE: + getContext()->checkAccess(AccessType::SYSTEM_DROP_PUFFIN_FILES_CACHE); + system_context->clearPuffinFilesCache(); +>>>>>>> 4b7cecaa3cf (Merge pull request #2183 from Altinity/feature/antalya-26.6/iceberg-puffin-deletion-vectors-read-2) break; case Type::CLEAR_PRIMARY_INDEX_CACHE: getContext()->checkAccess(AccessType::SYSTEM_DROP_PRIMARY_INDEX_CACHE); @@ -2802,11 +2808,15 @@ AccessRightsElements InterpreterSystemQuery::getRequiredAccessForDDLOnCluster() required_access.emplace_back(AccessType::SYSTEM_DROP_AVRO_SCHEMA_CACHE); break; case Type::CLEAR_PARQUET_METADATA_CACHE: +<<<<<<< HEAD required_access.emplace_back(AccessType::SYSTEM_DROP_PARQUET_METADATA_CACHE); break; case Type::CLEAR_POINT_IN_POLYGON_CACHE: required_access.emplace_back(AccessType::SYSTEM_DROP_POINT_IN_POLYGON_CACHE); break; +======= + case Type::CLEAR_PUFFIN_FILES_CACHE: +>>>>>>> 4b7cecaa3cf (Merge pull request #2183 from Altinity/feature/antalya-26.6/iceberg-puffin-deletion-vectors-read-2) case Type::CLEAR_PRIMARY_INDEX_CACHE: required_access.emplace_back(AccessType::SYSTEM_DROP_PRIMARY_INDEX_CACHE); break; diff --git a/src/Interpreters/tests/gtest_cluster_function_read_task.cpp b/src/Interpreters/tests/gtest_cluster_function_read_task.cpp new file mode 100644 index 000000000000..b471b53ed4b6 --- /dev/null +++ b/src/Interpreters/tests/gtest_cluster_function_read_task.cpp @@ -0,0 +1,314 @@ +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#if USE_PARQUET +#include +#include +#endif + +using namespace DB; + +namespace DB::ErrorCodes +{ +extern const int UNKNOWN_PROTOCOL; +} + +/// Cluster-protocol fail-closed behavior for Iceberg deletion vectors / equality / position +/// deletes and file buckets. Master-only APIs (`read_source_index`, +/// `derive_file_name_from_url_path`, `getIdentifier(bool)`) are intentionally not covered here. + +TEST(ClusterFunctionReadTaskResponse, RejectsSchemaTransformOnProtocolBeforeDataLakeMetadata) +{ + ClusterFunctionReadTaskResponse response; + response.path = "/path/file.parquet"; + response.data_lake_metadata.schema_transform + = std::make_shared(NamesAndTypesList{{"x", std::make_shared()}}); + + String serialized; + WriteBufferFromString out(serialized); + try + { + response.serialize(out, DBMS_CLUSTER_INITIAL_PROCESSING_PROTOCOL_VERSION); + FAIL() << "Expected exception"; + } + catch (const Exception & e) + { + EXPECT_EQ(e.code(), ErrorCodes::UNKNOWN_PROTOCOL); + EXPECT_NE(e.message().find("schema_transform"), std::string::npos); + } +} + +TEST(ClusterFunctionReadTaskResponse, AllowsEmptySchemaTransformOnProtocolBeforeDataLakeMetadata) +{ + ClusterFunctionReadTaskResponse response; + response.path = "/path/file.parquet"; + response.data_lake_metadata.schema_transform = std::make_shared(); + + String serialized; + WriteBufferFromString out(serialized); + response.serialize(out, DBMS_CLUSTER_INITIAL_PROCESSING_PROTOCOL_VERSION); + out.finalize(); + EXPECT_FALSE(serialized.empty()); +} + +TEST(ClusterFunctionReadTaskResponse, RoundTripsSchemaTransformOnSupportedProtocol) +{ + ClusterFunctionReadTaskResponse response; + response.path = "/path/file.parquet"; + response.data_lake_metadata.schema_transform + = std::make_shared(NamesAndTypesList{{"x", std::make_shared()}}); + + String serialized; + WriteBufferFromString out(serialized); + response.serialize(out, DBMS_CLUSTER_PROCESSING_PROTOCOL_VERSION_WITH_DATA_LAKE_METADATA); + out.finalize(); + + ReadBufferFromString in(serialized); + ClusterFunctionReadTaskResponse deserialized; + deserialized.deserialize(in); + + ASSERT_TRUE(deserialized.data_lake_metadata.schema_transform); + EXPECT_FALSE(deserialized.data_lake_metadata.schema_transform->getInputs().empty()); +} + +TEST(ClusterFunctionReadTaskResponse, RejectsNonEmptyExcludedRowsOnOldProtocol) +{ + ClusterFunctionReadTaskResponse response; + response.path = "/path/file.parquet"; + response.data_lake_metadata.excluded_rows = std::make_shared(); + response.data_lake_metadata.excluded_rows->add(7); + + String serialized; + WriteBufferFromString out(serialized); + try + { + response.serialize(out, DBMS_CLUSTER_PROCESSING_PROTOCOL_VERSION_WITH_FILE_BUCKETS_INFO); + FAIL() << "Expected exception"; + } + catch (const Exception & e) + { + EXPECT_EQ(e.code(), ErrorCodes::UNKNOWN_PROTOCOL); + EXPECT_NE(e.message().find("excluded_rows"), std::string::npos); + } +} + +TEST(ClusterFunctionReadTaskResponse, AllowsEmptyExcludedRowsOnOldProtocol) +{ + ClusterFunctionReadTaskResponse response; + response.path = "/path/file.parquet"; + response.data_lake_metadata.excluded_rows = std::make_shared(); + + String serialized; + WriteBufferFromString out(serialized); + response.serialize(out, DBMS_CLUSTER_PROCESSING_PROTOCOL_VERSION_WITH_FILE_BUCKETS_INFO); + out.finalize(); + EXPECT_FALSE(serialized.empty()); +} + +TEST(ClusterFunctionReadTaskResponse, RoundTripsExcludedRowsOnSupportedProtocol) +{ + ClusterFunctionReadTaskResponse response; + response.path = "/path/file.parquet"; + response.data_lake_metadata.excluded_rows = std::make_shared(); + response.data_lake_metadata.excluded_rows->add(3); + response.data_lake_metadata.excluded_rows->add(9); + + String serialized; + WriteBufferFromString out(serialized); + response.serialize(out, DBMS_CLUSTER_PROCESSING_PROTOCOL_VERSION_WITH_EXCLUDED_ROWS); + out.finalize(); + + ReadBufferFromString in(serialized); + ClusterFunctionReadTaskResponse deserialized; + deserialized.deserialize(in); + + ASSERT_TRUE(deserialized.data_lake_metadata.excluded_rows); + EXPECT_EQ(deserialized.data_lake_metadata.excluded_rows->size(), 2u); + EXPECT_TRUE(deserialized.data_lake_metadata.excluded_rows->rb_contains(3)); + EXPECT_TRUE(deserialized.data_lake_metadata.excluded_rows->rb_contains(9)); +} + +TEST(ClusterFunctionReadTaskResponse, RejectsEqualityDeletesOnProtocolBeforeIcebergMetadata) +{ + ClusterFunctionReadTaskResponse response; + response.path = "/path/file.parquet"; + response.iceberg_info = Iceberg::IcebergObjectSerializableInfo{}; + response.iceberg_info->equality_deletes_objects.push_back( + Iceberg::EqualityDeleteObject{ + .file_path = "/path/eq.parquet", + .file_format = "PARQUET", + .equality_ids = std::vector{1}, + .schema_id = 0, + }); + + String serialized; + WriteBufferFromString out(serialized); + try + { + response.serialize(out, DBMS_CLUSTER_PROCESSING_PROTOCOL_VERSION_WITH_DATA_LAKE_METADATA); + FAIL() << "Expected exception"; + } + catch (const Exception & e) + { + EXPECT_EQ(e.code(), ErrorCodes::UNKNOWN_PROTOCOL); + EXPECT_NE(e.message().find("iceberg_info"), std::string::npos); + } +} + +TEST(ClusterFunctionReadTaskResponse, RejectsPositionDeletesOnProtocolBeforeIcebergMetadata) +{ + ClusterFunctionReadTaskResponse response; + response.path = "/path/file.parquet"; + response.iceberg_info = Iceberg::IcebergObjectSerializableInfo{}; + response.iceberg_info->position_deletes_objects.push_back( + Iceberg::PositionDeleteObject{ + .file_path = "/path/pos.parquet", + .file_format = "PARQUET", + .reference_data_file_path = std::nullopt, + .sequence_number = 1, + }); + + String serialized; + WriteBufferFromString out(serialized); + try + { + response.serialize(out, DBMS_CLUSTER_PROCESSING_PROTOCOL_VERSION_WITH_DATA_LAKE_METADATA); + FAIL() << "Expected exception"; + } + catch (const Exception & e) + { + EXPECT_EQ(e.code(), ErrorCodes::UNKNOWN_PROTOCOL); + EXPECT_NE(e.message().find("iceberg_info"), std::string::npos); + } +} + +TEST(ClusterFunctionReadTaskResponse, RejectsIcebergInfoWithoutDeletesOnProtocolBeforeIcebergMetadata) +{ + ClusterFunctionReadTaskResponse response; + response.path = "/path/file.parquet"; + response.iceberg_info = Iceberg::IcebergObjectSerializableInfo{}; + response.iceberg_info->underlying_format_read_schema_id = 1; + response.iceberg_info->schema_id_relevant_to_iterator = 2; + response.iceberg_info->file_format = "PARQUET"; + + String serialized; + WriteBufferFromString out(serialized); + try + { + response.serialize(out, DBMS_CLUSTER_PROCESSING_PROTOCOL_VERSION_WITH_DATA_LAKE_METADATA); + FAIL() << "Expected exception"; + } + catch (const Exception & e) + { + EXPECT_EQ(e.code(), ErrorCodes::UNKNOWN_PROTOCOL); + EXPECT_NE(e.message().find("iceberg_info"), std::string::npos); + } +} + +TEST(ClusterFunctionReadTaskResponse, RoundTripsIcebergDeletesOnSupportedProtocol) +{ + ClusterFunctionReadTaskResponse response; + response.path = "/path/file.parquet"; + response.iceberg_info = Iceberg::IcebergObjectSerializableInfo{}; + response.iceberg_info->data_object_file_path_key + = Iceberg::IcebergPathFromMetadata::deserialize("s3://bucket/path/file.parquet"); + response.iceberg_info->file_format = "PARQUET"; + response.iceberg_info->equality_deletes_objects.push_back( + Iceberg::EqualityDeleteObject{ + .file_path = "/path/eq.parquet", + .file_format = "PARQUET", + .equality_ids = std::vector{1, 2}, + .schema_id = 7, + }); + response.iceberg_info->position_deletes_objects.push_back( + Iceberg::PositionDeleteObject{ + .file_path = "/path/pos.parquet", + .file_format = "PARQUET", + .reference_data_file_path = std::nullopt, + .sequence_number = 42, + }); + + String serialized; + WriteBufferFromString out(serialized); + response.serialize(out, DBMS_CLUSTER_PROCESSING_PROTOCOL_VERSION_WITH_ICEBERG_METADATA); + out.finalize(); + + ReadBufferFromString in(serialized); + ClusterFunctionReadTaskResponse deserialized; + deserialized.deserialize(in); + + ASSERT_TRUE(deserialized.iceberg_info.has_value()); + ASSERT_EQ(deserialized.iceberg_info->equality_deletes_objects.size(), 1u); + EXPECT_EQ(deserialized.iceberg_info->equality_deletes_objects[0].file_path, "/path/eq.parquet"); + ASSERT_EQ(deserialized.iceberg_info->position_deletes_objects.size(), 1u); + EXPECT_EQ(deserialized.iceberg_info->position_deletes_objects[0].file_path, "/path/pos.parquet"); +} + +#if USE_PARQUET + +TEST(ClusterFunctionReadTaskResponse, RejectsFileBucketInfoOnProtocolBeforeFileBuckets) +{ + ClusterFunctionReadTaskResponse response; + response.path = "/path/file.parquet"; + response.file_bucket_info = std::make_shared(std::vector{0, 1}); + + String serialized; + WriteBufferFromString out(serialized); + try + { + response.serialize(out, DBMS_CLUSTER_PROCESSING_PROTOCOL_VERSION_WITH_ICEBERG_METADATA); + FAIL() << "Expected exception"; + } + catch (const Exception & e) + { + EXPECT_EQ(e.code(), ErrorCodes::UNKNOWN_PROTOCOL); + EXPECT_NE(e.message().find("file_bucket_info"), std::string::npos); + } +} + +TEST(ClusterFunctionReadTaskResponse, AllowsMissingFileBucketInfoOnProtocolBeforeFileBuckets) +{ + ClusterFunctionReadTaskResponse response; + response.path = "/path/file.parquet"; + + String serialized; + WriteBufferFromString out(serialized); + response.serialize(out, DBMS_CLUSTER_PROCESSING_PROTOCOL_VERSION_WITH_ICEBERG_METADATA); + out.finalize(); + EXPECT_FALSE(serialized.empty()); +} + +TEST(ClusterFunctionReadTaskResponse, RoundTripsFileBucketInfoOnSupportedProtocol) +{ + tryRegisterFormats(); + + ClusterFunctionReadTaskResponse response; + response.path = "/path/file.parquet"; + response.file_bucket_info = std::make_shared(std::vector{2, 5}); + + String serialized; + WriteBufferFromString out(serialized); + response.serialize(out, DBMS_CLUSTER_PROCESSING_PROTOCOL_VERSION_WITH_FILE_BUCKETS_INFO); + out.finalize(); + + ReadBufferFromString in(serialized); + ClusterFunctionReadTaskResponse deserialized; + deserialized.deserialize(in); + + ASSERT_TRUE(deserialized.file_bucket_info); + auto * parquet_buckets = dynamic_cast(deserialized.file_bucket_info.get()); + ASSERT_TRUE(parquet_buckets); + EXPECT_EQ(parquet_buckets->row_group_ids, (std::vector{2, 5})); +} + +#endif diff --git a/src/Parsers/ASTSystemQuery.cpp b/src/Parsers/ASTSystemQuery.cpp index 61fcc824e732..3da44cdf3f6b 100644 --- a/src/Parsers/ASTSystemQuery.cpp +++ b/src/Parsers/ASTSystemQuery.cpp @@ -635,6 +635,11 @@ void ASTSystemQuery::formatImpl(WriteBuffer & ostr, const FormatSettings & setti case Type::CLEAR_PARQUET_METADATA_CACHE: case Type::CLEAR_POINT_IN_POLYGON_CACHE: case Type::CLEAR_AVRO_SCHEMA_CACHE: +<<<<<<< HEAD +======= + case Type::CLEAR_PUFFIN_FILES_CACHE: + case Type::DROP_OBJECT_STORAGE_LIST_OBJECTS_CACHE: +>>>>>>> 4b7cecaa3cf (Merge pull request #2183 from Altinity/feature/antalya-26.6/iceberg-puffin-deletion-vectors-read-2) case Type::RESET_COVERAGE: case Type::RESTART_REPLICAS: case Type::JEMALLOC_PURGE: diff --git a/src/Parsers/ASTSystemQuery.h b/src/Parsers/ASTSystemQuery.h index 148421bcc275..65ab87adf720 100644 --- a/src/Parsers/ASTSystemQuery.h +++ b/src/Parsers/ASTSystemQuery.h @@ -49,7 +49,11 @@ class ASTSystemQuery : public IAST, public ASTQueryWithOnCluster CLEAR_ICEBERG_METADATA_CACHE, CLEAR_PAIMON_METADATA_CACHE, CLEAR_PARQUET_METADATA_CACHE, +<<<<<<< HEAD CLEAR_POINT_IN_POLYGON_CACHE, +======= + CLEAR_PUFFIN_FILES_CACHE, +>>>>>>> 4b7cecaa3cf (Merge pull request #2183 from Altinity/feature/antalya-26.6/iceberg-puffin-deletion-vectors-read-2) CLEAR_FILESYSTEM_CACHE, CLEAR_DISTRIBUTED_CACHE, CLEAR_DISK_METADATA_CACHE, diff --git a/src/Parsers/ParserSystemQuery.cpp b/src/Parsers/ParserSystemQuery.cpp index ad60ad571ee9..6393b38d4e94 100644 --- a/src/Parsers/ParserSystemQuery.cpp +++ b/src/Parsers/ParserSystemQuery.cpp @@ -322,7 +322,12 @@ bool ParserSystemQuery::parseImpl(IParser::Pos & pos, ASTPtr & node, Expected & {"DROP ICEBERG METADATA CACHE", Type::CLEAR_ICEBERG_METADATA_CACHE}, {"DROP PAIMON METADATA CACHE", Type::CLEAR_PAIMON_METADATA_CACHE}, {"DROP PARQUET METADATA CACHE", Type::CLEAR_PARQUET_METADATA_CACHE}, +<<<<<<< HEAD {"DROP POINT IN POLYGON CACHE", Type::CLEAR_POINT_IN_POLYGON_CACHE}, +======= + {"DROP PUFFIN FILES CACHE", Type::CLEAR_PUFFIN_FILES_CACHE}, + {"DROP PUFFIN_FILES_CACHE", Type::CLEAR_PUFFIN_FILES_CACHE}, +>>>>>>> 4b7cecaa3cf (Merge pull request #2183 from Altinity/feature/antalya-26.6/iceberg-puffin-deletion-vectors-read-2) {"DROP FILESYSTEM CACHE", Type::CLEAR_FILESYSTEM_CACHE}, {"DROP DISTRIBUTED CACHE", Type::CLEAR_DISTRIBUTED_CACHE}, {"DROP DISK METADATA CACHE", Type::CLEAR_DISK_METADATA_CACHE}, diff --git a/src/Processors/Formats/Impl/Parquet/Reader.cpp b/src/Processors/Formats/Impl/Parquet/Reader.cpp index 1ee01ab2bbd5..8461292e6891 100644 --- a/src/Processors/Formats/Impl/Parquet/Reader.cpp +++ b/src/Processors/Formats/Impl/Parquet/Reader.cpp @@ -13,7 +13,11 @@ #include #include #include +<<<<<<< HEAD #include +======= +#include +>>>>>>> 4b7cecaa3cf (Merge pull request #2183 from Altinity/feature/antalya-26.6/iceberg-puffin-deletion-vectors-read-2) #include #include #include @@ -360,6 +364,7 @@ void Reader::getHyperrectangleForRowGroup(const parq::RowGroup * meta, Hyperrect } } +<<<<<<< HEAD bool Reader::spatialBboxStatsHaveNoNulls(const parq::RowGroup & meta, size_t spatial_key_condition_idx) const { for (size_t bbox_pc_idx : spatial_key_condition_bbox_col_indices.at(spatial_key_condition_idx)) @@ -371,6 +376,42 @@ bool Reader::spatialBboxStatsHaveNoNulls(const parq::RowGroup & meta, size_t spa return false; } return true; +======= +std::vector buildRowGroupGlobalOffsets(const parq::FileMetaData & file_metadata) +{ + if (file_metadata.num_rows < 0) + throw Exception(ErrorCodes::INCORRECT_DATA, "Parquet file has negative row count: {}", file_metadata.num_rows); + + const size_t num_row_groups = file_metadata.row_groups.size(); + std::vector global_offsets(num_row_groups + 1, 0); + UInt64 total_rows = 0; + + for (size_t i = 0; i < num_row_groups; ++i) + { + const Int64 num_rows = file_metadata.row_groups[i].num_rows; + if (num_rows < 0) + throw Exception(ErrorCodes::INCORRECT_DATA, "Parquet row group {} has negative row count: {}", i, num_rows); + + UInt64 next_total = 0; + if (common::addOverflow(total_rows, static_cast(num_rows), next_total)) + { + throw Exception( + ErrorCodes::INCORRECT_DATA, + "Parquet row group row counts overflow when computing global offsets (at row group {})", + i); + } + + total_rows = next_total; + global_offsets[i + 1] = static_cast(total_rows); + } + + /// Do not require the row-group sum to equal `FileMetaData.num_rows`. Some writers leave a + /// stale or inconsistent file-level count; global offsets and deletion-vector positions are + /// defined by the row-group layout. This helper runs on every ParquetV3 read, so rejecting + /// mismatches would break previously readable files. + + return global_offsets; +>>>>>>> 4b7cecaa3cf (Merge pull request #2183 from Altinity/feature/antalya-26.6/iceberg-puffin-deletion-vectors-read-2) } void Reader::prefilterAndInitRowGroups(const std::optional> & row_groups_to_read) @@ -697,17 +738,16 @@ void Reader::prefilterAndInitRowGroups(const std::optional global_offsets = buildRowGroupGlobalOffsets(file_metadata); for (size_t row_group_idx = 0; row_group_idx < file_metadata.row_groups.size(); ++row_group_idx) { const auto * meta = &file_metadata.row_groups[row_group_idx]; - if (meta->num_rows < 0) - throw Exception(ErrorCodes::INCORRECT_DATA, "Row group {} has negative row count: {}", row_group_idx, meta->num_rows); if (meta->num_rows == 0) continue; /// Empty row groups are valid in Parquet; skip them. if (meta->columns.size() != total_primitive_columns_in_file) throw Exception(ErrorCodes::INCORRECT_DATA, "Row group {} has unexpected number of columns: {} != {}", row_group_idx, meta->columns.size(), total_primitive_columns_in_file); +<<<<<<< HEAD total_rows += size_t(meta->num_rows); // before potentially skipping the row group /// Lazy materialization: skip row groups that contain none of the requested rows. @@ -722,6 +762,8 @@ void Reader::prefilterAndInitRowGroups(const std::optionalbegin()), size_t(end_it - rows_to_read->begin())}; } +======= +>>>>>>> 4b7cecaa3cf (Merge pull request #2183 from Altinity/feature/antalya-26.6/iceberg-puffin-deletion-vectors-read-2) Hyperrectangle hyperrectangle(extended_sample_block.columns(), Range::createWholeUniverse()); if ((options.format.parquet.filter_push_down && format_filter_info->key_condition) || !spatial_key_conditions.empty()) @@ -773,7 +815,7 @@ void Reader::prefilterAndInitRowGroups(const std::optionalcontains(row_group_idx); row_group.requested_rows_slice = requested_rows_slice; row_group.row_group_idx = row_group_idx; - row_group.start_global_row_idx = total_rows - size_t(meta->num_rows); + row_group.start_global_row_idx = global_offsets[row_group_idx]; row_group.columns.resize(primitive_columns.size()); row_group.hyperrectangle = std::move(hyperrectangle); diff --git a/src/Processors/Formats/Impl/Parquet/Reader.h b/src/Processors/Formats/Impl/Parquet/Reader.h index 69a4b98c9c15..611105f13b14 100644 --- a/src/Processors/Formats/Impl/Parquet/Reader.h +++ b/src/Processors/Formats/Impl/Parquet/Reader.h @@ -666,4 +666,9 @@ struct Reader void readRowsInPage(size_t end_row_idx, ColumnSubchunk & subchunk, ColumnChunk & column, const PrimitiveColumnInfo & column_info, const RowSubgroup * row_subgroup = nullptr); }; +/// Prefix offsets of row groups in the file; result size is `row_groups.size() + 1`. +/// Throws `INCORRECT_DATA` on negative counts or size overflow. A mismatch between the +/// row-group sum and `FileMetaData.num_rows` is tolerated: offsets follow the row-group layout. +std::vector buildRowGroupGlobalOffsets(const parq::FileMetaData & file_metadata); + } diff --git a/src/Processors/Formats/Impl/ParquetV3BlockInputFormat.cpp b/src/Processors/Formats/Impl/ParquetV3BlockInputFormat.cpp index 8d8cbabdd35f..452a0a102c85 100644 --- a/src/Processors/Formats/Impl/ParquetV3BlockInputFormat.cpp +++ b/src/Processors/Formats/Impl/ParquetV3BlockInputFormat.cpp @@ -25,6 +25,7 @@ namespace DB namespace ErrorCodes { extern const int LOGICAL_ERROR; + extern const int INCORRECT_DATA; } static Parquet::ReadOptions convertReadOptions(const FormatSettings & format_settings) @@ -122,23 +123,65 @@ parquet::format::FileMetaData ParquetV3BlockInputFormat::getFileMetadata(Parquet } } -Chunk ParquetV3BlockInputFormat::read() +void ParquetV3BlockInputFormat::prepareNeedOnlyCountRowGroups(const parquet::format::FileMetaData & file_metadata) { - if (need_only_count) + need_only_count_row_groups.clear(); + need_only_count_next = 0; + + if (!buckets_to_read) { - if (reported_count) - return {}; + /// Unbucketed need_only_count historically returns one chunk from `FileMetaData.num_rows`. + /// Keep that for plain Parquet COUNT (and Iceberg without row-group buckets). Per-row-group + /// spans below are only for bucketed reads, where each task must count its assigned groups + /// instead of the whole file. + if (file_metadata.num_rows < 0) + throw Exception(ErrorCodes::INCORRECT_DATA, "Parquet file has negative row count: {}", file_metadata.num_rows); + + need_only_count_row_groups.push_back( + {.row_num_offset = 0, .num_rows = static_cast(file_metadata.num_rows)}); + return; + } - /// Don't init Reader and ReadManager if we only need file metadata. - Parquet::Prefetcher temp_prefetcher; - temp_prefetcher.init(in, read_options, parser_shared_resources); - parquet::format::FileMetaData file_metadata = getFileMetadata(temp_prefetcher); + const std::vector global_offsets = Parquet::buildRowGroupGlobalOffsets(file_metadata); + const size_t num_row_groups = file_metadata.row_groups.size(); + for (size_t row_group_id : buckets_to_read->row_group_ids) + { + if (row_group_id >= num_row_groups) + throw Exception( + ErrorCodes::INCORRECT_DATA, + "Parquet bucket row group {} is out of range (file has {} row groups)", + row_group_id, + num_row_groups); + + const size_t num_rows = static_cast(file_metadata.row_groups[row_group_id].num_rows); + if (num_rows == 0) + continue; - auto chunk = getChunkForCount(size_t(file_metadata.num_rows)); - chunk.getChunkInfos().add(std::make_shared(0)); + need_only_count_row_groups.push_back( + {.row_num_offset = global_offsets[row_group_id], .num_rows = num_rows}); + } +} + +Chunk ParquetV3BlockInputFormat::read() +{ + if (need_only_count) + { + if (!need_only_count_prepared) + { + /// Don't init Reader and ReadManager if we only need file metadata. + Parquet::Prefetcher temp_prefetcher; + temp_prefetcher.init(in, read_options, parser_shared_resources); + prepareNeedOnlyCountRowGroups(getFileMetadata(temp_prefetcher)); + need_only_count_prepared = true; + } + + if (need_only_count_next >= need_only_count_row_groups.size()) + return {}; - reported_count = true; + const auto & row_group = need_only_count_row_groups[need_only_count_next++]; + auto chunk = getChunkForCount(row_group.num_rows); + chunk.getChunkInfos().add(std::make_shared(row_group.row_num_offset)); return chunk; } @@ -214,6 +257,9 @@ void ParquetV3BlockInputFormat::resetParser() reader.reset(); } previous_block_missing_values.clear(); + need_only_count_prepared = false; + need_only_count_next = 0; + need_only_count_row_groups.clear(); IInputFormat::resetParser(); } diff --git a/src/Processors/Formats/Impl/ParquetV3BlockInputFormat.h b/src/Processors/Formats/Impl/ParquetV3BlockInputFormat.h index 0ee6d2226a55..77f3eb2ecd70 100644 --- a/src/Processors/Formats/Impl/ParquetV3BlockInputFormat.h +++ b/src/Processors/Formats/Impl/ParquetV3BlockInputFormat.h @@ -83,12 +83,21 @@ class ParquetV3BlockInputFormat final : public IInputFormat std::mutex reader_mutex; std::optional reader; - bool reported_count = false; // if need_only_count + bool need_only_count_prepared = false; + size_t need_only_count_next = 0; + + struct NeedOnlyCountRowGroup + { + size_t row_num_offset = 0; + size_t num_rows = 0; + }; + std::vector need_only_count_row_groups; BlockMissingValues previous_block_missing_values; size_t previous_approx_bytes_read_for_chunk = 0; void initializeIfNeeded(); + void prepareNeedOnlyCountRowGroups(const parquet::format::FileMetaData & file_metadata); std::shared_ptr buckets_to_read; parquet::format::FileMetaData getFileMetadata(Parquet::Prefetcher & prefetcher) const; diff --git a/src/Processors/Formats/Impl/PuffinBlockInputFormat.cpp b/src/Processors/Formats/Impl/PuffinBlockInputFormat.cpp index e742a400410d..db9559027330 100644 --- a/src/Processors/Formats/Impl/PuffinBlockInputFormat.cpp +++ b/src/Processors/Formats/Impl/PuffinBlockInputFormat.cpp @@ -24,7 +24,9 @@ #include #include #include +#include #include +#include #include #include #include @@ -32,9 +34,16 @@ #include #include #include +#include #include +namespace ProfileEvents +{ +extern const Event PuffinFilesRead; +extern const Event PuffinFileReadMicroseconds; +} + namespace DB { @@ -44,26 +53,33 @@ namespace ErrorCodes extern const int LZ4_DECODER_FAILED; } +constexpr Int64 DELETION_VECTOR_MAX_POSITION = 0x7FFFFFFE80000000LL; +constexpr Int32 DELETION_VECTOR_MAX_KEY = std::numeric_limits::max() - 1; + +static UInt64 positionFromKeyAndSubPosition(UInt32 key, UInt32 sub_position) +{ + return (static_cast(key) << 32) | static_cast(sub_position); +} + namespace { +struct ScopedPuffinFileReadProfileEvent +{ + ProfileEventTimeIncrement watch; + + ScopedPuffinFileReadProfileEvent() + : watch(ProfileEvents::PuffinFileReadMicroseconds) + { + ProfileEvents::increment(ProfileEvents::PuffinFilesRead); + } +}; + constexpr UInt8 PUFFIN_MAGIC[4] = {0x50, 0x46, 0x41, 0x31}; constexpr UInt8 PUFFIN_FOOTER_COMPRESSED_FLAG = 0x01; -constexpr size_t PUFFIN_FOOTER_TRAILER_SIZE = 12; constexpr size_t PUFFIN_FOOTER_LZ4_MAX_RATIO = 255; -constexpr size_t PUFFIN_FOOTER_MAX_PAYLOAD_SIZE = 16 * 1024 * 1024; -constexpr UInt64 PUFFIN_DV_MAX_MATERIALIZED_POSITIONS = 100'000'000; -constexpr size_t PUFFIN_DV_MAX_BLOB_SIZE = 2ULL * 1024 * 1024 * 1024; -constexpr UInt8 DELETION_VECTOR_MAGIC[4] = {0xD1, 0xD3, 0x39, 0x64}; -constexpr Int64 DELETION_VECTOR_MAX_POSITION = 0x7FFFFFFE80000000LL; -constexpr Int32 DELETION_VECTOR_MAX_KEY = std::numeric_limits::max() - 1; constexpr const char * PUFFIN_DELETION_VECTOR_BLOB_TYPE = "deletion-vector-v1"; -UInt64 positionFromKeyAndSubPosition(UInt32 key, UInt32 sub_position) -{ - return (static_cast(key) << 32) | static_cast(sub_position); -} - void checkMagic(const UInt8 * p, const char * context) { if (std::memcmp(p, PUFFIN_MAGIC, 4) != 0) @@ -253,28 +269,6 @@ String requireBlobMetadataString(const Poco::JSON::Object::Ptr & blob_obj, const return requireJSONStringValue(blob_obj->get(field_name), blob_index, field_name); } -void requireDeletionVectorV1Properties(const PuffinBlob & blob, size_t blob_index) -{ - static constexpr const char * required_properties[] = {"referenced-data-file", "cardinality"}; - for (const char * key : required_properties) - { - auto it = blob.properties.find(key); - if (it == blob.properties.end() || it->second.empty()) - throw Exception( - ErrorCodes::BAD_ARGUMENTS, - "Puffin blob {}: deletion-vector-v1 missing required property '{}'", - blob_index, - key); - } - - UInt64 cardinality = 0; - if (!tryParse(cardinality, blob.properties.at("cardinality"))) - throw Exception( - ErrorCodes::BAD_ARGUMENTS, - "Puffin blob {}: deletion-vector-v1 property 'cardinality' must be an unsigned integer", - blob_index); -} - void parseStringValuedProperties( const Poco::JSON::Object::Ptr & props_obj, std::map * out, @@ -387,12 +381,13 @@ std::vector parseFooterJSON(const String & footer_json, size_t blob_ } else { - throw Exception( - ErrorCodes::BAD_ARGUMENTS, - "Puffin blob {}: unsupported blob type '{}', only '{}' is supported", - i, - blob.type, - PUFFIN_DELETION_VECTOR_BLOB_TYPE); + /// Puffin allows arbitrary blob types in one file (indexes, sketches, DVs, ...). + /// Keep common metadata so Iceberg can bind a DV by offset/length; do not require + /// deletion-vector properties for non-DV entries. + if (blob_obj->has("compression-codec") && !blob_obj->isNull("compression-codec")) + blob.compression_codec = requireBlobMetadataString(blob_obj, "compression-codec", i); + + parseBlobProperties(blob_obj, blob, i, /*required=*/false); } requireBlobMetadataField(blob_obj, "fields", i); @@ -410,8 +405,10 @@ std::vector parseFooterJSON(const String & footer_json, size_t blob_ return blobs; } -std::vector readPuffinFooterFromSeekable(SeekableReadBuffer & seekable, size_t file_size) +std::vector readPuffinFooterFromSeekableImpl(SeekableReadBuffer & seekable, size_t file_size) { + ScopedPuffinFileReadProfileEvent profile_event; + if (file_size < 16) throw Exception(ErrorCodes::BAD_ARGUMENTS, "Puffin file too small"); @@ -485,7 +482,7 @@ PuffinFooter readPuffinFooter(ReadBuffer & buf, bool seekable_read) if (seekable_read && seekable && seekable->checkIfActuallySeekable() && file_size_opt) { - result.blobs = readPuffinFooterFromSeekable(*seekable, *file_size_opt); + result.blobs = readPuffinFooterFromSeekableImpl(*seekable, *file_size_opt); } else { @@ -495,15 +492,10 @@ PuffinFooter readPuffinFooter(ReadBuffer & buf, bool seekable_read) throw Exception(ErrorCodes::BAD_ARGUMENTS, "Puffin file too small"); checkMagic(result.data.data(), "header"); - std::vector tmp(DEFAULT_BLOCK_SIZE); - while (!buf.eof()) - { - size_t n = buf.read(reinterpret_cast(tmp.data()), tmp.size()); - result.data.insert(result.data.end(), tmp.data(), tmp.data() + n); - } + appendReadBufferWithAbsoluteSizeLimit(buf, result.data, PUFFIN_NON_SEEKABLE_MAX_BUFFERED_SIZE); ReadBufferFromMemory mem_buf(result.data.data(), result.data.size()); - result.blobs = readPuffinFooterFromSeekable(mem_buf, result.data.size()); + result.blobs = readPuffinFooterFromSeekableImpl(mem_buf, result.data.size()); } return result; @@ -554,60 +546,159 @@ void readDeletionVectorEnvelopePrefix( } String readDeletionVectorBlobBytes( - const PuffinBlob & blob, ReadBuffer & buf, const std::vector & data, bool seekable_read) + const PuffinBlob & blob, + ReadBuffer & buf, + const std::vector & data, + bool seekable_read, + UInt64 expected_cardinality) { - if (blob.length < 0) - throw Exception(ErrorCodes::BAD_ARGUMENTS, "Deletion vector blob length is negative"); - - if (static_cast(blob.length) > PUFFIN_DV_MAX_BLOB_SIZE) - throw Exception( - ErrorCodes::BAD_ARGUMENTS, - "Deletion vector blob length {} exceeds absolute limit {}", - blob.length, - PUFFIN_DV_MAX_BLOB_SIZE); + ScopedPuffinFileReadProfileEvent profile_event; - if (static_cast(blob.length) < 12) - throw Exception(ErrorCodes::BAD_ARGUMENTS, "Deletion vector blob is too small"); + /// Fail closed before envelope peek / full allocate — shared with Iceberg + /// `readDeletionVectorFromPuffin`. `deserializeDeletionVectorV1` still re-checks. + checkDeletionVectorBlobReadLimits(blob.length, expected_cardinality); UInt8 header[8]; readDeletionVectorEnvelopePrefix(blob, buf, data, seekable_read, header); + validateDeletionVectorEnvelope(header, blob.length); - ReadBufferFromMemory header_buf(reinterpret_cast(header), sizeof(header)); - UInt32 combined_length = 0; - readBinaryBigEndian(combined_length, header_buf); - if (std::memcmp(header + sizeof(UInt32), DELETION_VECTOR_MAGIC, sizeof(DELETION_VECTOR_MAGIC)) != 0) - throw Exception(ErrorCodes::BAD_ARGUMENTS, "Invalid deletion vector magic"); + return readPuffinBlobBytes(blob, buf, data, seekable_read); +} - if (combined_length < sizeof(DELETION_VECTOR_MAGIC)) - throw Exception(ErrorCodes::BAD_ARGUMENTS, "Invalid deletion vector combined length: {}", combined_length); +NamesAndTypesList getPuffinMetadataSchema() +{ + return { + {"blob_type", std::make_shared()}, + {"snapshot_id", std::make_shared()}, + {"sequence_number", std::make_shared()}, + {"fields", std::make_shared(std::make_shared())}, + {"offset", std::make_shared()}, + {"length", std::make_shared()}, + {"compression_codec", std::make_shared()}, + {"properties", std::make_shared(std::make_shared(), std::make_shared())}, + }; +} - UInt64 expected_blob_size = 0; - if (common::addOverflow(static_cast(combined_length), UInt64{8}, expected_blob_size)) - throw Exception(ErrorCodes::BAD_ARGUMENTS, "Invalid deletion vector combined length: {}", combined_length); +NamesAndTypesList getPuffinSchema() +{ + return { + {"referenced_data_file", std::make_shared()}, + {"deleted_rows", std::make_shared(std::make_shared())}, + }; +} + +void checkPuffinFormatHeader(const Block & header, const NamesAndTypesList & expected_schema, const char * format_name) +{ + std::unordered_map name_to_type; + for (const auto & [name, type] : expected_schema) + name_to_type[name] = type; + + String allowed_columns; + for (const auto & [name, type] : expected_schema) + { + if (!allowed_columns.empty()) + allowed_columns += ", "; + allowed_columns += name; + } + + for (const auto & [name, type] : header.getNamesAndTypes()) + { + auto it = name_to_type.find(name); + if (it == name_to_type.end()) + throw Exception( + ErrorCodes::BAD_ARGUMENTS, + "Unexpected column: {}. {} format allows only the next columns: {}", + name, + format_name, + allowed_columns); + + if (!it->second->equals(*type)) + throw Exception( + ErrorCodes::BAD_ARGUMENTS, + "Unexpected type {} for column {}. Expected type: {}", + type->getName(), + name, + it->second->getName()); + } +} + +void checkPuffinMetadataHeader(const Block & header) +{ + checkPuffinFormatHeader(header, getPuffinMetadataSchema(), "PuffinMetadata"); +} + +void checkPuffinHeader(const Block & header) +{ + checkPuffinFormatHeader(header, getPuffinSchema(), "Puffin"); +} - if (static_cast(blob.length) != expected_blob_size) +} + +UInt64 requireDeletionVectorV1Properties(const PuffinBlob & blob, size_t blob_index) +{ + /// Puffin v1: snapshot-id and sequence-number are unknown when the file is written and must be -1. + if (blob.snapshot_id != -1 || blob.sequence_number != -1) throw Exception( ErrorCodes::BAD_ARGUMENTS, - "Deletion vector blob size {} does not match combined length {}", - blob.length, - combined_length); + "Puffin blob {}: deletion-vector-v1 snapshot-id and sequence-number must be -1", + blob_index); - return readPuffinBlobBytes(blob, buf, data, seekable_read); + validateDeletionVectorV1Fields(blob.fields, blob_index); + + static constexpr const char * required_properties[] = {"referenced-data-file", "cardinality"}; + for (const char * key : required_properties) + { + auto it = blob.properties.find(key); + if (it == blob.properties.end() || it->second.empty()) + throw Exception( + ErrorCodes::BAD_ARGUMENTS, + "Puffin blob {}: deletion-vector-v1 missing required property '{}'", + blob_index, + key); + } + + UInt64 cardinality = 0; + if (!tryParse(cardinality, blob.properties.at("cardinality"))) + throw Exception( + ErrorCodes::BAD_ARGUMENTS, + "Puffin blob {}: deletion-vector-v1 property 'cardinality' must be an unsigned integer", + blob_index); + + return cardinality; } +namespace +{ + roaring::Roaring readRoaringPortableSafe(const char * data, size_t size, Int32 key) { + roaring::Roaring bitmap; try { - return roaring::Roaring::readSafe(data, size); + bitmap = roaring::Roaring::readSafe(data, size); } catch (const std::exception & e) { throw Exception(ErrorCodes::BAD_ARGUMENTS, "Failed to deserialize deletion vector roaring bitmap at key {}: {}", key, e.what()); } + + /// `readSafe` only bounds the read; CRoaring requires internal validation before use on untrusted input. + const char * reason = nullptr; + if (!roaring::api::roaring_bitmap_internal_validate(&bitmap.roaring, &reason)) + { + throw Exception( + ErrorCodes::BAD_ARGUMENTS, + "Deletion vector roaring bitmap at key {} failed internal validation: {}", + key, + reason ? reason : "unknown"); + } + + return bitmap; +} + } -void deserializeRoaringPositionBitmap(std::string_view bytes, UInt64 expected_cardinality, ColumnUInt64 & positions) +std::vector deserializeRoaringPositionBitmap(std::string_view bytes, std::optional expected_cardinality) { if (bytes.size() < sizeof(Int64)) throw Exception(ErrorCodes::BAD_ARGUMENTS, "Deletion vector bitmap is too small"); @@ -623,6 +714,10 @@ void deserializeRoaringPositionBitmap(std::string_view bytes, UInt64 expected_ca if (bitmap_count < 0 || bitmap_count > std::numeric_limits::max()) throw Exception(ErrorCodes::BAD_ARGUMENTS, "Invalid deletion vector bitmap count: {}", bitmap_count); + std::vector positions; + if (expected_cardinality.has_value()) + positions.reserve(*expected_cardinality); + Int32 last_key = -1; Int32 remaining_count = static_cast(bitmap_count); UInt64 running_cardinality = 0; @@ -647,29 +742,32 @@ void deserializeRoaringPositionBitmap(std::string_view bytes, UInt64 expected_ca if (bitmap_size > remaining) throw Exception(ErrorCodes::BAD_ARGUMENTS, "Deletion vector roaring bitmap at key {} exceeds blob size", key); - const UInt64 bitmap_cardinality = bitmap.cardinality(); - UInt64 new_running_cardinality = 0; - if (common::addOverflow(running_cardinality, bitmap_cardinality, new_running_cardinality)) - throw Exception( - ErrorCodes::BAD_ARGUMENTS, - "Deletion vector cardinality exceeds declared cardinality {}", - expected_cardinality); + if (expected_cardinality.has_value()) + { + const UInt64 bitmap_cardinality = bitmap.cardinality(); + UInt64 new_running_cardinality = 0; + if (common::addOverflow(running_cardinality, bitmap_cardinality, new_running_cardinality)) + throw Exception( + ErrorCodes::BAD_ARGUMENTS, + "Deletion vector cardinality exceeds declared cardinality {}", + *expected_cardinality); - if (new_running_cardinality > expected_cardinality) - throw Exception( - ErrorCodes::BAD_ARGUMENTS, - "Deletion vector cardinality {} exceeds declared cardinality {}", - new_running_cardinality, - expected_cardinality); + if (new_running_cardinality > *expected_cardinality) + throw Exception( + ErrorCodes::BAD_ARGUMENTS, + "Deletion vector cardinality {} exceeds declared cardinality {}", + new_running_cardinality, + *expected_cardinality); - running_cardinality = new_running_cardinality; + running_cardinality = new_running_cardinality; + } for (UInt32 sub_position : bitmap) { const UInt64 position = positionFromKeyAndSubPosition(static_cast(key), sub_position); if (position > static_cast(DELETION_VECTOR_MAX_POSITION)) throw Exception(ErrorCodes::BAD_ARGUMENTS, "Deletion vector position {} is out of supported range", position); - positions.insertValue(position); + positions.push_back(position); } ptr += bitmap_size; @@ -681,12 +779,21 @@ void deserializeRoaringPositionBitmap(std::string_view bytes, UInt64 expected_ca if (remaining != 0) throw Exception(ErrorCodes::BAD_ARGUMENTS, "Deletion vector bitmap has {} trailing bytes", remaining); - if (running_cardinality != expected_cardinality) + if (expected_cardinality.has_value() && running_cardinality != *expected_cardinality) throw Exception( ErrorCodes::BAD_ARGUMENTS, "Deletion vector cardinality {} does not match deserialized row count {}", - expected_cardinality, + *expected_cardinality, running_cardinality); + + return positions; +} + +static void deserializeRoaringPositionBitmap(std::string_view bytes, UInt64 expected_cardinality, ColumnUInt64 & positions) +{ + auto decoded = deserializeRoaringPositionBitmap(bytes, std::optional{expected_cardinality}); + for (UInt64 position : decoded) + positions.insertValue(position); } std::string_view extractDeletionVectorPayload(std::string_view blob) @@ -733,73 +840,9 @@ void deserializeDeletionVectorV1(std::string_view blob, UInt64 expected_cardinal deserializeRoaringPositionBitmap(extractDeletionVectorPayload(blob), expected_cardinality, positions); } -NamesAndTypesList getPuffinMetadataSchema() -{ - return { - {"blob_type", std::make_shared()}, - {"snapshot_id", std::make_shared()}, - {"sequence_number", std::make_shared()}, - {"fields", std::make_shared(std::make_shared())}, - {"offset", std::make_shared()}, - {"length", std::make_shared()}, - {"compression_codec", std::make_shared()}, - {"properties", std::make_shared(std::make_shared(), std::make_shared())}, - }; -} - -NamesAndTypesList getPuffinSchema() -{ - return { - {"referenced_data_file", std::make_shared()}, - {"deleted_rows", std::make_shared(std::make_shared())}, - }; -} - -void checkPuffinFormatHeader(const Block & header, const NamesAndTypesList & expected_schema, const char * format_name) -{ - std::unordered_map name_to_type; - for (const auto & [name, type] : expected_schema) - name_to_type[name] = type; - - String allowed_columns; - for (const auto & [name, type] : expected_schema) - { - if (!allowed_columns.empty()) - allowed_columns += ", "; - allowed_columns += name; - } - - for (const auto & [name, type] : header.getNamesAndTypes()) - { - auto it = name_to_type.find(name); - if (it == name_to_type.end()) - throw Exception( - ErrorCodes::BAD_ARGUMENTS, - "Unexpected column: {}. {} format allows only the next columns: {}", - name, - format_name, - allowed_columns); - - if (!it->second->equals(*type)) - throw Exception( - ErrorCodes::BAD_ARGUMENTS, - "Unexpected type {} for column {}. Expected type: {}", - type->getName(), - name, - it->second->getName()); - } -} - -void checkPuffinMetadataHeader(const Block & header) -{ - checkPuffinFormatHeader(header, getPuffinMetadataSchema(), "PuffinMetadata"); -} - -void checkPuffinHeader(const Block & header) +std::vector readPuffinFooterFromSeekable(SeekableReadBuffer & seekable, size_t file_size) { - checkPuffinFormatHeader(header, getPuffinSchema(), "Puffin"); -} - + return readPuffinFooterFromSeekableImpl(seekable, file_size); } PuffinMetadataInputFormat::PuffinMetadataInputFormat(ReadBuffer & buf, SharedHeader header_, const FormatSettings & format_settings_) @@ -906,29 +949,19 @@ Chunk PuffinInputFormat::read() const auto & blob = footer.blobs[blob_index++]; if (blob.type != PUFFIN_DELETION_VECTOR_BLOB_TYPE) - throw Exception( - ErrorCodes::BAD_ARGUMENTS, - "Puffin blob {}: unexpected blob type '{}', only '{}' is supported", - current_blob_index, - blob.type, - PUFFIN_DELETION_VECTOR_BLOB_TYPE); + continue; + const UInt64 expected_cardinality = requireDeletionVectorV1Properties(blob, current_blob_index); const auto & referenced_data_file = blob.properties.at("referenced-data-file"); - UInt64 expected_cardinality = 0; - if (!tryParse(expected_cardinality, blob.properties.at("cardinality"))) - throw Exception( - ErrorCodes::BAD_ARGUMENTS, - "Puffin blob {}: deletion-vector-v1 property 'cardinality' must be an unsigned integer", - current_blob_index); - auto col_file = ColumnString::create(); col_file->insertData(referenced_data_file.data(), referenced_data_file.size()); MutableColumnPtr col_rows; if (need_deleted_rows) { - const String blob_data = readDeletionVectorBlobBytes(blob, *in, footer.data, seekable_read); + const String blob_data + = readDeletionVectorBlobBytes(blob, *in, footer.data, seekable_read, expected_cardinality); auto col_rows_data = ColumnUInt64::create(); deserializeDeletionVectorV1(blob_data, expected_cardinality, *col_rows_data); @@ -1002,9 +1035,7 @@ void registerInputFormatPuffin(FormatFactory & factory) ## Description {#description} Special input format for reading [Apache Iceberg Puffin](https://iceberg.apache.org/puffin-spec/) file footer metadata. -It outputs one row per blob entry from the footer `BlobMetadata` list. - -`deletion-vector-v1` is the only supported blob type: a file containing any other blob type (for example `apache-datasketches-theta-v1`) is rejected. +It outputs one row per blob entry from the footer `BlobMetadata` list, including non-deletion-vector types (for example `apache-datasketches-theta-v1`). Full deletion-vector property validation applies only to `deletion-vector-v1` entries. Fixed output columns: - `blob_type` (`String`) - blob type, for example `deletion-vector-v1` @@ -1047,16 +1078,16 @@ Pair with the `Puffin` format to read `deletion-vector-v1` blob payloads. Input format for reading [Apache Iceberg Puffin](https://iceberg.apache.org/puffin-spec/) files. -The format exposes deleted row positions from `deletion-vector-v1` blobs. It is the only supported blob type: a file containing any other blob type (for example `apache-datasketches-theta-v1`) is rejected. +The format exposes deleted row positions from `deletion-vector-v1` blobs. Other blob types (for example `apache-datasketches-theta-v1`) are skipped. If a puffin file contains multiple `deletion-vector-v1` blobs, the format outputs one row per such blob. Fixed output columns: - `referenced_data_file` (`String`) - location of the data file the deletion vector applies to (`referenced-data-file` blob property) - `deleted_rows` (`Array(UInt64)`) - 64-bit row positions deleted according to the deletion vector roaring bitmap -Deletion vectors whose declared `cardinality` exceeds an absolute materialization ceiling are rejected when `deleted_rows` is requested. Footer `deletion-vector-v1` properties (including that `cardinality` parses as an unsigned integer) are always validated. Selecting only `referenced_data_file` skips on-disk payload I/O and therefore also skips envelope, CRC, roaring deserialize, and the materialization ceiling — intentionally, so a path-only projection does not read up to the blob-size cap. +Deletion vectors whose declared `cardinality` exceeds an absolute materialization ceiling are rejected when `deleted_rows` is requested, **before** envelope peek or full blob allocation (same fail-closed order as the Iceberg deletion-vector reader). Footer `deletion-vector-v1` properties are always validated: `cardinality` must parse as an unsigned integer, `snapshot-id` / `sequence-number` must be `-1`, and `fields` must be either empty or the singleton Iceberg reserved `_pos` id (`2147483645`) that Spark writes for file-scoped DVs — other `fields` lists (column-scoped DVs) are rejected. Selecting only `referenced_data_file` skips on-disk payload I/O and therefore also skips envelope, CRC, roaring deserialize, and the materialization ceiling — intentionally, so a path-only projection does not read up to the blob-size cap. -On-disk `deletion-vector-v1` blob length is bounded by an absolute ceiling (aligned with Iceberg's 2 GiB content-size check). When `deleted_rows` is requested, the reader peeks the envelope header (combined length and magic) before allocating the full payload; CRC is verified after the bounded read. +On-disk `deletion-vector-v1` blob length is bounded by an absolute ceiling (aligned with Iceberg's 2 GiB content-size check). When `deleted_rows` is requested and cardinality is within the materialization ceiling, the reader peeks the envelope header (combined length and magic) before allocating the full payload; CRC is verified after the bounded read. LZ4-compressed and uncompressed puffin footers are supported. Footer payload size (and declared LZ4 content size) is bounded by a compression ratio where applicable and an absolute ceiling; oversized footers are rejected before allocation. diff --git a/src/Processors/Formats/Impl/PuffinBlockInputFormat.h b/src/Processors/Formats/Impl/PuffinBlockInputFormat.h index 203dcd368544..251e96ef60f5 100644 --- a/src/Processors/Formats/Impl/PuffinBlockInputFormat.h +++ b/src/Processors/Formats/Impl/PuffinBlockInputFormat.h @@ -1,10 +1,16 @@ #pragma once +#include #include +#include #include #include +#include +#include +#include + namespace DB { @@ -26,6 +32,20 @@ struct PuffinFooter std::vector data; }; +/// Shared with the Iceberg deletion-vector loader (seekable object-storage path). +std::vector readPuffinFooterFromSeekable(SeekableReadBuffer & seekable, size_t file_size); + +/// Shared deletion-vector-v1 payload helpers (also used by `PuffinDeletionVectorReader`). +std::string_view extractDeletionVectorPayload(std::string_view blob); +std::vector deserializeRoaringPositionBitmap( + std::string_view bytes, std::optional expected_cardinality = std::nullopt); +void deserializeDeletionVectorV1(std::string_view blob, UInt64 expected_cardinality, ColumnUInt64 & positions); + +/// Validate deletion-vector-v1 footer identity (`snapshot-id` / `sequence-number` / `fields` / +/// required string properties). Returns parsed `cardinality`. Used by the SQL `Puffin` path and +/// by `bindDeletionVectorBlob`. +UInt64 requireDeletionVectorV1Properties(const PuffinBlob & blob, size_t blob_index); + class PuffinMetadataInputFormat : public IInputFormat { public: diff --git a/src/Processors/tests/gtest_parquet_row_group_global_offsets.cpp b/src/Processors/tests/gtest_parquet_row_group_global_offsets.cpp new file mode 100644 index 000000000000..664a8a43defa --- /dev/null +++ b/src/Processors/tests/gtest_parquet_row_group_global_offsets.cpp @@ -0,0 +1,85 @@ +#include + +#include + +#if USE_PARQUET + +#include +#include +#include + +#include + +using namespace DB; +using namespace DB::Parquet; + +namespace DB::ErrorCodes +{ +extern const int INCORRECT_DATA; +} + +namespace +{ + +parq::FileMetaData makeFileMetaData(Int64 file_num_rows, const std::vector & row_group_num_rows) +{ + parq::FileMetaData meta; + meta.__set_num_rows(file_num_rows); + meta.row_groups.reserve(row_group_num_rows.size()); + for (Int64 num_rows : row_group_num_rows) + { + parq::RowGroup row_group; + row_group.__set_num_rows(num_rows); + meta.row_groups.push_back(std::move(row_group)); + } + return meta; +} + +} + +TEST(ParquetRowGroupGlobalOffsets, BuildsPrefixOffsets) +{ + const auto offsets = buildRowGroupGlobalOffsets(makeFileMetaData(30, {10, 0, 20})); + ASSERT_EQ(offsets.size(), 4u); + EXPECT_EQ(offsets[0], 0u); + EXPECT_EQ(offsets[1], 10u); + EXPECT_EQ(offsets[2], 10u); + EXPECT_EQ(offsets[3], 30u); +} + +TEST(ParquetRowGroupGlobalOffsets, RejectsNegativeFileRowCount) +{ + EXPECT_THROW(buildRowGroupGlobalOffsets(makeFileMetaData(-1, {})), Exception); +} + +TEST(ParquetRowGroupGlobalOffsets, RejectsNegativeRowGroupRowCount) +{ + EXPECT_THROW(buildRowGroupGlobalOffsets(makeFileMetaData(10, {10, -1})), Exception); +} + +TEST(ParquetRowGroupGlobalOffsets, ToleratesMismatchWithFileNumRows) +{ + /// Offsets follow row-group counts even when FileMetaData.num_rows is stale/wrong. + const auto offsets = buildRowGroupGlobalOffsets(makeFileMetaData(11, {10, 0})); + ASSERT_EQ(offsets.size(), 3u); + EXPECT_EQ(offsets[0], 0u); + EXPECT_EQ(offsets[1], 10u); + EXPECT_EQ(offsets[2], 10u); +} + +TEST(ParquetRowGroupGlobalOffsets, RejectsOverflowingCumulativeCounts) +{ + constexpr Int64 max_rows = std::numeric_limits::max(); + /// Two Int64::max values fit in UInt64; the third overflows checked addition. + try + { + buildRowGroupGlobalOffsets(makeFileMetaData(max_rows, {max_rows, max_rows, max_rows})); + FAIL() << "Expected INCORRECT_DATA"; + } + catch (const Exception & e) + { + EXPECT_EQ(e.code(), ErrorCodes::INCORRECT_DATA); + } +} + +#endif diff --git a/src/Processors/tests/gtest_parquet_v3_need_only_count_buckets.cpp b/src/Processors/tests/gtest_parquet_v3_need_only_count_buckets.cpp new file mode 100644 index 000000000000..ea0f28a5c81a --- /dev/null +++ b/src/Processors/tests/gtest_parquet_v3_need_only_count_buckets.cpp @@ -0,0 +1,191 @@ +#include + +#include + +#if USE_PARQUET + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +using namespace DB; + +namespace DB::ErrorCodes +{ +extern const int LOGICAL_ERROR; +} + +namespace +{ + +void writeMultiRowGroupParquet(const String & path, size_t rows_per_group, size_t num_groups) +{ + FormatSettings format_settings; + format_settings.parquet.row_group_rows = rows_per_group; + format_settings.parquet.parallel_encoding = false; + + Block header; + header.insert(ColumnWithTypeAndName(std::make_shared(), "x")); + + Chunks chunks; + for (size_t group = 0; group < num_groups; ++group) + { + auto column = ColumnUInt64::create(); + for (size_t i = 0; i < rows_per_group; ++i) + column->insert(group * rows_per_group + i); + chunks.emplace_back(Columns{std::move(column)}, rows_per_group); + } + + auto source = std::make_shared(std::make_shared(header), std::move(chunks)); + QueryPipelineBuilder pipeline_builder; + pipeline_builder.init(Pipe(source)); + auto pipeline = QueryPipelineBuilder::getPipeline(std::move(pipeline_builder)); + + WriteBufferFromFile write_buffer(path); + auto output = std::make_shared(write_buffer, pipeline.getSharedHeader(), format_settings, nullptr); + pipeline.complete(output); + CompletedPipelineExecutor executor(pipeline); + executor.execute(); + output->finalize(); + write_buffer.finalize(); +} + +size_t readNeedOnlyCountTotal( + ReadBuffer & in, + const SharedHeader & header, + const FormatSettings & format_settings, + FormatParserSharedResourcesPtr parser_shared_resources, + const FileBucketInfoPtr & buckets) +{ + auto input = std::make_shared( + in, + header, + format_settings, + parser_shared_resources, + std::make_shared(), + /*min_bytes_for_seek=*/ 1024); + if (buckets) + input->setBucketsToRead(buckets); + input->needOnlyCount(); + + size_t total = 0; + while (true) + { + Chunk chunk = input->generate(); + if (!chunk) + break; + total += chunk.getNumRows(); + } + return total; +} + +std::vector> readNeedOnlyCountOffsetsAndRows( + ReadBuffer & in, + const SharedHeader & header, + const FormatSettings & format_settings, + FormatParserSharedResourcesPtr parser_shared_resources, + const FileBucketInfoPtr & buckets) +{ + auto input = std::make_shared( + in, + header, + format_settings, + parser_shared_resources, + std::make_shared(), + /*min_bytes_for_seek=*/ 1024); + if (buckets) + input->setBucketsToRead(buckets); + input->needOnlyCount(); + + std::vector> result; + while (true) + { + Chunk chunk = input->generate(); + if (!chunk) + break; + auto info = chunk.getChunkInfos().get(); + if (!info) + throw Exception(ErrorCodes::LOGICAL_ERROR, "Expected ChunkInfoRowNumbers on need_only_count chunk"); + result.emplace_back(info->row_num_offset, chunk.getNumRows()); + } + return result; +} + +} + +TEST(ParquetV3NeedOnlyCountBuckets, CountsOnlyAssignedRowGroups) +{ + tryRegisterFormats(); + const auto context = getContext().context; + + Poco::TemporaryFile temp_file; + const String path = temp_file.path(); + constexpr size_t rows_per_group = 10; + constexpr size_t num_groups = 3; + writeMultiRowGroupParquet(path, rows_per_group, num_groups); + + Block header; + header.insert(ColumnWithTypeAndName(std::make_shared(), "x")); + auto shared_header = std::make_shared(header); + FormatSettings format_settings; + auto parser_shared_resources = FormatParserSharedResources::singleThreaded(context->getSettingsRef()); + + { + ReadBufferFromFile in(path); + EXPECT_EQ( + readNeedOnlyCountTotal(in, shared_header, format_settings, parser_shared_resources, /*buckets=*/ nullptr), + rows_per_group * num_groups); + } + + { + /// Unbucketed need-only-count uses `FileMetaData.num_rows` as a single whole-file span. + ReadBufferFromFile in(path); + const auto offsets_and_rows + = readNeedOnlyCountOffsetsAndRows(in, shared_header, format_settings, parser_shared_resources, /*buckets=*/ nullptr); + ASSERT_EQ(offsets_and_rows.size(), 1u); + EXPECT_EQ(offsets_and_rows[0].first, 0u); + EXPECT_EQ(offsets_and_rows[0].second, rows_per_group * num_groups); + } + + { + ReadBufferFromFile in(path); + auto buckets = std::make_shared(std::vector{1}); + EXPECT_EQ(readNeedOnlyCountTotal(in, shared_header, format_settings, parser_shared_resources, buckets), rows_per_group); + } + + { + /// Bucketed tasks must count only assigned row groups (not the whole file) and keep + /// file-global row offsets for downstream row-number consumers. + ReadBufferFromFile in(path); + auto buckets = std::make_shared(std::vector{0, 2}); + const auto offsets_and_rows + = readNeedOnlyCountOffsetsAndRows(in, shared_header, format_settings, parser_shared_resources, buckets); + ASSERT_EQ(offsets_and_rows.size(), 2u); + EXPECT_EQ(offsets_and_rows[0].first, 0u); + EXPECT_EQ(offsets_and_rows[0].second, rows_per_group); + EXPECT_EQ(offsets_and_rows[1].first, 2 * rows_per_group); + EXPECT_EQ(offsets_and_rows[1].second, rows_per_group); + } + + { + ReadBufferFromFile in(path); + auto buckets = std::make_shared(std::vector{}); + EXPECT_EQ(readNeedOnlyCountTotal(in, shared_header, format_settings, parser_shared_resources, buckets), 0u); + } +} + +#endif diff --git a/src/Storages/ObjectStorage/DataLakes/Common/AvroForIcebergDeserializer.cpp b/src/Storages/ObjectStorage/DataLakes/Common/AvroForIcebergDeserializer.cpp index f383336c9596..fd912f27c012 100644 --- a/src/Storages/ObjectStorage/DataLakes/Common/AvroForIcebergDeserializer.cpp +++ b/src/Storages/ObjectStorage/DataLakes/Common/AvroForIcebergDeserializer.cpp @@ -5,6 +5,7 @@ #include #include #include +#include #include #include #include @@ -16,6 +17,11 @@ #include #include #include +<<<<<<< HEAD +======= +#include +#include +>>>>>>> 4b7cecaa3cf (Merge pull request #2183 from Altinity/feature/antalya-26.6/iceberg-puffin-deletion-vectors-read-2) namespace DB::ErrorCodes { @@ -279,6 +285,22 @@ ParsedManifestFileEntryPtr AvroForIcebergDeserializer::createParsedManifestFileE const auto record_count = getValueFromRowByName(row_index, c_data_file_record_count, TypeIndex::Int64).safeGet(); const auto file_size_in_bytes = getValueFromRowByName(row_index, c_data_file_file_size_in_bytes, TypeIndex::Int64).safeGet(); + std::optional content_offset; + if (hasPath(c_data_file_content_offset)) + { + const auto content_offset_value = getValueFromRowByName(row_index, c_data_file_content_offset); + if (!content_offset_value.isNull()) + content_offset = content_offset_value.safeGet(); + } + + std::optional content_size_in_bytes; + if (hasPath(c_data_file_content_size_in_bytes)) + { + const auto content_size_value = getValueFromRowByName(row_index, c_data_file_content_size_in_bytes); + if (!content_size_value.isNull()) + content_size_in_bytes = content_size_value.safeGet(); + } + switch (content_type) { case FileContentType::DATA: { @@ -303,6 +325,7 @@ ParsedManifestFileEntryPtr AvroForIcebergDeserializer::createParsedManifestFileE } case FileContentType::POSITION_DELETE: { /// reference_file_path can be absent in schema for some reason, though it is present in specification: https://iceberg.apache.org/spec/#manifests + const bool is_puffin = Poco::toLower(file_format) == "puffin"; std::optional lower_reference_data_file_path; std::optional upper_reference_data_file_path; bool bounds_set_by_referenced_data_file = false; @@ -318,7 +341,9 @@ ParsedManifestFileEntryPtr AvroForIcebergDeserializer::createParsedManifestFileE bounds_set_by_referenced_data_file = true; } } - if (!bounds_set_by_referenced_data_file) + /// Parquet position deletes may fall back to file-path column bounds. Puffin deletion + /// vectors must use the dedicated referenced_data_file field only. + if (!bounds_set_by_referenced_data_file && !is_puffin) { if (auto it = value_for_bounds.find(IcebergPositionDeleteTransform::data_file_path_column_field_id); it != value_for_bounds.end()) @@ -330,6 +355,20 @@ ParsedManifestFileEntryPtr AvroForIcebergDeserializer::createParsedManifestFileE upper_reference_data_file_path.emplace(Iceberg::IcebergPathFromMetadata::deserialize(upper.safeGet())); } } + + if (is_puffin) + { + if (!content_offset.has_value() || !content_size_in_bytes.has_value()) + { + throw Exception( + DB::ErrorCodes::ICEBERG_SPECIFICATION_VIOLATION, + "Puffin deletion vector entry in manifest file '{}' is missing content_offset or content_size_in_bytes", + manifest_file_path); + } + requireDirectReferencedDataFileForPuffinDeletionVector( + bounds_set_by_referenced_data_file, lower_reference_data_file_path, manifest_file_path); + } + return std::make_shared( FileContentType::POSITION_DELETE, file_path_key, @@ -347,7 +386,9 @@ ParsedManifestFileEntryPtr AvroForIcebergDeserializer::createParsedManifestFileE /*equality_ids*/ std::nullopt, /*sort_order_id = */ std::nullopt, record_count, - file_size_in_bytes); + file_size_in_bytes, + content_offset, + content_size_in_bytes); } case FileContentType::EQUALITY_DELETE: { std::vector equality_ids; diff --git a/src/Storages/ObjectStorage/DataLakes/DataLakeObjectMetadata.h b/src/Storages/ObjectStorage/DataLakes/DataLakeObjectMetadata.h index fcc2c0495e18..566b1965e0df 100644 --- a/src/Storages/ObjectStorage/DataLakes/DataLakeObjectMetadata.h +++ b/src/Storages/ObjectStorage/DataLakes/DataLakeObjectMetadata.h @@ -1,6 +1,8 @@ #pragma once #include +#include + namespace DB { template @@ -17,4 +19,11 @@ struct DataLakeObjectMetadata ExcludedRowsPtr excluded_rows; }; +/// True when a deletion vector (or similar) will filter rows via DeletionVectorTransform. +/// Count-from-files cache must skip these objects: the cache key is data-file identity only, +/// while excluded_rows can change independently (Iceberg puffin DVs, DeltaLake selection vectors). +/// Cluster task serialization must also fail closed when these cannot be carried on the wire. +bool hasNonEmptyExcludedRows(const DataLakeObjectMetadata & metadata); +bool hasNonEmptyExcludedRows(const std::optional & metadata); + } diff --git a/src/Storages/ObjectStorage/DataLakes/DeletionVectorTransform.cpp b/src/Storages/ObjectStorage/DataLakes/DeletionVectorTransform.cpp index 3fd353ef99a1..eb1077ca4eeb 100644 --- a/src/Storages/ObjectStorage/DataLakes/DeletionVectorTransform.cpp +++ b/src/Storages/ObjectStorage/DataLakes/DeletionVectorTransform.cpp @@ -3,6 +3,7 @@ #include #include #include +#include namespace DB::ErrorCodes @@ -12,6 +13,76 @@ namespace DB::ErrorCodes namespace DB { +namespace +{ + +bool isConstCountOnlyChunk(const Chunk & chunk, const ChunkInfoRowNumbers & chunk_info) +{ + /// need_only_count emits ColumnConst defaults with no prior filter. Dense filtering is still + /// required when real column values or an applied_filter (PREWHERE / prior filters) are present. + if (chunk_info.applied_filter.has_value()) + return false; + + for (const auto & column : chunk.getColumns()) + { + if (!isColumnConst(*column)) + return false; + } + return true; +} + +void transformConstCountOnlyChunk(Chunk & chunk, ChunkInfoRowNumbers & chunk_info, const DataLakeObjectMetadata::ExcludedRows & excluded_rows) +{ + /// Const need_only_count chunks have no prior applied_filter (`isConstCountOnlyChunk`). + /// Iceberg does not attach parquet position deletes when a DV is present, and equality deletes + /// disable need_only_count — so no later row-number consumer follows this path today. Keep the + /// cheap cardinality + cloneResized path (no Filter / rb_range materialization). + chassert(!chunk_info.applied_filter.has_value()); + + const size_t num_rows_before = chunk.getNumRows(); + size_t range_end = 0; + if (common::addOverflow(chunk_info.row_num_offset, num_rows_before, range_end)) + { + throw Exception( + ErrorCodes::LOGICAL_ERROR, + "Deletion vector count range overflows size_t: offset {} num_rows {}", + chunk_info.row_num_offset, + num_rows_before); + } + + const UInt64 deleted = excluded_rows.rb_range_cardinality(chunk_info.row_num_offset, range_end); + if (deleted > num_rows_before) + { + throw Exception( + ErrorCodes::LOGICAL_ERROR, + "Deletion vector reports {} deletes in a count chunk of {} rows (offset {})", + deleted, + num_rows_before, + chunk_info.row_num_offset); + } + + const size_t num_rows_after = num_rows_before - static_cast(deleted); + if (num_rows_after == num_rows_before) + return; + + auto columns = chunk.detachColumns(); + for (auto & column : columns) + column = column->cloneResized(num_rows_after); + + chunk.setColumns(std::move(columns), num_rows_after); +} + +} + +bool hasNonEmptyExcludedRows(const DataLakeObjectMetadata & metadata) +{ + return metadata.excluded_rows && metadata.excluded_rows->size() > 0; +} + +bool hasNonEmptyExcludedRows(const std::optional & metadata) +{ + return metadata.has_value() && hasNonEmptyExcludedRows(*metadata); +} DeletionVectorTransform::DeletionVectorTransform( const DB::SharedHeader & header_, @@ -32,6 +103,12 @@ void DeletionVectorTransform::transform(DB::Chunk & chunk, const ExcludedRows & if (!chunk_info) throw DB::Exception(DB::ErrorCodes::LOGICAL_ERROR, "ChunkInfoRowNumbers does not exist"); + if (isConstCountOnlyChunk(chunk, *chunk_info)) + { + transformConstCountOnlyChunk(chunk, *chunk_info, excluded_rows); + return; + } + const size_t num_rows_before = chunk.getNumRows(); size_t num_rows_after = num_rows_before; size_t idx_in_chunk = 0; diff --git a/src/Storages/ObjectStorage/DataLakes/Iceberg/Constant.h b/src/Storages/ObjectStorage/DataLakes/Iceberg/Constant.h index b58d769a373d..f823774b1cff 100644 --- a/src/Storages/ObjectStorage/DataLakes/Iceberg/Constant.h +++ b/src/Storages/ObjectStorage/DataLakes/Iceberg/Constant.h @@ -184,6 +184,8 @@ DEFINE_ICEBERG_FIELD_COMPOUND(data_file, null_value_counts); DEFINE_ICEBERG_FIELD_COMPOUND(data_file, lower_bounds); DEFINE_ICEBERG_FIELD_COMPOUND(data_file, upper_bounds); DEFINE_ICEBERG_FIELD_COMPOUND(data_file, referenced_data_file); +DEFINE_ICEBERG_FIELD_COMPOUND(data_file, content_offset); +DEFINE_ICEBERG_FIELD_COMPOUND(data_file, content_size_in_bytes); DEFINE_ICEBERG_FIELD_COMPOUND(data_file, sort_order_id); DEFINE_ICEBERG_FIELD_COMPOUND(data_file, record_count); DEFINE_ICEBERG_FIELD_COMPOUND(data_file, file_size_in_bytes); diff --git a/src/Storages/ObjectStorage/DataLakes/Iceberg/IcebergDataObjectInfo.cpp b/src/Storages/ObjectStorage/DataLakes/Iceberg/IcebergDataObjectInfo.cpp index 53710d72093d..8adf207896e2 100644 --- a/src/Storages/ObjectStorage/DataLakes/Iceberg/IcebergDataObjectInfo.cpp +++ b/src/Storages/ObjectStorage/DataLakes/Iceberg/IcebergDataObjectInfo.cpp @@ -26,6 +26,23 @@ extern const int UNKNOWN_PROTOCOL; using namespace DB::Iceberg; +namespace DB::Iceberg +{ + +void requireParquetDataFileForRowDeletes(const String & file_format, std::string_view feature_name) +{ + if (Poco::toUpper(file_format) != "PARQUET") + { + throw Exception( + DB::ErrorCodes::NOT_IMPLEMENTED, + "{} are only supported for data files of Parquet format in Iceberg, but got {}", + feature_name, + file_format); + } +} + +} + namespace DB { @@ -102,13 +119,7 @@ std::shared_ptr IcebergDataObjectInfo::getPositionDeleteTransf void IcebergDataObjectInfo::addPositionDeleteObject(Iceberg::ProcessedManifestFileEntryPtr position_delete_object, const String & resolved_storage_path) { - if (Poco::toUpper(info.file_format) != "PARQUET") - { - throw Exception( - ErrorCodes::NOT_IMPLEMENTED, - "Position deletes are only supported for data files of Parquet format in Iceberg, but got {}", - info.file_format); - } + Iceberg::requireParquetDataFileForRowDeletes(info.file_format, "Position deletes"); info.position_deletes_objects.emplace_back( resolved_storage_path, position_delete_object->parsed_entry->file_format, std::nullopt, position_delete_object->sequence_number); @@ -123,6 +134,18 @@ void IcebergDataObjectInfo::addEqualityDeleteObject(const Iceberg::ProcessedMani equality_delete_object->resolved_schema_id); } +bool hasIcebergEqualityDeletes(const ObjectInfoPtr & object_info) +{ + const auto * iceberg = dynamic_cast(object_info.get()); + return iceberg && !iceberg->info.equality_deletes_objects.empty(); +} + +bool hasIcebergPositionDeletes(const ObjectInfoPtr & object_info) +{ + const auto * iceberg = dynamic_cast(object_info.get()); + return iceberg && !iceberg->info.position_deletes_objects.empty(); +} + #endif void IcebergObjectSerializableInfo::serializeForClusterFunctionProtocol(WriteBuffer & out, size_t protocol_version) const diff --git a/src/Storages/ObjectStorage/DataLakes/Iceberg/IcebergDataObjectInfo.h b/src/Storages/ObjectStorage/DataLakes/Iceberg/IcebergDataObjectInfo.h index 3caf8de87958..f8da4f7da7f2 100644 --- a/src/Storages/ObjectStorage/DataLakes/Iceberg/IcebergDataObjectInfo.h +++ b/src/Storages/ObjectStorage/DataLakes/Iceberg/IcebergDataObjectInfo.h @@ -11,12 +11,18 @@ #include #include +#include + namespace DB::Iceberg { String computePartitionId(const Row & partition_key_value); +/// Position deletes and deletion vectors need file-relative row numbers (`ChunkInfoRowNumbers`), +/// which Parquet readers emit. Reject other data-file formats early with NOT_IMPLEMENTED. +void requireParquetDataFileForRowDeletes(const String & file_format, std::string_view feature_name); + struct IcebergObjectSerializableInfo { @@ -91,10 +97,16 @@ struct IcebergDataObjectInfo : public ObjectInfo, std::enable_shared_from_this; + +/// Equality deletes need real column values — disable need_only_count / count-from-files cache. +bool hasIcebergEqualityDeletes(const ObjectInfoPtr & object_info); +/// Position-delete attachment changes independently of data-file identity — skip count-from-files cache. +bool hasIcebergPositionDeletes(const ObjectInfoPtr & object_info); } #endif diff --git a/src/Storages/ObjectStorage/DataLakes/Iceberg/IcebergDeletionVector.cpp b/src/Storages/ObjectStorage/DataLakes/Iceberg/IcebergDeletionVector.cpp new file mode 100644 index 000000000000..91d40f6df66f --- /dev/null +++ b/src/Storages/ObjectStorage/DataLakes/Iceberg/IcebergDeletionVector.cpp @@ -0,0 +1,395 @@ +#include "config.h" + +#if USE_AVRO + +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include + +namespace DB +{ + +namespace ErrorCodes +{ + extern const int ICEBERG_SPECIFICATION_VIOLATION; + extern const int BAD_ARGUMENTS; + extern const int LOGICAL_ERROR; +} + +namespace Setting +{ +extern const SettingsBool use_puffin_files_cache; +} + +} + +namespace DB::Iceberg +{ + +void validateDeletionVectorPositionsAgainstDataFile( + std::span deleted_positions, + UInt64 expected_cardinality, + Int64 data_file_record_count) +{ + if (data_file_record_count < 0) + { + throw Exception( + ErrorCodes::ICEBERG_SPECIFICATION_VIOLATION, + "Data file record_count {} must be non-negative", + data_file_record_count); + } + + const UInt64 data_file_rows = static_cast(data_file_record_count); + + if (expected_cardinality > data_file_rows) + { + throw Exception( + ErrorCodes::ICEBERG_SPECIFICATION_VIOLATION, + "Deletion vector cardinality {} exceeds data file record_count {}", + expected_cardinality, + data_file_record_count); + } + + for (UInt64 position : deleted_positions) + { + if (position >= data_file_rows) + { + throw Exception( + ErrorCodes::ICEBERG_SPECIFICATION_VIOLATION, + "Deletion vector position {} is out of range for data file record_count {}", + position, + data_file_record_count); + } + } +} + +namespace +{ + +using FooterBlobsPtr = PuffinFilesCache::FooterBlobsPtr; + +void logUndersizedPuffinFilesCacheOnce(LoggerPtr log, size_t max_size_in_bytes, UInt64 minimum_entry_weight) +{ + static std::atomic_flag logged = ATOMIC_FLAG_INIT; + if (!logged.test_and_set()) + { + LOG_WARNING( + log, + "Not using Puffin files cache because puffin_files_cache_size ({}) is smaller than the " + "minimum deletion-vector entry weight (at least {}); falling back to filesystem-cache-enabled reads", + max_size_in_bytes, + minimum_entry_weight); + } +} + +FooterBlobsPtr readFooterBlobs( + ObjectStoragePtr object_storage, + const String & puffin_path, + ContextPtr context, + LoggerPtr log, + bool disable_filesystem_cache) +{ + RelativePathWithMetadata puffin_object{puffin_path}; + auto read_settings = context->getReadSettings(); + if (disable_filesystem_cache) + read_settings.enable_filesystem_cache = false; + + auto read_buffer = createReadBuffer(puffin_object, object_storage, context, log, read_settings); + + auto * seekable = dynamic_cast(read_buffer.get()); + if (!seekable) + throw Exception(ErrorCodes::BAD_ARGUMENTS, "Puffin deletion vector read requires a seekable buffer"); + + auto file_size = tryGetFileSizeFromReadBuffer(*read_buffer); + if (!file_size) + throw Exception(ErrorCodes::BAD_ARGUMENTS, "Cannot determine Puffin file size for '{}'", puffin_path); + + return std::make_shared>(readPuffinFooterBlobsFromSeekable(*seekable, *file_size)); +} + +DataLakeObjectMetadata::ExcludedRowsPtr loadDeletionVectorUncached( + ObjectStoragePtr object_storage, + const String & puffin_path, + Int64 content_offset, + Int64 content_size_in_bytes, + const IcebergPathFromMetadata & expected_data_file, + UInt64 expected_cardinality, + Int64 data_file_record_count, + ContextPtr context, + LoggerPtr log, + bool disable_filesystem_cache, + FooterBlobsPtr preloaded_footer) +{ + RelativePathWithMetadata puffin_object{puffin_path}; + auto read_settings = context->getReadSettings(); + if (disable_filesystem_cache) + read_settings.enable_filesystem_cache = false; + + auto read_buffer = createReadBuffer(puffin_object, object_storage, context, log, read_settings); + + auto * seekable = dynamic_cast(read_buffer.get()); + if (!seekable) + throw Exception(ErrorCodes::BAD_ARGUMENTS, "Puffin deletion vector read requires a seekable buffer"); + + auto file_size = tryGetFileSizeFromReadBuffer(*read_buffer); + if (!file_size) + throw Exception(ErrorCodes::BAD_ARGUMENTS, "Cannot determine Puffin file size for '{}'", puffin_path); + + FooterBlobsPtr footer_owner = preloaded_footer; + if (!footer_owner) + footer_owner = std::make_shared>(readPuffinFooterBlobsFromSeekable(*seekable, *file_size)); + + bindDeletionVectorBlob( + *footer_owner, + content_offset, + content_size_in_bytes, + expected_data_file.serialize(), + expected_cardinality); + + auto deleted_positions = readDeletionVectorFromPuffin( + *read_buffer, content_offset, content_size_in_bytes, expected_cardinality); + + validateDeletionVectorPositionsAgainstDataFile(deleted_positions, expected_cardinality, data_file_record_count); + + if (deleted_positions.empty()) + return nullptr; + + auto bitmap = std::make_shared(); + for (UInt64 position : deleted_positions) + bitmap->add(static_cast(position)); + + LOG_DEBUG( + log, + "Loaded deletion vector from puffin file '{}' for data file '{}': {} deleted rows", + puffin_path, + expected_data_file.serialize(), + deleted_positions.size()); + + return bitmap; +} + +} + +DataLakeObjectMetadata::ExcludedRowsPtr loadDeletionVector( + ObjectStoragePtr object_storage, + const String & puffin_path, + Int64 content_offset, + Int64 content_size_in_bytes, + const IcebergPathFromMetadata & expected_data_file, + const std::optional & referenced_data_file, + Int64 expected_cardinality, + Int64 data_file_record_count, + ContextPtr context, + LoggerPtr log) +{ + if (referenced_data_file.has_value() && referenced_data_file.value() != expected_data_file) + { + throw Exception( + ErrorCodes::ICEBERG_SPECIFICATION_VIOLATION, + "Deletion vector referenced_data_file '{}' does not match data file '{}'", + referenced_data_file->serialize(), + expected_data_file.serialize()); + } + + if (expected_cardinality < 0) + { + throw Exception( + ErrorCodes::ICEBERG_SPECIFICATION_VIOLATION, + "Deletion vector record_count {} must be non-negative", + expected_cardinality); + } + + if (data_file_record_count < 0) + { + throw Exception( + ErrorCodes::ICEBERG_SPECIFICATION_VIOLATION, + "Data file record_count {} must be non-negative", + data_file_record_count); + } + + const UInt64 expected_cardinality_u64 = static_cast(expected_cardinality); + + /// Fail closed before I/O when declared DV cardinality cannot fit in the data file. + if (expected_cardinality_u64 > static_cast(data_file_record_count)) + { + throw Exception( + ErrorCodes::ICEBERG_SPECIFICATION_VIOLATION, + "Deletion vector cardinality {} exceeds data file record_count {}", + expected_cardinality, + data_file_record_count); + } + + const bool use_cache_setting = context->getSettingsRef()[Setting::use_puffin_files_cache]; + auto cache = use_cache_setting ? context->getPuffinFilesCache() : nullptr; + + /// puffin_files_cache_size=0 means the LRU accepts no entries ("disabled" in server settings). + /// Take the same uncached path as use_puffin_files_cache=0 so we keep filesystem cache and + /// skip etag HEAD / getOrSet (which would disable filesystem cache on the miss loader). + if (!cache || cache->maxSizeInBytes() == 0) + { + if (!use_cache_setting) + { + LOG_TRACE(log, "Not using Puffin files cache for '{}', because the setting use_puffin_files_cache is false", puffin_path); + } + else + { + LOG_TRACE( + log, + "Not using Puffin files cache for '{}', because puffin_files_cache_size is 0", + puffin_path); + } + return loadDeletionVectorUncached( + object_storage, + puffin_path, + content_offset, + content_size_in_bytes, + expected_data_file, + expected_cardinality_u64, + data_file_record_count, + context, + log, + false, + nullptr); + } + + const String storage_identity = PuffinFilesCache::makeStorageIdentity(*object_storage); + const String referenced_data_file_key = expected_data_file.serialize(); + + /// Before etag HEAD: if the budget cannot hold even the key/overhead lower bound, skip the + /// cached path entirely (keeps filesystem cache, avoids a useless HEAD + immediate eviction). + { + const PuffinFilesCacheKey provisional_key{ + storage_identity, + puffin_path, + /*etag=*/"", + content_offset, + content_size_in_bytes, + referenced_data_file_key, + expected_cardinality_u64, + static_cast(data_file_record_count)}; + const UInt64 minimum_entry_weight + = PuffinFilesCacheCell::estimateMinimumMemorySize(provisional_key.approximateMemoryBytes()); + if (minimum_entry_weight > cache->maxSizeInBytes()) + { + logUndersizedPuffinFilesCacheOnce(log, cache->maxSizeInBytes(), minimum_entry_weight); + return loadDeletionVectorUncached( + object_storage, + puffin_path, + content_offset, + content_size_in_bytes, + expected_data_file, + expected_cardinality_u64, + data_file_record_count, + context, + log, + false, + nullptr); + } + } + + RelativePathWithMetadata puffin_object{puffin_path}; + if (!puffin_object.metadata) + puffin_object.metadata = object_storage->getObjectMetadata(puffin_object.getPath(), /*with_tags=*/ false); + + if (puffin_object.metadata->etag.empty()) + { + LOG_TRACE( + log, + "Not using Puffin files cache for '{}', because etag is empty", + puffin_path); + return loadDeletionVectorUncached( + object_storage, + puffin_path, + content_offset, + content_size_in_bytes, + expected_data_file, + expected_cardinality_u64, + data_file_record_count, + context, + log, + false, + nullptr); + } + + auto footer_key = PuffinFilesCache::tryCreateFooterKey(storage_identity, puffin_path, puffin_object.metadata->etag); + auto cache_key = PuffinFilesCache::tryCreateKey( + storage_identity, + puffin_path, + puffin_object.metadata->etag, + content_offset, + content_size_in_bytes, + referenced_data_file_key, + expected_cardinality_u64, + static_cast(data_file_record_count)); + + /// Empty etag is the only reason tryCreate* returns nullopt; that case is handled above. + if (!footer_key || !cache_key) + { + throw Exception( + ErrorCodes::LOGICAL_ERROR, + "PuffinFilesCache::tryCreate* returned nullopt for non-empty etag on '{}'", + puffin_path); + } + + /// Recheck with the real etag (provisional estimate used an empty etag string). + { + const UInt64 minimum_entry_weight + = PuffinFilesCacheCell::estimateMinimumMemorySize(cache_key->approximateMemoryBytes()); + if (minimum_entry_weight > cache->maxSizeInBytes()) + { + logUndersizedPuffinFilesCacheOnce(log, cache->maxSizeInBytes(), minimum_entry_weight); + return loadDeletionVectorUncached( + object_storage, + puffin_path, + content_offset, + content_size_in_bytes, + expected_data_file, + expected_cardinality_u64, + data_file_record_count, + context, + log, + false, + nullptr); + } + } + + /// Footer is keyed by file identity only, so N DV slices in one coalesced Puffin share one parse. + /// Resolve the footer only on a deletion-vector cache miss (nested memo lookup). + return cache->getOrSetDeletionVector(*cache_key, [&]() + { + auto footer = cache->getOrSetFooter(*footer_key, [&]() + { + return readFooterBlobs(object_storage, puffin_path, context, log, /*disable_filesystem_cache=*/ true); + }); + + return loadDeletionVectorUncached( + object_storage, + puffin_path, + content_offset, + content_size_in_bytes, + expected_data_file, + expected_cardinality_u64, + data_file_record_count, + context, + log, + true, + footer); + }); +} + +} + +#endif diff --git a/src/Storages/ObjectStorage/DataLakes/Iceberg/IcebergDeletionVector.h b/src/Storages/ObjectStorage/DataLakes/Iceberg/IcebergDeletionVector.h new file mode 100644 index 000000000000..a0323730564d --- /dev/null +++ b/src/Storages/ObjectStorage/DataLakes/Iceberg/IcebergDeletionVector.h @@ -0,0 +1,38 @@ +#pragma once +#include "config.h" + +#if USE_AVRO + +#include +#include +#include +#include +#include + +#include +#include + +namespace DB::Iceberg +{ + +/// Reject DV positions outside `[0, data_file_record_count)` and cardinality that cannot fit. +void validateDeletionVectorPositionsAgainstDataFile( + std::span deleted_positions, + UInt64 expected_cardinality, + Int64 data_file_record_count); + +DataLakeObjectMetadata::ExcludedRowsPtr loadDeletionVector( + ObjectStoragePtr object_storage, + const String & puffin_path, + Int64 content_offset, + Int64 content_size_in_bytes, + const IcebergPathFromMetadata & expected_data_file, + const std::optional & referenced_data_file, + Int64 expected_cardinality, + Int64 data_file_record_count, + ContextPtr context, + LoggerPtr log); + +} + +#endif diff --git a/src/Storages/ObjectStorage/DataLakes/Iceberg/IcebergIterator.cpp b/src/Storages/ObjectStorage/DataLakes/Iceberg/IcebergIterator.cpp index ada57d40ae42..569899006f85 100644 --- a/src/Storages/ObjectStorage/DataLakes/Iceberg/IcebergIterator.cpp +++ b/src/Storages/ObjectStorage/DataLakes/Iceberg/IcebergIterator.cpp @@ -46,6 +46,7 @@ #include #include #include +#include #include #include @@ -75,6 +76,7 @@ namespace DB namespace ErrorCodes { extern const int LOGICAL_ERROR; +extern const int ICEBERG_SPECIFICATION_VIOLATION; } namespace Setting { @@ -314,10 +316,39 @@ IcebergIterator::IcebergIterator( , blocking_queue(100) , callback(std::move(callback_)) { +<<<<<<< HEAD /// Decoding any manifest reads settings from the context, so a missing one is fatal either way. if (!local_context) throw DB::Exception(DB::ErrorCodes::LOGICAL_ERROR, "Context is required to construct IcebergIterator"); +======= + auto delete_file = deletes_iterator.next(); + while (delete_file.has_value()) + { + if (delete_file.value()->parsed_entry->equality_ids.has_value()) + { + equality_deletes_files.emplace_back(std::move(delete_file.value())); + } + else if (delete_file.value()->parsed_entry->isDeletionVector()) + { + deletion_vector_files.emplace_back(std::move(delete_file.value())); + } + else + { + parquet_position_deletes_files.emplace_back(std::move(delete_file.value())); + } + delete_file = deletes_iterator.next(); + } + LOG_DEBUG( + logger, + "Taken {} deletion vector files, {} parquet position delete files and {} equality delete files in iceberg iterator", + deletion_vector_files.size(), + parquet_position_deletes_files.size(), + equality_deletes_files.size()); + std::sort(equality_deletes_files.begin(), equality_deletes_files.end()); + std::sort(deletion_vector_files.begin(), deletion_vector_files.end()); + std::sort(parquet_position_deletes_files.begin(), parquet_position_deletes_files.end()); +>>>>>>> 4b7cecaa3cf (Merge pull request #2183 from Altinity/feature/antalya-26.6/iceberg-puffin-deletion-vectors-read-2) producer_task = std::make_unique( [this, thread_group = CurrentThread::getGroup()]() { @@ -469,6 +500,7 @@ ObjectInfoPtr IcebergIterator::next(size_t) Iceberg::ProcessedManifestFileEntryPtr manifest_file_entry; if (blocking_queue.pop(manifest_file_entry)) { +<<<<<<< HEAD IcebergDataObjectInfoPtr object_info = std::make_shared( manifest_file_entry, @@ -477,29 +509,42 @@ ObjectInfoPtr IcebergIterator::next(size_t) Iceberg::getIdentityPartitionColumnValues(*manifest_file_entry, *persistent_components.schema_processor)); for (const auto & position_delete : defineDeletesSpan(manifest_file_entry, position_deletes_files, /* is_equality_delete */ false, logger)) +======= + const auto & raw_metadata_path = manifest_file_entry->parsed_entry->file_path_key.serialize(); + auto [storage_to_use, resolved_key] = resolveObjectStorageForPath( + persistent_components.table_location, raw_metadata_path, + object_storage, *secondary_storages, local_context, + persistent_components.path_resolver); + + IcebergDataObjectInfoPtr object_info = std::make_shared( + manifest_file_entry, raw_metadata_path, table_state_snapshot->schema_id, storage_to_use, resolved_key); + + object_info->info.requires_external_storage = (storage_to_use != object_storage); + + const auto & data_file_path = object_info->info.data_object_file_path_key; + bool has_deletion_vector = false; + + for (const auto & deletion_vector : + defineDeletesSpan(manifest_file_entry, deletion_vector_files, /* is_equality_delete */ false, logger)) +>>>>>>> 4b7cecaa3cf (Merge pull request #2183 from Altinity/feature/antalya-26.6/iceberg-puffin-deletion-vectors-read-2) { - const auto & data_file_path = object_info->info.data_object_file_path_key; - const auto & lower = position_delete->parsed_entry->lower_reference_data_file_path; - const auto & upper = position_delete->parsed_entry->upper_reference_data_file_path; - bool can_contain_data_file_deletes - = (!lower.has_value() || *lower <= data_file_path) - && (!upper.has_value() || *upper >= data_file_path); - /// Skip position deletes that do not match the data file path. - if (!can_contain_data_file_deletes) + const auto & referenced_data_file = deletion_vector->parsed_entry->lower_reference_data_file_path; + if (!referenced_data_file.has_value() || referenced_data_file.value() != data_file_path) + continue; + + if (has_deletion_vector) { - ProfileEvents::increment(ProfileEvents::IcebergMinMaxPrunedDeleteFiles); - LOG_TEST( - logger, - "Skipping position delete file `{}` for data file `{}` because position delete has out of bounds reference data file " - "bounds: " - "(lower bound: `{}`, upper bound: `{}`)", - position_delete->parsed_entry->file_path_key, - data_file_path, - lower.has_value() ? lower->serialize() : "[no lower bound]", - upper.has_value() ? upper->serialize() : "[no upper bound]"); + throw Exception( + ErrorCodes::ICEBERG_SPECIFICATION_VIOLATION, + "Multiple deletion vectors match data file '{}'", + data_file_path); } - else + + Iceberg::requireParquetDataFileForRowDeletes(object_info->info.file_format, "Deletion vectors"); + + if (!object_info->info.record_count.has_value()) { +<<<<<<< HEAD ProfileEvents::increment(ProfileEvents::IcebergMinMaxNonPrunedDeleteFiles); LOG_TEST( logger, @@ -511,6 +556,85 @@ ObjectInfoPtr IcebergIterator::next(size_t) upper.has_value() ? upper->serialize() : "[no upper bound]"); object_info->addPositionDeleteObject( position_delete, persistent_components.path_resolver.resolve(position_delete->parsed_entry->file_path_key)); +======= + throw Exception( + ErrorCodes::ICEBERG_SPECIFICATION_VIOLATION, + "Data file '{}' is missing record_count required to validate deletion vector positions", + data_file_path); + } + + const auto & parsed_entry = deletion_vector->parsed_entry; + const auto puffin_metadata_path = parsed_entry->file_path_key.serialize(); + auto [puffin_storage, puffin_key] = resolveObjectStorageForPath( + persistent_components.table_location, puffin_metadata_path, + object_storage, *secondary_storages, local_context, + persistent_components.path_resolver); + + /// For icebergCluster, next() runs on the initiator's task-distribution path: DV + /// I/O / CRC / roaring materialization happen here, then excluded_rows is sent on + /// the wire per task. Workers apply the bitmap and do not re-read the Puffin blob. + auto excluded_rows = Iceberg::loadDeletionVector( + puffin_storage, + puffin_key, + parsed_entry->content_offset.value(), + parsed_entry->content_size_in_bytes.value(), + data_file_path, + referenced_data_file, + parsed_entry->record_count, + *object_info->info.record_count, + local_context, + logger); + + object_info->data_lake_metadata.emplace(); + if (excluded_rows) + object_info->data_lake_metadata->excluded_rows = std::move(excluded_rows); + has_deletion_vector = true; + LOG_DEBUG( + logger, + "Attached deletion vector from puffin file `{}` to data file `{}`", + parsed_entry->file_path_key, + data_file_path); + } + + if (!has_deletion_vector) + { + for (const auto & position_delete : + defineDeletesSpan(manifest_file_entry, parquet_position_deletes_files, /* is_equality_delete */ false, logger)) + { + const auto & lower = position_delete->parsed_entry->lower_reference_data_file_path; + const auto & upper = position_delete->parsed_entry->upper_reference_data_file_path; + bool can_contain_data_file_deletes + = (!lower.has_value() || *lower <= data_file_path) + && (!upper.has_value() || *upper >= data_file_path); + /// Skip position deletes that do not match the data file path. + if (!can_contain_data_file_deletes) + { + ProfileEvents::increment(ProfileEvents::IcebergMinMaxPrunedDeleteFiles); + LOG_TEST( + logger, + "Skipping position delete file `{}` for data file `{}` because position delete has out of bounds reference data file " + "bounds: " + "(lower bound: `{}`, upper bound: `{}`)", + position_delete->parsed_entry->file_path_key, + data_file_path, + lower.has_value() ? lower->serialize() : "[no lower bound]", + upper.has_value() ? upper->serialize() : "[no upper bound]"); + } + else + { + ProfileEvents::increment(ProfileEvents::IcebergMinMaxNonPrunedDeleteFiles); + LOG_TEST( + logger, + "Processing position delete file `{}` for data file `{}` with reference data file bounds: " + "(lower bound: `{}`, upper bound: `{}`)", + position_delete->parsed_entry->file_path_key, + data_file_path, + lower.has_value() ? lower->serialize() : "[no lower bound]", + upper.has_value() ? upper->serialize() : "[no upper bound]"); + object_info->addPositionDeleteObject( + position_delete, position_delete->parsed_entry->file_path_key.serialize()); + } +>>>>>>> 4b7cecaa3cf (Merge pull request #2183 from Altinity/feature/antalya-26.6/iceberg-puffin-deletion-vectors-read-2) } } diff --git a/src/Storages/ObjectStorage/DataLakes/Iceberg/IcebergIterator.h b/src/Storages/ObjectStorage/DataLakes/Iceberg/IcebergIterator.h index bb86db7cfe77..caebff5b865d 100644 --- a/src/Storages/ObjectStorage/DataLakes/Iceberg/IcebergIterator.h +++ b/src/Storages/ObjectStorage/DataLakes/Iceberg/IcebergIterator.h @@ -110,9 +110,14 @@ class IcebergIterator : public IObjectIterator ConcurrentBoundedQueue blocking_queue; std::unique_ptr producer_task; IDataLakeMetadata::FileProgressCallback callback; +<<<<<<< HEAD /// Filled once under `deletes_mutex` and never mutated afterwards, so `next` may read them /// unguarded once it has gone through `ensureDeletesReady`. std::vector position_deletes_files; +======= + std::vector deletion_vector_files; + std::vector parquet_position_deletes_files; +>>>>>>> 4b7cecaa3cf (Merge pull request #2183 from Altinity/feature/antalya-26.6/iceberg-puffin-deletion-vectors-read-2) std::vector equality_deletes_files; std::mutex deletes_mutex; bool deletes_ready TSA_GUARDED_BY(deletes_mutex) = false; diff --git a/src/Storages/ObjectStorage/DataLakes/Iceberg/IcebergMetadata.cpp b/src/Storages/ObjectStorage/DataLakes/Iceberg/IcebergMetadata.cpp index 2af675f6cf10..e3c1f6f6a12f 100644 --- a/src/Storages/ObjectStorage/DataLakes/Iceberg/IcebergMetadata.cpp +++ b/src/Storages/ObjectStorage/DataLakes/Iceberg/IcebergMetadata.cpp @@ -10,6 +10,11 @@ #include #include #include +<<<<<<< HEAD +======= +#include +#include +>>>>>>> 4b7cecaa3cf (Merge pull request #2183 from Altinity/feature/antalya-26.6/iceberg-puffin-deletion-vectors-read-2) #include #include #include @@ -410,6 +415,7 @@ IcebergDataSnapshotPtr IcebergMetadata::createIcebergDataSnapshotFromSnapshotJSO std::optional total_rows; std::optional total_bytes; std::optional total_position_deletes; + std::optional total_equality_deletes; if (snapshot_object->has(f_summary)) { @@ -424,6 +430,9 @@ IcebergDataSnapshotPtr IcebergMetadata::createIcebergDataSnapshotFromSnapshotJSO { total_position_deletes = summary_object->getValue(f_total_position_deletes); } + + if (summary_object->has(f_total_equality_deletes)) + total_equality_deletes = summary_object->getValue(f_total_equality_deletes); } if (!snapshot_object->has(f_schema_id)) @@ -437,7 +446,8 @@ IcebergDataSnapshotPtr IcebergMetadata::createIcebergDataSnapshotFromSnapshotJSO schema_id, total_rows, total_bytes, - total_position_deletes); + total_position_deletes, + total_equality_deletes); } IcebergDataSnapshotPtr @@ -1230,7 +1240,14 @@ std::optional IcebergMetadata::totalRows(ContextPtr local_context) const return 0; } + /// Equality deletes remove data rows by value match; summary `total-equality-deletes` counts + /// rows in delete files, not deleted data rows. Fail closed when the field is present and > 0. + /// If the field is absent, skip to manifests for EQUALITY_DELETE files. + if (actual_data_snapshot->total_equality_delete_rows.has_value() + && *actual_data_snapshot->total_equality_delete_rows > 0) + return {}; +<<<<<<< HEAD /// Row counts stored in the metadata layers above the manifest files are not used as /// data sources, because writers derive them instead of measuring them against the data: /// - the snapshot summary's `total-records` is maintained incrementally (parent total @@ -1246,10 +1263,22 @@ std::optional IcebergMetadata::totalRows(ContextPtr local_context) const /// required field in every format version, so summing it over the live data files is /// exact, at the cost of opening the manifest files (served from the Iceberg metadata /// cache on repeated queries). +======= + /// Do not trust snapshot-summary `total-records` for the answer. Those totals are optional, + /// writer-maintained incrementally, and a single bad commit can poison every later snapshot. + /// Sum required per-data-file `record_count` from manifests when there are no live delete + /// files; otherwise fail closed to a real scan. Summary is compared only for a mismatch warning. + /// + /// Manifest-list `added_rows_count`/`existing_rows_count` are not used (some writers stamp them + /// from snapshot summary and can report 0 after compaction). Subtracting live position-delete / + /// deletion-vector `record_count` from data-file totals is also unsafe (duplicates, stale + /// references, DV supersession of parquet position deletes). +>>>>>>> 4b7cecaa3cf (Merge pull request #2183 from Altinity/feature/antalya-26.6/iceberg-puffin-deletion-vectors-read-2) UInt64 result = 0; for (const auto & manifest_list_entry : actual_data_snapshot->manifest_list_entries) { auto manifest_file_ptr = getManifestFileEntriesHandle( +<<<<<<< HEAD object_storage, persistent_components, local_context, log, manifest_list_entry, actual_table_state_snapshot.schema_id); /// Live delete files make an exact metadata-only count impossible: @@ -1271,6 +1300,35 @@ std::optional IcebergMetadata::totalRows(ContextPtr local_context) const if (!manifest_rows.has_value()) return {}; result += *manifest_rows; +======= + object_storage, persistent_components, local_context, log, manifest_list_entry, actual_table_state_snapshot.schema_id, *secondary_storages); + + if (!manifest_file_ptr.getFilesWithoutDeleted(FileContentType::EQUALITY_DELETE).empty() + || !manifest_file_ptr.getFilesWithoutDeleted(FileContentType::POSITION_DELETE).empty()) + return {}; + /// nullopt means a negative / overflowing per-file `record_count`: fail closed to a + /// real scan instead of returning a wrong count. Do not use optional column + /// `value_counts` here — nested fields can report element counts larger than rows. + auto manifest_rows = manifest_file_ptr.getRowsCountInAllFilesExcludingDeleted(FileContentType::DATA); + if (!manifest_rows.has_value()) + return {}; + /// Per-manifest sums are capped at Int64::max; still guard the cross-manifest total. + if (common::addOverflow(result, static_cast(*manifest_rows), result)) + return {}; + } + + if (auto summary_total_rows = actual_data_snapshot->getTotalRows(); + summary_total_rows.has_value() && *summary_total_rows != result) + { + LOG_WARNING( + log, + "Iceberg snapshot summary of table {} claims {} total rows, but its manifest files describe {} rows. " + "The snapshot summary is inconsistent with the table data (possibly a corrupted commit in the table " + "history), using the row count from the manifest files", + persistent_components.table_location, + *summary_total_rows, + result); +>>>>>>> 4b7cecaa3cf (Merge pull request #2183 from Altinity/feature/antalya-26.6/iceberg-puffin-deletion-vectors-read-2) } const auto summary_total_rows = actual_data_snapshot->getTotalRows(); @@ -1285,7 +1343,7 @@ std::optional IcebergMetadata::totalRows(ContextPtr local_context) const result); ProfileEvents::increment(ProfileEvents::IcebergTrivialCountOptimizationApplied); - return result; + return static_cast(result); } std::optional IcebergMetadata::totalBytes(ContextPtr local_context) const @@ -1300,7 +1358,9 @@ std::optional IcebergMetadata::totalBytes(ContextPtr local_context) cons if (actual_data_snapshot->total_bytes.has_value()) return actual_data_snapshot->total_bytes; - Int64 result = 0; + /// Per-manifest sums are capped at Int64::max; still guard the cross-manifest total + /// (same fail-closed contract as `totalRows`). + UInt64 result = 0; for (const auto & manifest_list_entry : actual_data_snapshot->manifest_list_entries) { auto manifest_file_ptr = getManifestFileEntriesHandle( @@ -1309,10 +1369,11 @@ std::optional IcebergMetadata::totalBytes(ContextPtr local_context) cons if (!count.has_value()) return {}; - result += count.value(); + if (common::addOverflow(result, static_cast(*count), result)) + return {}; } - return result; + return static_cast(result); } ObjectIterator IcebergMetadata::iterate( diff --git a/src/Storages/ObjectStorage/DataLakes/Iceberg/ManifestFile.cpp b/src/Storages/ObjectStorage/DataLakes/Iceberg/ManifestFile.cpp index f289f3d80d40..bffd91213282 100644 --- a/src/Storages/ObjectStorage/DataLakes/Iceberg/ManifestFile.cpp +++ b/src/Storages/ObjectStorage/DataLakes/Iceberg/ManifestFile.cpp @@ -3,6 +3,7 @@ #if USE_AVRO #include +#include #include @@ -13,6 +14,7 @@ namespace DB::ErrorCodes { extern const int LOGICAL_ERROR; + extern const int ICEBERG_SPECIFICATION_VIOLATION; } namespace DB::Iceberg @@ -32,6 +34,69 @@ String FileContentTypeToString(FileContentType type) throw DB::Exception(DB::ErrorCodes::LOGICAL_ERROR, "Unsupported content type: {}", static_cast(type)); } +std::optional getRecordCountInAllFilesExcludingDeleted( + const std::vector & files) +{ + Int64 result = 0; + for (const auto & file : files) + { + const Int64 record_count = file->parsed_entry->record_count; + if (record_count < 0) + return std::nullopt; + + const UInt64 record_count_u = static_cast(record_count); + const UInt64 result_u = static_cast(result); + if (result_u > static_cast(std::numeric_limits::max()) - record_count_u) + return std::nullopt; + + result += record_count; + } + return result; +} + +std::optional getBytesSizeInAllDataFilesExcludingDeleted( + const std::vector & files) +{ + Int64 result = 0; + for (const auto & file : files) + { + std::optional file_bytes; + for (const auto & [column, column_info] : file->parsed_entry->columns_infos) + { + if (column_info.bytes_size.has_value()) + { + file_bytes = *column_info.bytes_size; + break; + } + } + + if (!file_bytes.has_value() || *file_bytes < 0) + return std::nullopt; + + const UInt64 file_bytes_u = static_cast(*file_bytes); + const UInt64 result_u = static_cast(result); + if (result_u > static_cast(std::numeric_limits::max()) - file_bytes_u) + return std::nullopt; + + result += *file_bytes; + } + return result; +} + +void requireDirectReferencedDataFileForPuffinDeletionVector( + bool set_from_referenced_data_file_field, + const std::optional & referenced_path, + const IcebergPathFromMetadata & manifest_file_path) +{ + if (!set_from_referenced_data_file_field || !referenced_path.has_value() || referenced_path->empty()) + { + throw DB::Exception( + DB::ErrorCodes::ICEBERG_SPECIFICATION_VIOLATION, + "Puffin deletion vector entry in manifest file '{}' is missing referenced_data_file", + manifest_file_path); + } +} + static std::strong_ordering operator<=>(const PartitionSpecsEntry & lhs, const PartitionSpecsEntry & rhs) { return std::tie(lhs.source_id, lhs.transform_name, lhs.partition_name) diff --git a/src/Storages/ObjectStorage/DataLakes/Iceberg/ManifestFile.h b/src/Storages/ObjectStorage/DataLakes/Iceberg/ManifestFile.h index 5b363f43e35e..96b4897818e6 100644 --- a/src/Storages/ObjectStorage/DataLakes/Iceberg/ManifestFile.h +++ b/src/Storages/ObjectStorage/DataLakes/Iceberg/ManifestFile.h @@ -15,6 +15,7 @@ #include #include +#include namespace DB::Iceberg { @@ -96,6 +97,17 @@ struct ParsedManifestFileEntry : boost::noncopyable Int64 record_count; Int64 file_size_in_bytes; + /// Iceberg v3 deletion vector metadata (position delete entries with puffin format) + std::optional content_offset; + std::optional content_size_in_bytes; + + bool isDeletionVector() const + { + return Poco::toLower(file_format) == "puffin" + && content_offset.has_value() + && content_size_in_bytes.has_value(); + } + ParsedManifestFileEntry( FileContentType content_type_, IcebergPathFromMetadata file_path_key_, @@ -113,7 +125,9 @@ struct ParsedManifestFileEntry : boost::noncopyable std::optional> equality_ids_, std::optional sort_order_id_, Int64 record_count_, - Int64 file_size_in_bytes_) + Int64 file_size_in_bytes_, + std::optional content_offset_ = std::nullopt, + std::optional content_size_in_bytes_ = std::nullopt) : content_type(content_type_) , file_path_key(std::move(file_path_key_)) , row_number(row_number_) @@ -131,6 +145,8 @@ struct ParsedManifestFileEntry : boost::noncopyable , sort_order_id(sort_order_id_) , record_count(record_count_) , file_size_in_bytes(file_size_in_bytes_) + , content_offset(content_offset_) + , content_size_in_bytes(content_size_in_bytes_) { } }; @@ -150,6 +166,25 @@ struct ProcessedManifestFileEntry using ProcessedManifestFileEntryPtr = std::shared_ptr; +/// Sum required per-file `record_count` over live manifest entries. +/// Returns nullopt if any entry has a negative `record_count` or the sum would overflow `Int64` +/// (fail closed — do not use optional column `value_counts`, which can disagree for nested fields). +std::optional getRecordCountInAllFilesExcludingDeleted( + const std::vector & files); + +/// Sum one declared column `bytes_size` per live data-file entry (first column that has it). +/// Returns nullopt if any file lacks a bytes size, has a negative size, or the sum would overflow +/// `Int64` (fail closed — same contract as `getRecordCountInAllFilesExcludingDeleted`). +std::optional getBytesSizeInAllDataFilesExcludingDeleted( + const std::vector & files); + +/// Puffin deletion vectors must identify the data file via the dedicated `referenced_data_file` +/// manifest field (non-empty). Position-delete lower/upper bounds must not be used as a fallback. +void requireDirectReferencedDataFileForPuffinDeletionVector( + bool set_from_referenced_data_file_field, + const std::optional & referenced_path, + const IcebergPathFromMetadata & manifest_file_path); + bool operator<(const PartitionSpecification & lhs, const PartitionSpecification & rhs); bool operator<(const DB::Row & lhs, const DB::Row & rhs); diff --git a/src/Storages/ObjectStorage/DataLakes/Iceberg/ManifestFileIterator.cpp b/src/Storages/ObjectStorage/DataLakes/Iceberg/ManifestFileIterator.cpp index 81fcde12362b..f50307841899 100644 --- a/src/Storages/ObjectStorage/DataLakes/Iceberg/ManifestFileIterator.cpp +++ b/src/Storages/ObjectStorage/DataLakes/Iceberg/ManifestFileIterator.cpp @@ -169,6 +169,7 @@ bool ManifestFileIterator::ManifestFileEntriesHandle::areAllDataFilesSortedBySor bool ManifestFileIterator::ManifestFileEntriesHandle::areAllDataFilesEligibleForLazyMaterialization(Int32 table_schema_id) const { +<<<<<<< HEAD /// Equality deletes force reading all physical columns of the data files they apply to /// (see IcebergMetadata::getInitialSchemaByPath), so the pruned main read is impossible. if (!equality_delete_files->empty()) @@ -206,29 +207,14 @@ std::optional ManifestFileIterator::ManifestFileEntriesHandle::getRowsCo result += static_cast(file->parsed_entry->record_count); } return result; +======= + return getRecordCountInAllFilesExcludingDeleted(getFilesWithoutDeleted(content)); +>>>>>>> 4b7cecaa3cf (Merge pull request #2183 from Altinity/feature/antalya-26.6/iceberg-puffin-deletion-vectors-read-2) } std::optional ManifestFileIterator::ManifestFileEntriesHandle::getBytesCountInAllDataFilesExcludingDeleted() const { - size_t result = 0; - for (const auto & file : getFilesWithoutDeleted(FileContentType::DATA)) - { - /// Have at least one column with bytes count - bool found = false; - for (const auto & [column, column_info] : file->parsed_entry->columns_infos) - { - if (column_info.bytes_size.has_value()) - { - result += *column_info.bytes_size; - found = true; - break; - } - } - - if (!found) - return std::nullopt; - } - return result; + return getBytesSizeInAllDataFilesExcludingDeleted(getFilesWithoutDeleted(FileContentType::DATA)); } ManifestFileIterator::ManifestFileEntriesHandle ManifestFileIterator::getFilesWithoutDeletedHandle() const @@ -633,28 +619,14 @@ bool ManifestFileIterator::areAllDataFilesSortedBySortOrderID(Int32 sort_order_i return true; } -std::optional ManifestFileIterator::getBytesCountInAllDataFilesExcludingDeleted() const +std::optional ManifestFileIterator::getRowsCountInAllFilesExcludingDeleted(FileContentType content) const { - Int64 result = 0; - auto handle = getFilesWithoutDeletedHandle(); - for (const auto & file : handle.getFilesWithoutDeleted(FileContentType::DATA)) - { - /// Have at least one column with bytes count - bool found = false; - for (const auto & [column, column_info] : file->parsed_entry->columns_infos) - { - if (column_info.bytes_size.has_value()) - { - result += *column_info.bytes_size; - found = true; - break; - } - } + return getFilesWithoutDeletedHandle().getRowsCountInAllFilesExcludingDeleted(content); +} - if (!found) - return std::nullopt; - } - return result; +std::optional ManifestFileIterator::getBytesCountInAllDataFilesExcludingDeleted() const +{ + return getFilesWithoutDeletedHandle().getBytesCountInAllDataFilesExcludingDeleted(); } } diff --git a/src/Storages/ObjectStorage/DataLakes/Iceberg/ManifestFileIterator.h b/src/Storages/ObjectStorage/DataLakes/Iceberg/ManifestFileIterator.h index 71e5ac320e83..229ef01cc888 100644 --- a/src/Storages/ObjectStorage/DataLakes/Iceberg/ManifestFileIterator.h +++ b/src/Storages/ObjectStorage/DataLakes/Iceberg/ManifestFileIterator.h @@ -98,8 +98,8 @@ class ManifestFileIterator : public boost::noncopyable bool hasPartitionKey() const; const DB::KeyDescription & getPartitionKeyDescription() const; - /// Fields with rows count in manifest files are optional - /// they can be absent. + /// Sums required per-file `record_count` for live entries of the given content type. + /// Returns nullopt on negative `record_count` or Int64 overflow (fail closed). std::optional getRowsCountInAllFilesExcludingDeleted(FileContentType content) const; std::optional getBytesCountInAllDataFilesExcludingDeleted() const; diff --git a/src/Storages/ObjectStorage/DataLakes/Iceberg/Mutations.cpp b/src/Storages/ObjectStorage/DataLakes/Iceberg/Mutations.cpp index 5eea075634b1..b224c27e11f3 100644 --- a/src/Storages/ObjectStorage/DataLakes/Iceberg/Mutations.cpp +++ b/src/Storages/ObjectStorage/DataLakes/Iceberg/Mutations.cpp @@ -45,7 +45,12 @@ namespace DB::ErrorCodes extern const int BAD_ARGUMENTS; extern const int LOGICAL_ERROR; extern const int LIMIT_EXCEEDED; +<<<<<<< HEAD extern const int NOT_IMPLEMENTED; +======= +extern const int SUPPORT_IS_DISABLED; +extern const int QUERY_WAS_CANCELLED; +>>>>>>> 4b7cecaa3cf (Merge pull request #2183 from Altinity/feature/antalya-26.6/iceberg-puffin-deletion-vectors-read-2) } namespace DB::DataLakeStorageSetting @@ -685,8 +690,20 @@ void mutate( auto metadata = getMetadataJSONObject(metadata_path, object_storage, persistent_table_components.metadata_cache, context, log, compression_method, persistent_table_components.table_uuid); - if (metadata->getValue(f_format_version) < 2) - throw Exception(ErrorCodes::BAD_ARGUMENTS, "Mutations are supported only for the second version of iceberg format"); + /// Iceberg v3 writers must not add new position-delete files; row-level deletes require + /// deletion vectors. Fail closed before any object writes until ClickHouse can write DVs. + const Int32 format_version = metadata->getValue(f_format_version); + if (format_version < 2) + throw Exception(ErrorCodes::BAD_ARGUMENTS, "Mutations are supported only for Iceberg format version 2"); + if (format_version >= 3) + { + throw Exception( + ErrorCodes::SUPPORT_IS_DISABLED, + "Iceberg DELETE and UPDATE mutations are not supported for format version {}. " + "ClickHouse writes parquet position-delete files, which Iceberg v3+ writers must not add; " + "writing deletion vectors is not implemented yet", + format_version); + } auto partition_spec_id = metadata->getValue(Iceberg::f_default_spec_id); auto partitions_specs = metadata->getArray(Iceberg::f_partition_specs); Poco::JSON::Object::Ptr partititon_spec; diff --git a/src/Storages/ObjectStorage/DataLakes/Iceberg/Snapshot.h b/src/Storages/ObjectStorage/DataLakes/Iceberg/Snapshot.h index e828fe38d92d..c204a8215bf6 100644 --- a/src/Storages/ObjectStorage/DataLakes/Iceberg/Snapshot.h +++ b/src/Storages/ObjectStorage/DataLakes/Iceberg/Snapshot.h @@ -16,6 +16,7 @@ struct IcebergDataSnapshot DB::ManifestFileCacheKeys manifest_list_entries; Int64 snapshot_id; Int64 schema_id_on_snapshot_commit; +<<<<<<< HEAD /// Row-count hint from the snapshot summary (`total-records`). Only used to log a /// warning when it disagrees with the row count derived from the manifest files; never /// used as a data source, because the summary is maintained incrementally by writers @@ -23,12 +24,28 @@ struct IcebergDataSnapshot std::optional total_rows; std::optional total_bytes; std::optional total_position_delete_rows; +======= + /// From snapshot summary (`total-records`). Compared to the manifest-derived count for a + /// mismatch warning only — never used as the trivial COUNT answer. Summary totals are + /// maintained incrementally by writers and can be poisoned by a bad commit in table history. + std::optional total_rows; + std::optional total_bytes; + std::optional total_position_delete_rows; + /// Rows in equality-delete files (snapshot summary). Not a count of deleted data rows; + /// used only to fail closed early when present and > 0. + std::optional total_equality_delete_rows; + std::optional partition_key; + std::optional sorting_key; +>>>>>>> 4b7cecaa3cf (Merge pull request #2183 from Altinity/feature/antalya-26.6/iceberg-puffin-deletion-vectors-read-2) std::optional getTotalRows() const { - if (total_rows.has_value() && total_position_delete_rows.has_value()) - return *total_rows - *total_position_delete_rows; - return std::nullopt; + if (!total_rows.has_value() || !total_position_delete_rows.has_value()) + return std::nullopt; + /// Fail closed on inconsistent summary: unsigned subtract would wrap to a huge COUNT. + if (*total_position_delete_rows > *total_rows) + return std::nullopt; + return *total_rows - *total_position_delete_rows; } }; diff --git a/src/Storages/ObjectStorage/DataLakes/Iceberg/tests/gtest_iceberg_count_shortcuts.cpp b/src/Storages/ObjectStorage/DataLakes/Iceberg/tests/gtest_iceberg_count_shortcuts.cpp new file mode 100644 index 000000000000..1c1a894e8018 --- /dev/null +++ b/src/Storages/ObjectStorage/DataLakes/Iceberg/tests/gtest_iceberg_count_shortcuts.cpp @@ -0,0 +1,249 @@ +#include + +#include "config.h" + +#if USE_AVRO + +#include +#include +#include +#include + +#include + +using namespace DB; +using namespace DB::Iceberg; + +TEST(IcebergCountShortcuts, HasEqualityAndPositionDeleteHelpers) +{ + Iceberg::IcebergObjectSerializableInfo info; + info.data_object_file_path_key = Iceberg::IcebergPathFromMetadata::deserialize("s3://bucket/data/file.parquet"); + info.file_format = "PARQUET"; + + auto plain = std::make_shared(RelativePathWithMetadata{"data/file.parquet"}); + EXPECT_FALSE(hasIcebergEqualityDeletes(plain)); + EXPECT_FALSE(hasIcebergPositionDeletes(plain)); + + auto iceberg = std::make_shared(RelativePathWithMetadata{"data/file.parquet"}, info); + EXPECT_FALSE(hasIcebergEqualityDeletes(iceberg)); + EXPECT_FALSE(hasIcebergPositionDeletes(iceberg)); + + iceberg->info.equality_deletes_objects.push_back( + Iceberg::EqualityDeleteObject{ + .file_path = "s3://bucket/deletes/eq.parquet", + .file_format = "PARQUET", + .equality_ids = std::vector{1}, + .schema_id = 0, + }); + EXPECT_TRUE(hasIcebergEqualityDeletes(iceberg)); + EXPECT_FALSE(hasIcebergPositionDeletes(iceberg)); + + iceberg->info.position_deletes_objects.push_back( + Iceberg::PositionDeleteObject{ + .file_path = "s3://bucket/deletes/pos.parquet", + .file_format = "PARQUET", + .reference_data_file_path = std::nullopt, + .sequence_number = 1, + }); + EXPECT_TRUE(hasIcebergEqualityDeletes(iceberg)); + EXPECT_TRUE(hasIcebergPositionDeletes(iceberg)); +} + +TEST(IcebergCountShortcuts, GetTotalRowsRequiresBothSummaryFields) +{ + Iceberg::IcebergDataSnapshot snapshot; + snapshot.total_rows = 100; + snapshot.total_position_delete_rows = 0; + snapshot.total_equality_delete_rows = 0; + + ASSERT_TRUE(snapshot.getTotalRows().has_value()); + EXPECT_EQ(*snapshot.getTotalRows(), 100u); + + snapshot.total_position_delete_rows = 10; + ASSERT_TRUE(snapshot.getTotalRows().has_value()); + EXPECT_EQ(*snapshot.getTotalRows(), 90u); + + snapshot.total_position_delete_rows = std::nullopt; + EXPECT_FALSE(snapshot.getTotalRows().has_value()); + + snapshot.total_position_delete_rows = 0; + snapshot.total_rows = std::nullopt; + EXPECT_FALSE(snapshot.getTotalRows().has_value()); +} + +/// Snapshot-summary arithmetic can disagree with manifests (poisoned incremental totals, +/// stale total-position-deletes after rewrite). getTotalRows is only for mismatch warnings; +/// IcebergMetadata::totalRows must prefer the manifest record_count sum. +TEST(IcebergCountShortcuts, SummaryArithmeticCanDisagreeWithManifestTruth) +{ + Iceberg::IcebergDataSnapshot snapshot; + snapshot.total_rows = 999999; + snapshot.total_position_delete_rows = 0; + snapshot.total_equality_delete_rows = 0; + + ASSERT_TRUE(snapshot.getTotalRows().has_value()); + EXPECT_EQ(*snapshot.getTotalRows(), 999999u); + + snapshot.total_rows = 90; + snapshot.total_position_delete_rows = 10; + ASSERT_TRUE(snapshot.getTotalRows().has_value()); + EXPECT_EQ(*snapshot.getTotalRows(), 80u); +} + +/// Live deletion vectors: summary subtraction may look plausible while DVs are live. +/// totalRows must open manifests, see POSITION_DELETE files, and fail closed to a scan. +TEST(IcebergCountShortcuts, LiveDeletionVectorSummaryLooksPlausible) +{ + Iceberg::IcebergDataSnapshot snapshot; + snapshot.total_rows = 100; + snapshot.total_position_delete_rows = 10; + snapshot.total_equality_delete_rows = 0; + + ASSERT_TRUE(snapshot.getTotalRows().has_value()); + EXPECT_EQ(*snapshot.getTotalRows(), 90u); +} + +TEST(IcebergCountShortcuts, GetTotalRowsFailsClosedWhenPositionDeletesExceedRows) +{ + Iceberg::IcebergDataSnapshot snapshot; + snapshot.total_rows = 5; + snapshot.total_position_delete_rows = 6; + EXPECT_FALSE(snapshot.getTotalRows().has_value()); +} + +namespace +{ + +ProcessedManifestFileEntryPtr makeDataEntryForRecordCount( + Int64 record_count, + std::unordered_map columns_infos = {}) +{ + auto parsed = std::make_shared( + FileContentType::DATA, + IcebergPathFromMetadata::deserialize("s3://bucket/data/file.parquet"), + /*row_number=*/0, + ManifestEntryStatus::ADDED, + /*written_sequence_number=*/std::nullopt, + /*written_snapshot_id=*/std::nullopt, + DB::Row{}, + std::move(columns_infos), + std::unordered_map>{}, + /*file_format=*/"PARQUET", + /*lower_reference_data_file_path=*/std::nullopt, + /*upper_reference_data_file_path=*/std::nullopt, + /*equality_ids=*/std::nullopt, + /*sort_order_id=*/std::nullopt, + record_count, + /*file_size_in_bytes=*/100); + + auto processed = std::make_shared(); + processed->parsed_entry = std::move(parsed); + processed->common_partition_specification = std::make_shared(); + processed->sequence_number = 0; + processed->resolved_schema_id = 0; + processed->manifest_file_path = "s3://bucket/metadata/manifest.avro"; + return processed; +} + +} + +TEST(IcebergRecordCountAggregate, SumsRecordCountIgnoringValueCounts) +{ + ColumnInfo nested_list_stats; + nested_list_stats.rows_count = 1000; /// nested element count, not row count + + const auto total = getRecordCountInAllFilesExcludingDeleted({ + makeDataEntryForRecordCount(/*record_count=*/10, {{/*column_id=*/2, nested_list_stats}}), + makeDataEntryForRecordCount(/*record_count=*/5), + }); + + ASSERT_TRUE(total.has_value()); + EXPECT_EQ(*total, 15); +} + +TEST(IcebergRecordCountAggregate, SucceedsWithoutValueCounts) +{ + const auto total = getRecordCountInAllFilesExcludingDeleted({ + makeDataEntryForRecordCount(/*record_count=*/42), + }); + + ASSERT_TRUE(total.has_value()); + EXPECT_EQ(*total, 42); +} + +TEST(IcebergRecordCountAggregate, EmptyManifestIsZero) +{ + const auto total = getRecordCountInAllFilesExcludingDeleted({}); + ASSERT_TRUE(total.has_value()); + EXPECT_EQ(*total, 0); +} + +TEST(IcebergRecordCountAggregate, NegativeRecordCountFailsClosed) +{ + const auto total = getRecordCountInAllFilesExcludingDeleted({ + makeDataEntryForRecordCount(/*record_count=*/10), + makeDataEntryForRecordCount(/*record_count=*/-1), + }); + EXPECT_FALSE(total.has_value()); +} + +TEST(IcebergRecordCountAggregate, OverflowFailsClosed) +{ + const auto total = getRecordCountInAllFilesExcludingDeleted({ + makeDataEntryForRecordCount(/*record_count=*/std::numeric_limits::max()), + makeDataEntryForRecordCount(/*record_count=*/1), + }); + EXPECT_FALSE(total.has_value()); +} + +namespace +{ + +ColumnInfo columnBytes(Int64 bytes) +{ + ColumnInfo info; + info.bytes_size = bytes; + return info; +} + +} + +TEST(IcebergBytesSizeAggregate, SumsFirstDeclaredColumnBytesSize) +{ + const auto total = getBytesSizeInAllDataFilesExcludingDeleted({ + makeDataEntryForRecordCount(/*record_count=*/1, {{/*column_id=*/1, columnBytes(10)}}), + makeDataEntryForRecordCount(/*record_count=*/1, {{/*column_id=*/1, columnBytes(5)}}), + }); + + ASSERT_TRUE(total.has_value()); + EXPECT_EQ(*total, 15); +} + +TEST(IcebergBytesSizeAggregate, MissingBytesSizeFailsClosed) +{ + const auto total = getBytesSizeInAllDataFilesExcludingDeleted({ + makeDataEntryForRecordCount(/*record_count=*/1, {{/*column_id=*/1, columnBytes(10)}}), + makeDataEntryForRecordCount(/*record_count=*/1), + }); + EXPECT_FALSE(total.has_value()); +} + +TEST(IcebergBytesSizeAggregate, NegativeBytesSizeFailsClosed) +{ + const auto total = getBytesSizeInAllDataFilesExcludingDeleted({ + makeDataEntryForRecordCount(/*record_count=*/1, {{/*column_id=*/1, columnBytes(-1)}}), + }); + EXPECT_FALSE(total.has_value()); +} + +TEST(IcebergBytesSizeAggregate, OverflowFailsClosed) +{ + const auto total = getBytesSizeInAllDataFilesExcludingDeleted({ + makeDataEntryForRecordCount( + /*record_count=*/1, {{/*column_id=*/1, columnBytes(std::numeric_limits::max())}}), + makeDataEntryForRecordCount(/*record_count=*/1, {{/*column_id=*/1, columnBytes(1)}}), + }); + EXPECT_FALSE(total.has_value()); +} + +#endif diff --git a/src/Storages/ObjectStorage/DataLakes/Iceberg/tests/gtest_iceberg_data_object_info_clone.cpp b/src/Storages/ObjectStorage/DataLakes/Iceberg/tests/gtest_iceberg_data_object_info_clone.cpp new file mode 100644 index 000000000000..b863ee605446 --- /dev/null +++ b/src/Storages/ObjectStorage/DataLakes/Iceberg/tests/gtest_iceberg_data_object_info_clone.cpp @@ -0,0 +1,64 @@ +#include + +#include "config.h" + +#if USE_AVRO + +#include +#include +#include +#include + +using namespace DB; + +TEST(IcebergDataObjectInfoClone, PreservesEqualityAndPositionDeletes) +{ + Iceberg::IcebergObjectSerializableInfo info; + info.data_object_file_path_key = Iceberg::IcebergPathFromMetadata::deserialize("s3://bucket/data/file.parquet"); + info.file_format = "PARQUET"; + info.equality_deletes_objects.push_back( + Iceberg::EqualityDeleteObject{ + .file_path = "s3://bucket/deletes/eq.parquet", + .file_format = "PARQUET", + .equality_ids = std::vector{1, 2}, + .schema_id = 7, + }); + info.position_deletes_objects.push_back( + Iceberg::PositionDeleteObject{ + .file_path = "s3://bucket/deletes/pos.parquet", + .file_format = "PARQUET", + .reference_data_file_path = std::nullopt, + .sequence_number = 42, + }); + + auto original = std::make_shared( + RelativePathWithMetadata{"data/file.parquet"}, info); + original->data_lake_metadata.emplace(); + original->data_lake_metadata->excluded_rows = std::make_shared(); + original->data_lake_metadata->excluded_rows->add(11); + + /// Object-slicing (the old ObjectIteratorSplitByBuckets path) drops Iceberg metadata. + ObjectInfo sliced_value = *original; + auto sliced = std::make_shared(sliced_value); + EXPECT_FALSE(std::dynamic_pointer_cast(sliced)); + ASSERT_TRUE(sliced->data_lake_metadata.has_value()); + ASSERT_TRUE(sliced->data_lake_metadata->excluded_rows); + EXPECT_EQ(sliced->data_lake_metadata->excluded_rows->size(), 1u); + + auto cloned = original->clone(); + auto iceberg_cloned = std::dynamic_pointer_cast(cloned); + ASSERT_TRUE(iceberg_cloned); + ASSERT_EQ(iceberg_cloned->info.equality_deletes_objects.size(), 1u); + EXPECT_EQ(iceberg_cloned->info.equality_deletes_objects[0].file_path, "s3://bucket/deletes/eq.parquet"); + ASSERT_TRUE(iceberg_cloned->info.equality_deletes_objects[0].equality_ids.has_value()); + EXPECT_EQ(iceberg_cloned->info.equality_deletes_objects[0].equality_ids->size(), 2u); + ASSERT_EQ(iceberg_cloned->info.position_deletes_objects.size(), 1u); + EXPECT_EQ(iceberg_cloned->info.position_deletes_objects[0].file_path, "s3://bucket/deletes/pos.parquet"); + EXPECT_EQ(iceberg_cloned->info.position_deletes_objects[0].sequence_number, 42); + ASSERT_TRUE(iceberg_cloned->data_lake_metadata.has_value()); + ASSERT_TRUE(iceberg_cloned->data_lake_metadata->excluded_rows); + EXPECT_EQ(iceberg_cloned->data_lake_metadata->excluded_rows->size(), 1u); + EXPECT_TRUE(iceberg_cloned->data_lake_metadata->excluded_rows->rb_contains(11)); +} + +#endif diff --git a/src/Storages/ObjectStorage/DataLakes/Iceberg/tests/gtest_iceberg_deletion_vector_positions.cpp b/src/Storages/ObjectStorage/DataLakes/Iceberg/tests/gtest_iceberg_deletion_vector_positions.cpp new file mode 100644 index 000000000000..fa3442c4fad7 --- /dev/null +++ b/src/Storages/ObjectStorage/DataLakes/Iceberg/tests/gtest_iceberg_deletion_vector_positions.cpp @@ -0,0 +1,90 @@ +#include + +#include "config.h" + +#if USE_AVRO + +#include +#include + +#include + +using namespace DB; +using namespace DB::Iceberg; + +namespace DB +{ +namespace ErrorCodes +{ +extern const int ICEBERG_SPECIFICATION_VIOLATION; +} +} + +TEST(IcebergDeletionVectorPositions, AcceptsBoundaryPosition) +{ + /// Valid file-local positions are in [0, record_count). + const std::vector positions = {0, 9}; + EXPECT_NO_THROW(validateDeletionVectorPositionsAgainstDataFile(positions, /*expected_cardinality=*/2, /*data_file_record_count=*/10)); +} + +TEST(IcebergDeletionVectorPositions, RejectsPositionEqualToRecordCount) +{ + const std::vector positions = {10}; + try + { + validateDeletionVectorPositionsAgainstDataFile(positions, /*expected_cardinality=*/1, /*data_file_record_count=*/10); + FAIL() << "Expected exception"; + } + catch (const Exception & e) + { + EXPECT_EQ(e.code(), ErrorCodes::ICEBERG_SPECIFICATION_VIOLATION); + EXPECT_NE(e.message().find("out of range"), std::string::npos); + } +} + +TEST(IcebergDeletionVectorPositions, RejectsPositionAboveRecordCount) +{ + const std::vector positions = {11}; + try + { + validateDeletionVectorPositionsAgainstDataFile(positions, /*expected_cardinality=*/1, /*data_file_record_count=*/10); + FAIL() << "Expected exception"; + } + catch (const Exception & e) + { + EXPECT_EQ(e.code(), ErrorCodes::ICEBERG_SPECIFICATION_VIOLATION); + EXPECT_NE(e.message().find("out of range"), std::string::npos); + } +} + +TEST(IcebergDeletionVectorPositions, RejectsCardinalityExceedingRecordCount) +{ + const std::vector positions; + try + { + validateDeletionVectorPositionsAgainstDataFile(positions, /*expected_cardinality=*/11, /*data_file_record_count=*/10); + FAIL() << "Expected exception"; + } + catch (const Exception & e) + { + EXPECT_EQ(e.code(), ErrorCodes::ICEBERG_SPECIFICATION_VIOLATION); + EXPECT_NE(e.message().find("exceeds data file record_count"), std::string::npos); + } +} + +TEST(IcebergDeletionVectorPositions, RejectsNegativeDataFileRecordCount) +{ + const std::vector positions; + try + { + validateDeletionVectorPositionsAgainstDataFile(positions, /*expected_cardinality=*/0, /*data_file_record_count=*/-1); + FAIL() << "Expected exception"; + } + catch (const Exception & e) + { + EXPECT_EQ(e.code(), ErrorCodes::ICEBERG_SPECIFICATION_VIOLATION); + EXPECT_NE(e.message().find("non-negative"), std::string::npos); + } +} + +#endif diff --git a/src/Storages/ObjectStorage/DataLakes/Iceberg/tests/gtest_iceberg_parquet_row_deletes_guard.cpp b/src/Storages/ObjectStorage/DataLakes/Iceberg/tests/gtest_iceberg_parquet_row_deletes_guard.cpp new file mode 100644 index 000000000000..ce5262728d1b --- /dev/null +++ b/src/Storages/ObjectStorage/DataLakes/Iceberg/tests/gtest_iceberg_parquet_row_deletes_guard.cpp @@ -0,0 +1,47 @@ +#include + +#include +#include + +using namespace DB; + +namespace DB +{ +namespace ErrorCodes +{ +extern const int NOT_IMPLEMENTED; +} +} + +TEST(IcebergParquetRowDeletesGuard, AcceptsParquet) +{ + EXPECT_NO_THROW(Iceberg::requireParquetDataFileForRowDeletes("parquet", "Deletion vectors")); + EXPECT_NO_THROW(Iceberg::requireParquetDataFileForRowDeletes("PARQUET", "Position deletes")); + EXPECT_NO_THROW(Iceberg::requireParquetDataFileForRowDeletes("Parquet", "Deletion vectors")); +} + +TEST(IcebergParquetRowDeletesGuard, RejectsNonParquet) +{ + try + { + Iceberg::requireParquetDataFileForRowDeletes("ORC", "Deletion vectors"); + FAIL() << "Expected exception"; + } + catch (const Exception & e) + { + EXPECT_EQ(e.code(), ErrorCodes::NOT_IMPLEMENTED); + EXPECT_NE(e.message().find("Deletion vectors are only supported"), std::string::npos); + EXPECT_NE(e.message().find("ORC"), std::string::npos); + } + + try + { + Iceberg::requireParquetDataFileForRowDeletes("AVRO", "Position deletes"); + FAIL() << "Expected exception"; + } + catch (const Exception & e) + { + EXPECT_EQ(e.code(), ErrorCodes::NOT_IMPLEMENTED); + EXPECT_NE(e.message().find("Position deletes are only supported"), std::string::npos); + } +} diff --git a/src/Storages/ObjectStorage/DataLakes/PuffinDeletionVectorReader.cpp b/src/Storages/ObjectStorage/DataLakes/PuffinDeletionVectorReader.cpp new file mode 100644 index 000000000000..b91a87fc6269 --- /dev/null +++ b/src/Storages/ObjectStorage/DataLakes/PuffinDeletionVectorReader.cpp @@ -0,0 +1,297 @@ +#include +#include + +#include +#include +#include +#include +#include +#include +#include + +#include +#include + +namespace ProfileEvents +{ +extern const Event PuffinFilesRead; +extern const Event PuffinFileReadMicroseconds; +} + +namespace DB +{ + +namespace ErrorCodes +{ + extern const int BAD_ARGUMENTS; +} + +void validateDeletionVectorV1Fields(const std::vector & fields, size_t blob_index) +{ + if (fields.empty()) + return; + + /// Spark / Iceberg file-scoped DVs use the reserved `_pos` id as a singleton marker. + if (fields.size() == 1 && fields[0] == ICEBERG_ROW_POSITION_FIELD_ID) + return; + + throw Exception( + ErrorCodes::BAD_ARGUMENTS, + "Puffin blob {}: deletion-vector-v1 has unsupported non-empty 'fields' " + "(only [] or [{}] / Iceberg _pos are accepted; column-scoped DVs are not supported)", + blob_index, + ICEBERG_ROW_POSITION_FIELD_ID); +} + +namespace +{ + +struct ScopedPuffinFileReadProfileEvent +{ + ProfileEventTimeIncrement watch; + + ScopedPuffinFileReadProfileEvent() + : watch(ProfileEvents::PuffinFileReadMicroseconds) + { + ProfileEvents::increment(ProfileEvents::PuffinFilesRead); + } +}; + +UInt32 readBigEndianUInt32(const UInt8 * data) +{ + return (static_cast(data[0]) << 24) + | (static_cast(data[1]) << 16) + | (static_cast(data[2]) << 8) + | static_cast(data[3]); +} + +} + +void validatePuffinBlobBounds(Int64 offset, Int64 length, size_t file_size, std::string_view context) +{ + if (offset < 0 || length < 0) + throw Exception(ErrorCodes::BAD_ARGUMENTS, "{}: offset/length out of bounds", context); + + if (offset > static_cast(file_size) || length > static_cast(file_size)) + throw Exception(ErrorCodes::BAD_ARGUMENTS, "{}: offset/length out of bounds", context); + + Int64 end = 0; + if (common::addOverflow(offset, length, end)) + throw Exception(ErrorCodes::BAD_ARGUMENTS, "{}: offset/length out of bounds", context); + + if (static_cast(end) > file_size) + throw Exception(ErrorCodes::BAD_ARGUMENTS, "{}: offset/length out of bounds", context); +} + +void checkDeletionVectorBlobReadLimits(Int64 length, std::optional expected_cardinality) +{ + /// Same fail-closed order as the SQL `Puffin` path: cardinality before blob length / allocate. + if (expected_cardinality.has_value() && *expected_cardinality > PUFFIN_DV_MAX_MATERIALIZED_POSITIONS) + throw Exception( + ErrorCodes::BAD_ARGUMENTS, + "Deletion vector cardinality {} exceeds materialization limit {}", + *expected_cardinality, + PUFFIN_DV_MAX_MATERIALIZED_POSITIONS); + + if (length < 0) + throw Exception(ErrorCodes::BAD_ARGUMENTS, "Deletion vector blob length is negative"); + + if (static_cast(length) > PUFFIN_DV_MAX_BLOB_SIZE) + throw Exception( + ErrorCodes::BAD_ARGUMENTS, + "Deletion vector blob length {} exceeds absolute limit {}", + length, + PUFFIN_DV_MAX_BLOB_SIZE); + + if (static_cast(length) < 12) + throw Exception(ErrorCodes::BAD_ARGUMENTS, "Deletion vector blob is too small"); +} + +void validateDeletionVectorEnvelope(const UInt8 * header, Int64 length) +{ + const UInt32 combined_length = readBigEndianUInt32(header); + if (std::memcmp(header + sizeof(UInt32), DELETION_VECTOR_MAGIC, sizeof(DELETION_VECTOR_MAGIC)) != 0) + throw Exception(ErrorCodes::BAD_ARGUMENTS, "Invalid deletion vector magic"); + + if (combined_length < sizeof(DELETION_VECTOR_MAGIC)) + throw Exception(ErrorCodes::BAD_ARGUMENTS, "Invalid deletion vector combined length: {}", combined_length); + + UInt64 expected_blob_size = 0; + if (common::addOverflow(static_cast(combined_length), UInt64{8}, expected_blob_size)) + throw Exception(ErrorCodes::BAD_ARGUMENTS, "Invalid deletion vector combined length: {}", combined_length); + + if (static_cast(length) != expected_blob_size) + throw Exception( + ErrorCodes::BAD_ARGUMENTS, + "Deletion vector blob size {} does not match combined length {}", + length, + combined_length); +} + +std::vector deserializeDeletionVectorV1Blob(std::string_view blob_bytes, std::optional expected_cardinality) +{ + if (expected_cardinality.has_value() && *expected_cardinality > PUFFIN_DV_MAX_MATERIALIZED_POSITIONS) + throw Exception( + ErrorCodes::BAD_ARGUMENTS, + "Deletion vector cardinality {} exceeds materialization limit {}", + *expected_cardinality, + PUFFIN_DV_MAX_MATERIALIZED_POSITIONS); + + return deserializeRoaringPositionBitmap(extractDeletionVectorPayload(blob_bytes), expected_cardinality); +} + +std::vector readDeletionVectorFromPuffin(ReadBuffer & file, Int64 offset, Int64 length, std::optional expected_cardinality) +{ + ScopedPuffinFileReadProfileEvent profile_event; + + checkDeletionVectorBlobReadLimits(length, expected_cardinality); + + if (auto file_size = tryGetFileSizeFromReadBuffer(file)) + validatePuffinBlobBounds(offset, length, *file_size); + else if (offset < 0) + throw Exception(ErrorCodes::BAD_ARGUMENTS, "Invalid Puffin deletion vector offset {} or length {}", offset, length); + + auto * seekable = dynamic_cast(&file); + if (!seekable) + throw Exception(ErrorCodes::BAD_ARGUMENTS, "Puffin deletion vector read requires a seekable buffer"); + + /// Peek combined_length + magic before allocating `length` (up to 2 GiB). Matches + /// `readDeletionVectorBlobBytes` in the SQL Puffin format path. + seekable->seek(offset, SEEK_SET); + + UInt8 header[8]; + file.readStrict(reinterpret_cast(header), sizeof(header)); + validateDeletionVectorEnvelope(header, length); + + String blob_data(static_cast(length), '\0'); + std::memcpy(blob_data.data(), header, sizeof(header)); + file.readStrict(blob_data.data() + sizeof(header), blob_data.size() - sizeof(header)); + + return deserializeDeletionVectorV1Blob(blob_data, expected_cardinality); +} + +void appendReadBufferWithAbsoluteSizeLimit(ReadBuffer & buf, std::vector & out, size_t max_buffered_size) +{ + if (out.size() > max_buffered_size) + { + throw Exception( + ErrorCodes::BAD_ARGUMENTS, + "Puffin non-seekable buffer size {} exceeds absolute limit {}", + out.size(), + max_buffered_size); + } + + std::vector tmp(DBMS_DEFAULT_BUFFER_SIZE); + while (!buf.eof()) + { + const size_t capacity = max_buffered_size - out.size(); + if (capacity == 0) + { + throw Exception( + ErrorCodes::BAD_ARGUMENTS, + "Puffin non-seekable input exceeds absolute buffer limit {} bytes; use seekable input for larger files", + max_buffered_size); + } + + const size_t to_read = std::min(tmp.size(), capacity); + const size_t n = buf.read(reinterpret_cast(tmp.data()), to_read); + if (n == 0) + break; + + out.insert(out.end(), tmp.data(), tmp.data() + n); + + /// If we filled the remaining capacity and the stream still has data, fail closed. + if (n == capacity && !buf.eof()) + { + throw Exception( + ErrorCodes::BAD_ARGUMENTS, + "Puffin non-seekable input exceeds absolute buffer limit {} bytes; use seekable input for larger files", + max_buffered_size); + } + } +} + +const PuffinBlob & bindDeletionVectorBlob( + const std::vector & blobs, + Int64 content_offset, + Int64 content_size_in_bytes, + std::string_view expected_referenced_data_file, + UInt64 expected_cardinality) +{ + const PuffinBlob * matched = nullptr; + size_t matched_index = 0; + + for (size_t i = 0; i < blobs.size(); ++i) + { + if (blobs[i].offset != content_offset || blobs[i].length != content_size_in_bytes) + continue; + + if (matched) + { + throw Exception( + ErrorCodes::BAD_ARGUMENTS, + "Multiple Puffin blobs claim offset {} length {}", + content_offset, + content_size_in_bytes); + } + + matched = &blobs[i]; + matched_index = i; + } + + if (!matched) + { + throw Exception( + ErrorCodes::BAD_ARGUMENTS, + "No Puffin footer blob at offset {} length {}", + content_offset, + content_size_in_bytes); + } + + if (matched->type != "deletion-vector-v1") + { + throw Exception( + ErrorCodes::BAD_ARGUMENTS, + "Puffin blob {} at offset {} length {} has type '{}', expected deletion-vector-v1", + matched_index, + content_offset, + content_size_in_bytes, + matched->type); + } + + if (!matched->compression_codec.empty()) + { + throw Exception( + ErrorCodes::BAD_ARGUMENTS, + "Puffin blob {}: deletion-vector-v1 must omit compression-codec", + matched_index); + } + + /// Structural DV footer checks live in the pre-existing SQL `Puffin` helper. + const UInt64 footer_cardinality = requireDeletionVectorV1Properties(*matched, matched_index); + + const auto & referenced_data_file = matched->properties.at("referenced-data-file"); + if (referenced_data_file != expected_referenced_data_file) + { + throw Exception( + ErrorCodes::BAD_ARGUMENTS, + "Puffin blob {} referenced-data-file '{}' does not match expected data file '{}'", + matched_index, + referenced_data_file, + expected_referenced_data_file); + } + + if (footer_cardinality != expected_cardinality) + { + throw Exception( + ErrorCodes::BAD_ARGUMENTS, + "Puffin blob {} cardinality {} does not match expected cardinality {}", + matched_index, + footer_cardinality, + expected_cardinality); + } + + return *matched; +} + +} diff --git a/src/Storages/ObjectStorage/DataLakes/PuffinDeletionVectorReader.h b/src/Storages/ObjectStorage/DataLakes/PuffinDeletionVectorReader.h new file mode 100644 index 000000000000..b78f57041d02 --- /dev/null +++ b/src/Storages/ObjectStorage/DataLakes/PuffinDeletionVectorReader.h @@ -0,0 +1,80 @@ +#pragma once + +#include +#include +#include + +#include +#include +#include +#include + +namespace DB +{ + +/// Absolute cap on on-disk deletion-vector-v1 blob length before reading bytes into memory. +/// Aligns with Iceberg DeleteLoader's 2 GiB content-size check. +constexpr size_t PUFFIN_DV_MAX_BLOB_SIZE = 2ULL * 1024 * 1024 * 1024; +/// Absolute cap on materialized deleted positions (~800 MiB of UInt64s at this limit). +constexpr UInt64 PUFFIN_DV_MAX_MATERIALIZED_POSITIONS = 100'000'000; + +/// Leading / footer-open magic is always 4 bytes (`PFA1`). +constexpr size_t PUFFIN_MAGIC_SIZE = 4; +constexpr size_t PUFFIN_FOOTER_TRAILER_SIZE = 12; +/// Absolute cap on footer payload size (uncompressed JSON bytes, or declared LZ4 contentSize). +constexpr size_t PUFFIN_FOOTER_MAX_PAYLOAD_SIZE = 16 * 1024 * 1024; + +/// Non-seekable SQL path must buffer the whole file to reach the trailer. Cap total buffered size +/// so a crafted pipe cannot allocate unbounded memory before footer-length validation. +/// Sized for one max DV blob + max footer (header magic + blob + footer magic + payload + trailer). +constexpr size_t PUFFIN_NON_SEEKABLE_MAX_BUFFERED_SIZE = PUFFIN_MAGIC_SIZE + PUFFIN_DV_MAX_BLOB_SIZE + + PUFFIN_MAGIC_SIZE + PUFFIN_FOOTER_MAX_PAYLOAD_SIZE + PUFFIN_FOOTER_TRAILER_SIZE; + +/// Iceberg reserved `_pos` field id (`std::numeric_limits::max() - 2`). Spark / Iceberg +/// writers put this singleton in deletion-vector-v1 puffin `fields` for file-scoped DVs. +constexpr Int32 ICEBERG_ROW_POSITION_FIELD_ID = std::numeric_limits::max() - 2; + +/// File-scoped DVs may use `fields=[]` or `fields=[ICEBERG_ROW_POSITION_FIELD_ID]`. +/// Any other list is treated as unsupported column-scoped deletion vectors. +void validateDeletionVectorV1Fields(const std::vector & fields, size_t blob_index); + +/// Validate that [offset, offset + length) fits within file_size. +void validatePuffinBlobBounds(Int64 offset, Int64 length, size_t file_size, std::string_view context = "Puffin deletion vector"); + +/// Iceberg deletion-vector-v1 envelope magic (`0xD1D33964`), shared with SQL `Puffin` decode. +inline constexpr UInt8 DELETION_VECTOR_MAGIC[4] = {0xD1, 0xD3, 0x39, 0x64}; + +/// Fail closed before envelope peek / full allocate. Shared by SQL `Puffin` and Iceberg loaders. +/// Order: cardinality ceiling, then length bounds (`length < 0`, absolute blob cap, min envelope). +void checkDeletionVectorBlobReadLimits(Int64 length, std::optional expected_cardinality); + +/// Validate deletion-vector-v1 envelope (combined_length + magic) against declared blob `length`. +/// `header` must point at the first 8 bytes of the blob. Throws on mismatch before a full allocate. +void validateDeletionVectorEnvelope(const UInt8 * header, Int64 length); + +/// Deserialize a deletion-vector-v1 blob (magic + CRC wrapper + roaring bitmap payload). +std::vector deserializeDeletionVectorV1Blob(std::string_view blob_bytes, std::optional expected_cardinality = std::nullopt); + +/// Read a deletion-vector-v1 blob from a Puffin file at the given offset and length. +std::vector readDeletionVectorFromPuffin(ReadBuffer & file, Int64 offset, Int64 length, std::optional expected_cardinality = std::nullopt); + +/// Append bytes from `buf` into `out` until EOF. Throws if `out` would exceed `max_buffered_size`. +void appendReadBufferWithAbsoluteSizeLimit(ReadBuffer & buf, std::vector & out, size_t max_buffered_size); + +/// Compatibility alias for Iceberg / tests: same as `readPuffinFooterFromSeekable`. +inline std::vector readPuffinFooterBlobsFromSeekable(SeekableReadBuffer & seekable, size_t file_size) +{ + return readPuffinFooterFromSeekable(seekable, file_size); +} + +/// Find the unique footer blob at (`content_offset`, `content_size_in_bytes`) and bind it as a +/// deletion-vector-v1 for `expected_referenced_data_file` with `expected_cardinality`. +/// Throws if the slice is missing, ambiguous, or does not match the expected DV identity. +const PuffinBlob & bindDeletionVectorBlob( + const std::vector & blobs, + Int64 content_offset, + Int64 content_size_in_bytes, + std::string_view expected_referenced_data_file, + UInt64 expected_cardinality); + +} diff --git a/src/Storages/ObjectStorage/DataLakes/PuffinFilesCache.cpp b/src/Storages/ObjectStorage/DataLakes/PuffinFilesCache.cpp new file mode 100644 index 000000000000..720df8a7d447 --- /dev/null +++ b/src/Storages/ObjectStorage/DataLakes/PuffinFilesCache.cpp @@ -0,0 +1,338 @@ +#include + +#include +#include +#include +#include + +#include +#include + +namespace CurrentMetrics +{ +extern const Metric PuffinFilesCacheBytes; +extern const Metric PuffinFilesCacheFiles; +} + +namespace ProfileEvents +{ +extern const Event PuffinFilesCacheWeightLost; +} + +namespace DB +{ + +namespace +{ + +constexpr size_t FOOTER_MEMO_ENTRY_OVERHEAD = 256; +constexpr size_t FOOTER_BLOB_OVERHEAD = 64; + +UInt64 saturatingAdd(UInt64 left, UInt64 right) +{ + UInt64 result = 0; + if (common::addOverflow(left, right, result)) + return std::numeric_limits::max(); + return result; +} + +UInt64 saturatingAdd(std::initializer_list values) +{ + UInt64 result = 0; + for (UInt64 value : values) + result = saturatingAdd(result, value); + return result; +} + +} + +DataLakeObjectMetadata::ExcludedRowsPtr PuffinFilesCache::cloneExcludedRows(const PuffinFilesCacheCell & cell) +{ + if (cell.is_empty_deletion_vector) + return nullptr; + + auto cloned = std::make_shared(); + cloned->merge(*cell.excluded_rows); + return cloned; +} + +bool PuffinFilesCacheKey::operator==(const PuffinFilesCacheKey & other) const +{ + return storage_identity == other.storage_identity + && file_path == other.file_path + && etag == other.etag + && content_offset == other.content_offset + && content_size_in_bytes == other.content_size_in_bytes + && referenced_data_file == other.referenced_data_file + && expected_cardinality == other.expected_cardinality + && data_file_record_count == other.data_file_record_count; +} + +UInt64 PuffinFilesCacheKey::approximateMemoryBytes() const +{ + /// Charge string payloads plus the key object / hash-map slot baseline. + return saturatingAdd( + {static_cast(sizeof(PuffinFilesCacheKey)), + static_cast(storage_identity.size()), + static_cast(file_path.size()), + static_cast(etag.size()), + static_cast(referenced_data_file.size())}); +} + +size_t PuffinFilesCacheKeyHash::operator()(const PuffinFilesCacheKey & key) const +{ + size_t hash = 0; + boost::hash_combine(hash, CityHash_v1_0_2::CityHash64(key.storage_identity.data(), key.storage_identity.size())); + boost::hash_combine(hash, CityHash_v1_0_2::CityHash64(key.file_path.data(), key.file_path.size())); + boost::hash_combine(hash, CityHash_v1_0_2::CityHash64(key.etag.data(), key.etag.size())); + boost::hash_combine(hash, key.content_offset); + boost::hash_combine(hash, key.content_size_in_bytes); + boost::hash_combine(hash, CityHash_v1_0_2::CityHash64(key.referenced_data_file.data(), key.referenced_data_file.size())); + boost::hash_combine(hash, key.expected_cardinality); + boost::hash_combine(hash, key.data_file_record_count); + return hash; +} + +UInt64 PuffinFilesCacheCell::calculateMemorySize( + bool is_empty_deletion_vector_, + const DataLakeObjectMetadata::ExcludedRowsPtr & excluded_rows_, + UInt64 key_memory_bytes_) +{ + const UInt64 payload_bytes = is_empty_deletion_vector_ + ? 0 + : (excluded_rows_ ? excluded_rows_->getAllocatedBytes() : 0); + + return saturatingAdd( + {key_memory_bytes_, + payload_bytes, + static_cast(sizeof(PuffinFilesCacheCell)), + static_cast(SIZE_IN_MEMORY_OVERHEAD)}); +} + +PuffinFilesCacheCell::PuffinFilesCacheCell(DataLakeObjectMetadata::ExcludedRowsPtr excluded_rows_, UInt64 key_memory_bytes_) + : excluded_rows(std::move(excluded_rows_)) + , is_empty_deletion_vector(!excluded_rows) + , memory_bytes(calculateMemorySize(is_empty_deletion_vector, excluded_rows, key_memory_bytes_)) +{ +} + +size_t PuffinFilesCacheWeightFunction::operator()(const PuffinFilesCacheCell & cell) const +{ + return cell.memory_bytes; +} + +bool PuffinFooterCacheKey::operator==(const PuffinFooterCacheKey & other) const +{ + return storage_identity == other.storage_identity && file_path == other.file_path && etag == other.etag; +} + +size_t PuffinFooterCacheKeyHash::operator()(const PuffinFooterCacheKey & key) const +{ + size_t hash = 0; + boost::hash_combine(hash, CityHash_v1_0_2::CityHash64(key.storage_identity.data(), key.storage_identity.size())); + boost::hash_combine(hash, CityHash_v1_0_2::CityHash64(key.file_path.data(), key.file_path.size())); + boost::hash_combine(hash, CityHash_v1_0_2::CityHash64(key.etag.data(), key.etag.size())); + return hash; +} + +UInt64 PuffinFilesCache::approximateFooterEntryBytes(const PuffinFooterCacheKey & key, const FooterBlobsPtr & blobs) +{ + UInt64 bytes = saturatingAdd( + {static_cast(sizeof(PuffinFooterCacheKey)), + static_cast(key.storage_identity.size()), + static_cast(key.file_path.size()), + static_cast(key.etag.size()), + static_cast(sizeof(FooterMemoEntry)), + static_cast(FOOTER_MEMO_ENTRY_OVERHEAD)}); + + if (!blobs) + return bytes; + + bytes = saturatingAdd(bytes, static_cast(blobs->capacity() * sizeof(PuffinBlob))); + for (const auto & blob : *blobs) + { + bytes = saturatingAdd( + {bytes, + static_cast(blob.type.size()), + static_cast(blob.compression_codec.size()), + static_cast(blob.fields.capacity() * sizeof(Int32)), + static_cast(FOOTER_BLOB_OVERHEAD)}); + for (const auto & [prop_key, value] : blob.properties) + bytes = saturatingAdd(bytes, saturatingAdd(static_cast(prop_key.size()), static_cast(value.size()))); + } + return bytes; +} + +void PuffinFilesCache::clearFooterMemoUnlocked() +{ + if (footer_memo_bytes != 0) + CurrentMetrics::sub(CurrentMetrics::PuffinFilesCacheBytes, static_cast(footer_memo_bytes)); + if (!footer_memo.empty()) + CurrentMetrics::sub(CurrentMetrics::PuffinFilesCacheFiles, static_cast(footer_memo.size())); + footer_memo.clear(); + footer_memo_bytes = 0; +} + +void PuffinFilesCache::accountFooterMemoInsertUnlocked(UInt64 entry_bytes) +{ + footer_memo_bytes += entry_bytes; + CurrentMetrics::add(CurrentMetrics::PuffinFilesCacheBytes, static_cast(entry_bytes)); + CurrentMetrics::add(CurrentMetrics::PuffinFilesCacheFiles); +} + +void PuffinFilesCache::eraseFooterMemoVictimUnlocked() +{ + auto victim = footer_memo.begin(); + const UInt64 victim_bytes = victim->second.memory_bytes; + footer_memo.erase(victim); + if (footer_memo_bytes >= victim_bytes) + footer_memo_bytes -= victim_bytes; + else + footer_memo_bytes = 0; + CurrentMetrics::sub(CurrentMetrics::PuffinFilesCacheBytes, static_cast(victim_bytes)); + CurrentMetrics::sub(CurrentMetrics::PuffinFilesCacheFiles); +} + +bool PuffinFilesCache::needsFooterEvictionForInsertUnlocked(size_t dv_bytes, size_t dv_count, UInt64 entry_bytes) const +{ + if (shared_max_count > 0 && dv_count + footer_memo.size() >= shared_max_count) + return true; + + /// Room left for memo under the shared byte budget after DV weight. + if (dv_bytes > shared_max_bytes) + return !footer_memo.empty(); + + const size_t room_for_memo = shared_max_bytes - dv_bytes; + return footer_memo_bytes > room_for_memo - entry_bytes; +} + +void PuffinFilesCache::trimFooterMemoToSharedBudget() +{ + const size_t dv_bytes = Base::sizeInBytes(); + const size_t dv_count = Base::count(); + + std::lock_guard lock(footer_mutex); + if (shared_max_bytes == 0) + { + clearFooterMemoUnlocked(); + return; + } + + while (!footer_memo.empty()) + { + const bool over_bytes = dv_bytes > shared_max_bytes + || footer_memo_bytes > shared_max_bytes - dv_bytes; + const bool over_count = shared_max_count > 0 && dv_count + footer_memo.size() > shared_max_count; + if (!over_bytes && !over_count) + break; + eraseFooterMemoVictimUnlocked(); + } +} + +PuffinFilesCache::PuffinFilesCache( + const String & cache_policy, + size_t max_size_in_bytes, + size_t max_count, + double size_ratio) + : Base( + cache_policy, + CurrentMetrics::PuffinFilesCacheBytes, + CurrentMetrics::PuffinFilesCacheFiles, + max_size_in_bytes, + max_count, + size_ratio) + , log(getLogger("PuffinFilesCache")) + , shared_max_count(max_count) + , shared_max_bytes(max_size_in_bytes) +{ +} + +void PuffinFilesCache::clear() +{ + Base::clear(); + std::lock_guard lock(footer_mutex); + clearFooterMemoUnlocked(); +} + +void PuffinFilesCache::setMaxSizeInBytes(size_t max_size_in_bytes) +{ + Base::setMaxSizeInBytes(max_size_in_bytes); + { + std::lock_guard lock(footer_mutex); + shared_max_bytes = max_size_in_bytes; + } + trimFooterMemoToSharedBudget(); +} + +void PuffinFilesCache::setMaxCount(size_t max_count) +{ + Base::setMaxCount(max_count); + { + std::lock_guard lock(footer_mutex); + shared_max_count = max_count; + } + trimFooterMemoToSharedBudget(); +} + +size_t PuffinFilesCache::footerMemoEntries() const +{ + std::lock_guard lock(footer_mutex); + return footer_memo.size(); +} + +UInt64 PuffinFilesCache::footerMemoBytes() const +{ + std::lock_guard lock(footer_mutex); + return footer_memo_bytes; +} + +String PuffinFilesCache::makeStorageIdentity(const IObjectStorage & object_storage) +{ + /// Include getDescription() (S3 endpoint, Azure account URL, Local path, ...) so two + /// backends with the same bucket/prefix on different hosts do not share cache entries. + return object_storage.getName() + "://" + object_storage.getDescription() + "/" + + object_storage.getObjectsNamespace() + "/" + object_storage.getCommonKeyPrefix(); +} + +std::optional PuffinFilesCache::tryCreateKey( + const String & storage_identity, + const String & file_path, + const String & etag, + Int64 content_offset, + Int64 content_size_in_bytes, + const String & referenced_data_file, + UInt64 expected_cardinality, + UInt64 data_file_record_count) +{ + if (etag.empty()) + return std::nullopt; + + return PuffinFilesCacheKey{ + storage_identity, + file_path, + etag, + content_offset, + content_size_in_bytes, + referenced_data_file, + expected_cardinality, + data_file_record_count}; +} + +std::optional PuffinFilesCache::tryCreateFooterKey( + const String & storage_identity, + const String & file_path, + const String & etag) +{ + if (etag.empty()) + return std::nullopt; + + return PuffinFooterCacheKey{storage_identity, file_path, etag}; +} + +void PuffinFilesCache::onEntryRemoval(const size_t weight_loss, const MappedPtr &) +{ + LOG_TRACE(log, "Puffin files cache eviction"); + ProfileEvents::increment(ProfileEvents::PuffinFilesCacheWeightLost, weight_loss); +} + +} diff --git a/src/Storages/ObjectStorage/DataLakes/PuffinFilesCache.h b/src/Storages/ObjectStorage/DataLakes/PuffinFilesCache.h new file mode 100644 index 000000000000..8196380ae50d --- /dev/null +++ b/src/Storages/ObjectStorage/DataLakes/PuffinFilesCache.h @@ -0,0 +1,341 @@ +#pragma once + +#include +#include + +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include + +namespace ProfileEvents +{ +extern const Event PuffinFilesCacheHits; +extern const Event PuffinFilesCacheMisses; +extern const Event PuffinFilesCacheWeightLost; +} + +namespace DB +{ + +class IObjectStorage; + +struct PuffinFilesCacheKey +{ + /// Distinguishes object-storage backends that share the same relative path (and possibly etag). + /// Built via `makeStorageIdentity` from storage type + description (endpoint) + namespace + prefix. + String storage_identity; + String file_path; + String etag; + Int64 content_offset = 0; + Int64 content_size_in_bytes = 0; + String referenced_data_file; + /// Manifest DV record_count / expected roaring cardinality. Included so a cache hit cannot + /// skip re-validation when a later request declares a different cardinality for the same slice. + UInt64 expected_cardinality = 0; + /// Data-file manifest record_count used to bound DV positions. Same rationale as cardinality. + UInt64 data_file_record_count = 0; + + bool operator==(const PuffinFilesCacheKey & other) const; + + /// Approximate bytes for key strings + fixed key fields (used in entry weight). + UInt64 approximateMemoryBytes() const; +}; + +struct PuffinFilesCacheKeyHash +{ + size_t operator()(const PuffinFilesCacheKey & key) const; +}; + +struct PuffinFilesCacheCell : private boost::noncopyable +{ + DataLakeObjectMetadata::ExcludedRowsPtr excluded_rows; + bool is_empty_deletion_vector = false; + UInt64 memory_bytes = 0; + + PuffinFilesCacheCell(DataLakeObjectMetadata::ExcludedRowsPtr excluded_rows_, UInt64 key_memory_bytes_); + + static UInt64 calculateMemorySize( + bool is_empty_deletion_vector_, + const DataLakeObjectMetadata::ExcludedRowsPtr & excluded_rows_, + UInt64 key_memory_bytes_); + + /// Lower-bound resident weight before the DV payload is known (key + cell object + overhead). + /// Used to skip the in-memory cache path when `puffin_files_cache_size` cannot hold even one entry. + static UInt64 estimateMinimumMemorySize(UInt64 key_memory_bytes_) + { + return calculateMemorySize(/*is_empty_deletion_vector_=*/true, nullptr, key_memory_bytes_); + } + +private: + /// Hash-map node + LRU list node + shared_ptr control block underestimates are absorbed here. + static constexpr size_t SIZE_IN_MEMORY_OVERHEAD = 256; +}; + +struct PuffinFilesCacheWeightFunction +{ + size_t operator()(const PuffinFilesCacheCell & cell) const; +}; + +/// File-level footer identity (shared by all DV slices in one coalesced Puffin). +struct PuffinFooterCacheKey +{ + String storage_identity; + String file_path; + String etag; + + bool operator==(const PuffinFooterCacheKey & other) const; +}; + +struct PuffinFooterCacheKeyHash +{ + size_t operator()(const PuffinFooterCacheKey & key) const; +}; + +/// Cache for parsed content loaded from Puffin files (deletion vectors today, indexes later). +/// Also memoizes parsed footers so coalesced multi-DV Puffins parse the footer once per file. +/// +/// Footer memo and DV LRU share one byte budget and one entry-count budget (`puffin_files_cache_size` +/// / max entries). Memo weight is charged into `CurrentMetrics::PuffinFilesCacheBytes` / +/// `PuffinFilesCacheFiles` so resident usage stays observable and cannot reach ~2× the configured max. +class PuffinFilesCache : public CacheBase +{ +public: + using Base = CacheBase; + using FooterBlobsPtr = std::shared_ptr>; + + PuffinFilesCache(const String & cache_policy, size_t max_size_in_bytes, size_t max_count, double size_ratio); + + /// Stable backend identity for cache keys: + /// `getName()://getDescription()/getObjectsNamespace()/getCommonKeyPrefix()`. + static String makeStorageIdentity(const IObjectStorage & object_storage); + + static std::optional tryCreateKey( + const String & storage_identity, + const String & file_path, + const String & etag, + Int64 content_offset, + Int64 content_size_in_bytes, + const String & referenced_data_file, + UInt64 expected_cardinality, + UInt64 data_file_record_count); + + static std::optional tryCreateFooterKey( + const String & storage_identity, + const String & file_path, + const String & etag); + + /// Clears deletion-vector entries and footer memo. + void clear(); + + void setMaxSizeInBytes(size_t max_size_in_bytes); + void setMaxCount(size_t max_count); + + /// Test/observability helpers for the footer memo. + size_t footerMemoEntries() const; + UInt64 footerMemoBytes() const; + + /// Small memo (not a weighted LRU): shares the configured byte/count limits with the DV cache. + /// Concurrent misses on the same key each run `load_fn` (no stampede / waiter token); + /// only coalesced sequential slice loads share one parse. On insert, entries are dropped + /// one-by-one until the new entry fits beside current DV weight — not a full memo clear. + template + FooterBlobsPtr getOrSetFooter(const PuffinFooterCacheKey & key, LoadFunc && load_fn) + { + { + std::lock_guard lock(footer_mutex); + if (shared_max_bytes != 0) + { + if (auto it = footer_memo.find(key); it != footer_memo.end()) + return it->second.blobs; + } + } + + auto blobs = load_fn(); + + /// Snapshot DV occupancy outside `footer_mutex` to avoid Base↔memo lock-order inversion + /// with `clear()` / `setMax*` (those take Base first, then memo). + const size_t dv_bytes = Base::sizeInBytes(); + const size_t dv_count = Base::count(); + + std::lock_guard lock(footer_mutex); + if (shared_max_bytes == 0) + return blobs; + + if (auto it = footer_memo.find(key); it != footer_memo.end()) + return it->second.blobs; + + const UInt64 entry_bytes = approximateFooterEntryBytes(key, blobs); + /// Must fit beside current DV weight under the single shared byte budget. + if (entry_bytes > shared_max_bytes || dv_bytes > shared_max_bytes - entry_bytes) + return blobs; + + while (needsFooterEvictionForInsertUnlocked(dv_bytes, dv_count, entry_bytes)) + { + if (footer_memo.empty()) + break; + eraseFooterMemoVictimUnlocked(); + } + + if (needsFooterEvictionForInsertUnlocked(dv_bytes, dv_count, entry_bytes)) + return blobs; + + auto [it, inserted] = footer_memo.emplace(key, FooterMemoEntry{blobs, entry_bytes}); + if (inserted) + accountFooterMemoInsertUnlocked(entry_bytes); + return it->second.blobs; + } + + template + DataLakeObjectMetadata::ExcludedRowsPtr getOrSetDeletionVector(const PuffinFilesCacheKey & key, LoadFunc && load_fn) + { + /// True if this caller's load_fn ran. Used only for tracing; hit/miss metrics use the + /// atomic CacheGetOrSetOutcome from getOrSetWithOutcome (not a follow-up contains()). + bool loaded = false; + auto load_fn_wrapper = [&]() + { + loaded = true; + auto excluded_rows = load_fn(); + const bool is_empty_deletion_vector = !excluded_rows; + if (is_empty_deletion_vector) + { + LOG_TRACE( + log, + "Cached empty puffin deletion vector for {} | {} | {} at offset {} length {} for data file {}", + key.storage_identity, + key.file_path, + key.etag, + key.content_offset, + key.content_size_in_bytes, + key.referenced_data_file); + } + else + { + LOG_TRACE( + log, + "Loaded puffin deletion vector into cache for {} | {} | {} at offset {} length {} for data file {}", + key.storage_identity, + key.file_path, + key.etag, + key.content_offset, + key.content_size_in_bytes, + key.referenced_data_file); + } + return std::make_shared(std::move(excluded_rows), key.approximateMemoryBytes()); + }; + + auto [cell, outcome] = Base::getOrSetWithOutcome(key, load_fn_wrapper); + /// DV weight may have grown; reclaim footer memo so DV + memo stay within one budget. + trimFooterMemoToSharedBudget(); + + const bool served_from_cache = outcome == CacheGetOrSetOutcome::Hit; + if (!served_from_cache) + { + if (loaded && outcome == CacheGetOrSetOutcome::MissNotResident) + { + LOG_TRACE( + log, + "Puffin files cache miss (load discarded by concurrent clear) for {} | {} | {} at offset {} length {} for data file {}", + key.storage_identity, + key.file_path, + key.etag, + key.content_offset, + key.content_size_in_bytes, + key.referenced_data_file); + } + else if (!loaded && outcome == CacheGetOrSetOutcome::MissNotResident) + { + LOG_TRACE( + log, + "Puffin files cache miss (waited for load discarded by concurrent clear) for {} | {} | {} at offset {} length {} for data file {}", + key.storage_identity, + key.file_path, + key.etag, + key.content_offset, + key.content_size_in_bytes, + key.referenced_data_file); + } + else + { + LOG_TRACE( + log, + "Puffin files cache miss for {} | {} | {} at offset {} length {} for data file {}", + key.storage_identity, + key.file_path, + key.etag, + key.content_offset, + key.content_size_in_bytes, + key.referenced_data_file); + } + ProfileEvents::increment(ProfileEvents::PuffinFilesCacheMisses); + } + else if (cell->is_empty_deletion_vector) + { + LOG_TRACE( + log, + "Puffin files cache hit (empty deletion vector) for {} | {} | {} at offset {} length {} for data file {}", + key.storage_identity, + key.file_path, + key.etag, + key.content_offset, + key.content_size_in_bytes, + key.referenced_data_file); + ProfileEvents::increment(ProfileEvents::PuffinFilesCacheHits); + } + else + { + LOG_TRACE( + log, + "Puffin files cache hit for {} | {} | {} at offset {} length {} for data file {}", + key.storage_identity, + key.file_path, + key.etag, + key.content_offset, + key.content_size_in_bytes, + key.referenced_data_file); + ProfileEvents::increment(ProfileEvents::PuffinFilesCacheHits); + } + + return cloneExcludedRows(*cell); + } + +private: + struct FooterMemoEntry + { + FooterBlobsPtr blobs; + UInt64 memory_bytes = 0; + }; + + static DataLakeObjectMetadata::ExcludedRowsPtr cloneExcludedRows(const PuffinFilesCacheCell & cell); + static UInt64 approximateFooterEntryBytes(const PuffinFooterCacheKey & key, const FooterBlobsPtr & blobs); + + void clearFooterMemoUnlocked(); + void accountFooterMemoInsertUnlocked(UInt64 entry_bytes); + void eraseFooterMemoVictimUnlocked(); + bool needsFooterEvictionForInsertUnlocked(size_t dv_bytes, size_t dv_count, UInt64 entry_bytes) const; + void trimFooterMemoToSharedBudget(); + + LoggerPtr log; + mutable std::mutex footer_mutex; + std::unordered_map footer_memo; + /// Configured shared limits (same values passed to Base). Copied here so memo insert/trim can + /// decide without calling back into Base while holding `footer_mutex`. + size_t shared_max_count = 0; + size_t shared_max_bytes = 0; + UInt64 footer_memo_bytes = 0; + + void onEntryRemoval(size_t weight_loss, const MappedPtr &) override; +}; + +using PuffinFilesCachePtr = std::shared_ptr; + +} diff --git a/src/Storages/ObjectStorage/DataLakes/tests/gtest_deletion_vector_before_equality_filter.cpp b/src/Storages/ObjectStorage/DataLakes/tests/gtest_deletion_vector_before_equality_filter.cpp new file mode 100644 index 000000000000..3b379381ca76 --- /dev/null +++ b/src/Storages/ObjectStorage/DataLakes/tests/gtest_deletion_vector_before_equality_filter.cpp @@ -0,0 +1,86 @@ +#include + +#include +#include +#include +#include + +using namespace DB; + +namespace +{ + +Chunk makeChunkWithFileRowNumbers(const std::vector & values, size_t row_num_offset = 0) +{ + auto column = ColumnUInt64::create(); + for (UInt64 value : values) + column->insert(value); + + Chunk chunk(Columns{std::move(column)}, values.size()); + chunk.getChunkInfos().add(std::make_shared(row_num_offset)); + return chunk; +} + +/// Mimics Iceberg equality-delete `FilterTransform`: shrink columns, leave `applied_filter` unset. +void shrinkWithoutAppliedFilter(Chunk & chunk, const IColumn::Filter & filter) +{ + size_t result_size = 0; + for (UInt8 keep : filter) + result_size += keep != 0; + + auto columns = chunk.detachColumns(); + for (auto & column : columns) + column = column->filter(filter, -1); + chunk.setColumns(std::move(columns), result_size); +} + +std::vector readValues(const Chunk & chunk) +{ + const auto & column = assert_cast(*chunk.getColumns().at(0)); + std::vector values; + values.reserve(column.size()); + for (size_t i = 0; i < column.size(); ++i) + values.push_back(column.getData()[i]); + return values; +} + +std::shared_ptr makeExcludedRows(std::initializer_list positions) +{ + auto excluded_rows = std::make_shared(); + for (UInt64 position : positions) + excluded_rows->add(position); + return excluded_rows; +} + +} + +/// Equality removes file row 1; DV deletes file position 2. Survivors must be {0, 3}. +TEST(DeletionVectorBeforeEqualityFilter, CorrectOrderKeepsFileRowMapping) +{ + Chunk chunk = makeChunkWithFileRowNumbers({0, 1, 2, 3}); + + DeletionVectorTransform::transform(chunk, *makeExcludedRows({2})); + ASSERT_EQ(readValues(chunk), (std::vector{0, 1, 3})); + + /// Drop equality-deleted value 1 without updating applied_filter (as FilterTransform does). + shrinkWithoutAppliedFilter(chunk, IColumn::Filter{1, 0, 1}); + EXPECT_EQ(readValues(chunk), (std::vector{0, 3})); +} + +/// Documents why StorageObjectStorageSource must run DV before equality FilterTransform: +/// reversing the order maps DV positions onto dense post-equality indices and keeps the +/// wrong survivors ({0, 2} instead of {0, 3}). +TEST(DeletionVectorBeforeEqualityFilter, WrongOrderProducesWrongSurvivors) +{ + Chunk chunk = makeChunkWithFileRowNumbers({0, 1, 2, 3}); + + /// Equality first: drop value 1 → dense [0, 2, 3], applied_filter unset. + shrinkWithoutAppliedFilter(chunk, IColumn::Filter{1, 0, 1, 1}); + ASSERT_EQ(readValues(chunk), (std::vector{0, 2, 3})); + + /// DV still targets file position 2, but ChunkInfoRowNumbers still starts at 0 over the + /// shrunk chunk, so index 2 (value 3) is removed instead of original file row 2. + DeletionVectorTransform::transform(chunk, *makeExcludedRows({2})); + EXPECT_EQ(readValues(chunk), (std::vector{0, 2})); + EXPECT_NE(readValues(chunk), (std::vector{0, 3})); +} diff --git a/src/Storages/ObjectStorage/DataLakes/tests/gtest_deletion_vector_need_only_count.cpp b/src/Storages/ObjectStorage/DataLakes/tests/gtest_deletion_vector_need_only_count.cpp new file mode 100644 index 000000000000..2e614ae34a6a --- /dev/null +++ b/src/Storages/ObjectStorage/DataLakes/tests/gtest_deletion_vector_need_only_count.cpp @@ -0,0 +1,127 @@ +#include + +#include +#include +#include +#include +#include +#include + +using namespace DB; + +namespace +{ + +Chunk makeConstCountChunk(size_t num_rows, size_t row_num_offset) +{ + auto nested = ColumnUInt64::create(); + nested->insertDefault(); + Columns columns; + columns.emplace_back(ColumnConst::create(std::move(nested), num_rows)); + Chunk chunk(std::move(columns), num_rows); + chunk.getChunkInfos().add(std::make_shared(row_num_offset)); + return chunk; +} + +Chunk makeMaterializedChunk(const std::vector & values, size_t row_num_offset = 0) +{ + auto column = ColumnUInt64::create(); + for (UInt64 value : values) + column->insert(value); + + Chunk chunk(Columns{std::move(column)}, values.size()); + chunk.getChunkInfos().add(std::make_shared(row_num_offset)); + return chunk; +} + +std::shared_ptr makeExcludedRows(std::initializer_list positions) +{ + auto excluded_rows = std::make_shared(); + for (UInt64 position : positions) + excluded_rows->add(position); + return excluded_rows; +} + +} + +TEST(RoaringBitmapRangeCardinality, CountsInclusiveStartExclusiveEnd) +{ + DataLakeObjectMetadata::ExcludedRows bitmap; + bitmap.add(1); + bitmap.add(5); + bitmap.add(10); + + EXPECT_EQ(bitmap.rb_range_cardinality(0, 0), 0u); + EXPECT_EQ(bitmap.rb_range_cardinality(5, 5), 0u); + EXPECT_EQ(bitmap.rb_range_cardinality(0, 5), 1u); + EXPECT_EQ(bitmap.rb_range_cardinality(0, 6), 2u); + EXPECT_EQ(bitmap.rb_range_cardinality(1, 11), 3u); + EXPECT_EQ(bitmap.rb_range_cardinality(2, 10), 1u); + EXPECT_EQ(bitmap.rb_range_cardinality(11, 100), 0u); +} + +TEST(RoaringBitmapRangeCardinality, MatchesRbRangeCountForLargeBitmap) +{ + DataLakeObjectMetadata::ExcludedRows bitmap; + /// Force large roaring path (small-set threshold is 32). + for (UInt64 i = 0; i < 64; ++i) + bitmap.add(i * 3); + ASSERT_TRUE(bitmap.isLarge()); + + DataLakeObjectMetadata::ExcludedRows subset; + const UInt64 via_range = bitmap.rb_range(10, 100, subset); + EXPECT_EQ(bitmap.rb_range_cardinality(10, 100), via_range); + EXPECT_EQ(subset.size(), via_range); +} + +TEST(RoaringBitmapRangeCardinality, SumOfRowGroupRangesMatchesWholeFile) +{ + DataLakeObjectMetadata::ExcludedRows bitmap; + for (UInt64 i = 0; i < 10'000; ++i) + bitmap.add(i * 7); + ASSERT_TRUE(bitmap.isLarge()); + + constexpr size_t rows_per_group = 1'000; + constexpr size_t num_groups = 200; + UInt64 summed = 0; + for (size_t group = 0; group < num_groups; ++group) + { + const UInt64 start = group * rows_per_group; + summed += bitmap.rb_range_cardinality(start, start + rows_per_group); + } + + EXPECT_EQ(summed, bitmap.rb_range_cardinality(0, num_groups * rows_per_group)); + EXPECT_EQ(summed, bitmap.size()); +} + +TEST(DeletionVectorNeedOnlyCount, ConstChunkUsesRangeCardinality) +{ + /// Large enough that a dense Filter would be expensive; const columns stay O(1) via + /// cloneResized + rb_range_cardinality. No applied_filter: Iceberg does not chain another + /// row-number consumer after a DV on the need_only_count path. + constexpr size_t num_rows = 5'000'000; + Chunk chunk = makeConstCountChunk(num_rows, /*row_num_offset=*/0); + + DeletionVectorTransform::transform(chunk, *makeExcludedRows({0, 1, num_rows - 1, num_rows + 10})); + EXPECT_EQ(chunk.getNumRows(), num_rows - 3); + ASSERT_EQ(chunk.getNumColumns(), 1u); + EXPECT_TRUE(isColumnConst(*chunk.getColumns()[0])); + EXPECT_FALSE(chunk.getChunkInfos().get()->applied_filter.has_value()); +} + +TEST(DeletionVectorNeedOnlyCount, ConstChunkHonorsRowNumOffset) +{ + Chunk chunk = makeConstCountChunk(/*num_rows=*/10, /*row_num_offset=*/100); + /// Deletes at absolute positions 99 (before), 105 (inside), 110 (end exclusive / outside). + DeletionVectorTransform::transform(chunk, *makeExcludedRows({99, 105, 110})); + EXPECT_EQ(chunk.getNumRows(), 9u); + EXPECT_FALSE(chunk.getChunkInfos().get()->applied_filter.has_value()); +} + +TEST(DeletionVectorNeedOnlyCount, MaterializedChunkStillUsesDenseFilter) +{ + Chunk chunk = makeMaterializedChunk({10, 11, 12, 13}, /*row_num_offset=*/10); + DeletionVectorTransform::transform(chunk, *makeExcludedRows({11, 13})); + EXPECT_EQ(chunk.getNumRows(), 2u); + ASSERT_TRUE(chunk.getChunkInfos().get()->applied_filter.has_value()); +} diff --git a/src/Storages/ObjectStorage/DataLakes/tests/gtest_has_non_empty_excluded_rows.cpp b/src/Storages/ObjectStorage/DataLakes/tests/gtest_has_non_empty_excluded_rows.cpp new file mode 100644 index 000000000000..9903493c3413 --- /dev/null +++ b/src/Storages/ObjectStorage/DataLakes/tests/gtest_has_non_empty_excluded_rows.cpp @@ -0,0 +1,28 @@ +#include + +#include +#include + +using namespace DB; + +TEST(HasNonEmptyExcludedRows, EmptyOptionalIsFalse) +{ + EXPECT_FALSE(hasNonEmptyExcludedRows(std::nullopt)); +} + +TEST(HasNonEmptyExcludedRows, MissingOrEmptyBitmapIsFalse) +{ + DataLakeObjectMetadata metadata; + EXPECT_FALSE(hasNonEmptyExcludedRows(metadata)); + + metadata.excluded_rows = std::make_shared(); + EXPECT_FALSE(hasNonEmptyExcludedRows(metadata)); +} + +TEST(HasNonEmptyExcludedRows, NonEmptyBitmapIsTrue) +{ + DataLakeObjectMetadata metadata; + metadata.excluded_rows = std::make_shared(); + metadata.excluded_rows->add(7); + EXPECT_TRUE(hasNonEmptyExcludedRows(metadata)); +} diff --git a/src/Storages/ObjectStorage/DataLakes/tests/gtest_puffin_deletion_vector_bounds.cpp b/src/Storages/ObjectStorage/DataLakes/tests/gtest_puffin_deletion_vector_bounds.cpp new file mode 100644 index 000000000000..0c036309b73b --- /dev/null +++ b/src/Storages/ObjectStorage/DataLakes/tests/gtest_puffin_deletion_vector_bounds.cpp @@ -0,0 +1,63 @@ +#include + +#include +#include +#include + +using namespace DB; + +namespace DB +{ +namespace ErrorCodes +{ +extern const int BAD_ARGUMENTS; +} +} + +TEST(PuffinDeletionVectorBounds, RejectsLengthExceedingFileSize) +{ + const String data(64, '\0'); + ReadBufferFromOutsideMemoryFile file("test.puffin", data); + + try + { + readDeletionVectorFromPuffin(file, 0, 1000); + FAIL() << "Expected exception"; + } + catch (const Exception & e) + { + EXPECT_EQ(e.code(), ErrorCodes::BAD_ARGUMENTS); + EXPECT_NE(e.message().find("offset/length out of bounds"), std::string::npos); + } +} + +TEST(PuffinDeletionVectorBounds, RejectsOffsetPlusLengthOverflow) +{ + const String data(64, '\0'); + ReadBufferFromOutsideMemoryFile file("test.puffin", data); + + try + { + readDeletionVectorFromPuffin(file, 60, 12); + FAIL() << "Expected exception"; + } + catch (const Exception & e) + { + EXPECT_EQ(e.code(), ErrorCodes::BAD_ARGUMENTS); + EXPECT_NE(e.message().find("offset/length out of bounds"), std::string::npos); + } +} + +TEST(PuffinDeletionVectorBounds, ValidatePuffinBlobBoundsRejectsNegativeOffset) +{ + try + { + validatePuffinBlobBounds(-1, 10, 64); + FAIL() << "Expected exception"; + } + catch (const Exception & e) + { + EXPECT_EQ(e.code(), ErrorCodes::BAD_ARGUMENTS); + EXPECT_NE(e.message().find("offset/length out of bounds"), std::string::npos); + } +} diff --git a/src/Storages/ObjectStorage/DataLakes/tests/gtest_puffin_deletion_vector_cardinality.cpp b/src/Storages/ObjectStorage/DataLakes/tests/gtest_puffin_deletion_vector_cardinality.cpp new file mode 100644 index 000000000000..2253f4ac548d --- /dev/null +++ b/src/Storages/ObjectStorage/DataLakes/tests/gtest_puffin_deletion_vector_cardinality.cpp @@ -0,0 +1,96 @@ +#include + +#include +#include +#include + +#include + +using namespace DB; + +namespace DB +{ +namespace ErrorCodes +{ +extern const int BAD_ARGUMENTS; +} +} + +namespace +{ + +/// deletion-vector-v1 blob for positions {2, 5} (cardinality 2). +constexpr UInt8 two_position_dv_blob[] = { + 0x00, 0x00, 0x00, 0x24, 0xD1, 0xD3, 0x39, 0x64, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x3A, 0x30, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0x00, + 0x10, 0x00, 0x00, 0x00, 0x02, 0x00, 0x05, 0x00, 0x2C, 0xDB, 0x9F, 0xC1, +}; + +constexpr Int64 large_declared_length = 64 * 1024 * 1024; + +} + +TEST(PuffinDeletionVectorCardinality, RejectsCardinalityAboveMaterializationLimitBeforeParse) +{ + /// Ceiling is checked before payload validation, so even an empty blob must fail closed. + try + { + deserializeDeletionVectorV1Blob(std::string_view{}, PUFFIN_DV_MAX_MATERIALIZED_POSITIONS + 1); + FAIL() << "Expected exception"; + } + catch (const Exception & e) + { + EXPECT_EQ(e.code(), ErrorCodes::BAD_ARGUMENTS); + EXPECT_NE(e.message().find("exceeds materialization limit"), std::string::npos); + } +} + +TEST(PuffinDeletionVectorCardinality, RejectsCardinalityAboveLimitBeforeFullAllocate) +{ + /// Huge declared length with only a tiny buffer: without an early ceiling check this would + /// allocate `large_declared_length` (or fail mid-read after that allocate). ReadBufferFromMemory + /// does not expose file size, so bounds checks alone do not stop this. + /// The SQL `Puffin` format applies the same early check in `readDeletionVectorBlobBytes`. + const char header[8] = {}; + ReadBufferFromMemory file(header, sizeof(header)); + + try + { + readDeletionVectorFromPuffin( + file, 0, large_declared_length, PUFFIN_DV_MAX_MATERIALIZED_POSITIONS + 1); + FAIL() << "Expected exception"; + } + catch (const Exception & e) + { + EXPECT_EQ(e.code(), ErrorCodes::BAD_ARGUMENTS); + EXPECT_NE(e.message().find("exceeds materialization limit"), std::string::npos); + } +} + +TEST(PuffinDeletionVectorCardinality, RejectsBitmapExceedingDeclaredCardinality) +{ + const std::string_view blob( + reinterpret_cast(two_position_dv_blob), sizeof(two_position_dv_blob)); + + try + { + deserializeDeletionVectorV1Blob(blob, /*expected_cardinality=*/1); + FAIL() << "Expected exception"; + } + catch (const Exception & e) + { + EXPECT_EQ(e.code(), ErrorCodes::BAD_ARGUMENTS); + EXPECT_NE(e.message().find("exceeds declared cardinality"), std::string::npos); + } +} + +TEST(PuffinDeletionVectorCardinality, AcceptsMatchingCardinality) +{ + const std::string_view blob( + reinterpret_cast(two_position_dv_blob), sizeof(two_position_dv_blob)); + + const auto positions = deserializeDeletionVectorV1Blob(blob, /*expected_cardinality=*/2); + ASSERT_EQ(positions.size(), 2u); + EXPECT_EQ(positions[0], 2u); + EXPECT_EQ(positions[1], 5u); +} diff --git a/src/Storages/ObjectStorage/DataLakes/tests/gtest_puffin_deletion_vector_envelope.cpp b/src/Storages/ObjectStorage/DataLakes/tests/gtest_puffin_deletion_vector_envelope.cpp new file mode 100644 index 000000000000..4db35bad5ad4 --- /dev/null +++ b/src/Storages/ObjectStorage/DataLakes/tests/gtest_puffin_deletion_vector_envelope.cpp @@ -0,0 +1,124 @@ +#include + +#include +#include +#include + +#include + +using namespace DB; + +namespace DB +{ +namespace ErrorCodes +{ +extern const int BAD_ARGUMENTS; +} +} + +namespace +{ + +/// deletion-vector-v1 blob for positions {2, 5} (cardinality 2). +constexpr UInt8 two_position_dv_blob[] = { + 0x00, 0x00, 0x00, 0x24, 0xD1, 0xD3, 0x39, 0x64, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x3A, 0x30, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0x00, + 0x10, 0x00, 0x00, 0x00, 0x02, 0x00, 0x05, 0x00, 0x2C, 0xDB, 0x9F, 0xC1, +}; + +/// Large declared length that would force a huge allocate if peeked after full read. +constexpr Int64 large_declared_length = 64 * 1024 * 1024; + +} + +TEST(PuffinDeletionVectorEnvelope, RejectsLengthBelowEnvelopeMinimum) +{ + const char zeros[16] = {}; + ReadBufferFromMemory file(zeros, sizeof(zeros)); + + try + { + readDeletionVectorFromPuffin(file, 0, 11); + FAIL() << "Expected exception"; + } + catch (const Exception & e) + { + EXPECT_EQ(e.code(), ErrorCodes::BAD_ARGUMENTS); + EXPECT_NE(e.message().find("too small"), std::string::npos); + } +} + +TEST(PuffinDeletionVectorEnvelope, RejectsInvalidMagicBeforeFullAllocate) +{ + /// Only 8 bytes available; declared length is huge. Without envelope peek this would allocate + /// `large_declared_length` (or fail mid-read after that allocate). ReadBufferFromMemory does not + /// expose file size, so the absolute 2 GiB / bounds checks alone do not stop this. + const UInt8 header[8] = {0x00, 0x00, 0x00, 0x24, 0x00, 0x00, 0x00, 0x00}; // wrong magic + ReadBufferFromMemory file(header, sizeof(header)); + + try + { + readDeletionVectorFromPuffin(file, 0, large_declared_length); + FAIL() << "Expected exception"; + } + catch (const Exception & e) + { + EXPECT_EQ(e.code(), ErrorCodes::BAD_ARGUMENTS); + EXPECT_NE(e.message().find("Invalid deletion vector magic"), std::string::npos); + } +} + +TEST(PuffinDeletionVectorEnvelope, RejectsCombinedLengthMismatchBeforeFullAllocate) +{ + /// Valid magic, but combined_length implies blob size 0x24+8=44, while caller length is huge. + const UInt8 header[8] = {0x00, 0x00, 0x00, 0x24, 0xD1, 0xD3, 0x39, 0x64}; + ReadBufferFromMemory file(header, sizeof(header)); + + try + { + readDeletionVectorFromPuffin(file, 0, large_declared_length); + FAIL() << "Expected exception"; + } + catch (const Exception & e) + { + EXPECT_EQ(e.code(), ErrorCodes::BAD_ARGUMENTS); + EXPECT_NE(e.message().find("does not match combined length"), std::string::npos); + } +} + +TEST(PuffinDeletionVectorEnvelope, ReadsValidBlobAfterEnvelopePeek) +{ + ReadBufferFromMemory file(two_position_dv_blob, sizeof(two_position_dv_blob)); + const auto positions = readDeletionVectorFromPuffin( + file, 0, static_cast(sizeof(two_position_dv_blob)), /*expected_cardinality=*/2); + + ASSERT_EQ(positions.size(), 2u); + EXPECT_EQ(positions[0], 2u); + EXPECT_EQ(positions[1], 5u); +} + +TEST(PuffinDeletionVectorEnvelope, RejectsInternallyInconsistentRoaringAfterReadSafe) +{ + /// Valid DV envelope + CRC wrapping a portable roaring that `readSafe` accepts but + /// `roaring_bitmap_internal_validate` rejects (duplicate values in an array container). + /// Bytes from croaring's robust_deserialization_unit `deserialize_unsorted_array` fixture. + constexpr UInt8 inconsistent_roaring_dv_blob[] = { + 0x00, 0x00, 0x00, 0x1D, 0xD1, 0xD3, 0x39, 0x64, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x3B, 0x30, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x6E, 0x0E, 0x9B, 0x12, + }; + + const std::string_view blob( + reinterpret_cast(inconsistent_roaring_dv_blob), sizeof(inconsistent_roaring_dv_blob)); + + try + { + deserializeDeletionVectorV1Blob(blob, /*expected_cardinality=*/2); + FAIL() << "Expected exception"; + } + catch (const Exception & e) + { + EXPECT_EQ(e.code(), ErrorCodes::BAD_ARGUMENTS); + EXPECT_NE(e.message().find("failed internal validation"), std::string::npos); + } +} diff --git a/src/Storages/ObjectStorage/DataLakes/tests/gtest_puffin_deletion_vector_footer_bind.cpp b/src/Storages/ObjectStorage/DataLakes/tests/gtest_puffin_deletion_vector_footer_bind.cpp new file mode 100644 index 000000000000..66dcd1e7ad8f --- /dev/null +++ b/src/Storages/ObjectStorage/DataLakes/tests/gtest_puffin_deletion_vector_footer_bind.cpp @@ -0,0 +1,258 @@ +#include + +#include +#include +#include + +using namespace DB; + +namespace DB +{ +namespace ErrorCodes +{ +extern const int BAD_ARGUMENTS; +} +} + +namespace +{ + +/// Two equal-cardinality deletion-vector-v1 blobs for different data files. +/// Blob A at offset 4 length 44 -> /data/file_a.parquet positions {2, 5} +/// Blob B at offset 48 length 44 -> /data/file_b.parquet positions {7, 9} +constexpr UInt8 two_equal_cardinality_dvs_puffin[] = { + 0x50, 0x46, 0x41, 0x31, 0x00, 0x00, 0x00, 0x24, 0xD1, 0xD3, 0x39, 0x64, 0x01, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x3A, 0x30, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x01, 0x00, 0x10, 0x00, 0x00, 0x00, 0x02, 0x00, 0x05, 0x00, 0x2C, 0xDB, 0x9F, 0xC1, + 0x00, 0x00, 0x00, 0x24, 0xD1, 0xD3, 0x39, 0x64, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x3A, 0x30, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0x00, + 0x10, 0x00, 0x00, 0x00, 0x07, 0x00, 0x09, 0x00, 0xB7, 0xB0, 0x20, 0xFF, 0x50, 0x46, 0x41, 0x31, + 0x7B, 0x22, 0x62, 0x6C, 0x6F, 0x62, 0x73, 0x22, 0x3A, 0x20, 0x5B, 0x7B, 0x22, 0x74, 0x79, 0x70, + 0x65, 0x22, 0x3A, 0x20, 0x22, 0x64, 0x65, 0x6C, 0x65, 0x74, 0x69, 0x6F, 0x6E, 0x2D, 0x76, 0x65, + 0x63, 0x74, 0x6F, 0x72, 0x2D, 0x76, 0x31, 0x22, 0x2C, 0x20, 0x22, 0x66, 0x69, 0x65, 0x6C, 0x64, + 0x73, 0x22, 0x3A, 0x20, 0x5B, 0x5D, 0x2C, 0x20, 0x22, 0x73, 0x6E, 0x61, 0x70, 0x73, 0x68, 0x6F, + 0x74, 0x2D, 0x69, 0x64, 0x22, 0x3A, 0x20, 0x2D, 0x31, 0x2C, 0x20, 0x22, 0x73, 0x65, 0x71, 0x75, + 0x65, 0x6E, 0x63, 0x65, 0x2D, 0x6E, 0x75, 0x6D, 0x62, 0x65, 0x72, 0x22, 0x3A, 0x20, 0x2D, 0x31, + 0x2C, 0x20, 0x22, 0x6F, 0x66, 0x66, 0x73, 0x65, 0x74, 0x22, 0x3A, 0x20, 0x34, 0x2C, 0x20, 0x22, + 0x6C, 0x65, 0x6E, 0x67, 0x74, 0x68, 0x22, 0x3A, 0x20, 0x34, 0x34, 0x2C, 0x20, 0x22, 0x70, 0x72, + 0x6F, 0x70, 0x65, 0x72, 0x74, 0x69, 0x65, 0x73, 0x22, 0x3A, 0x20, 0x7B, 0x22, 0x72, 0x65, 0x66, + 0x65, 0x72, 0x65, 0x6E, 0x63, 0x65, 0x64, 0x2D, 0x64, 0x61, 0x74, 0x61, 0x2D, 0x66, 0x69, 0x6C, + 0x65, 0x22, 0x3A, 0x20, 0x22, 0x2F, 0x64, 0x61, 0x74, 0x61, 0x2F, 0x66, 0x69, 0x6C, 0x65, 0x5F, + 0x61, 0x2E, 0x70, 0x61, 0x72, 0x71, 0x75, 0x65, 0x74, 0x22, 0x2C, 0x20, 0x22, 0x63, 0x61, 0x72, + 0x64, 0x69, 0x6E, 0x61, 0x6C, 0x69, 0x74, 0x79, 0x22, 0x3A, 0x20, 0x22, 0x32, 0x22, 0x7D, 0x7D, + 0x2C, 0x20, 0x7B, 0x22, 0x74, 0x79, 0x70, 0x65, 0x22, 0x3A, 0x20, 0x22, 0x64, 0x65, 0x6C, 0x65, + 0x74, 0x69, 0x6F, 0x6E, 0x2D, 0x76, 0x65, 0x63, 0x74, 0x6F, 0x72, 0x2D, 0x76, 0x31, 0x22, 0x2C, + 0x20, 0x22, 0x66, 0x69, 0x65, 0x6C, 0x64, 0x73, 0x22, 0x3A, 0x20, 0x5B, 0x5D, 0x2C, 0x20, 0x22, + 0x73, 0x6E, 0x61, 0x70, 0x73, 0x68, 0x6F, 0x74, 0x2D, 0x69, 0x64, 0x22, 0x3A, 0x20, 0x2D, 0x31, + 0x2C, 0x20, 0x22, 0x73, 0x65, 0x71, 0x75, 0x65, 0x6E, 0x63, 0x65, 0x2D, 0x6E, 0x75, 0x6D, 0x62, + 0x65, 0x72, 0x22, 0x3A, 0x20, 0x2D, 0x31, 0x2C, 0x20, 0x22, 0x6F, 0x66, 0x66, 0x73, 0x65, 0x74, + 0x22, 0x3A, 0x20, 0x34, 0x38, 0x2C, 0x20, 0x22, 0x6C, 0x65, 0x6E, 0x67, 0x74, 0x68, 0x22, 0x3A, + 0x20, 0x34, 0x34, 0x2C, 0x20, 0x22, 0x70, 0x72, 0x6F, 0x70, 0x65, 0x72, 0x74, 0x69, 0x65, 0x73, + 0x22, 0x3A, 0x20, 0x7B, 0x22, 0x72, 0x65, 0x66, 0x65, 0x72, 0x65, 0x6E, 0x63, 0x65, 0x64, 0x2D, + 0x64, 0x61, 0x74, 0x61, 0x2D, 0x66, 0x69, 0x6C, 0x65, 0x22, 0x3A, 0x20, 0x22, 0x2F, 0x64, 0x61, + 0x74, 0x61, 0x2F, 0x66, 0x69, 0x6C, 0x65, 0x5F, 0x62, 0x2E, 0x70, 0x61, 0x72, 0x71, 0x75, 0x65, + 0x74, 0x22, 0x2C, 0x20, 0x22, 0x63, 0x61, 0x72, 0x64, 0x69, 0x6E, 0x61, 0x6C, 0x69, 0x74, 0x79, + 0x22, 0x3A, 0x20, 0x22, 0x32, 0x22, 0x7D, 0x7D, 0x5D, 0x7D, 0x9A, 0x01, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x50, 0x46, 0x41, 0x31, +}; + +std::vector readFixtureBlobs() +{ + ReadBufferFromMemory file(two_equal_cardinality_dvs_puffin, sizeof(two_equal_cardinality_dvs_puffin)); + return readPuffinFooterBlobsFromSeekable(file, sizeof(two_equal_cardinality_dvs_puffin)); +} + +} + +TEST(PuffinDeletionVectorFooterBind, AcceptsMatchingOffsetReferencedFileAndCardinality) +{ + const auto blobs = readFixtureBlobs(); + const auto & blob = bindDeletionVectorBlob(blobs, /*content_offset=*/4, /*content_size_in_bytes=*/44, "/data/file_a.parquet", 2); + EXPECT_EQ(blob.type, "deletion-vector-v1"); + EXPECT_EQ(blob.properties.at("referenced-data-file"), "/data/file_a.parquet"); + + ReadBufferFromMemory file(two_equal_cardinality_dvs_puffin, sizeof(two_equal_cardinality_dvs_puffin)); + const auto positions = readDeletionVectorFromPuffin(file, 4, 44, /*expected_cardinality=*/2); + ASSERT_EQ(positions.size(), 2u); + EXPECT_EQ(positions[0], 2u); + EXPECT_EQ(positions[1], 5u); +} + +TEST(PuffinDeletionVectorFooterBind, RejectsSwappedOffsetForOtherFilesEqualCardinalityBlob) +{ + /// Manifest for file A points at file B's slice (same length and cardinality). + const auto blobs = readFixtureBlobs(); + try + { + bindDeletionVectorBlob(blobs, /*content_offset=*/48, /*content_size_in_bytes=*/44, "/data/file_a.parquet", 2); + FAIL() << "Expected exception"; + } + catch (const Exception & e) + { + EXPECT_EQ(e.code(), ErrorCodes::BAD_ARGUMENTS); + EXPECT_NE(e.message().find("referenced-data-file"), std::string::npos); + } +} + +TEST(PuffinDeletionVectorFooterBind, RejectsCardinalityMismatch) +{ + const auto blobs = readFixtureBlobs(); + try + { + bindDeletionVectorBlob(blobs, /*content_offset=*/4, /*content_size_in_bytes=*/44, "/data/file_a.parquet", 3); + FAIL() << "Expected exception"; + } + catch (const Exception & e) + { + EXPECT_EQ(e.code(), ErrorCodes::BAD_ARGUMENTS); + EXPECT_NE(e.message().find("cardinality"), std::string::npos); + } +} + +TEST(PuffinDeletionVectorFooterBind, RejectsMissingOffset) +{ + const auto blobs = readFixtureBlobs(); + try + { + bindDeletionVectorBlob(blobs, /*content_offset=*/99, /*content_size_in_bytes=*/44, "/data/file_a.parquet", 2); + FAIL() << "Expected exception"; + } + catch (const Exception & e) + { + EXPECT_EQ(e.code(), ErrorCodes::BAD_ARGUMENTS); + EXPECT_NE(e.message().find("No Puffin footer blob"), std::string::npos); + } +} + +namespace +{ + +/// Sketch placeholder (16 bytes) then deletion-vector-v1 for positions {2, 5} (44 bytes). +/// Keep in sync with tests/queries/0_stateless/data_puffin/mixed_blob_types.puffin. +constexpr UInt8 mixed_sketch_and_dv_puffin[] = { + 0x50, 0x46, 0x41, 0x31, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x24, 0xD1, 0xD3, 0x39, 0x64, 0x01, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x3A, 0x30, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x01, 0x00, 0x10, 0x00, 0x00, 0x00, 0x02, 0x00, 0x05, 0x00, 0x2C, 0xDB, 0x9F, 0xC1, + 0x50, 0x46, 0x41, 0x31, 0x7B, 0x22, 0x62, 0x6C, 0x6F, 0x62, 0x73, 0x22, 0x3A, 0x20, 0x5B, 0x7B, + 0x22, 0x74, 0x79, 0x70, 0x65, 0x22, 0x3A, 0x20, 0x22, 0x61, 0x70, 0x61, 0x63, 0x68, 0x65, 0x2D, + 0x64, 0x61, 0x74, 0x61, 0x73, 0x6B, 0x65, 0x74, 0x63, 0x68, 0x65, 0x73, 0x2D, 0x74, 0x68, 0x65, + 0x74, 0x61, 0x2D, 0x76, 0x31, 0x22, 0x2C, 0x20, 0x22, 0x66, 0x69, 0x65, 0x6C, 0x64, 0x73, 0x22, + 0x3A, 0x20, 0x5B, 0x5D, 0x2C, 0x20, 0x22, 0x73, 0x6E, 0x61, 0x70, 0x73, 0x68, 0x6F, 0x74, 0x2D, + 0x69, 0x64, 0x22, 0x3A, 0x20, 0x2D, 0x31, 0x2C, 0x20, 0x22, 0x73, 0x65, 0x71, 0x75, 0x65, 0x6E, + 0x63, 0x65, 0x2D, 0x6E, 0x75, 0x6D, 0x62, 0x65, 0x72, 0x22, 0x3A, 0x20, 0x2D, 0x31, 0x2C, 0x20, + 0x22, 0x6F, 0x66, 0x66, 0x73, 0x65, 0x74, 0x22, 0x3A, 0x20, 0x34, 0x2C, 0x20, 0x22, 0x6C, 0x65, + 0x6E, 0x67, 0x74, 0x68, 0x22, 0x3A, 0x20, 0x31, 0x36, 0x2C, 0x20, 0x22, 0x70, 0x72, 0x6F, 0x70, + 0x65, 0x72, 0x74, 0x69, 0x65, 0x73, 0x22, 0x3A, 0x20, 0x7B, 0x7D, 0x7D, 0x2C, 0x20, 0x7B, 0x22, + 0x74, 0x79, 0x70, 0x65, 0x22, 0x3A, 0x20, 0x22, 0x64, 0x65, 0x6C, 0x65, 0x74, 0x69, 0x6F, 0x6E, + 0x2D, 0x76, 0x65, 0x63, 0x74, 0x6F, 0x72, 0x2D, 0x76, 0x31, 0x22, 0x2C, 0x20, 0x22, 0x66, 0x69, + 0x65, 0x6C, 0x64, 0x73, 0x22, 0x3A, 0x20, 0x5B, 0x5D, 0x2C, 0x20, 0x22, 0x73, 0x6E, 0x61, 0x70, + 0x73, 0x68, 0x6F, 0x74, 0x2D, 0x69, 0x64, 0x22, 0x3A, 0x20, 0x2D, 0x31, 0x2C, 0x20, 0x22, 0x73, + 0x65, 0x71, 0x75, 0x65, 0x6E, 0x63, 0x65, 0x2D, 0x6E, 0x75, 0x6D, 0x62, 0x65, 0x72, 0x22, 0x3A, + 0x20, 0x2D, 0x31, 0x2C, 0x20, 0x22, 0x6F, 0x66, 0x66, 0x73, 0x65, 0x74, 0x22, 0x3A, 0x20, 0x32, + 0x30, 0x2C, 0x20, 0x22, 0x6C, 0x65, 0x6E, 0x67, 0x74, 0x68, 0x22, 0x3A, 0x20, 0x34, 0x34, 0x2C, + 0x20, 0x22, 0x70, 0x72, 0x6F, 0x70, 0x65, 0x72, 0x74, 0x69, 0x65, 0x73, 0x22, 0x3A, 0x20, 0x7B, + 0x22, 0x72, 0x65, 0x66, 0x65, 0x72, 0x65, 0x6E, 0x63, 0x65, 0x64, 0x2D, 0x64, 0x61, 0x74, 0x61, + 0x2D, 0x66, 0x69, 0x6C, 0x65, 0x22, 0x3A, 0x20, 0x22, 0x2F, 0x64, 0x61, 0x74, 0x61, 0x2F, 0x74, + 0x61, 0x62, 0x6C, 0x65, 0x2F, 0x70, 0x61, 0x72, 0x74, 0x2D, 0x30, 0x30, 0x30, 0x30, 0x30, 0x2E, + 0x70, 0x61, 0x72, 0x71, 0x75, 0x65, 0x74, 0x22, 0x2C, 0x20, 0x22, 0x63, 0x61, 0x72, 0x64, 0x69, + 0x6E, 0x61, 0x6C, 0x69, 0x74, 0x79, 0x22, 0x3A, 0x20, 0x22, 0x32, 0x22, 0x7D, 0x7D, 0x5D, 0x7D, + 0x6C, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x50, 0x46, 0x41, 0x31, +}; + +} + +TEST(PuffinDeletionVectorFooterBind, AcceptsMixedBlobTypesAndBindsDeletionVector) +{ + ReadBufferFromMemory file(mixed_sketch_and_dv_puffin, sizeof(mixed_sketch_and_dv_puffin)); + const auto blobs = readPuffinFooterBlobsFromSeekable(file, sizeof(mixed_sketch_and_dv_puffin)); + ASSERT_EQ(blobs.size(), 2u); + EXPECT_EQ(blobs[0].type, "apache-datasketches-theta-v1"); + EXPECT_EQ(blobs[1].type, "deletion-vector-v1"); + + const auto & blob = bindDeletionVectorBlob( + blobs, /*content_offset=*/20, /*content_size_in_bytes=*/44, "/data/table/part-00000.parquet", 2); + EXPECT_EQ(blob.type, "deletion-vector-v1"); + + ReadBufferFromMemory payload(mixed_sketch_and_dv_puffin, sizeof(mixed_sketch_and_dv_puffin)); + const auto positions = readDeletionVectorFromPuffin(payload, 20, 44, /*expected_cardinality=*/2); + ASSERT_EQ(positions.size(), 2u); + EXPECT_EQ(positions[0], 2u); + EXPECT_EQ(positions[1], 5u); +} + +TEST(PuffinDeletionVectorFooterBind, RejectsBindToNonDeletionVectorBlob) +{ + ReadBufferFromMemory file(mixed_sketch_and_dv_puffin, sizeof(mixed_sketch_and_dv_puffin)); + const auto blobs = readPuffinFooterBlobsFromSeekable(file, sizeof(mixed_sketch_and_dv_puffin)); + try + { + bindDeletionVectorBlob(blobs, /*content_offset=*/4, /*content_size_in_bytes=*/16, "/data/table/part-00000.parquet", 2); + FAIL() << "Expected exception"; + } + catch (const Exception & e) + { + EXPECT_EQ(e.code(), ErrorCodes::BAD_ARGUMENTS); + EXPECT_NE(e.message().find("expected deletion-vector-v1"), std::string::npos); + } +} + +TEST(PuffinDeletionVectorFooterBind, RejectsColumnScopedFields) +{ + auto blobs = readFixtureBlobs(); + ASSERT_FALSE(blobs.empty()); + blobs[0].fields = {1}; + + try + { + bindDeletionVectorBlob(blobs, /*content_offset=*/4, /*content_size_in_bytes=*/44, "/data/file_a.parquet", 2); + FAIL() << "Expected exception"; + } + catch (const Exception & e) + { + EXPECT_EQ(e.code(), ErrorCodes::BAD_ARGUMENTS); + EXPECT_NE(e.message().find("unsupported non-empty 'fields'"), std::string::npos); + } +} + +TEST(PuffinDeletionVectorFooterBind, AcceptsIcebergRowPositionFieldMarker) +{ + auto blobs = readFixtureBlobs(); + ASSERT_FALSE(blobs.empty()); + /// Spark-written file-scoped DVs use fields=[_pos] (2147483645), not []. + blobs[0].fields = {ICEBERG_ROW_POSITION_FIELD_ID}; + + EXPECT_NO_THROW( + bindDeletionVectorBlob(blobs, /*content_offset=*/4, /*content_size_in_bytes=*/44, "/data/file_a.parquet", 2)); +} + +TEST(PuffinDeletionVectorFooterBind, RejectsNonMinusOneSnapshotOrSequence) +{ + auto blobs = readFixtureBlobs(); + ASSERT_FALSE(blobs.empty()); + blobs[0].snapshot_id = 0; + + try + { + bindDeletionVectorBlob(blobs, /*content_offset=*/4, /*content_size_in_bytes=*/44, "/data/file_a.parquet", 2); + FAIL() << "Expected exception"; + } + catch (const Exception & e) + { + EXPECT_EQ(e.code(), ErrorCodes::BAD_ARGUMENTS); + EXPECT_NE(e.message().find("snapshot-id and sequence-number must be -1"), std::string::npos); + } + + blobs = readFixtureBlobs(); + blobs[0].sequence_number = 0; + try + { + bindDeletionVectorBlob(blobs, /*content_offset=*/4, /*content_size_in_bytes=*/44, "/data/file_a.parquet", 2); + FAIL() << "Expected exception"; + } + catch (const Exception & e) + { + EXPECT_EQ(e.code(), ErrorCodes::BAD_ARGUMENTS); + EXPECT_NE(e.message().find("snapshot-id and sequence-number must be -1"), std::string::npos); + } +} diff --git a/src/Storages/ObjectStorage/DataLakes/tests/gtest_puffin_dv_referenced_data_file.cpp b/src/Storages/ObjectStorage/DataLakes/tests/gtest_puffin_dv_referenced_data_file.cpp new file mode 100644 index 000000000000..dba4b03f5244 --- /dev/null +++ b/src/Storages/ObjectStorage/DataLakes/tests/gtest_puffin_dv_referenced_data_file.cpp @@ -0,0 +1,71 @@ +#include + +#include + +#if USE_AVRO + +#include +#include +#include + +using namespace DB; +using namespace DB::Iceberg; + +namespace DB::ErrorCodes +{ +extern const int ICEBERG_SPECIFICATION_VIOLATION; +} + +TEST(PuffinDeletionVectorReferencedDataFile, AcceptsNonEmptyDirectField) +{ + const auto path = IcebergPathFromMetadata::deserialize("/data/file.parquet"); + const auto manifest = IcebergPathFromMetadata::deserialize("/meta/manifest.avro"); + EXPECT_NO_THROW(requireDirectReferencedDataFileForPuffinDeletionVector(/*set_from_referenced_data_file_field=*/true, path, manifest)); +} + +TEST(PuffinDeletionVectorReferencedDataFile, RejectsBoundsOnlyFallback) +{ + const auto path = IcebergPathFromMetadata::deserialize("/data/file.parquet"); + const auto manifest = IcebergPathFromMetadata::deserialize("/meta/manifest.avro"); + try + { + requireDirectReferencedDataFileForPuffinDeletionVector(/*set_from_referenced_data_file_field=*/false, path, manifest); + FAIL() << "Expected ICEBERG_SPECIFICATION_VIOLATION"; + } + catch (const Exception & e) + { + EXPECT_EQ(e.code(), ErrorCodes::ICEBERG_SPECIFICATION_VIOLATION); + EXPECT_TRUE(e.message().find("referenced_data_file") != std::string::npos); + } +} + +TEST(PuffinDeletionVectorReferencedDataFile, RejectsMissingPath) +{ + const auto manifest = IcebergPathFromMetadata::deserialize("/meta/manifest.avro"); + try + { + requireDirectReferencedDataFileForPuffinDeletionVector(/*set_from_referenced_data_file_field=*/true, std::nullopt, manifest); + FAIL() << "Expected ICEBERG_SPECIFICATION_VIOLATION"; + } + catch (const Exception & e) + { + EXPECT_EQ(e.code(), ErrorCodes::ICEBERG_SPECIFICATION_VIOLATION); + } +} + +TEST(PuffinDeletionVectorReferencedDataFile, RejectsEmptyPath) +{ + const auto empty_path = IcebergPathFromMetadata::deserialize(""); + const auto manifest = IcebergPathFromMetadata::deserialize("/meta/manifest.avro"); + try + { + requireDirectReferencedDataFileForPuffinDeletionVector(/*set_from_referenced_data_file_field=*/true, empty_path, manifest); + FAIL() << "Expected ICEBERG_SPECIFICATION_VIOLATION"; + } + catch (const Exception & e) + { + EXPECT_EQ(e.code(), ErrorCodes::ICEBERG_SPECIFICATION_VIOLATION); + } +} + +#endif diff --git a/src/Storages/ObjectStorage/DataLakes/tests/gtest_puffin_files_cache_clone.cpp b/src/Storages/ObjectStorage/DataLakes/tests/gtest_puffin_files_cache_clone.cpp new file mode 100644 index 000000000000..584a8f76b984 --- /dev/null +++ b/src/Storages/ObjectStorage/DataLakes/tests/gtest_puffin_files_cache_clone.cpp @@ -0,0 +1,92 @@ +#include + +#include +#include +#include + +using namespace DB; + +namespace CurrentMetrics +{ +extern const Metric PuffinFilesCacheBytes; +extern const Metric PuffinFilesCacheFiles; +} + +namespace +{ + +DataLakeObjectMetadata::ExcludedRowsPtr makeExcludedRows(const std::vector & positions) +{ + auto excluded_rows = std::make_shared(); + for (size_t position : positions) + excluded_rows->add(position); + return excluded_rows; +} + +} + +TEST(PuffinFilesCacheClone, CacheHitReturnsIndependentCopy) +{ + PuffinFilesCache cache("SLRU", 1'000'000, 100, 0.5); + + const auto key = PuffinFilesCache::tryCreateKey( + "Local:////test-prefix", "puffin.bin", "etag-1", 100, 200, "data/file-a.parquet", 3, 100); + ASSERT_TRUE(key.has_value()); + + size_t load_calls = 0; + auto load_fn = [&]() + { + ++load_calls; + return makeExcludedRows({1, 5, 10}); + }; + + auto first = cache.getOrSetDeletionVector(*key, load_fn); + auto second = cache.getOrSetDeletionVector(*key, load_fn); + + ASSERT_EQ(load_calls, 1); + ASSERT_NE(first, second); + EXPECT_TRUE(first->rb_contains(1)); + EXPECT_TRUE(first->rb_contains(5)); + EXPECT_TRUE(first->rb_contains(10)); + EXPECT_FALSE(first->rb_contains(99)); + + first->add(99); + + EXPECT_FALSE(second->rb_contains(99)); + + auto third = cache.getOrSetDeletionVector(*key, load_fn); + ASSERT_NE(third, first); + EXPECT_FALSE(third->rb_contains(99)); + EXPECT_TRUE(third->rb_contains(10)); +} + +TEST(PuffinFilesCacheClone, EmptyExcludedRowsReturnsNullptr) +{ + PuffinFilesCache cache("SLRU", 1'000'000, 100, 0.5); + + const auto key = PuffinFilesCache::tryCreateKey( + "Local:////test-prefix", "puffin.bin", "etag-1", 100, 200, "data/file-a.parquet", 3, 100); + ASSERT_TRUE(key.has_value()); + + const auto files_before = CurrentMetrics::get(CurrentMetrics::PuffinFilesCacheFiles); + const auto bytes_before = CurrentMetrics::get(CurrentMetrics::PuffinFilesCacheBytes); + + size_t load_calls = 0; + auto load_fn = [&]() + { + ++load_calls; + return DataLakeObjectMetadata::ExcludedRowsPtr{}; + }; + + auto first = cache.getOrSetDeletionVector(*key, load_fn); + auto second = cache.getOrSetDeletionVector(*key, load_fn); + + const auto expected_weight = PuffinFilesCacheCell::calculateMemorySize( + /*is_empty_deletion_vector_=*/true, nullptr, key->approximateMemoryBytes()); + + EXPECT_EQ(first, nullptr); + EXPECT_EQ(second, nullptr); + EXPECT_EQ(load_calls, 1); + EXPECT_EQ(CurrentMetrics::get(CurrentMetrics::PuffinFilesCacheFiles), files_before + 1); + EXPECT_EQ(CurrentMetrics::get(CurrentMetrics::PuffinFilesCacheBytes), bytes_before + static_cast(expected_weight)); +} diff --git a/src/Storages/ObjectStorage/DataLakes/tests/gtest_puffin_files_cache_key.cpp b/src/Storages/ObjectStorage/DataLakes/tests/gtest_puffin_files_cache_key.cpp new file mode 100644 index 000000000000..0c5352ba741d --- /dev/null +++ b/src/Storages/ObjectStorage/DataLakes/tests/gtest_puffin_files_cache_key.cpp @@ -0,0 +1,199 @@ +#include + +#include +#include +#include + +using namespace DB; + +namespace +{ + +constexpr const char * kDefaultStorageIdentity = "Local:////test-prefix"; + +std::optional makeKey( + const String & referenced_data_file, + UInt64 expected_cardinality = 2, + UInt64 data_file_record_count = 100, + const String & storage_identity = kDefaultStorageIdentity) +{ + return PuffinFilesCache::tryCreateKey( + storage_identity, + "puffin.bin", + "etag-1", + 100, + 200, + referenced_data_file, + expected_cardinality, + data_file_record_count); +} + +DataLakeObjectMetadata::ExcludedRowsPtr makeExcludedRows(const std::vector & positions) +{ + auto excluded_rows = std::make_shared(); + for (size_t position : positions) + excluded_rows->add(position); + return excluded_rows; +} + +} + +TEST(PuffinFilesCacheKey, SamePuffinSliceDifferentReferencedDataFile) +{ + const auto key1 = makeKey("data/file-a.parquet"); + const auto key2 = makeKey("data/file-b.parquet"); + + ASSERT_TRUE(key1.has_value()); + ASSERT_TRUE(key2.has_value()); + EXPECT_NE(*key1, *key2); + EXPECT_NE(PuffinFilesCacheKeyHash{}(*key1), PuffinFilesCacheKeyHash{}(*key2)); +} + +TEST(PuffinFilesCacheKey, SameReferencedDataFileProducesEqualKeys) +{ + const auto key1 = makeKey("data/file-a.parquet"); + const auto key2 = makeKey("data/file-a.parquet"); + + ASSERT_TRUE(key1.has_value()); + ASSERT_TRUE(key2.has_value()); + EXPECT_EQ(*key1, *key2); + EXPECT_EQ(PuffinFilesCacheKeyHash{}(*key1), PuffinFilesCacheKeyHash{}(*key2)); +} + +TEST(PuffinFilesCacheKey, DifferentExpectedCardinalityProducesUnequalKeys) +{ + const auto key1 = makeKey("data/file-a.parquet", /*expected_cardinality=*/2); + const auto key2 = makeKey("data/file-a.parquet", /*expected_cardinality=*/3); + + ASSERT_TRUE(key1.has_value()); + ASSERT_TRUE(key2.has_value()); + EXPECT_NE(*key1, *key2); + EXPECT_NE(PuffinFilesCacheKeyHash{}(*key1), PuffinFilesCacheKeyHash{}(*key2)); +} + +TEST(PuffinFilesCacheKey, DifferentDataFileRecordCountProducesUnequalKeys) +{ + const auto key1 = makeKey("data/file-a.parquet", /*expected_cardinality=*/2, /*data_file_record_count=*/100); + const auto key2 = makeKey("data/file-a.parquet", /*expected_cardinality=*/2, /*data_file_record_count=*/50); + + ASSERT_TRUE(key1.has_value()); + ASSERT_TRUE(key2.has_value()); + EXPECT_NE(*key1, *key2); + EXPECT_NE(PuffinFilesCacheKeyHash{}(*key1), PuffinFilesCacheKeyHash{}(*key2)); +} + +TEST(PuffinFilesCacheKey, DifferentStorageIdentityProducesUnequalKeys) +{ + const auto key1 = makeKey("data/file-a.parquet", 2, 100, "S3://http://minio-a:9001/bucket/warehouse"); + const auto key2 = makeKey("data/file-a.parquet", 2, 100, "S3://http://minio-a:9001/bucket-b/warehouse"); + + ASSERT_TRUE(key1.has_value()); + ASSERT_TRUE(key2.has_value()); + EXPECT_NE(*key1, *key2); + EXPECT_NE(PuffinFilesCacheKeyHash{}(*key1), PuffinFilesCacheKeyHash{}(*key2)); +} + +TEST(PuffinFilesCacheKey, DifferentEndpointSameBucketPrefixProducesUnequalKeys) +{ + /// Same bucket + prefix, different getDescription() (S3 endpoint). + const auto key1 = makeKey("data/file-a.parquet", 2, 100, "S3://http://minio-a:9001/bucket/warehouse"); + const auto key2 = makeKey("data/file-a.parquet", 2, 100, "S3://http://minio-b:9001/bucket/warehouse"); + + ASSERT_TRUE(key1.has_value()); + ASSERT_TRUE(key2.has_value()); + EXPECT_NE(*key1, *key2); + EXPECT_NE(PuffinFilesCacheKeyHash{}(*key1), PuffinFilesCacheKeyHash{}(*key2)); +} + +TEST(PuffinFilesCacheKey, EmptyEtagReturnsNullopt) +{ + const auto key = PuffinFilesCache::tryCreateKey( + kDefaultStorageIdentity, "puffin.bin", "", 100, 200, "data/file-a.parquet", 2, 100); + EXPECT_FALSE(key.has_value()); +} + +TEST(PuffinFilesCacheKey, DifferentStorageIdentityDoesNotHitShare) +{ + PuffinFilesCache cache("SLRU", 1'000'000, 100, 0.5); + + const auto key_a = makeKey("data/file-a.parquet", 2, 100, "S3://http://minio-a:9001/bucket/warehouse"); + const auto key_b = makeKey("data/file-a.parquet", 2, 100, "S3://http://minio-a:9001/bucket-b/warehouse"); + ASSERT_TRUE(key_a.has_value()); + ASSERT_TRUE(key_b.has_value()); + + size_t load_a_calls = 0; + size_t load_b_calls = 0; + + auto first = cache.getOrSetDeletionVector(*key_a, [&]() + { + ++load_a_calls; + return makeExcludedRows({1, 2}); + }); + + auto second = cache.getOrSetDeletionVector(*key_b, [&]() + { + ++load_b_calls; + return makeExcludedRows({10, 20}); + }); + + ASSERT_EQ(load_a_calls, 1); + ASSERT_EQ(load_b_calls, 1); + ASSERT_TRUE(first); + ASSERT_TRUE(second); + EXPECT_TRUE(first->rb_contains(1)); + EXPECT_FALSE(first->rb_contains(10)); + EXPECT_TRUE(second->rb_contains(10)); + EXPECT_FALSE(second->rb_contains(1)); + + auto third = cache.getOrSetDeletionVector(*key_b, [&]() + { + ++load_b_calls; + return makeExcludedRows({99}); + }); + + ASSERT_EQ(load_b_calls, 1); + ASSERT_TRUE(third); + EXPECT_TRUE(third->rb_contains(10)); + EXPECT_FALSE(third->rb_contains(99)); +} + +TEST(PuffinFilesCacheKey, DifferentEndpointDoesNotHitShare) +{ + PuffinFilesCache cache("SLRU", 1'000'000, 100, 0.5); + + const auto key_a = makeKey("data/file-a.parquet", 2, 100, "S3://http://minio-a:9001/bucket/warehouse"); + const auto key_b = makeKey("data/file-a.parquet", 2, 100, "S3://http://minio-b:9001/bucket/warehouse"); + ASSERT_TRUE(key_a.has_value()); + ASSERT_TRUE(key_b.has_value()); + + size_t load_a_calls = 0; + size_t load_b_calls = 0; + + auto first = cache.getOrSetDeletionVector(*key_a, [&]() + { + ++load_a_calls; + return makeExcludedRows({1, 2}); + }); + + auto second = cache.getOrSetDeletionVector(*key_b, [&]() + { + ++load_b_calls; + return makeExcludedRows({10, 20}); + }); + + ASSERT_EQ(load_a_calls, 1); + ASSERT_EQ(load_b_calls, 1); + ASSERT_TRUE(first); + ASSERT_TRUE(second); + EXPECT_TRUE(first->rb_contains(1)); + EXPECT_FALSE(first->rb_contains(10)); + EXPECT_TRUE(second->rb_contains(10)); + EXPECT_FALSE(second->rb_contains(1)); +} + +TEST(PuffinFilesCacheKey, MakeStorageIdentityIncludesDescription) +{ + LocalObjectStorage storage(LocalObjectStorageSettings("disk", "warehouse", /*read_only=*/true)); + const auto identity = PuffinFilesCache::makeStorageIdentity(storage); + EXPECT_EQ(identity, "Local://" + storage.getDescription() + "//warehouse"); +} diff --git a/src/Storages/ObjectStorage/DataLakes/tests/gtest_puffin_files_cache_metrics.cpp b/src/Storages/ObjectStorage/DataLakes/tests/gtest_puffin_files_cache_metrics.cpp new file mode 100644 index 000000000000..91b730bf83d7 --- /dev/null +++ b/src/Storages/ObjectStorage/DataLakes/tests/gtest_puffin_files_cache_metrics.cpp @@ -0,0 +1,203 @@ +#include + +#include +#include +#include +#include + +#include +#include +#include +#include + +using namespace DB; + +namespace ProfileEvents +{ +extern const Event PuffinFilesCacheHits; +extern const Event PuffinFilesCacheMisses; +} + +namespace +{ + +DataLakeObjectMetadata::ExcludedRowsPtr makeExcludedRows(const std::vector & positions) +{ + auto excluded_rows = std::make_shared(); + for (size_t position : positions) + excluded_rows->add(position); + return excluded_rows; +} + +} + +TEST(PuffinFilesCacheMetrics, ClearDuringLoadCountsAsMissNotHit) +{ + PuffinFilesCache cache("SLRU", 1'000'000, 100, 0.5); + + const auto key = PuffinFilesCache::tryCreateKey( + "Local:////test-prefix", "puffin.bin", "etag-1", 100, 200, "data/file-a.parquet", 1, 100); + ASSERT_TRUE(key.has_value()); + + auto & counters = CurrentThread::getProfileEvents(); + const auto hits_before = counters[ProfileEvents::PuffinFilesCacheHits].load(); + const auto misses_before = counters[ProfileEvents::PuffinFilesCacheMisses].load(); + + size_t load_calls = 0; + const auto result = cache.getOrSetDeletionVector( + *key, + [&]() + { + ++load_calls; + /// Simulate concurrent SYSTEM DROP while this key is loading: CacheBase then returns + /// `{value, false}` because the insert token was discarded. + cache.clear(); + return makeExcludedRows({1}); + }); + + ASSERT_EQ(load_calls, 1u); + ASSERT_NE(result, nullptr); + EXPECT_TRUE(result->rb_contains(1)); + EXPECT_EQ(counters[ProfileEvents::PuffinFilesCacheHits].load() - hits_before, 0u); + EXPECT_EQ(counters[ProfileEvents::PuffinFilesCacheMisses].load() - misses_before, 1u); + + /// Entry was not inserted after clear; the next lookup must load again. + const auto second = cache.getOrSetDeletionVector(*key, [&]() + { + ++load_calls; + return makeExcludedRows({1}); + }); + ASSERT_EQ(load_calls, 2u); + ASSERT_NE(second, nullptr); + EXPECT_EQ(counters[ProfileEvents::PuffinFilesCacheMisses].load() - misses_before, 2u); + EXPECT_EQ(counters[ProfileEvents::PuffinFilesCacheHits].load() - hits_before, 0u); +} + +TEST(PuffinFilesCacheMetrics, WaiterOfClearDiscardedLoadCountsAsMiss) +{ + PuffinFilesCache cache("SLRU", 1'000'000, 100, 0.5); + + const auto key = PuffinFilesCache::tryCreateKey( + "Local:////test-prefix", "puffin.bin", "etag-waiter", 100, 200, "data/file-w.parquet", 1, 100); + ASSERT_TRUE(key.has_value()); + + const auto hits_before = ProfileEvents::global_counters[ProfileEvents::PuffinFilesCacheHits].load(); + const auto misses_before = ProfileEvents::global_counters[ProfileEvents::PuffinFilesCacheMisses].load(); + + std::promise load_started; + auto load_started_future = load_started.get_future(); + + std::atomic load_calls{0}; + std::atomic waiter_load_called{false}; + std::atomic waiter_joined_insert_token{false}; + + std::thread producer( + [&]() + { + cache.getOrSetDeletionVector( + *key, + [&]() + { + ++load_calls; + load_started.set_value(); + + /// Wait until the waiter has acquired the same insert token (refcount >= 2) + /// before clear()+finish. A fixed sleep raced: if the producer finished first, + /// the waiter started a fresh load and this test became flaky. + const auto deadline = std::chrono::steady_clock::now() + std::chrono::seconds(5); + while (cache.getInsertTokenRefcount(*key) < 2) + { + if (std::chrono::steady_clock::now() >= deadline) + return makeExcludedRows({42}); + std::this_thread::yield(); + } + waiter_joined_insert_token.store(true); + + cache.clear(); + return makeExcludedRows({42}); + }); + }); + + ASSERT_EQ(load_started_future.wait_for(std::chrono::seconds(5)), std::future_status::ready); + + std::thread waiter( + [&]() + { + cache.getOrSetDeletionVector( + *key, + [&]() + { + waiter_load_called.store(true); + return makeExcludedRows({99}); + }); + }); + + producer.join(); + waiter.join(); + + ASSERT_TRUE(waiter_joined_insert_token.load()); + EXPECT_EQ(load_calls.load(), 1u); + EXPECT_FALSE(waiter_load_called.load()); + EXPECT_EQ(ProfileEvents::global_counters[ProfileEvents::PuffinFilesCacheHits] - hits_before, 0u); + EXPECT_EQ(ProfileEvents::global_counters[ProfileEvents::PuffinFilesCacheMisses] - misses_before, 2u); +} + +TEST(PuffinFilesCacheMetrics, OrdinaryHitAndMissCounters) +{ + PuffinFilesCache cache("SLRU", 1'000'000, 100, 0.5); + + const auto key = PuffinFilesCache::tryCreateKey( + "Local:////test-prefix", "puffin.bin", "etag-2", 100, 200, "data/file-b.parquet", 1, 100); + ASSERT_TRUE(key.has_value()); + + auto & counters = CurrentThread::getProfileEvents(); + const auto hits_before = counters[ProfileEvents::PuffinFilesCacheHits].load(); + const auto misses_before = counters[ProfileEvents::PuffinFilesCacheMisses].load(); + + size_t load_calls = 0; + auto load_fn = [&]() + { + ++load_calls; + return makeExcludedRows({7}); + }; + + ASSERT_NE(cache.getOrSetDeletionVector(*key, load_fn), nullptr); + ASSERT_NE(cache.getOrSetDeletionVector(*key, load_fn), nullptr); + + EXPECT_EQ(load_calls, 1u); + EXPECT_EQ(counters[ProfileEvents::PuffinFilesCacheMisses].load() - misses_before, 1u); + EXPECT_EQ(counters[ProfileEvents::PuffinFilesCacheHits].load() - hits_before, 1u); +} + +TEST(PuffinFilesCacheMetrics, HitRemainsHitWhenCacheClearedAfterLookup) +{ + PuffinFilesCache cache("SLRU", 1'000'000, 100, 0.5); + + const auto key = PuffinFilesCache::tryCreateKey( + "Local:////test-prefix", "puffin.bin", "etag-hit-clear", 100, 200, "data/file-c.parquet", 1, 100); + ASSERT_TRUE(key.has_value()); + + auto & counters = CurrentThread::getProfileEvents(); + size_t load_calls = 0; + auto load_fn = [&]() + { + ++load_calls; + return makeExcludedRows({3}); + }; + + ASSERT_NE(cache.getOrSetDeletionVector(*key, load_fn), nullptr); + + const auto hits_before = counters[ProfileEvents::PuffinFilesCacheHits].load(); + const auto misses_before = counters[ProfileEvents::PuffinFilesCacheMisses].load(); + + ASSERT_NE(cache.getOrSetDeletionVector(*key, load_fn), nullptr); + EXPECT_EQ(load_calls, 1u); + EXPECT_EQ(counters[ProfileEvents::PuffinFilesCacheHits].load() - hits_before, 1u); + EXPECT_EQ(counters[ProfileEvents::PuffinFilesCacheMisses].load() - misses_before, 0u); + + /// Clearing after the hit must not rewrite the already-recorded hit as a miss. The old + /// contains()-after-getOrSet path could race here with SYSTEM DROP PUFFIN FILES CACHE. + cache.clear(); + EXPECT_EQ(counters[ProfileEvents::PuffinFilesCacheHits].load() - hits_before, 1u); + EXPECT_EQ(counters[ProfileEvents::PuffinFilesCacheMisses].load() - misses_before, 0u); +} diff --git a/src/Storages/ObjectStorage/DataLakes/tests/gtest_puffin_files_cache_weight.cpp b/src/Storages/ObjectStorage/DataLakes/tests/gtest_puffin_files_cache_weight.cpp new file mode 100644 index 000000000000..c9916bdebeee --- /dev/null +++ b/src/Storages/ObjectStorage/DataLakes/tests/gtest_puffin_files_cache_weight.cpp @@ -0,0 +1,157 @@ +#include + +#include +#include +#include +#include +#include + +using namespace DB; + +namespace CurrentMetrics +{ +extern const Metric PuffinFilesCacheBytes; +extern const Metric PuffinFilesCacheFiles; +} + +namespace +{ + +DataLakeObjectMetadata::ExcludedRowsPtr makeLargeSparseExcludedRows(size_t keys = 33) +{ + auto excluded_rows = std::make_shared(); + for (size_t i = 0; i < keys; ++i) + excluded_rows->add((static_cast(i) + 1) << 32); + return excluded_rows; +} + +roaring::Roaring64Map makeLargeSparseRoaring64(size_t keys = 33) +{ + roaring::Roaring64Map bitmap; + for (size_t i = 0; i < keys; ++i) + bitmap.add((static_cast(i) + 1) << 32); + return bitmap; +} + +PuffinFilesCacheKey makeUniqueKey(size_t index, const String & long_suffix = "") +{ + auto key = PuffinFilesCache::tryCreateKey( + "Local:////test-prefix", + "puffin.bin", + "etag-" + std::to_string(index) + long_suffix, + 100, + 200, + "data/file-" + std::to_string(index) + ".parquet" + long_suffix, + /*expected_cardinality=*/0, + /*data_file_record_count=*/1000); + EXPECT_TRUE(key.has_value()); + return *key; +} + +} + +TEST(RoaringBitmapWithSmallSetMemory, LargeSparseBitmapAllocatedBytesExceedCardinalityEstimate) +{ + const auto excluded_rows = makeLargeSparseExcludedRows(); + + ASSERT_TRUE(excluded_rows->isLarge()); + EXPECT_GT(excluded_rows->getAllocatedBytes(), excluded_rows->size() * sizeof(size_t)); +} + +TEST(RoaringBitmapWithSmallSetMemory, LargeSparseBitmapAllocatedBytesExceedSerializedSize) +{ + const auto excluded_rows = makeLargeSparseExcludedRows(); + const auto serialized = makeLargeSparseRoaring64().getSizeInBytes(/*portable=*/true); + + ASSERT_TRUE(excluded_rows->isLarge()); + EXPECT_GT(excluded_rows->getAllocatedBytes(), serialized); +} + +TEST(PuffinFilesCacheWeight, UsesRoaringAllocatedBytesForWeight) +{ + const auto excluded_rows = makeLargeSparseExcludedRows(); + const auto key = PuffinFilesCache::tryCreateKey( + "Local:////test-prefix", "puffin.bin", "etag-1", 100, 200, "data/file-a.parquet", 33, 1000); + ASSERT_TRUE(key.has_value()); + + PuffinFilesCache cache("SLRU", 1'000'000, 100, 0.5); + cache.getOrSetDeletionVector(*key, [&]() { return excluded_rows; }); + + EXPECT_GE(CurrentMetrics::get(CurrentMetrics::PuffinFilesCacheBytes), excluded_rows->getAllocatedBytes()); +} + +TEST(PuffinFilesCacheWeight, EvictsSparseBitmapsUnderSmallByteLimit) +{ + /// Each sparse entry weighs far more than cardinality × sizeof(size_t); a tiny byte + /// limit must not retain many of them if weight uses allocated bytes. + const auto sample_key = makeUniqueKey(0); + const auto sample_rows = makeLargeSparseExcludedRows(16); + const auto entry_weight = PuffinFilesCacheCell::calculateMemorySize( + /*is_empty_deletion_vector_=*/false, sample_rows, sample_key.approximateMemoryBytes()); + ASSERT_GT(entry_weight, 0u); + + const size_t max_bytes = static_cast(entry_weight) + 64; // roughly one entry + PuffinFilesCache cache("SLRU", max_bytes, /*max_count=*/100, /*size_ratio=*/0.5); + + for (size_t i = 0; i < 8; ++i) + { + const auto key = makeUniqueKey(i); + cache.getOrSetDeletionVector(key, [&]() { return makeLargeSparseExcludedRows(16); }); + } + + EXPECT_LE(CurrentMetrics::get(CurrentMetrics::PuffinFilesCacheBytes), static_cast(max_bytes)); + EXPECT_LT(CurrentMetrics::get(CurrentMetrics::PuffinFilesCacheFiles), 8); +} + +TEST(PuffinFilesCacheWeight, LongKeyEmptyEntriesEvictAtByteLimit) +{ + /// Empty DVs used to weigh 1 byte and ignore key strings. With an unlimited entry + /// count, long unique keys must still be bounded by the configured byte limit. + const String long_suffix(8 * 1024, 'x'); + const auto sample_key = makeUniqueKey(0, long_suffix); + const auto entry_weight = PuffinFilesCacheCell::calculateMemorySize( + /*is_empty_deletion_vector_=*/true, nullptr, sample_key.approximateMemoryBytes()); + ASSERT_GT(entry_weight, sample_key.approximateMemoryBytes()); + ASSERT_GT(entry_weight, 8 * 1024u); + + /// Allow roughly two long-key empty entries; inserting many more must evict. + const size_t max_bytes = static_cast(entry_weight) * 2 + 128; + PuffinFilesCache cache("SLRU", max_bytes, /*max_count=*/0, /*size_ratio=*/0.5); + + for (size_t i = 0; i < 32; ++i) + { + const auto key = makeUniqueKey(i, long_suffix); + cache.getOrSetDeletionVector(key, []() -> DataLakeObjectMetadata::ExcludedRowsPtr { return nullptr; }); + } + + EXPECT_LE(CurrentMetrics::get(CurrentMetrics::PuffinFilesCacheBytes), static_cast(max_bytes)); + EXPECT_LE(CurrentMetrics::get(CurrentMetrics::PuffinFilesCacheFiles), 3); +} + +TEST(PuffinFilesCacheWeight, EmptyEntryChargesKeyNotOneByte) +{ + const String long_path(4096, 'p'); + const auto key = PuffinFilesCache::tryCreateKey( + "Local:////test-prefix", long_path, "etag-empty", 0, 0, long_path, 0, 0); + ASSERT_TRUE(key.has_value()); + + const auto weight = PuffinFilesCacheCell::calculateMemorySize( + /*is_empty_deletion_vector_=*/true, nullptr, key->approximateMemoryBytes()); + EXPECT_GT(weight, key->approximateMemoryBytes()); + EXPECT_GE(weight, 2 * 4096u); +} + +TEST(PuffinFilesCacheWeight, EstimateMinimumMatchesEmptyEntryWeight) +{ + const auto key = PuffinFilesCache::tryCreateKey( + "Local:////test-prefix", "data/file.parquet", "etag-1", 4, 44, "/data/file.parquet", 2, 100); + ASSERT_TRUE(key.has_value()); + + const UInt64 key_bytes = key->approximateMemoryBytes(); + EXPECT_EQ( + PuffinFilesCacheCell::estimateMinimumMemorySize(key_bytes), + PuffinFilesCacheCell::calculateMemorySize(/*is_empty_deletion_vector_=*/true, nullptr, key_bytes)); + + /// A 1-byte cache cannot hold the key/overhead lower bound for a realistic path. + EXPECT_GT(PuffinFilesCacheCell::estimateMinimumMemorySize(key_bytes), 1u); +} diff --git a/src/Storages/ObjectStorage/DataLakes/tests/gtest_puffin_footer_cache.cpp b/src/Storages/ObjectStorage/DataLakes/tests/gtest_puffin_footer_cache.cpp new file mode 100644 index 000000000000..5aea99489283 --- /dev/null +++ b/src/Storages/ObjectStorage/DataLakes/tests/gtest_puffin_footer_cache.cpp @@ -0,0 +1,295 @@ +#include + +#include +#include +#include +#include +#include +#include + +using namespace DB; + +namespace ProfileEvents +{ +extern const Event PuffinFilesRead; +} + +namespace CurrentMetrics +{ +extern const Metric PuffinFilesCacheBytes; +extern const Metric PuffinFilesCacheFiles; +} + +namespace +{ + +/// Two equal-cardinality deletion-vector-v1 blobs for different data files. +constexpr UInt8 two_equal_cardinality_dvs_puffin[] = { + 0x50, 0x46, 0x41, 0x31, 0x00, 0x00, 0x00, 0x24, 0xD1, 0xD3, 0x39, 0x64, 0x01, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x3A, 0x30, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x01, 0x00, 0x10, 0x00, 0x00, 0x00, 0x02, 0x00, 0x05, 0x00, 0x2C, 0xDB, 0x9F, 0xC1, + 0x00, 0x00, 0x00, 0x24, 0xD1, 0xD3, 0x39, 0x64, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x3A, 0x30, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0x00, + 0x10, 0x00, 0x00, 0x00, 0x07, 0x00, 0x09, 0x00, 0xB7, 0xB0, 0x20, 0xFF, 0x50, 0x46, 0x41, 0x31, + 0x7B, 0x22, 0x62, 0x6C, 0x6F, 0x62, 0x73, 0x22, 0x3A, 0x20, 0x5B, 0x7B, 0x22, 0x74, 0x79, 0x70, + 0x65, 0x22, 0x3A, 0x20, 0x22, 0x64, 0x65, 0x6C, 0x65, 0x74, 0x69, 0x6F, 0x6E, 0x2D, 0x76, 0x65, + 0x63, 0x74, 0x6F, 0x72, 0x2D, 0x76, 0x31, 0x22, 0x2C, 0x20, 0x22, 0x66, 0x69, 0x65, 0x6C, 0x64, + 0x73, 0x22, 0x3A, 0x20, 0x5B, 0x5D, 0x2C, 0x20, 0x22, 0x73, 0x6E, 0x61, 0x70, 0x73, 0x68, 0x6F, + 0x74, 0x2D, 0x69, 0x64, 0x22, 0x3A, 0x20, 0x2D, 0x31, 0x2C, 0x20, 0x22, 0x73, 0x65, 0x71, 0x75, + 0x65, 0x6E, 0x63, 0x65, 0x2D, 0x6E, 0x75, 0x6D, 0x62, 0x65, 0x72, 0x22, 0x3A, 0x20, 0x2D, 0x31, + 0x2C, 0x20, 0x22, 0x6F, 0x66, 0x66, 0x73, 0x65, 0x74, 0x22, 0x3A, 0x20, 0x34, 0x2C, 0x20, 0x22, + 0x6C, 0x65, 0x6E, 0x67, 0x74, 0x68, 0x22, 0x3A, 0x20, 0x34, 0x34, 0x2C, 0x20, 0x22, 0x70, 0x72, + 0x6F, 0x70, 0x65, 0x72, 0x74, 0x69, 0x65, 0x73, 0x22, 0x3A, 0x20, 0x7B, 0x22, 0x72, 0x65, 0x66, + 0x65, 0x72, 0x65, 0x6E, 0x63, 0x65, 0x64, 0x2D, 0x64, 0x61, 0x74, 0x61, 0x2D, 0x66, 0x69, 0x6C, + 0x65, 0x22, 0x3A, 0x20, 0x22, 0x2F, 0x64, 0x61, 0x74, 0x61, 0x2F, 0x66, 0x69, 0x6C, 0x65, 0x5F, + 0x61, 0x2E, 0x70, 0x61, 0x72, 0x71, 0x75, 0x65, 0x74, 0x22, 0x2C, 0x20, 0x22, 0x63, 0x61, 0x72, + 0x64, 0x69, 0x6E, 0x61, 0x6C, 0x69, 0x74, 0x79, 0x22, 0x3A, 0x20, 0x22, 0x32, 0x22, 0x7D, 0x7D, + 0x2C, 0x20, 0x7B, 0x22, 0x74, 0x79, 0x70, 0x65, 0x22, 0x3A, 0x20, 0x22, 0x64, 0x65, 0x6C, 0x65, + 0x74, 0x69, 0x6F, 0x6E, 0x2D, 0x76, 0x65, 0x63, 0x74, 0x6F, 0x72, 0x2D, 0x76, 0x31, 0x22, 0x2C, + 0x20, 0x22, 0x66, 0x69, 0x65, 0x6C, 0x64, 0x73, 0x22, 0x3A, 0x20, 0x5B, 0x5D, 0x2C, 0x20, 0x22, + 0x73, 0x6E, 0x61, 0x70, 0x73, 0x68, 0x6F, 0x74, 0x2D, 0x69, 0x64, 0x22, 0x3A, 0x20, 0x2D, 0x31, + 0x2C, 0x20, 0x22, 0x73, 0x65, 0x71, 0x75, 0x65, 0x6E, 0x63, 0x65, 0x2D, 0x6E, 0x75, 0x6D, 0x62, + 0x65, 0x72, 0x22, 0x3A, 0x20, 0x2D, 0x31, 0x2C, 0x20, 0x22, 0x6F, 0x66, 0x66, 0x73, 0x65, 0x74, + 0x22, 0x3A, 0x20, 0x34, 0x38, 0x2C, 0x20, 0x22, 0x6C, 0x65, 0x6E, 0x67, 0x74, 0x68, 0x22, 0x3A, + 0x20, 0x34, 0x34, 0x2C, 0x20, 0x22, 0x70, 0x72, 0x6F, 0x70, 0x65, 0x72, 0x74, 0x69, 0x65, 0x73, + 0x22, 0x3A, 0x20, 0x7B, 0x22, 0x72, 0x65, 0x66, 0x65, 0x72, 0x65, 0x6E, 0x63, 0x65, 0x64, 0x2D, + 0x64, 0x61, 0x74, 0x61, 0x2D, 0x66, 0x69, 0x6C, 0x65, 0x22, 0x3A, 0x20, 0x22, 0x2F, 0x64, 0x61, + 0x74, 0x61, 0x2F, 0x66, 0x69, 0x6C, 0x65, 0x5F, 0x62, 0x2E, 0x70, 0x61, 0x72, 0x71, 0x75, 0x65, + 0x74, 0x22, 0x2C, 0x20, 0x22, 0x63, 0x61, 0x72, 0x64, 0x69, 0x6E, 0x61, 0x6C, 0x69, 0x74, 0x79, + 0x22, 0x3A, 0x20, 0x22, 0x32, 0x22, 0x7D, 0x7D, 0x5D, 0x7D, 0x9A, 0x01, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x50, 0x46, 0x41, 0x31, +}; + +PuffinFilesCache::FooterBlobsPtr loadFixtureFooter() +{ + ReadBufferFromMemory file(two_equal_cardinality_dvs_puffin, sizeof(two_equal_cardinality_dvs_puffin)); + return std::make_shared>( + readPuffinFooterBlobsFromSeekable(file, sizeof(two_equal_cardinality_dvs_puffin))); +} + +} + +TEST(PuffinFooterMemo, CoalescedSlicesShareOneFooterParse) +{ + PuffinFilesCache cache("SLRU", 1'000'000, 100, 0.5); + + const auto footer_key = PuffinFilesCache::tryCreateFooterKey("Local:////test", "coalesced.puffin", "etag-1"); + ASSERT_TRUE(footer_key.has_value()); + + const auto key_a = PuffinFilesCache::tryCreateKey( + "Local:////test", "coalesced.puffin", "etag-1", 4, 44, "/data/file_a.parquet", 2, 100); + const auto key_b = PuffinFilesCache::tryCreateKey( + "Local:////test", "coalesced.puffin", "etag-1", 48, 44, "/data/file_b.parquet", 2, 100); + ASSERT_TRUE(key_a.has_value()); + ASSERT_TRUE(key_b.has_value()); + + auto & counters = ProfileEvents::global_counters; + const auto files_read_before = counters[ProfileEvents::PuffinFilesRead].load(); + + size_t footer_loads = 0; + auto load_footer = [&]() + { + ++footer_loads; + return loadFixtureFooter(); + }; + + /// Cold scan of two DV slices from one coalesced Puffin: one footer parse, two bitmap loads. + cache.getOrSetDeletionVector(*key_a, [&]() + { + auto footer = cache.getOrSetFooter(*footer_key, load_footer); + EXPECT_EQ(footer->size(), 2u); + bindDeletionVectorBlob(*footer, 4, 44, "/data/file_a.parquet", 2); + ReadBufferFromMemory file(two_equal_cardinality_dvs_puffin, sizeof(two_equal_cardinality_dvs_puffin)); + const auto positions = readDeletionVectorFromPuffin(file, 4, 44, 2); + auto excluded = std::make_shared(); + for (UInt64 position : positions) + excluded->add(static_cast(position)); + return excluded; + }); + + cache.getOrSetDeletionVector(*key_b, [&]() + { + auto footer = cache.getOrSetFooter(*footer_key, load_footer); + EXPECT_EQ(footer->size(), 2u); + bindDeletionVectorBlob(*footer, 48, 44, "/data/file_b.parquet", 2); + ReadBufferFromMemory file(two_equal_cardinality_dvs_puffin, sizeof(two_equal_cardinality_dvs_puffin)); + const auto positions = readDeletionVectorFromPuffin(file, 48, 44, 2); + auto excluded = std::make_shared(); + for (UInt64 position : positions) + excluded->add(static_cast(position)); + return excluded; + }); + + EXPECT_EQ(footer_loads, 1u); + /// One footer parse (`PuffinFilesRead` in readPuffinFooter) plus two blob reads. + EXPECT_EQ(counters[ProfileEvents::PuffinFilesRead].load() - files_read_before, 3u); +} + +TEST(PuffinFooterMemo, ClearDropsFooterEntries) +{ + PuffinFilesCache cache("SLRU", 1'000'000, 100, 0.5); + const auto footer_key = PuffinFilesCache::tryCreateFooterKey("Local:////test", "coalesced.puffin", "etag-1"); + ASSERT_TRUE(footer_key.has_value()); + + size_t footer_loads = 0; + auto load_footer = [&]() + { + ++footer_loads; + return loadFixtureFooter(); + }; + + ASSERT_NE(cache.getOrSetFooter(*footer_key, load_footer), nullptr); + cache.clear(); + ASSERT_NE(cache.getOrSetFooter(*footer_key, load_footer), nullptr); + EXPECT_EQ(footer_loads, 2u); +} + +TEST(PuffinFooterMemo, SizeZeroDoesNotInsert) +{ + PuffinFilesCache cache("SLRU", /*max_size_in_bytes=*/0, 100, 0.5); + const auto footer_key = PuffinFilesCache::tryCreateFooterKey("Local:////test", "coalesced.puffin", "etag-1"); + ASSERT_TRUE(footer_key.has_value()); + + size_t footer_loads = 0; + auto load_footer = [&]() + { + ++footer_loads; + return loadFixtureFooter(); + }; + + ASSERT_NE(cache.getOrSetFooter(*footer_key, load_footer), nullptr); + ASSERT_NE(cache.getOrSetFooter(*footer_key, load_footer), nullptr); + EXPECT_EQ(footer_loads, 2u); + EXPECT_EQ(cache.footerMemoEntries(), 0u); + EXPECT_EQ(cache.footerMemoBytes(), 0u); +} + +TEST(PuffinFooterMemo, SizeZeroClearsMemo) +{ + PuffinFilesCache cache("SLRU", 1'000'000, 100, 0.5); + const auto footer_key = PuffinFilesCache::tryCreateFooterKey("Local:////test", "coalesced.puffin", "etag-1"); + ASSERT_TRUE(footer_key.has_value()); + + size_t footer_loads = 0; + auto load_footer = [&]() + { + ++footer_loads; + return loadFixtureFooter(); + }; + + ASSERT_NE(cache.getOrSetFooter(*footer_key, load_footer), nullptr); + EXPECT_EQ(footer_loads, 1u); + EXPECT_EQ(cache.footerMemoEntries(), 1u); + EXPECT_GT(cache.footerMemoBytes(), 0u); + + cache.setMaxSizeInBytes(0); + EXPECT_EQ(cache.footerMemoEntries(), 0u); + EXPECT_EQ(cache.footerMemoBytes(), 0u); + + ASSERT_NE(cache.getOrSetFooter(*footer_key, load_footer), nullptr); + EXPECT_EQ(footer_loads, 2u); + EXPECT_EQ(cache.footerMemoEntries(), 0u); + + cache.setMaxSizeInBytes(1'000'000); + ASSERT_NE(cache.getOrSetFooter(*footer_key, load_footer), nullptr); + ASSERT_NE(cache.getOrSetFooter(*footer_key, load_footer), nullptr); + EXPECT_EQ(footer_loads, 3u); + EXPECT_EQ(cache.footerMemoEntries(), 1u); +} + +TEST(PuffinFooterMemo, ByteBudgetEvictsOnShrink) +{ + PuffinFilesCache cache("SLRU", 1'000'000, 100, 0.5); + const auto footer_key = PuffinFilesCache::tryCreateFooterKey("Local:////test", "coalesced.puffin", "etag-1"); + ASSERT_TRUE(footer_key.has_value()); + + size_t footer_loads = 0; + auto load_footer = [&]() + { + ++footer_loads; + return loadFixtureFooter(); + }; + + ASSERT_NE(cache.getOrSetFooter(*footer_key, load_footer), nullptr); + const UInt64 memo_bytes = cache.footerMemoBytes(); + ASSERT_GT(memo_bytes, 1u); + + /// Shrinking below current memo weight must drop retained footers. + cache.setMaxSizeInBytes(1); + EXPECT_EQ(cache.footerMemoEntries(), 0u); + EXPECT_EQ(cache.footerMemoBytes(), 0u); + + ASSERT_NE(cache.getOrSetFooter(*footer_key, load_footer), nullptr); + EXPECT_EQ(footer_loads, 2u); + /// Single fixture footer exceeds a 1-byte budget, so it must not be re-inserted. + EXPECT_EQ(cache.footerMemoEntries(), 0u); +} + +TEST(PuffinFooterMemo, CountLimitEvictsOneEntryNotAll) +{ + PuffinFilesCache cache("SLRU", 1'000'000, /*max_count=*/1, 0.5); + const auto key_a = PuffinFilesCache::tryCreateFooterKey("Local:////test", "a.puffin", "etag-a"); + const auto key_b = PuffinFilesCache::tryCreateFooterKey("Local:////test", "b.puffin", "etag-b"); + ASSERT_TRUE(key_a.has_value()); + ASSERT_TRUE(key_b.has_value()); + + size_t footer_loads = 0; + auto load_footer = [&]() + { + ++footer_loads; + return loadFixtureFooter(); + }; + + ASSERT_NE(cache.getOrSetFooter(*key_a, load_footer), nullptr); + EXPECT_EQ(cache.footerMemoEntries(), 1u); + + ASSERT_NE(cache.getOrSetFooter(*key_b, load_footer), nullptr); + /// Evict one victim for room — do not wipe the memo to empty before insert. + EXPECT_EQ(cache.footerMemoEntries(), 1u); + EXPECT_EQ(footer_loads, 2u); + + /// The retained entry must be key_b (key_a was the only victim). + ASSERT_NE(cache.getOrSetFooter(*key_b, load_footer), nullptr); + EXPECT_EQ(footer_loads, 2u); +} + +TEST(PuffinFooterMemo, SharesBudgetAndMetricsWithDeletionVectors) +{ + /// Tiny shared budget: a resident DV must leave no room for a second full budget of footers. + PuffinFilesCache cache("SLRU", /*max_size_in_bytes=*/50'000, /*max_count=*/100, 0.5); + + const auto footer_key = PuffinFilesCache::tryCreateFooterKey("Local:////test", "coalesced.puffin", "etag-1"); + ASSERT_TRUE(footer_key.has_value()); + + const auto bytes_before = CurrentMetrics::get(CurrentMetrics::PuffinFilesCacheBytes); + const auto files_before = CurrentMetrics::get(CurrentMetrics::PuffinFilesCacheFiles); + + ASSERT_NE(cache.getOrSetFooter(*footer_key, loadFixtureFooter), nullptr); + const UInt64 memo_bytes = cache.footerMemoBytes(); + ASSERT_GT(memo_bytes, 0u); + EXPECT_EQ(cache.footerMemoEntries(), 1u); + EXPECT_EQ(CurrentMetrics::get(CurrentMetrics::PuffinFilesCacheBytes), bytes_before + static_cast(memo_bytes)); + EXPECT_EQ(CurrentMetrics::get(CurrentMetrics::PuffinFilesCacheFiles), files_before + 1); + + /// Fill almost all remaining shared budget with a DV so the memo must be trimmed. + auto excluded = std::make_shared(); + for (size_t i = 0; i < 2000; ++i) + excluded->add(i); + + const auto dv_key = PuffinFilesCache::tryCreateKey( + "Local:////test", "coalesced.puffin", "etag-1", 4, 44, "/data/file_a.parquet", 2, 100); + ASSERT_TRUE(dv_key.has_value()); + + cache.getOrSetDeletionVector(*dv_key, [&]() { return excluded; }); + + /// After DV insert, footer memo is trimmed so DV + memo stay within configured max. + EXPECT_LE( + static_cast(CurrentMetrics::get(CurrentMetrics::PuffinFilesCacheBytes) - bytes_before), + 50'000u); + EXPECT_LE(cache.footerMemoBytes() + cache.sizeInBytes(), 50'000u); + + cache.clear(); + EXPECT_EQ(cache.footerMemoEntries(), 0u); + EXPECT_EQ(CurrentMetrics::get(CurrentMetrics::PuffinFilesCacheBytes), bytes_before); + EXPECT_EQ(CurrentMetrics::get(CurrentMetrics::PuffinFilesCacheFiles), files_before); +} diff --git a/src/Storages/ObjectStorage/DataLakes/tests/gtest_puffin_non_seekable_buffer_limit.cpp b/src/Storages/ObjectStorage/DataLakes/tests/gtest_puffin_non_seekable_buffer_limit.cpp new file mode 100644 index 000000000000..643613e6232f --- /dev/null +++ b/src/Storages/ObjectStorage/DataLakes/tests/gtest_puffin_non_seekable_buffer_limit.cpp @@ -0,0 +1,53 @@ +#include + +#include + +#include +#include + +#include + +using namespace DB; + +TEST(PuffinNonSeekableBufferLimit, StopsBeforeExceedingAbsoluteLimit) +{ + constexpr size_t max_buffered_size = 100; + std::vector out = {'P', 'F', 'A', '1'}; + + /// Stream after the already-buffered header would push the total over the limit. + const String rest(max_buffered_size, 'x'); + ReadBufferFromString buf(rest); + + EXPECT_THROW(appendReadBufferWithAbsoluteSizeLimit(buf, out, max_buffered_size), Exception); + EXPECT_LE(out.size(), max_buffered_size); +} + +TEST(PuffinNonSeekableBufferLimit, AcceptsInputWithinLimit) +{ + constexpr size_t max_buffered_size = 100; + std::vector out = {'P', 'F', 'A', '1'}; + + const String rest(50, 'y'); + ReadBufferFromString buf(rest); + + ASSERT_NO_THROW(appendReadBufferWithAbsoluteSizeLimit(buf, out, max_buffered_size)); + EXPECT_EQ(out.size(), 54u); + EXPECT_EQ(out[0], 'P'); + EXPECT_EQ(out[4], 'y'); +} + +TEST(PuffinNonSeekableBufferLimit, RejectsAlreadyOversizedPrefix) +{ + std::vector out(101, 'z'); + ReadBufferFromString buf(String{}); + + EXPECT_THROW(appendReadBufferWithAbsoluteSizeLimit(buf, out, /*max_buffered_size=*/100), Exception); +} + +TEST(PuffinNonSeekableBufferLimit, ProductionCeilingCoversMaxDvPlusFooter) +{ + EXPECT_EQ( + PUFFIN_NON_SEEKABLE_MAX_BUFFERED_SIZE, + PUFFIN_MAGIC_SIZE + PUFFIN_DV_MAX_BLOB_SIZE + PUFFIN_MAGIC_SIZE + PUFFIN_FOOTER_MAX_PAYLOAD_SIZE + + PUFFIN_FOOTER_TRAILER_SIZE); +} diff --git a/src/Storages/ObjectStorage/IObjectIterator.h b/src/Storages/ObjectStorage/IObjectIterator.h index 22f776a6ac35..aa0029552216 100644 --- a/src/Storages/ObjectStorage/IObjectIterator.h +++ b/src/Storages/ObjectStorage/IObjectIterator.h @@ -55,9 +55,15 @@ struct ObjectInfo FileBucketInfoPtr file_bucket_info; +<<<<<<< HEAD /// Lazy materialization: if set, read only these rows of the file. /// Sorted absolute row indexes within the file, see FormatFilterInfo::rows_to_read. std::shared_ptr> rows_to_read; +======= + /// Polymorphic copy: preserves the dynamic type (e.g. `IcebergDataObjectInfo` with equality / + /// position deletes, resolved storage, and metadata path) where a plain copy would slice it. + virtual std::shared_ptr clone() const { return std::make_shared(*this); } +>>>>>>> 4b7cecaa3cf (Merge pull request #2183 from Altinity/feature/antalya-26.6/iceberg-puffin-deletion-vectors-read-2) String getIdentifier(bool include_file_bucket_info = true) const; String getIdentifierForPath(const String & path, bool include_file_bucket_info = true) const; diff --git a/src/Storages/ObjectStorage/StorageObjectStorageSource.cpp b/src/Storages/ObjectStorage/StorageObjectStorageSource.cpp index ee69a32f134f..0de0832349a5 100644 --- a/src/Storages/ObjectStorage/StorageObjectStorageSource.cpp +++ b/src/Storages/ObjectStorage/StorageObjectStorageSource.cpp @@ -222,6 +222,7 @@ static void logIcebergFileStats(const ObjectInfoPtr & object_info, const LoggerP #endif } +<<<<<<< HEAD /// Whether reading this object goes through row-level delete transformers (Iceberg /// position/equality deletes, Delta Lake deletion vectors). The count-from-files cache /// is keyed only by the file path and its modification time, but delete files change the @@ -304,6 +305,21 @@ static std::optional buildIdentityPartitionColumnsDag( outputs.push_back(&dag.materializeNode(constant)); } return dag; +======= +/// Count-from-files cache key is data-file identity only. Skip when row filtering can change +/// independently (DVs / selection vectors, Iceberg eq/pos deletes) or when the task is a bucket subset. +/// Cache must stay fail-closed even when need_only_count is allowed for position deletes / DVs: +/// the key is path + mtime only, and deletes change the contributed row count without touching the file. +static bool canUseCountFromFilesCache(const ObjectInfoPtr & object_info) +{ + if (hasNonEmptyExcludedRows(object_info->data_lake_metadata) || object_info->file_bucket_info) + return false; +#if USE_AVRO + if (hasIcebergEqualityDeletes(object_info) || hasIcebergPositionDeletes(object_info)) + return false; +#endif + return true; +>>>>>>> 4b7cecaa3cf (Merge pull request #2183 from Altinity/feature/antalya-26.6/iceberg-puffin-deletion-vectors-read-2) } StorageObjectStorageSource::StorageObjectStorageSource( @@ -879,12 +895,20 @@ Chunk StorageObjectStorageSource::generate() } } + /// Do not cache filtered cardinality: filter DAG, PREWHERE, and row policies all + /// reduce rows seen by generate(), while the cache key is file identity only. if (reader.getInputFormat() && read_context->getSettingsRef()[Setting::use_cache_for_count_from_files] +<<<<<<< HEAD && !format_filter_info->filter_actions_dag && !hasAttachedDeletes(*reader.getObjectInfo()) && !reader.getObjectInfo()->rows_to_read) addNumRowsToCache(*reader.getObjectInfo(), total_rows_in_file); +======= + && format_filter_info && !format_filter_info->hasFilter() + && canUseCountFromFilesCache(reader.getObjectInfo())) + addNumRowsToCache(reader.getObjectInfo(), total_rows_in_file); +>>>>>>> 4b7cecaa3cf (Merge pull request #2183 from Altinity/feature/antalya-26.6/iceberg-puffin-deletion-vectors-read-2) total_rows_in_file = 0; chassert(reader_future.valid()); @@ -1074,9 +1098,29 @@ StorageObjectStorageSource::ReaderHolder StorageObjectStorageSource::createReade /// response. Skip the shortcut when `_headers` is requested so the real `GET` headers are used. const bool headers_requested = read_from_format_info.requested_virtual_columns.contains("_headers"); +<<<<<<< HEAD std::optional num_rows_from_cache = need_only_count && !headers_requested && context_->getSettingsRef()[Setting::use_cache_for_count_from_files] ? try_get_num_rows_from_cache() : std::nullopt; +======= + /// Equality-delete FilterTransform evaluates predicates against column values, but need_only_count + /// emits default-filled chunks — so disable the fast path for equality deletes only. + /// Position deletes and deletion vectors filter by row index (preserved on synthetic chunks); + /// DeletionVectorTransform adjusts const count chunks via roaring range cardinality, and Parquet + /// needOnlyCount reads footer/row-group metadata only (including bucketed reads). Count-from-files + /// cache stays separately fail-closed in canUseCountFromFilesCache. +#if USE_AVRO + const bool effective_need_only_count = need_only_count && !hasIcebergEqualityDeletes(object_info); +#else + const bool effective_need_only_count = need_only_count; +#endif + + const bool can_use_count_cache = effective_need_only_count + && context_->getSettingsRef()[Setting::use_cache_for_count_from_files] + && canUseCountFromFilesCache(object_info); + + std::optional num_rows_from_cache = can_use_count_cache ? try_get_num_rows_from_cache() : std::nullopt; +>>>>>>> 4b7cecaa3cf (Merge pull request #2183 from Altinity/feature/antalya-26.6/iceberg-puffin-deletion-vectors-read-2) if (num_rows_from_cache) { @@ -1331,7 +1375,7 @@ StorageObjectStorageSource::ReaderHolder StorageObjectStorageSource::createReade filter_info, true /* is_remote_fs */, compression_method, - need_only_count, + effective_need_only_count, std::nullopt /*min_block_size_bytes*/, std::nullopt /*min_block_size_rows*/, std::nullopt /*max_block_size_bytes*/); @@ -1349,17 +1393,18 @@ StorageObjectStorageSource::ReaderHolder StorageObjectStorageSource::createReade filter_info, true /* is_remote_fs */, compression_method, - need_only_count); + effective_need_only_count); } input_format->setBucketsToRead(object_info->file_bucket_info); input_format->setSerializationHints(read_from_format_info.serialization_hints); - if (need_only_count) + if (effective_need_only_count) input_format->needOnlyCount(); builder.init(Pipe(input_format)); +<<<<<<< HEAD if (!identity_partition_columns.empty()) { if (auto dag = buildIdentityPartitionColumnsDag(builder.getHeader(), identity_partition_columns)) @@ -1377,6 +1422,13 @@ StorageObjectStorageSource::ReaderHolder StorageObjectStorageSource::createReade if (object_info->data_lake_metadata && object_info->data_lake_metadata->excluded_rows && object_info->data_lake_metadata->excluded_rows->size() > 0) +======= + /// Deletion vectors (and selection vectors) address absolute file row numbers via + /// `ChunkInfoRowNumbers`. Iceberg equality deletes use a plain `FilterTransform` that + /// shrinks the chunk without maintaining `applied_filter`, so DV must run first — + /// otherwise later DV filtering maps dense post-equality indices to the wrong file rows. + if (hasNonEmptyExcludedRows(object_info->data_lake_metadata)) +>>>>>>> 4b7cecaa3cf (Merge pull request #2183 from Altinity/feature/antalya-26.6/iceberg-puffin-deletion-vectors-read-2) { builder.addSimpleTransform([&](const SharedHeader & header) { @@ -1384,6 +1436,8 @@ StorageObjectStorageSource::ReaderHolder StorageObjectStorageSource::createReade }); } + configuration->addDeleteTransformers(object_info, builder, format_settings, parser_shared_resources, context_); + std::optional schema_transform; if (object_info->data_lake_metadata && object_info->data_lake_metadata->schema_transform) { diff --git a/src/Storages/ObjectStorage/tests/gtest_rendezvous_hashing.cpp b/src/Storages/ObjectStorage/tests/gtest_rendezvous_hashing.cpp index bca5e8f982a2..65c6a02783cf 100644 --- a/src/Storages/ObjectStorage/tests/gtest_rendezvous_hashing.cpp +++ b/src/Storages/ObjectStorage/tests/gtest_rendezvous_hashing.cpp @@ -1,10 +1,15 @@ #include +<<<<<<< HEAD #include #include #include #include #include +======= + +>>>>>>> 4b7cecaa3cf (Merge pull request #2183 from Altinity/feature/antalya-26.6/iceberg-puffin-deletion-vectors-read-2) #include +#include using namespace DB; diff --git a/tests/integration/test_storage_iceberg_with_spark/test_deletion_vectors.py b/tests/integration/test_storage_iceberg_with_spark/test_deletion_vectors.py new file mode 100644 index 000000000000..b6b42786c07a --- /dev/null +++ b/tests/integration/test_storage_iceberg_with_spark/test_deletion_vectors.py @@ -0,0 +1,1015 @@ +import uuid + +import pytest + +from helpers.iceberg_utils import ( + default_upload_directory, + get_uuid_str, + get_creation_expression, +) + + +def get_array(query_result: str): + return sorted([int(x) for x in query_result.strip().split("\n") if x]) + + +def upload_table(cluster, storage_type, table_name): + default_upload_directory( + cluster, + storage_type, + f"/iceberg_data/default/{table_name}/", + f"/iceberg_data/default/{table_name}/", + ) + + +def _strip_file_uri_scheme(path): + """Return a bare absolute path ClickHouse can remap via IcebergPathResolver. + + Hadoop `Path.toString()` yields `file:/...` / `file:///...`. For s3/azure + table functions, those URIs can be resolved onto a secondary + LocalObjectStorage (ClickHouse node local disk) instead of the uploaded + object-storage tree. A scheme-less absolute path makes + `tryResolveObjectStorageForPath` return nullopt so `IcebergPathResolver` + remaps onto the table's base storage — matching how this harness reads + Spark data files after `upload_table`. + """ + if path.startswith("file://"): + return path[len("file://") :] + if path.startswith("file:"): + return path[len("file:") :] + return path + + +def add_equality_deletes_by_id(spark, table_name, ids): + """Commit an Iceberg equality-delete file for the given `id` values. + + Spark SQL DELETE on v3 only writes deletion vectors, so equality deletes are + produced by writing a Parquet file with Spark (correct long boxing) and + registering it via Iceberg RowDelta. + """ + jvm = spark._jvm + ice = jvm.org.apache.iceberg + table = ice.spark.Spark3Util.loadIcebergTable( + spark._jsparkSession, f"spark_catalog.default.{table_name}" + ) + + id_field_id = int(table.schema().findField("id").fieldId()) + staging_dir = table.locationProvider().newDataLocation( + f"eq-delete-staging-{uuid.uuid4()}" + ) + final_uri = table.locationProvider().newDataLocation( + f"eq-delete-{uuid.uuid4()}.parquet" + ) + + # Spark DataFrame write keeps BIGINT as Long — avoids py4j Integer boxing. + # coalesce(1) still writes a directory; flatten to a single Iceberg data file. + spark.createDataFrame([(int(v),) for v in ids], "id: long").coalesce(1).write.mode( + "overwrite" + ).parquet(staging_dir) + + staging_path = jvm.org.apache.hadoop.fs.Path(staging_dir) + fs = staging_path.getFileSystem(spark.sparkContext._jsc.hadoopConfiguration()) + part_hadoop_path = None + part_size = 0 + for status in fs.listStatus(staging_path): + name = status.getPath().getName() + if name.startswith("part-") and name.endswith(".parquet"): + part_hadoop_path = status.getPath() + part_size = int(status.getLen()) + break + if part_hadoop_path is None: + raise RuntimeError(f"No parquet part file written under {staging_dir}") + + final_hadoop_path = jvm.org.apache.hadoop.fs.Path(final_uri) + if not fs.rename(part_hadoop_path, final_hadoop_path): + raise RuntimeError(f"Failed to move {part_hadoop_path} to {final_hadoop_path}") + fs.delete(staging_path, True) + + equality_field_ids = spark.sparkContext._gateway.new_array(jvm.int, 1) + equality_field_ids[0] = id_field_id + + delete_file = ( + ice.FileMetadata.deleteFileBuilder(table.spec()) + .ofEqualityDeletes(equality_field_ids) + .withPath(_strip_file_uri_scheme(final_hadoop_path.toString())) + .withFileSizeInBytes(part_size) + .withRecordCount(len(ids)) + .withFormat(ice.FileFormat.PARQUET) + .build() + ) + table.newRowDelta().addDeletes(delete_file).commit() + + +@pytest.mark.parametrize("run_on_cluster", [False, True]) +@pytest.mark.parametrize("storage_type", ["s3", "azure", "local"]) +def test_deletion_vectors(started_cluster_iceberg_with_spark, storage_type, run_on_cluster): + if storage_type == "local" and run_on_cluster: + pytest.skip("Local storage with cluster execution is not supported") + + instance = started_cluster_iceberg_with_spark.instances["node1"] + spark = started_cluster_iceberg_with_spark.spark_session + table_name = "test_deletion_vectors_" + storage_type + "_" + get_uuid_str() + deleted_ids = [2, 5, 7, 100] + + spark.sql( + f""" + CREATE TABLE {table_name} (id bigint) USING iceberg + TBLPROPERTIES ( + 'format-version' = '3', + 'write.delete.mode' = 'merge-on-read', + 'write.update.mode' = 'merge-on-read', + 'write.merge.mode' = 'merge-on-read' + ) + """ + ) + spark.sql(f"INSERT INTO {table_name} SELECT id FROM range(0, 200)") + spark.sql( + f"DELETE FROM {table_name} WHERE id IN ({', '.join(str(x) for x in deleted_ids)})" + ) + + upload_table(started_cluster_iceberg_with_spark, storage_type, table_name) + + expression = get_creation_expression( + storage_type, + table_name, + started_cluster_iceberg_with_spark, + run_on_cluster=run_on_cluster, + table_function=True, + ) + + assert int(instance.query(f"SELECT count() FROM {expression}")) == 200 - len(deleted_ids) + assert get_array(instance.query(f"SELECT id FROM {expression}")) == [ + x for x in range(200) if x not in deleted_ids + ] + + +@pytest.mark.parametrize("run_on_cluster", [False, True]) +@pytest.mark.parametrize("storage_type", ["s3", "azure", "local"]) +def test_deletion_vectors_aggregates(started_cluster_iceberg_with_spark, storage_type, run_on_cluster): + """Aggregates over Iceberg v3 tables must ignore rows covered by deletion vectors.""" + if storage_type == "local" and run_on_cluster: + pytest.skip("Local storage with cluster execution is not supported") + + instance = started_cluster_iceberg_with_spark.instances["node1"] + spark = started_cluster_iceberg_with_spark.spark_session + table_name = "test_deletion_vectors_aggregates_" + storage_type + "_" + get_uuid_str() + deleted_ids = {2, 5, 7, 50, 99} + remaining_ids = [i for i in range(100) if i not in deleted_ids] + # value = 10 * id, so sum/avg expectations stay integer-friendly where possible. + remaining_values = [10 * i for i in remaining_ids] + + spark.sql( + f""" + CREATE TABLE {table_name} (id bigint, value bigint, group_id int) USING iceberg + TBLPROPERTIES ( + 'format-version' = '3', + 'write.delete.mode' = 'merge-on-read', + 'write.update.mode' = 'merge-on-read', + 'write.merge.mode' = 'merge-on-read' + ) + """ + ) + spark.sql( + f""" + INSERT INTO {table_name} + SELECT id, 10 * id, CAST(id % 3 AS INT) + FROM range(0, 100) + """ + ) + spark.sql( + f"DELETE FROM {table_name} WHERE id IN ({', '.join(str(x) for x in sorted(deleted_ids))})" + ) + + upload_table(started_cluster_iceberg_with_spark, storage_type, table_name) + + expression = get_creation_expression( + storage_type, + table_name, + started_cluster_iceberg_with_spark, + run_on_cluster=run_on_cluster, + table_function=True, + ) + + spark_row = spark.sql( + f""" + SELECT + count(*) AS cnt, + sum(id) AS sum_id, + sum(value) AS sum_value, + min(id) AS min_id, + max(id) AS max_id, + avg(value) AS avg_value + FROM {table_name} + """ + ).collect()[0] + + expected_count = len(remaining_ids) + expected_sum_id = sum(remaining_ids) + expected_sum_value = sum(remaining_values) + expected_min_id = min(remaining_ids) + expected_max_id = max(remaining_ids) + expected_avg_value = expected_sum_value / expected_count + + assert spark_row["cnt"] == expected_count + assert spark_row["sum_id"] == expected_sum_id + assert spark_row["sum_value"] == expected_sum_value + assert spark_row["min_id"] == expected_min_id + assert spark_row["max_id"] == expected_max_id + assert abs(float(spark_row["avg_value"]) - expected_avg_value) < 1e-9 + + ch_row = instance.query( + f""" + SELECT + count(), + count(id), + sum(id), + sum(value), + min(id), + max(id), + avg(value), + uniqExact(id), + countIf(id % 2 = 0), + sumIf(value, id % 2 = 0) + FROM {expression} + """ + ).strip().split("\t") + + assert int(ch_row[0]) == expected_count + assert int(ch_row[1]) == expected_count + assert int(ch_row[2]) == expected_sum_id + assert int(ch_row[3]) == expected_sum_value + assert int(ch_row[4]) == expected_min_id + assert int(ch_row[5]) == expected_max_id + assert abs(float(ch_row[6]) - expected_avg_value) < 1e-9 + assert int(ch_row[7]) == expected_count + + expected_even_ids = [i for i in remaining_ids if i % 2 == 0] + expected_count_if = len(expected_even_ids) + expected_sum_if = sum(10 * i for i in expected_even_ids) + assert int(ch_row[8]) == expected_count_if + assert int(ch_row[9]) == expected_sum_if + + # Trivial COUNT must match the full scan once deletion vectors are applied. + assert ( + int( + instance.query( + f"SELECT count() FROM {expression}", + settings={"optimize_trivial_count_query": 1}, + ) + ) + == expected_count + ) + assert ( + int( + instance.query( + f"SELECT count() FROM {expression}", + settings={"optimize_trivial_count_query": 0}, + ) + ) + == expected_count + ) + + # GROUP BY aggregates must also exclude deleted rows. + spark_groups = { + int(row["group_id"]): (int(row["cnt"]), int(row["sum_value"])) + for row in spark.sql( + f""" + SELECT group_id, count(*) AS cnt, sum(value) AS sum_value + FROM {table_name} + GROUP BY group_id + ORDER BY group_id + """ + ).collect() + } + ch_groups = {} + for line in instance.query( + f""" + SELECT group_id, count(), sum(value) + FROM {expression} + GROUP BY group_id + ORDER BY group_id + """ + ).strip().split("\n"): + group_id, cnt, sum_value = line.split("\t") + ch_groups[int(group_id)] = (int(cnt), int(sum_value)) + + expected_groups = {} + for group_id in (0, 1, 2): + ids = [i for i in remaining_ids if i % 3 == group_id] + expected_groups[group_id] = (len(ids), sum(10 * i for i in ids)) + + assert spark_groups == expected_groups + assert ch_groups == expected_groups + + +@pytest.mark.parametrize("storage_type", ["s3", "azure", "local"]) +def test_deletion_vectors_complex(started_cluster_iceberg_with_spark, storage_type): + instance = started_cluster_iceberg_with_spark.instances["node1"] + spark = started_cluster_iceberg_with_spark.spark_session + table_name = "test_deletion_vectors_complex_" + storage_type + "_" + get_uuid_str() + + def expected_complex_ids(): + ids = list(range(20, 90)) + list(range(100, 150)) + ids += [x for x in range(200, 250) if x not in {205, 210, 220}] + return sorted(ids) + + expected_ids = expected_complex_ids() + + spark.sql( + f""" + CREATE TABLE {table_name} (id bigint, data string) USING iceberg + PARTITIONED BY (bucket(5, id)) + TBLPROPERTIES ( + 'format-version' = '3', + 'write.delete.mode' = 'merge-on-read', + 'write.update.mode' = 'merge-on-read', + 'write.merge.mode' = 'merge-on-read' + ) + """ + ) + spark.sql( + f"INSERT INTO {table_name} SELECT id, char(id + ascii('a')) FROM range(10, 100)" + ) + upload_table(started_cluster_iceberg_with_spark, storage_type, table_name) + + expression = get_creation_expression( + storage_type, + table_name, + started_cluster_iceberg_with_spark, + table_function=True, + ) + + assert int(instance.query(f"SELECT count(id) FROM {expression}")) == 90 + assert get_array(instance.query(f"SELECT id FROM {expression}")) == list(range(10, 100)) + + spark.sql(f"DELETE FROM {table_name} WHERE id < 20") + upload_table(started_cluster_iceberg_with_spark, storage_type, table_name) + assert get_array(instance.query(f"SELECT id FROM {expression}")) == list(range(20, 100)) + + spark.sql(f"DELETE FROM {table_name} WHERE id >= 90") + upload_table(started_cluster_iceberg_with_spark, storage_type, table_name) + assert get_array(instance.query(f"SELECT id FROM {expression}")) == list(range(20, 90)) + + spark.sql( + f"INSERT INTO {table_name} SELECT id, char(id + ascii('a')) FROM range(100, 200)" + ) + upload_table(started_cluster_iceberg_with_spark, storage_type, table_name) + assert get_array(instance.query(f"SELECT id FROM {expression}")) == list(range(20, 90)) + list( + range(100, 200) + ) + + spark.sql(f"DELETE FROM {table_name} WHERE id >= 150") + upload_table(started_cluster_iceberg_with_spark, storage_type, table_name) + assert get_array(instance.query(f"SELECT id FROM {expression}")) == list(range(20, 90)) + list( + range(100, 150) + ) + + spark.sql(f"ALTER TABLE {table_name} ADD COLUMNS (label string)") + spark.sql( + f""" + INSERT INTO {table_name} + SELECT id, char(id + ascii('a')), 'new' + FROM range(200, 250) + """ + ) + upload_table(started_cluster_iceberg_with_spark, storage_type, table_name) + assert get_array(instance.query(f"SELECT id FROM {expression}")) == list(range(20, 90)) + list( + range(100, 150) + ) + list(range(200, 250)) + assert int(instance.query(f"SELECT count(id) FROM {expression} WHERE label = 'new'")) == 50 + + spark.sql(f"DELETE FROM {table_name} WHERE id IN (205, 210, 220)") + upload_table(started_cluster_iceberg_with_spark, storage_type, table_name) + assert get_array(instance.query(f"SELECT id FROM {expression}")) == expected_ids + assert int(instance.query(f"SELECT count(id) FROM {expression}")) == len(expected_ids) + + spark.sql(f"UPDATE {table_name} SET label = 'updated' WHERE id = 25") + upload_table(started_cluster_iceberg_with_spark, storage_type, table_name) + assert instance.query(f"SELECT label FROM {expression} WHERE id = 25").strip() == "updated" + assert int(instance.query(f"SELECT count(id) FROM {expression} WHERE label = 'updated'")) == 1 + + spark.sql(f"CALL system.rewrite_data_files(table => '{table_name}')") + upload_table(started_cluster_iceberg_with_spark, storage_type, table_name) + assert get_array(instance.query(f"SELECT id FROM {expression}")) == expected_ids + # After rewrite, snapshot summary may still report stale total-position-deletes while data + # files already have deletes applied. Trivial count must not subtract those and under-count. + assert ( + int( + instance.query( + f"SELECT count() FROM {expression}", + settings={ + "optimize_trivial_count_query": 1, + "use_iceberg_metadata_files_cache": 0, + }, + ) + ) + == len(expected_ids) + ) + assert ( + int( + instance.query( + f"SELECT count() FROM (SELECT * FROM {expression})", + settings={"use_iceberg_metadata_files_cache": 0}, + ) + ) + == len(expected_ids) + ) + + assert get_array( + instance.query( + f"SELECT id FROM {expression} WHERE id % 3 = 0" + ) + ) == sorted([x for x in expected_ids if x % 3 == 0]) + + +@pytest.mark.parametrize("storage_type", ["s3"]) +def test_deletion_vectors_count_after_rewrite_data_files( + started_cluster_iceberg_with_spark, storage_type +): + """Regression: Spark rewrite can leave total-position-deletes in the snapshot summary after + deletes were already applied into rewritten data files. SELECT count() must still match the scan. + """ + instance = started_cluster_iceberg_with_spark.instances["node1"] + spark = started_cluster_iceberg_with_spark.spark_session + table_name = "test_dv_count_after_rewrite_" + storage_type + "_" + get_uuid_str() + + spark.sql( + f""" + CREATE TABLE {table_name} ( + id bigint, + data string + ) USING iceberg + TBLPROPERTIES ( + 'format-version' = '3', + 'write.delete.mode' = 'merge-on-read', + 'write.update.mode' = 'merge-on-read', + 'write.merge.mode' = 'merge-on-read' + ) + """ + ) + spark.sql( + f""" + INSERT INTO {table_name} + SELECT id, concat('row-', CAST(id AS STRING)) FROM range(100) + """ + ) + spark.sql(f"DELETE FROM {table_name} WHERE id % 10 = 0") + upload_table(started_cluster_iceberg_with_spark, storage_type, table_name) + + expression = get_creation_expression( + storage_type, + table_name, + started_cluster_iceberg_with_spark, + table_function=True, + ) + expected = [x for x in range(100) if x % 10 != 0] + settings = {"use_iceberg_metadata_files_cache": 0} + + assert get_array(instance.query(f"SELECT id FROM {expression}", settings=settings)) == expected + assert ( + int( + instance.query( + f"SELECT count() FROM {expression}", + settings={**settings, "optimize_trivial_count_query": 1}, + ) + ) + == 90 + ) + + spark.sql( + f""" + CALL system.rewrite_data_files( + table => '{table_name}', + options => map('delete-file-threshold', '1') + ) + """ + ) + upload_table(started_cluster_iceberg_with_spark, storage_type, table_name) + + assert get_array(instance.query(f"SELECT id FROM {expression}", settings=settings)) == expected + assert ( + int( + instance.query( + f"SELECT count() FROM {expression}", + settings={**settings, "optimize_trivial_count_query": 1}, + ) + ) + == 90 + ) + assert ( + int( + instance.query( + f"SELECT count() FROM (SELECT * FROM {expression})", + settings=settings, + ) + ) + == 90 + ) + + +@pytest.mark.parametrize("storage_type", ["s3", "azure", "local"]) +def test_deletion_vectors_trivial_count_fails_closed_with_live_deletes( + started_cluster_iceberg_with_spark, storage_type +): + """Live DVs / equality deletes must not use snapshot-summary trivial COUNT. + + Summary fields are optional and can disagree with manifests. With any live + position deletes (puffin DVs) the summary shortcut must stay closed: + IcebergTrivialCountOptimizationApplied == 0, and count() must match a scan. + """ + instance = started_cluster_iceberg_with_spark.instances["node1"] + spark = started_cluster_iceberg_with_spark.spark_session + table_name = "test_dv_trivial_count_closed_" + storage_type + "_" + get_uuid_str() + + spark.sql( + f""" + CREATE TABLE {table_name} ( + id bigint, + data string + ) USING iceberg + TBLPROPERTIES ( + 'format-version' = '3', + 'write.delete.mode' = 'merge-on-read', + 'write.update.mode' = 'merge-on-read', + 'write.merge.mode' = 'merge-on-read' + ) + """ + ) + spark.sql( + f""" + INSERT INTO {table_name} + SELECT id, concat('row-', CAST(id AS STRING)) FROM range(100) + """ + ) + spark.sql(f"DELETE FROM {table_name} WHERE id % 10 = 0") + upload_table(started_cluster_iceberg_with_spark, storage_type, table_name) + + expression = get_creation_expression( + storage_type, + table_name, + started_cluster_iceberg_with_spark, + table_function=True, + ) + settings = { + "optimize_trivial_count_query": 1, + "use_iceberg_metadata_files_cache": 0, + } + expected_after_dv = [x for x in range(100) if x % 10 != 0] + + instance.query("SYSTEM DROP ICEBERG METADATA CACHE") + + query_id_dv = f"{table_name}-dv-{uuid.uuid4()}" + assert ( + int( + instance.query( + f"SELECT count() FROM {expression}", + query_id=query_id_dv, + settings=settings, + ) + ) + == len(expected_after_dv) + ) + assert ( + int( + instance.query( + f"SELECT count() FROM (SELECT * FROM {expression})", + settings={"use_iceberg_metadata_files_cache": 0}, + ) + ) + == len(expected_after_dv) + ) + assert get_array( + instance.query( + f"SELECT id FROM {expression}", + settings={"use_iceberg_metadata_files_cache": 0}, + ) + ) == expected_after_dv + + instance.query("SYSTEM FLUSH LOGS") + assert ( + int( + instance.query( + f""" + SELECT ProfileEvents['IcebergTrivialCountOptimizationApplied'] + FROM system.query_log + WHERE query_id = '{query_id_dv}' AND type = 'QueryFinish' + """ + ) + ) + == 0 + ) + + # Equality deletes whose optional summary field may stay 0 must still fail closed + # while the DV remains live (and after, while equality-delete files remain). + add_equality_deletes_by_id(spark, table_name, [1, 5]) + upload_table(started_cluster_iceberg_with_spark, storage_type, table_name) + expected_after_eq = [x for x in expected_after_dv if x not in (1, 5)] + + instance.query("SYSTEM DROP ICEBERG METADATA CACHE") + query_id_eq = f"{table_name}-eq-{uuid.uuid4()}" + assert ( + int( + instance.query( + f"SELECT count() FROM {expression}", + query_id=query_id_eq, + settings=settings, + ) + ) + == len(expected_after_eq) + ) + assert ( + int( + instance.query( + f"SELECT count() FROM (SELECT * FROM {expression})", + settings={"use_iceberg_metadata_files_cache": 0}, + ) + ) + == len(expected_after_eq) + ) + + instance.query("SYSTEM FLUSH LOGS") + assert ( + int( + instance.query( + f""" + SELECT ProfileEvents['IcebergTrivialCountOptimizationApplied'] + FROM system.query_log + WHERE query_id = '{query_id_eq}' AND type = 'QueryFinish' + """ + ) + ) + == 0 + ) + + +def _poison_snapshot_total_records(table_name, poisoned_total="999999"): + """Overwrite total-records in every snapshot summary of the latest metadata JSON. + + Simulates a corrupted / incorrectly-maintained incremental summary while leaving + data files and manifests consistent. + """ + import json + import glob as glob_mod + + metadata_dir = f"/var/lib/clickhouse/user_files/iceberg_data/default/{table_name}/metadata" + metadata_files = sorted(glob_mod.glob(f"{metadata_dir}/v*.metadata.json")) + if not metadata_files: + raise RuntimeError(f"No metadata JSON under {metadata_dir}") + path = metadata_files[-1] + with open(path, "r", encoding="utf-8") as f: + meta = json.load(f) + for snapshot in meta.get("snapshots", []): + summary = snapshot.setdefault("summary", {}) + summary["total-records"] = poisoned_total + with open(path, "w", encoding="utf-8") as f: + json.dump(meta, f, indent=2) + + +@pytest.mark.parametrize("storage_type", ["s3", "azure", "local"]) +def test_trivial_count_prefers_manifests_over_poisoned_summary( + started_cluster_iceberg_with_spark, storage_type +): + """Poisoned snapshot-summary total-records must not become SELECT count(). + + Manifest per-file record_count is ground truth; summary is warning-only. + """ + instance = started_cluster_iceberg_with_spark.instances["node1"] + spark = started_cluster_iceberg_with_spark.spark_session + table_name = "test_poisoned_summary_count_" + storage_type + "_" + get_uuid_str() + + spark.sql( + f""" + CREATE TABLE {table_name} ( + id bigint, + data string + ) USING iceberg + TBLPROPERTIES ('format-version' = '3') + """ + ) + spark.sql( + f""" + INSERT INTO {table_name} + SELECT id, concat('row-', CAST(id AS STRING)) FROM range(50) + """ + ) + _poison_snapshot_total_records(table_name, "999999") + upload_table(started_cluster_iceberg_with_spark, storage_type, table_name) + + expression = get_creation_expression( + storage_type, + table_name, + started_cluster_iceberg_with_spark, + table_function=True, + ) + settings = { + "optimize_trivial_count_query": 1, + "use_iceberg_metadata_files_cache": 0, + } + + instance.query("SYSTEM DROP ICEBERG METADATA CACHE") + query_id = f"{table_name}-{uuid.uuid4()}" + assert ( + int( + instance.query( + f"SELECT count() FROM {expression}", + query_id=query_id, + settings=settings, + ) + ) + == 50 + ) + assert ( + int( + instance.query( + f"SELECT count() FROM (SELECT * FROM {expression})", + settings={"use_iceberg_metadata_files_cache": 0}, + ) + ) + == 50 + ) + + instance.query("SYSTEM FLUSH LOGS") + # Manifest-sum path still counts as the trivial COUNT optimization. + assert ( + int( + instance.query( + f""" + SELECT ProfileEvents['IcebergTrivialCountOptimizationApplied'] + FROM system.query_log + WHERE query_id = '{query_id}' AND type = 'QueryFinish' + """ + ) + ) + == 1 + ) + + +@pytest.mark.parametrize("storage_type", ["s3"]) +def test_deletion_vectors_puffin_files_cache(started_cluster_iceberg_with_spark, storage_type): + instance = started_cluster_iceberg_with_spark.instances["node1"] + spark = started_cluster_iceberg_with_spark.spark_session + table_name = "test_deletion_vectors_cache_" + storage_type + "_" + get_uuid_str() + deleted_ids = [2, 5, 7, 100] + + spark.sql( + f""" + CREATE TABLE {table_name} (id bigint) USING iceberg + TBLPROPERTIES ( + 'format-version' = '3', + 'write.delete.mode' = 'merge-on-read', + 'write.update.mode' = 'merge-on-read', + 'write.merge.mode' = 'merge-on-read' + ) + """ + ) + spark.sql(f"INSERT INTO {table_name} SELECT id FROM range(0, 200)") + spark.sql( + f"DELETE FROM {table_name} WHERE id IN ({', '.join(str(x) for x in deleted_ids)})" + ) + + upload_table(started_cluster_iceberg_with_spark, storage_type, table_name) + + expression = get_creation_expression( + storage_type, + table_name, + started_cluster_iceberg_with_spark, + table_function=True, + ) + + instance.query("SYSTEM DROP PUFFIN FILES CACHE") + + query_id1 = f"{table_name}-{uuid.uuid4()}" + query_id2 = f"{table_name}-{uuid.uuid4()}" + query_id3 = f"{table_name}-{uuid.uuid4()}" + + assert int( + instance.query( + f"SELECT count(id) FROM {expression}", + query_id=query_id1, + settings={"use_puffin_files_cache": 1}, + ) + ) == 200 - len(deleted_ids) + + assert int( + instance.query( + f"SELECT count(id) FROM {expression}", + query_id=query_id2, + settings={"use_puffin_files_cache": 1}, + ) + ) == 200 - len(deleted_ids) + + instance.query("SYSTEM FLUSH LOGS") + + assert int( + instance.query( + f"SELECT ProfileEvents['PuffinFilesCacheMisses'] FROM system.query_log WHERE query_id = '{query_id1}' AND type = 'QueryFinish'" + ) + ) > 0 + assert int( + instance.query( + f"SELECT ProfileEvents['PuffinFilesCacheHits'] FROM system.query_log WHERE query_id = '{query_id2}' AND type = 'QueryFinish'" + ) + ) > 0 + + puffin_reads_first = int( + instance.query( + f"SELECT ProfileEvents['PuffinFilesRead'] FROM system.query_log WHERE query_id = '{query_id1}' AND type = 'QueryFinish'" + ) + ) + puffin_reads_second = int( + instance.query( + f"SELECT ProfileEvents['PuffinFilesRead'] FROM system.query_log WHERE query_id = '{query_id2}' AND type = 'QueryFinish'" + ) + ) + assert puffin_reads_first > 0 + assert puffin_reads_second == 0 + + instance.query("SYSTEM DROP PUFFIN FILES CACHE") + + assert int( + instance.query( + f"SELECT count(id) FROM {expression}", + query_id=query_id3, + settings={"use_puffin_files_cache": 1}, + ) + ) == 200 - len(deleted_ids) + + instance.query("SYSTEM FLUSH LOGS") + + assert int( + instance.query( + f"SELECT ProfileEvents['PuffinFilesCacheMisses'] FROM system.query_log WHERE query_id = '{query_id3}' AND type = 'QueryFinish'" + ) + ) > int( + instance.query( + f"SELECT ProfileEvents['PuffinFilesCacheMisses'] FROM system.query_log WHERE query_id = '{query_id2}' AND type = 'QueryFinish'" + ) + ) + + +@pytest.mark.parametrize("storage_type", ["s3", "azure", "local"]) +def test_deletion_vectors_reject_mutations(started_cluster_iceberg_with_spark, storage_type): + """DELETE/UPDATE must fail closed on tables that already contain deletion vectors. + + ClickHouse mutations write parquet position-delete files, which Iceberg readers ignore for + data files that have a matching DV — so a successful mutation would silently leave rows. + """ + instance = started_cluster_iceberg_with_spark.instances["node1"] + spark = started_cluster_iceberg_with_spark.spark_session + table_name = "test_deletion_vectors_reject_mutations_" + storage_type + "_" + get_uuid_str() + + spark.sql( + f""" + CREATE TABLE {table_name} (id bigint) USING iceberg + TBLPROPERTIES ( + 'format-version' = '3', + 'write.delete.mode' = 'merge-on-read', + 'write.update.mode' = 'merge-on-read', + 'write.merge.mode' = 'merge-on-read' + ) + """ + ) + spark.sql(f"INSERT INTO {table_name} SELECT id FROM range(0, 20)") + spark.sql(f"DELETE FROM {table_name} WHERE id IN (1, 2, 3)") + + upload_table(started_cluster_iceberg_with_spark, storage_type, table_name) + + instance.query( + get_creation_expression( + storage_type, + table_name, + started_cluster_iceberg_with_spark, + table_function=False, + ) + ) + + assert int(instance.query(f"SELECT count() FROM {table_name}")) == 17 + + delete_error = instance.query_and_get_error( + f"ALTER TABLE {table_name} DELETE WHERE id = 4", + settings={"allow_insert_into_iceberg": 1}, + ) + assert "deletion vectors" in delete_error.lower() + + update_error = instance.query_and_get_error( + f"ALTER TABLE {table_name} UPDATE id = 0 WHERE id = 4", + settings={"allow_insert_into_iceberg": 1}, + ) + assert "deletion vectors" in update_error.lower() + + # Rows must be unchanged after rejected mutations. + assert int(instance.query(f"SELECT count() FROM {table_name}")) == 17 + assert get_array(instance.query(f"SELECT id FROM {table_name}")) == [ + x for x in range(20) if x not in (1, 2, 3) + ] + + +@pytest.mark.parametrize("run_on_cluster", [False, True]) +@pytest.mark.parametrize("storage_type", ["s3", "azure", "local"]) +def test_deletion_vectors_with_equality_deletes( + started_cluster_iceberg_with_spark, storage_type, run_on_cluster +): + """DV + equality deletes on the same table must keep correct survivors. + + Guards StorageObjectStorageSource transform order: DV must run before equality + FilterTransform (see gtest_deletion_vector_before_equality_filter). + """ + if storage_type == "local" and run_on_cluster: + pytest.skip("Local storage with cluster execution is not supported") + + instance = started_cluster_iceberg_with_spark.instances["node1"] + spark = started_cluster_iceberg_with_spark.spark_session + table_name = "test_dv_with_eq_" + storage_type + "_" + get_uuid_str() + + # Small unpartitioned file so DV and equality deletes both apply to the same data file. + spark.sql( + f""" + CREATE TABLE {table_name} (id bigint) USING iceberg + TBLPROPERTIES ( + 'format-version' = '3', + 'write.delete.mode' = 'merge-on-read', + 'write.update.mode' = 'merge-on-read', + 'write.merge.mode' = 'merge-on-read' + ) + """ + ) + spark.sql(f"INSERT INTO {table_name} SELECT id FROM range(0, 20)") + # Deletion vector removes file positions for these ids (values equal positions here). + spark.sql(f"DELETE FROM {table_name} WHERE id IN (2, 7)") + # Equality deletes remove by value after DV materialization. + add_equality_deletes_by_id(spark, table_name, [1, 5]) + + upload_table(started_cluster_iceberg_with_spark, storage_type, table_name) + + expression = get_creation_expression( + storage_type, + table_name, + started_cluster_iceberg_with_spark, + run_on_cluster=run_on_cluster, + table_function=True, + ) + + deleted = {1, 2, 5, 7} + expected = [x for x in range(20) if x not in deleted] + spark_ids = sorted(int(r[0]) for r in spark.sql(f"SELECT id FROM {table_name}").collect()) + assert spark_ids == expected + + assert int(instance.query(f"SELECT count() FROM {expression}")) == len(expected) + assert get_array(instance.query(f"SELECT id FROM {expression}")) == expected + + +@pytest.mark.parametrize("storage_type", ["s3", "azure"]) +def test_deletion_vectors_cluster_bucket_split(started_cluster_iceberg_with_spark, storage_type): + """icebergCluster bucket splitting must preserve DV (and clone) metadata end-to-end.""" + instance = started_cluster_iceberg_with_spark.instances["node1"] + spark = started_cluster_iceberg_with_spark.spark_session + table_name = "test_dv_cluster_bucket_" + storage_type + "_" + get_uuid_str() + deleted_ids = [2, 5, 7, 100] + + spark.sql( + f""" + CREATE TABLE {table_name} (id bigint) USING iceberg + TBLPROPERTIES ( + 'format-version' = '3', + 'write.delete.mode' = 'merge-on-read', + 'write.update.mode' = 'merge-on-read', + 'write.merge.mode' = 'merge-on-read', + 'write.parquet.row-group-size-bytes' = '1' + ) + """ + ) + spark.sql(f"INSERT INTO {table_name} SELECT id FROM range(0, 200)") + spark.sql( + f"DELETE FROM {table_name} WHERE id IN ({', '.join(str(x) for x in deleted_ids)})" + ) + + upload_table(started_cluster_iceberg_with_spark, storage_type, table_name) + + expression = get_creation_expression( + storage_type, + table_name, + started_cluster_iceberg_with_spark, + run_on_cluster=True, + table_function=True, + ) + expected = [x for x in range(200) if x not in deleted_ids] + settings = { + "cluster_table_function_split_granularity": "bucket", + "cluster_table_function_buckets_batch_size": 1, + } + + assert ( + int(instance.query(f"SELECT count() FROM {expression}", settings=settings)) + == len(expected) + ) + assert get_array(instance.query(f"SELECT id FROM {expression}", settings=settings)) == expected + # Trivial count path must match under bucket splits (need_only_count + DV cardinality). + assert ( + int( + instance.query( + f"SELECT count() FROM {expression}", + settings={**settings, "optimize_trivial_count_query": 1}, + ) + ) + == len(expected) + ) diff --git a/tests/queries/0_stateless/01271_show_privileges.reference b/tests/queries/0_stateless/01271_show_privileges.reference index 083ace33ce32..e874c73a0b39 100644 --- a/tests/queries/0_stateless/01271_show_privileges.reference +++ b/tests/queries/0_stateless/01271_show_privileges.reference @@ -135,7 +135,11 @@ SYSTEM DROP ICEBERG METADATA CACHE ['SYSTEM CLEAR ICEBERG_METADATA_CACHE','SYSTE SYSTEM DROP PAIMON METADATA CACHE ['SYSTEM CLEAR PAIMON_METADATA_CACHE','SYSTEM DROP PAIMON_METADATA_CACHE'] GLOBAL SYSTEM DROP CACHE SYSTEM DROP AVRO SCHEMA CACHE ['SYSTEM CLEAR AVRO SCHEMA CACHE','SYSTEM DROP AVRO SCHEMA CACHE','DROP AVRO SCHEMA CACHE'] GLOBAL SYSTEM DROP CACHE SYSTEM DROP PARQUET METADATA CACHE ['SYSTEM DROP PARQUET_METADATA_CACHE'] GLOBAL SYSTEM DROP CACHE +<<<<<<< HEAD SYSTEM DROP POINT IN POLYGON CACHE ['SYSTEM CLEAR POINT IN POLYGON CACHE','SYSTEM DROP POINT IN POLYGON CACHE','DROP POINT IN POLYGON CACHE'] GLOBAL SYSTEM DROP CACHE +======= +SYSTEM DROP PUFFIN FILES CACHE ['SYSTEM DROP PUFFIN_FILES_CACHE'] GLOBAL SYSTEM DROP CACHE +>>>>>>> 4b7cecaa3cf (Merge pull request #2183 from Altinity/feature/antalya-26.6/iceberg-puffin-deletion-vectors-read-2) SYSTEM PREWARM PRIMARY INDEX CACHE ['SYSTEM PREWARM PRIMARY INDEX','PREWARM PRIMARY INDEX CACHE','PREWARM PRIMARY INDEX'] GLOBAL SYSTEM DROP CACHE SYSTEM DROP PRIMARY INDEX CACHE ['SYSTEM CLEAR PRIMARY INDEX CACHE','SYSTEM DROP PRIMARY INDEX','DROP PRIMARY INDEX CACHE','DROP PRIMARY INDEX'] GLOBAL SYSTEM DROP CACHE SYSTEM DROP UNCOMPRESSED CACHE ['SYSTEM CLEAR UNCOMPRESSED CACHE','SYSTEM DROP UNCOMPRESSED','DROP UNCOMPRESSED CACHE','DROP UNCOMPRESSED'] GLOBAL SYSTEM DROP CACHE diff --git a/tests/queries/0_stateless/04077_puffin_happy_path.reference b/tests/queries/0_stateless/04077_puffin_happy_path.reference index 66cf85ce28f3..dc6d6814d154 100644 --- a/tests/queries/0_stateless/04077_puffin_happy_path.reference +++ b/tests/queries/0_stateless/04077_puffin_happy_path.reference @@ -19,6 +19,12 @@ deletion-vector-v1 -1 -1 4 44 ['cardinality','referenced-data-file'] ['2','/dat /data/table/part-00000.parquet [2,5] /data/table/part-00000.parquet 2 /data/table/part-00000.parquet 5 +--- mixed_blob_types.puffin --- +apache-datasketches-theta-v1 -1 -1 4 16 [] [] +deletion-vector-v1 -1 -1 20 44 ['cardinality','referenced-data-file'] ['2','/data/table/part-00000.parquet'] +/data/table/part-00000.parquet [2,5] +/data/table/part-00000.parquet 2 +/data/table/part-00000.parquet 5 --- dense_range_100k.puffin --- 100000 0 99999 --- subset without deleted_rows --- diff --git a/tests/queries/0_stateless/04077_puffin_happy_path.sh b/tests/queries/0_stateless/04077_puffin_happy_path.sh index eedc30b57880..d86afa63a37c 100755 --- a/tests/queries/0_stateless/04077_puffin_happy_path.sh +++ b/tests/queries/0_stateless/04077_puffin_happy_path.sh @@ -32,6 +32,7 @@ run_happy_path() { run_happy_path "spark_deletion_vector.puffin" "$DATA/spark_deletion_vector.puffin" run_happy_path "compressed_footer.puffin" "$DATA/compressed_footer.puffin" run_happy_path "file_properties_ok.puffin" "$DATA/file_properties_ok.puffin" +run_happy_path "mixed_blob_types.puffin" "$DATA/mixed_blob_types.puffin" echo "--- dense_range_100k.puffin ---" $CLICKHOUSE_LOCAL -q " diff --git a/tests/queries/0_stateless/04117_parser_system_query_variants.reference b/tests/queries/0_stateless/04117_parser_system_query_variants.reference index c74441dfee20..7a4a135737ac 100644 --- a/tests/queries/0_stateless/04117_parser_system_query_variants.reference +++ b/tests/queries/0_stateless/04117_parser_system_query_variants.reference @@ -16,7 +16,12 @@ SYSTEM CLEAR ENCRYPTION HEADERS CACHE SYSTEM CLEAR COMPILED EXPRESSION CACHE SYSTEM CLEAR ICEBERG METADATA CACHE SYSTEM CLEAR PARQUET METADATA CACHE +<<<<<<< HEAD SYSTEM CLEAR POINT IN POLYGON CACHE +======= +SYSTEM CLEAR PUFFIN FILES CACHE +SYSTEM CLEAR PUFFIN FILES CACHE +>>>>>>> 4b7cecaa3cf (Merge pull request #2183 from Altinity/feature/antalya-26.6/iceberg-puffin-deletion-vectors-read-2) SYSTEM CLEAR FILESYSTEM CACHE SYSTEM CLEAR DISTRIBUTED CACHE SYSTEM CLEAR DISTRIBUTED CACHE CONNECTIONS ON CLUSTER cluster diff --git a/tests/queries/0_stateless/04117_parser_system_query_variants.sql b/tests/queries/0_stateless/04117_parser_system_query_variants.sql index 6814e1c6b3f9..344cdcb4a900 100644 --- a/tests/queries/0_stateless/04117_parser_system_query_variants.sql +++ b/tests/queries/0_stateless/04117_parser_system_query_variants.sql @@ -26,7 +26,12 @@ EXPLAIN SYNTAX SYSTEM DROP ENCRYPTION HEADERS CACHE; EXPLAIN SYNTAX SYSTEM DROP COMPILED EXPRESSION CACHE; EXPLAIN SYNTAX SYSTEM DROP ICEBERG METADATA CACHE; EXPLAIN SYNTAX SYSTEM DROP PARQUET METADATA CACHE; +<<<<<<< HEAD EXPLAIN SYNTAX SYSTEM DROP POINT IN POLYGON CACHE; +======= +EXPLAIN SYNTAX SYSTEM DROP PUFFIN FILES CACHE; +EXPLAIN SYNTAX SYSTEM DROP PUFFIN_FILES_CACHE; +>>>>>>> 4b7cecaa3cf (Merge pull request #2183 from Altinity/feature/antalya-26.6/iceberg-puffin-deletion-vectors-read-2) EXPLAIN SYNTAX SYSTEM DROP FILESYSTEM CACHE; EXPLAIN SYNTAX SYSTEM DROP DISTRIBUTED CACHE; EXPLAIN SYNTAX SYSTEM DROP DISTRIBUTED CACHE CONNECTIONS ON CLUSTER cluster; diff --git a/tests/queries/0_stateless/04261_iceberg_deletion_vector.reference b/tests/queries/0_stateless/04261_iceberg_deletion_vector.reference new file mode 100644 index 000000000000..fe7ad97c7e35 --- /dev/null +++ b/tests/queries/0_stateless/04261_iceberg_deletion_vector.reference @@ -0,0 +1,197 @@ +196 +0 +1 +3 +4 +6 +8 +9 +10 +11 +12 +13 +14 +15 +16 +17 +18 +19 +20 +21 +22 +23 +24 +25 +26 +27 +28 +29 +30 +31 +32 +33 +34 +35 +36 +37 +38 +39 +40 +41 +42 +43 +44 +45 +46 +47 +48 +49 +50 +51 +52 +53 +54 +55 +56 +57 +58 +59 +60 +61 +62 +63 +64 +65 +66 +67 +68 +69 +70 +71 +72 +73 +74 +75 +76 +77 +78 +79 +80 +81 +82 +83 +84 +85 +86 +87 +88 +89 +90 +91 +92 +93 +94 +95 +96 +97 +98 +99 +101 +102 +103 +104 +105 +106 +107 +108 +109 +110 +111 +112 +113 +114 +115 +116 +117 +118 +119 +120 +121 +122 +123 +124 +125 +126 +127 +128 +129 +130 +131 +132 +133 +134 +135 +136 +137 +138 +139 +140 +141 +142 +143 +144 +145 +146 +147 +148 +149 +150 +151 +152 +153 +154 +155 +156 +157 +158 +159 +160 +161 +162 +163 +164 +165 +166 +167 +168 +169 +170 +171 +172 +173 +174 +175 +176 +177 +178 +179 +180 +181 +182 +183 +184 +185 +186 +187 +188 +189 +190 +191 +192 +193 +194 +195 +196 +197 +198 +199 diff --git a/tests/queries/0_stateless/04261_iceberg_deletion_vector.sh b/tests/queries/0_stateless/04261_iceberg_deletion_vector.sh new file mode 100755 index 000000000000..b09a64d3e88d --- /dev/null +++ b/tests/queries/0_stateless/04261_iceberg_deletion_vector.sh @@ -0,0 +1,16 @@ +#!/usr/bin/env bash +# Tags: no-fasttest + +CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CURDIR"/../shell_config.sh + +TABLE_PATH="${CURDIR}/data_minio/dv_puffin_warehouse/default/dv_puffin_source" + +$CLICKHOUSE_LOCAL -q " +SELECT count() FROM icebergLocal('${TABLE_PATH}') +" + +$CLICKHOUSE_LOCAL -q " +SELECT id FROM icebergLocal('${TABLE_PATH}') ORDER BY id +" diff --git a/tests/queries/0_stateless/04262_iceberg_deletion_vector_complex.reference b/tests/queries/0_stateless/04262_iceberg_deletion_vector_complex.reference new file mode 100644 index 000000000000..32bd071ad3c5 --- /dev/null +++ b/tests/queries/0_stateless/04262_iceberg_deletion_vector_complex.reference @@ -0,0 +1,172 @@ +167 +47 +1 +updated +119 +20 +21 +22 +23 +24 +25 +26 +27 +28 +29 +30 +31 +32 +33 +34 +35 +36 +37 +38 +39 +40 +41 +42 +43 +44 +45 +46 +47 +48 +49 +50 +51 +52 +53 +54 +55 +56 +57 +58 +59 +60 +61 +62 +63 +64 +65 +66 +67 +68 +69 +70 +71 +72 +73 +74 +75 +76 +77 +78 +79 +80 +81 +82 +83 +84 +85 +86 +87 +88 +89 +100 +101 +102 +103 +104 +105 +106 +107 +108 +109 +110 +111 +112 +113 +114 +115 +116 +117 +118 +119 +120 +121 +122 +123 +124 +125 +126 +127 +128 +129 +130 +131 +132 +133 +134 +135 +136 +137 +138 +139 +140 +141 +142 +143 +144 +145 +146 +147 +148 +149 +200 +201 +202 +203 +204 +206 +207 +208 +209 +211 +212 +213 +214 +215 +216 +217 +218 +219 +221 +222 +223 +224 +225 +226 +227 +228 +229 +230 +231 +232 +233 +234 +235 +236 +237 +238 +239 +240 +241 +242 +243 +244 +245 +246 +247 +248 +249 diff --git a/tests/queries/0_stateless/04262_iceberg_deletion_vector_complex.sh b/tests/queries/0_stateless/04262_iceberg_deletion_vector_complex.sh new file mode 100755 index 000000000000..0fe7da881e31 --- /dev/null +++ b/tests/queries/0_stateless/04262_iceberg_deletion_vector_complex.sh @@ -0,0 +1,32 @@ +#!/usr/bin/env bash +# Tags: no-fasttest + +CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CURDIR"/../shell_config.sh + +TABLE_PATH="${CURDIR}/data_minio/dv_puffin_warehouse/default/dv_puffin_complex" + +$CLICKHOUSE_LOCAL -q " +SELECT count(id) FROM icebergLocal('${TABLE_PATH}') +" + +$CLICKHOUSE_LOCAL -q " +SELECT count(id) FROM icebergLocal('${TABLE_PATH}') WHERE label = 'new' +" + +$CLICKHOUSE_LOCAL -q " +SELECT count(id) FROM icebergLocal('${TABLE_PATH}') WHERE label = 'updated' +" + +$CLICKHOUSE_LOCAL -q " +SELECT label FROM icebergLocal('${TABLE_PATH}') WHERE id = 25 +" + +$CLICKHOUSE_LOCAL -q " +SELECT count(id) FROM icebergLocal('${TABLE_PATH}') WHERE label IS NULL +" + +$CLICKHOUSE_LOCAL -q " +SELECT id FROM icebergLocal('${TABLE_PATH}') ORDER BY id +" diff --git a/tests/queries/0_stateless/04263_iceberg_puffin_files_cache.reference b/tests/queries/0_stateless/04263_iceberg_puffin_files_cache.reference new file mode 100644 index 000000000000..50a04fbf6628 --- /dev/null +++ b/tests/queries/0_stateless/04263_iceberg_puffin_files_cache.reference @@ -0,0 +1,19 @@ +196 +PuffinFilesCacheMisses 1 +PuffinFilesRead 2 +196 +PuffinFilesCacheHits 1 +PuffinFilesCacheMisses 1 +PuffinFilesRead 2 +196 +PuffinFilesCacheHits 1 +PuffinFilesCacheMisses 2 +PuffinFilesRead 4 +196 +PuffinFilesCacheHits 1 +PuffinFilesCacheMisses 2 +PuffinFilesRead 6 +196 +PuffinFilesCacheHits 1 +PuffinFilesCacheMisses 2 +PuffinFilesRead 8 diff --git a/tests/queries/0_stateless/04263_iceberg_puffin_files_cache.sh b/tests/queries/0_stateless/04263_iceberg_puffin_files_cache.sh new file mode 100755 index 000000000000..f9565b8cf146 --- /dev/null +++ b/tests/queries/0_stateless/04263_iceberg_puffin_files_cache.sh @@ -0,0 +1,65 @@ +#!/usr/bin/env bash +# Tags: no-fasttest, no-parallel, no-parallel-replicas, no-random-settings +# no-fasttest: depends on local iceberg fixture +# no-parallel: cache is system-wide and tests can affect each other in unexpected way +# no-parallel-replicas: profile events are not available on the second replica +# no-random-settings: we need to test the interaction of specific setting combinations + +CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CURDIR"/../shell_config.sh + +TABLE_PATH="${CURDIR}/data_minio/dv_puffin_warehouse/default/dv_puffin_source" + +$CLICKHOUSE_LOCAL -q " +SYSTEM DROP PUFFIN FILES CACHE; + +SELECT count(id) +FROM icebergLocal('${TABLE_PATH}') +SETTINGS use_puffin_files_cache = 1; + +SELECT event, value +FROM system.events +WHERE event IN ('PuffinFilesCacheHits', 'PuffinFilesCacheMisses', 'PuffinFilesRead') +ORDER BY event; + +SELECT count(id) +FROM icebergLocal('${TABLE_PATH}') +SETTINGS use_puffin_files_cache = 1; + +SELECT event, value +FROM system.events +WHERE event IN ('PuffinFilesCacheHits', 'PuffinFilesCacheMisses', 'PuffinFilesRead') +ORDER BY event; + +SYSTEM DROP PUFFIN FILES CACHE; + +SELECT count(id) +FROM icebergLocal('${TABLE_PATH}') +SETTINGS use_puffin_files_cache = 1; + +SELECT event, value +FROM system.events +WHERE event IN ('PuffinFilesCacheHits', 'PuffinFilesCacheMisses', 'PuffinFilesRead') +ORDER BY event; + +SYSTEM DROP PUFFIN FILES CACHE; + +SELECT count(id) +FROM icebergLocal('${TABLE_PATH}') +SETTINGS use_puffin_files_cache = 0; + +SELECT event, value +FROM system.events +WHERE event IN ('PuffinFilesCacheHits', 'PuffinFilesCacheMisses', 'PuffinFilesRead') +ORDER BY event; + +SELECT count(id) +FROM icebergLocal('${TABLE_PATH}') +SETTINGS use_puffin_files_cache = 0; + +SELECT event, value +FROM system.events +WHERE event IN ('PuffinFilesCacheHits', 'PuffinFilesCacheMisses', 'PuffinFilesRead') +ORDER BY event; +" diff --git a/tests/queries/0_stateless/04337_iceberg_v3_row_lineage_reserved_field_id.sh b/tests/queries/0_stateless/04337_iceberg_v3_row_lineage_reserved_field_id.sh index 1372f4d60f90..65710df27614 100755 --- a/tests/queries/0_stateless/04337_iceberg_v3_row_lineage_reserved_field_id.sh +++ b/tests/queries/0_stateless/04337_iceberg_v3_row_lineage_reserved_field_id.sh @@ -52,13 +52,17 @@ ${CLICKHOUSE_CLIENT} --query "SELECT x FROM icebergLocal('${ICEBERG_TABLE_PATH}' # Conversely, 2147483447 (Integer.MAX_VALUE - 200) is the highest field id a table may use, i.e. # NOT reserved. An unmapped column with that id is a genuine schema mismatch and must still be # rejected, so the reserved-range check must be strictly greater-than. +# +# Insert at least two distinct values so allow_experimental_iceberg_read_optimization cannot treat +# `x` as a constant from Iceberg column stats and take the need-only-count path (which never runs +# SchemaConverter / field-id mapping checks). ICEBERG_TABLE_PATH_UNMAPPED="${CLICKHOUSE_USER_FILES}/lakehouses/${CLICKHOUSE_DATABASE}_v3_unmapped" rm -rf "${ICEBERG_TABLE_PATH_UNMAPPED}" ${CLICKHOUSE_CLIENT} --query " SET allow_experimental_insert_into_iceberg = 1; CREATE TABLE t_v3_unmapped (x Int32) ENGINE = IcebergLocal('${ICEBERG_TABLE_PATH_UNMAPPED}'); - INSERT INTO t_v3_unmapped (x) VALUES (1); + INSERT INTO t_v3_unmapped (x) VALUES (1), (2); " DATAFILE_UNMAPPED=$(ls "${ICEBERG_TABLE_PATH_UNMAPPED}"/data/*.parquet 2>/dev/null | head -1) @@ -73,7 +77,7 @@ x = pa.field("x", pa.int32(), nullable=False, metadata={b"PARQUET:field_id": b"1 # 2147483447 = Integer.MAX_VALUE - 200: the highest id a table may use, so NOT reserved. extra = pa.field("extra", pa.int64(), nullable=True, metadata={b"PARQUET:field_id": b"2147483447"}) table = pa.table( - {"x": pa.array([1], pa.int32()), "extra": pa.array([0], pa.int64())}, + {"x": pa.array([1, 2], pa.int32()), "extra": pa.array([0, 1], pa.int64())}, schema=pa.schema([x, extra]), ) pq.write_table(table, path) diff --git a/tests/queries/0_stateless/04549_puffin_allow_seeks_off.sh b/tests/queries/0_stateless/04549_puffin_allow_seeks_off.sh index 238205a43c66..8ae4bfff924e 100755 --- a/tests/queries/0_stateless/04549_puffin_allow_seeks_off.sh +++ b/tests/queries/0_stateless/04549_puffin_allow_seeks_off.sh @@ -23,6 +23,8 @@ SETTINGS input_format_allow_seeks = 0 " # Non-seekable path must reject a wrong leading magic before buffering the rest of the stream. +# Oversized valid-magic streams are covered by gtest_puffin_non_seekable_buffer_limit (absolute +# buffer ceiling is ~2 GiB + footer; too large for a CI fixture). echo "--- non-Puffin magic allow_seeks=0 ---" NOT_PUFFIN="${CLICKHOUSE_TMP}/04549_not_puffin.bin" # Prefix is wrong; trailing bytes would only matter if we buffered first then validated. diff --git a/tests/queries/0_stateless/04613_puffin_metadata_type_errors.reference b/tests/queries/0_stateless/04613_puffin_metadata_type_errors.reference index 1bb9169aca6f..9ee286b03479 100644 --- a/tests/queries/0_stateless/04613_puffin_metadata_type_errors.reference +++ b/tests/queries/0_stateless/04613_puffin_metadata_type_errors.reference @@ -28,6 +28,10 @@ Cannot parse Puffin footer JSON Cannot parse Puffin footer JSON --- dv_with_compression_codec.puffin --- must omit 'compression-codec' +--- dv_snapshot_id_not_minus_one.puffin --- +snapshot-id and sequence-number must be -1 +--- dv_sequence_number_not_minus_one.puffin --- +snapshot-id and sequence-number must be -1 --- invalid_cardinality_non_numeric.puffin --- property 'cardinality' must be an unsigned integer --- invalid_cardinality_negative.puffin --- diff --git a/tests/queries/0_stateless/04613_puffin_metadata_type_errors.sh b/tests/queries/0_stateless/04613_puffin_metadata_type_errors.sh index a4915b9de77e..fb6e5429f7fe 100755 --- a/tests/queries/0_stateless/04613_puffin_metadata_type_errors.sh +++ b/tests/queries/0_stateless/04613_puffin_metadata_type_errors.sh @@ -47,6 +47,12 @@ done launch "$id" meta "$DATA/dv_with_compression_codec.puffin" "must omit 'compression-codec'" id=$((id + 1)) +for f in dv_snapshot_id_not_minus_one dv_sequence_number_not_minus_one +do + launch "$id" meta "$DATA/$f.puffin" 'snapshot-id and sequence-number must be -1' + id=$((id + 1)) +done + for f in invalid_cardinality_non_numeric invalid_cardinality_negative do launch "$id" meta "$DATA/$f.puffin" "property 'cardinality' must be an unsigned integer" diff --git a/tests/queries/0_stateless/04656_count_from_files_cache_row_policy.reference b/tests/queries/0_stateless/04656_count_from_files_cache_row_policy.reference new file mode 100644 index 000000000000..256b5a9ab73d --- /dev/null +++ b/tests/queries/0_stateless/04656_count_from_files_cache_row_policy.reference @@ -0,0 +1,2 @@ +with_policy 5 +after_drop_policy 10 diff --git a/tests/queries/0_stateless/04656_count_from_files_cache_row_policy.sh b/tests/queries/0_stateless/04656_count_from_files_cache_row_policy.sh new file mode 100755 index 000000000000..dce2e16028e2 --- /dev/null +++ b/tests/queries/0_stateless/04656_count_from_files_cache_row_policy.sh @@ -0,0 +1,30 @@ +#!/usr/bin/env bash + +CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CURDIR"/../shell_config.sh + +# Filtered reads (row policy) must not populate count-from-files cache with a reduced +# cardinality that later unrestricted counts would reuse. + +$CLICKHOUSE_CLIENT -q " +DROP TABLE IF EXISTS ${CLICKHOUSE_DATABASE}.t_04656; +DROP ROW POLICY IF EXISTS p_04656 ON ${CLICKHOUSE_DATABASE}.t_04656; + +CREATE TABLE ${CLICKHOUSE_DATABASE}.t_04656 (x UInt8) ENGINE = File(TSV); +INSERT INTO ${CLICKHOUSE_DATABASE}.t_04656 SELECT number FROM numbers(10); + +CREATE ROW POLICY p_04656 ON ${CLICKHOUSE_DATABASE}.t_04656 USING x < 5 TO ALL; + +SELECT 'with_policy', count() +FROM ${CLICKHOUSE_DATABASE}.t_04656 +SETTINGS use_cache_for_count_from_files = 1, optimize_count_from_files = 1; + +DROP ROW POLICY p_04656 ON ${CLICKHOUSE_DATABASE}.t_04656; + +SELECT 'after_drop_policy', count() +FROM ${CLICKHOUSE_DATABASE}.t_04656 +SETTINGS use_cache_for_count_from_files = 1, optimize_count_from_files = 1; + +DROP TABLE ${CLICKHOUSE_DATABASE}.t_04656; +" diff --git a/tests/queries/0_stateless/04671_iceberg_v3_mutations_rejected.reference b/tests/queries/0_stateless/04671_iceberg_v3_mutations_rejected.reference new file mode 100644 index 000000000000..7dc9f22985a6 --- /dev/null +++ b/tests/queries/0_stateless/04671_iceberg_v3_mutations_rejected.reference @@ -0,0 +1,3 @@ +SUPPORT_IS_DISABLED +SUPPORT_IS_DISABLED +3 [1,2,3] diff --git a/tests/queries/0_stateless/04671_iceberg_v3_mutations_rejected.sh b/tests/queries/0_stateless/04671_iceberg_v3_mutations_rejected.sh new file mode 100755 index 000000000000..e5e9b0f9b045 --- /dev/null +++ b/tests/queries/0_stateless/04671_iceberg_v3_mutations_rejected.sh @@ -0,0 +1,41 @@ +#!/usr/bin/env bash +# Tags: no-fasttest + +# Iceberg v3 writers must not add new position-delete files. ClickHouse mutations +# still write parquet position deletes, so DELETE/UPDATE on a format-version-3 +# table (even without existing deletion vectors) must fail closed before any +# object writes. + +# Force quieter logs: CI often pre-sets CLICKHOUSE_CLIENT_SERVER_LOGS_LEVEL=warning, +# and ${VAR:-error} would keep that. IcebergMetadata logs a Warning when reading +# CH-written v3 metadata (v1 `schema` missing → v2 `schemas` fallback). +CLICKHOUSE_CLIENT_SERVER_LOGS_LEVEL=error + +CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CUR_DIR"/../shell_config.sh + +TABLE="t_${CLICKHOUSE_DATABASE}_${RANDOM}" +TABLE_PATH="${USER_FILES_PATH}/${TABLE}/" + +trap 'rm -rf "${TABLE_PATH}" 2>/dev/null' EXIT + +${CLICKHOUSE_CLIENT} --query "DROP TABLE IF EXISTS ${TABLE}" +${CLICKHOUSE_CLIENT} --query " + CREATE TABLE ${TABLE} (c0 Int32) + ENGINE = IcebergLocal('${TABLE_PATH}', 'Parquet') + SETTINGS iceberg_format_version = 3 +" + +${CLICKHOUSE_CLIENT} --allow_insert_into_iceberg=1 --query "INSERT INTO ${TABLE} VALUES (1), (2), (3)" + +# Fail closed: no position-delete files may be written for v3. +${CLICKHOUSE_CLIENT} --allow_insert_into_iceberg=1 --query \ + "ALTER TABLE ${TABLE} DELETE WHERE c0 = 1" 2>&1 | grep -o 'SUPPORT_IS_DISABLED' | head -n1 +${CLICKHOUSE_CLIENT} --allow_insert_into_iceberg=1 --query \ + "ALTER TABLE ${TABLE} UPDATE c0 = 10 WHERE c0 = 2" 2>&1 | grep -o 'SUPPORT_IS_DISABLED' | head -n1 + +# Rows must be unchanged (rejection happened before object writes). +${CLICKHOUSE_CLIENT} --query "SELECT count(), groupArray(c0) FROM (SELECT c0 FROM ${TABLE} ORDER BY c0)" + +${CLICKHOUSE_CLIENT} --query "DROP TABLE IF EXISTS ${TABLE}" diff --git a/tests/queries/0_stateless/04672_iceberg_dv_count_metadata_only.reference b/tests/queries/0_stateless/04672_iceberg_dv_count_metadata_only.reference new file mode 100644 index 000000000000..b3e9d2ae831b --- /dev/null +++ b/tests/queries/0_stateless/04672_iceberg_dv_count_metadata_only.reference @@ -0,0 +1,4 @@ +196 +count: metadata_only +optimize_off: decoded +row_group_spans: metadata_only diff --git a/tests/queries/0_stateless/04672_iceberg_dv_count_metadata_only.sh b/tests/queries/0_stateless/04672_iceberg_dv_count_metadata_only.sh new file mode 100755 index 000000000000..94d403ab99fa --- /dev/null +++ b/tests/queries/0_stateless/04672_iceberg_dv_count_metadata_only.sh @@ -0,0 +1,79 @@ +#!/usr/bin/env bash +# Tags: no-fasttest + +# Regression: SELECT count() over Iceberg deletion vectors must use Parquet +# needOnlyCount (footer / row-group metadata) plus roaring range cardinality, +# not decode data pages. Previously hasAttachedDeletes cleared need_only_count +# for any DV, making DeletionVectorTransform's const-count path unreachable. +# Bucketed / per-row-group spans use the same const-chunk + row_num_offset shape. + +CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CUR_DIR"/../shell_config.sh + +TABLE_PATH="${CUR_DIR}/data_minio/dv_puffin_warehouse/default/dv_puffin_source" + +profile_sum() +{ + local event="$1" + awk -v e="$event" '$0 ~ e ":" { sum += $(NF-1) } END { print sum+0 }' +} + +assert_metadata_only() +{ + local label="$1" + local output="$2" + local decode read_rg + decode=$(printf '%s\n' "$output" | profile_sum ParquetDecodingTasks) + read_rg=$(printf '%s\n' "$output" | profile_sum ParquetReadRowGroups) + if [ "$decode" -eq 0 ] && [ "$read_rg" -eq 0 ]; then + echo "${label}: metadata_only" + else + echo "${label}: decoded (ParquetDecodingTasks=${decode} ParquetReadRowGroups=${read_rg})" + fi +} + +# Correct surviving row count (4 deletes from 200). +$CLICKHOUSE_LOCAL -q " +SELECT count() FROM icebergLocal('${TABLE_PATH}') +SETTINGS + optimize_count_from_files = 1, + use_cache_for_count_from_files = 0 +" + +# Count must not decode column pages / open row-group readers. +out=$($CLICKHOUSE_LOCAL --print-profile-events -q " +SELECT count() FROM icebergLocal('${TABLE_PATH}') +SETTINGS + optimize_count_from_files = 1, + use_cache_for_count_from_files = 0 +" 2>&1) +assert_metadata_only "count" "$out" + +# Contrast: disabling count-from-files must decode (proves counters are live). +# Also set optimize_trivial_count_query=0 so a future summary shortcut cannot +# answer without opening Parquet (redundant once position deletes gate the shortcut). +out_full=$($CLICKHOUSE_LOCAL --print-profile-events -q " +SELECT count() FROM icebergLocal('${TABLE_PATH}') +SETTINGS + optimize_count_from_files = 0, + optimize_trivial_count_query = 0, + use_cache_for_count_from_files = 0 +" 2>&1) +decode_full=$(printf '%s\n' "$out_full" | profile_sum ParquetDecodingTasks) +if [ "$decode_full" -gt 0 ]; then + echo "optimize_off: decoded" +else + echo "optimize_off: unexpectedly_metadata_only" +fi + +# Per-row-group needOnlyCount spans (same chunk shape as cluster bucket splits): +# absolute row_num_offset + const defaults, adjusted by DV range cardinality. +out_spans=$($CLICKHOUSE_LOCAL --print-profile-events -q " +SELECT count() FROM icebergLocal('${TABLE_PATH}') +SETTINGS + optimize_count_from_files = 1, + use_cache_for_count_from_files = 0, + max_block_size = 1 +" 2>&1) +assert_metadata_only "row_group_spans" "$out_spans" diff --git a/tests/queries/0_stateless/data_minio/dv_puffin_warehouse/README.md b/tests/queries/0_stateless/data_minio/dv_puffin_warehouse/README.md new file mode 100644 index 000000000000..f4cf3a4c173b --- /dev/null +++ b/tests/queries/0_stateless/data_minio/dv_puffin_warehouse/README.md @@ -0,0 +1,35 @@ +# Iceberg v3 deletion vector fixtures + +Generated with `generate_iceberg_dv_fixture.py` (Spark + Iceberg 1.9.0). + +Warehouse: `dv_puffin_warehouse/` + +## `default/dv_puffin_source` + +Simple table with column `id BIGINT`: + +- 200 rows (`id` from 0 to 199) +- Deleted rows via Puffin deletion vector: 2, 5, 7, 100 + +## `default/dv_puffin_complex` + +Table with columns `id BIGINT`, `data STRING`, `label STRING` and multiple snapshots: + +- Initial insert `id` 10-99, then deletes `id < 20` and `id >= 90` +- Insert `id` 100-199, delete `id >= 150` +- Schema evolution (`label` column), insert `id` 200-249 with `label = 'new'` +- Delete `id` in (205, 210, 220), update `id = 25` to `label = 'updated'` +- Spark `rewrite_data_files` compaction + +Regenerate: + +```bash +python3 generate_iceberg_dv_fixture.py +``` + +Query paths for tests: + +```text +data_minio/dv_puffin_warehouse/default/dv_puffin_source +data_minio/dv_puffin_warehouse/default/dv_puffin_complex +``` diff --git a/tests/queries/0_stateless/data_minio/dv_puffin_warehouse/default/dv_puffin_complex/data/00000-14-ba106366-1379-49df-a585-047546b341d9-00001-deletes.puffin b/tests/queries/0_stateless/data_minio/dv_puffin_warehouse/default/dv_puffin_complex/data/00000-14-ba106366-1379-49df-a585-047546b341d9-00001-deletes.puffin new file mode 100644 index 0000000000000000000000000000000000000000..956042e717a548bc296df89f69e032150c44dd42 GIT binary patch literal 2141 zcmdUw&x+JA9LFmjW>=gy5FBz+nAoOG|BWoT2&>-q;z?x4pI>*vcG_-|jxNJqeGK2l zClJ|_2)@Cdd;x!55%e&7n8U#QQc9bA^UE(ELcXUb$1Dhf$KS3?eGfnW#`7o$@c#f` z2iQ)6U$1azlWdH#2f;Y_{`t$z9kL&c+)m?1r?agp2cFFMl+@6i`30FJ<&<=`?ZA2in08GpRA7gxnrh(% ztxP=>6E4~%ghn*JZCoeAm$gynuiM^1=)rjx;(8WSgwfo2;jwh*!Xd$YX&TdpdNuF* zsxr;|gRro;ov5J_eeM4lQ|+>=;p6SZ(CE4B)j9a1D2#@nj8R^eS|@o)6CRh8YgN#a zr#UTUq*IZJ3`$8OI>^&!ov>JzM{G)1Z5mM^Qw>?6UegVocXfQD0SO#w&X z00@bSo+HrWywyt30NMhyBNyZG%y|6C-*^1vh$cybK7HP*ThY0)HcOJL{k=o*`^EbY z(fMQM?d0RzFZJJK-buc``L;Wb6YP-~t?;OQzDMC~2Gk-2BLr;?-hm9(;T=UO>NSjN zP0J~(SyNQ3e32q=`0NK8uvVzWGzQ@10!9Kh%Ot>E25sMa2&iQ#8o``|A-Z{rW@l$` zK|^f2M-F;$YpKGD2abC^uEUVQPCzs_!5Tjc@_5wp^r>AukN`e-67O@d1<5)ec!;BW zPu2$Aq>Z*j?YepCdu_T`+`(`tRM7Kf3_qPoJIy1!J`@tII~dj4?cE{+ur4Igzli;tn`{5<^!I9E3f z==@qhv;Pfr)7&(m*|mT!ehYNefnTXC9YiK7EY?4NG;=w?=9z@32Pc~)vKXX}a-uf( uBkd*~HnkGG7p!OiC{&HgGFA%^Qj|@dSEZ^|6BnlsB7NT7z0}E7yM6$>E~FIz literal 0 HcmV?d00001 diff --git a/tests/queries/0_stateless/data_minio/dv_puffin_warehouse/default/dv_puffin_complex/data/00000-20-04b90311-de4e-4d07-9633-95670a571224-00001-deletes.puffin b/tests/queries/0_stateless/data_minio/dv_puffin_warehouse/default/dv_puffin_complex/data/00000-20-04b90311-de4e-4d07-9633-95670a571224-00001-deletes.puffin new file mode 100644 index 0000000000000000000000000000000000000000..ce4a05ab2431ee61dba5b610026f085560a79817 GIT binary patch literal 539 zcmY*WOK#jS40U>f9$}zW1B@!$kw0Ugi=aT!?y@feLmw&@VoQo7Pa6z!hhC}&C<2L4Krlx9X*1|KGI`F_QY72^#-de-tqpT<<0e;Bqu_ee5O{DfmWD+yA_-Q>efX< z{Fn5`B*f|X;+A|v&b*w$cNth1i!;}}+BtQ)_H2-8!c1|!xvzxq>t2{U|u zDr8oVa#gpGKa{N8jB;wAH|C%P<#p3i-UvYtpexGzx-2r4n2VmRA5NtSW`=R0o-fRqwxR$ literal 0 HcmV?d00001 diff --git a/tests/queries/0_stateless/data_minio/dv_puffin_warehouse/default/dv_puffin_complex/data/00000-9-dcc19f77-3570-494e-b336-f4d3301291b4-00001-deletes.puffin b/tests/queries/0_stateless/data_minio/dv_puffin_warehouse/default/dv_puffin_complex/data/00000-9-dcc19f77-3570-494e-b336-f4d3301291b4-00001-deletes.puffin new file mode 100644 index 0000000000000000000000000000000000000000..7df1cec9ff2a84f8a891ed255c8a40b01403463d GIT binary patch literal 1351 zcmdUv&2AGh5XapD2QEP21tNnGNHO~n?`9Lkp+e&0f+}$Xq2P~kV|BCM+TK7F^#OPU zUV=y91vnutJPxyokOERJJyfc-hw=D7#0(D=KiDIuHv9=U_g zyW}4E`0o4*nh$qt_t#oNZjsyM>&x#yP<{V@_0vt}n`9qPuaN_CgS`Fy=HLhmSy5R# z8R+!=ipH}R=#*-xAzIh4C8+2;TV`}fjfGnGvOMEsUX4m#d>GQuh&Ie!WLDED%g_O* z9W)A9(=8)Pn)vhiaxp6wKJVFD9A01teB9 z(KVr!$@AK(lUJ?_kVFVEBzT|46iGS@QA8{~grrX}TiC|BL@#G;XN+xTpM;0`I#7cV zT^+a1lv}hld|n48R?lRoP9Q$ZlC)opGXs?|6D=6ei<0p|2sQzoF<#|q%6XBN6T?#0 ztFtGq@K{wuYD#!*8&TVMhX0b&^Yh^)V6L_}msf!^zX<2iPR^st;XK)X&!elrIll;J zzLRs7B50ff|2DGnfPJW$?4SH$E7TkYuNB}lI39G^!=Nk<6Fqo@O}DUdFxFBS$ww7H i24$tOpT-iTl0`Mn%R-NJg`dbC<7MBj;{e0ePp&^TI-~Ib literal 0 HcmV?d00001 diff --git a/tests/queries/0_stateless/data_minio/dv_puffin_warehouse/default/dv_puffin_complex/data/id_bucket=0/00000-11-499dd379-3429-4dc8-9457-9b0d5a6a6e9b-0-00005.parquet b/tests/queries/0_stateless/data_minio/dv_puffin_warehouse/default/dv_puffin_complex/data/id_bucket=0/00000-11-499dd379-3429-4dc8-9457-9b0d5a6a6e9b-0-00005.parquet new file mode 100644 index 0000000000000000000000000000000000000000..fb364a125c2fa75b550e84d7a49497b2ed3e9d01 GIT binary patch literal 779 zcmZ`&&ubGw6n?Y2>K=k9bcS76N*J1u!Zv18^OGFvO?oP#Ac)}pN?l0OWU~>q1d8O~ zNvsMAVntCY9yFkW7fmDw@1A<~=FxvZ@J-eLguWv5_3r2i`KM`3f=HH-GufTuy~lgcUcP<*afDO=0kVJv0gD2-fF%JLRgu=l z*PZKja zY{H6vCIIY_LKoRLndX2E6~@yl+mRPUt#&+cea8{D#-%)l1u4oLDAr*-cT8JClcgrx%1iCXnr$vtEs@)+XMWCE&U!^bH9FSpr?QC0#e|{%YBa9z=_jA_}FVkYs0TV7qOzSw-63 zil-jLgU8~@b9?Y2_y-8yDtOt0-g*&xP+AXyZxXky6fuO4nK$3}-uLDu^VjaU2)K%K zxc6;$?}CL{+UlqS6y99@1q*$ESNre@zQ6%|g(LV0=`(y+0hKWTqTj%W`0n%jw-?W; z4F*JkvV^*X>k@8Bn3J#|VUYkeDxAlo{_*!r0~OQ;_3HpCaXR9OWJ#Wd$3G5tRar|r z>+}-QR3e#|71Pl7X(u0A4lJW$axi(8-ju}3rok4^g;gqM8WR6Hvbc^LT&A@67Q<~u z>m-%M`wGq`yKx}N?VCff=eGq8lig7;l(2bE0nE@+YkILa3Kh+=C{Yp`YJ0wDQ*1bi zkv+?7KP9tZ@K*=B<|1%fh&xX+AI~rDjA1geY*ZGb*A}`&@hJo-!v{dW{ zkHoTPd|>N#{B%{=Gj=zQdO>VEc6u}sHtm^`W4A)lZg%aN`|GxhxQ^ZNmL;Edjva}$ zUKj~FAhmD?x!Nn8C419JC6gx-Ik|s}$Ut~THgzIPru%>0gH14ISG{PhC*q0aDEC-I z-EgIobBk`J=;jK3FKjn+L2a^9^Xo0I)v8pdr-eJ^`$4VlR%_GEpfuGg*8|rrO?qQy QgTAlyg%=9IJpJas0pwQ6<^TWy literal 0 HcmV?d00001 diff --git a/tests/queries/0_stateless/data_minio/dv_puffin_warehouse/default/dv_puffin_complex/data/id_bucket=0/00000-20-04b90311-de4e-4d07-9633-95670a571224-00001.parquet b/tests/queries/0_stateless/data_minio/dv_puffin_warehouse/default/dv_puffin_complex/data/id_bucket=0/00000-20-04b90311-de4e-4d07-9633-95670a571224-00001.parquet new file mode 100644 index 0000000000000000000000000000000000000000..6dfde23d8eaeb51ba7d8cb37aefe5efe95801946 GIT binary patch literal 969 zcma)5&1(}u6n{HS(;zV*^bI?*hcM8PLK?G4)3hNM!9yuM1VJcR5Bo7`wrtXLvnklp zs|bQ8{{TVo;K7@>o(ooiLGS3%00_acjGtco{*zrG983uWg!3_4ViT{vy!?J^ z0sVZ6ULo{o;98+_i6ibXp=YKk=S*aRgWyaQsH_;JoeNz%UyQFz)Pz2@Zek3cL7P)n zVen%-B#UGt=UT&V?B#$M`@74bW%~-#fT=a6GtYg*0JCJKs9I|c>M6q@nK-oUTek8k zmX2dory*JeFfb+Rkj{vRj#>ChtC_&X;=I*HV?6yD`O8CGscjoXQ}+Pqu1)>_Fh6N7}O z15RS|NpiC!uTyC7cT2bn{F}yEA;aIY)#w29oaZQ81xMwf(^Wl7|Eyie@aaIwrW}UB z$O&ahj<+^ES2imp+3}R`4rTN4g^VI*N%pN?geR{g19dv`0!14g%O9%JRLno=AIZzo zm{UzS(Lw%$MJ0H1yjsGGPy0WJPcA{<8(6{VNQE1{Ab+NUp?A{Hn`N_JHuFV$MHZFK}Cd+2u2Y_vpcgjHEy<-By=gG z0;7xc)Wt3=Xwe-kDuR$K>%oXF4-o}DL=gq~AW=}%nO#?fAR2}_|KdL%L?Ipj-x2;#2&p!{0eX{P2^aZK*vnH|?*vItOHVS$R+WkTCP&nptljm0liLFU>1J;CV&E8t+e5dDWvnQ;=g_GAQ1eH@0>p|KM68N_Pz$@58c|?}Zv36#C?ReXlWqm19JPBNRM ziy<(4ise-Y?VQj?K?oBQGE8I;&4%F zz#}ds;a5%+0X*o&s^5*DJ7(L$3p31^5jB#VD^sy8Myn<~WjaWDSDQf^NGhGo#L`42 zg;a&9lZGmp#7)D}Q>0;i2eBiHOcGkFy`OqANg6vcX40UExMrmcx!BCwp4d#Xa-k+T zQDYz9KdLAMb4p$HL|y)j|D$k27KBx8T5?CmNLREbeLIb0%G{ptDV0j7Qt<_%8Pn2z zvASTWE*g$&@p!1VzTQx(qtRGhSgEb6*JD-H@tSZUDzwJ0z^n z4nBYUPBfOrSY|Rn{{5w&aLXmIMCU*O70$tVxBwR+4?0w#0ZY(?7Tkn*rIslq2mwI+ zTf&|Yw)OO2eEOe;9|$Z{EQ3*G)+6(PqS+fieDwI)^Osw%-n^T}JEtUi*PGb#=_2sfhJN7T2BJOvs8?sTw<0$67*aZhM5Fva z2nF~N?9L`A&At_^4ZU!o8)WW#!N6bbWsHJRE*P1-J@h+n#;KIb6}#HD+U>Ggt9izv kZ9A2!VODCcQ(SCct~!QMELroaju#i*-4A?1EqwC70NR(C!2kdN literal 0 HcmV?d00001 diff --git a/tests/queries/0_stateless/data_minio/dv_puffin_warehouse/default/dv_puffin_complex/data/id_bucket=1/00000-11-499dd379-3429-4dc8-9457-9b0d5a6a6e9b-0-00001.parquet b/tests/queries/0_stateless/data_minio/dv_puffin_warehouse/default/dv_puffin_complex/data/id_bucket=1/00000-11-499dd379-3429-4dc8-9457-9b0d5a6a6e9b-0-00001.parquet new file mode 100644 index 0000000000000000000000000000000000000000..43f7e9c338c66ca5c0cccdcd8aaf0090520bf278 GIT binary patch literal 766 zcmZ`%&ubGw6n?Y2>K;N#r8Dfx62d}40^7KonxEvL;6ZvSq9BOi{+P5YNt$jpf|MSM zU=LnO@gzA(LBVAe4W{VRjo@W zV%O_%ssL4Tph;nHRRWqqR|(wM%|S&m41^+wp<8({sG{dUMV2t2DVAX)iH%5JZB0c@ zl`<$COPxkh2F?uGp;b(z13Q!{W1l6u2!bM&P)rm|NInpDK=Qk!RfPgxPb29+nlStK z1c3dNX@F|82{evtWB^Va4co4P3{F_4&yD=H!#l1K(mq+{4j5`*_&M=}%W0%6E2$!v+x`5+FtWHue6ec`SI zi0Qu;uNkIi(-!pA1|sK{Z*o8r81iMKX0$<;mt`ahQ*qBQZ4YxTVGi}llAZ!|C$)hf z1!$UJGMnYt0!az_k)b1(4AO@-kFqEfM}&$%Z;xyP+vh1`8z>io7Kx}us48somaut} zS|WN5(qvF19mhCAioh_ixj4PZp71o&WZ_Y0WF(|<0+#7m>J2A{7a6`#WN@ir43dh| z<53#w3qCq49gGIp%Q%(U3Quzw@=Q(%Y$bLfxOTDCwQSS&hci{{u03Li>ts@?O^n$} zuTrh~b*rYRN_5oemX+QvRdHO~D_51?+mlL=(Nv{kj0E+#Qx)GHue-jD2Z*-SCYLu_ z8CIs$2-1BB31;_i5DIXQ!Fmp1;e7v(yE6r1-&8h@ZS~`&D=R4mW=J?SQ8UnP&qeTWizA~}Q$tX-50&)X*Zy)#%UaR9^`2D^4L$4-UE;N6B zUAZbZHU3;QvpV@p&F_2gokNq+(x~vmos%2KzB>Blk*TGWS@>|oC|DSdDD#)|yPlrD z_3P;EnvZHmX0P4)&ihxbngg6@3~1l%UhXmgr1=*dT4&I2?|k}a_AWMq92`4`ay#lj z(C*$)9$c6ZL+?%3eq2(~)2#pa(!korm_Cy;v^aY2vO1kwJ>5IdzFd6x>#h|~^B)Dh z@7DkAIy|+nZ+Q0Gms<}?-&+$6g(GA4v!3K$T4lir;;rI0?8}{nIQ<7q)U5P zNpxf4qz^Gy(iR|A!XV+Wq(V5I#gs@QdvAPqfFqqHk*+l6cR+p_BpJKWAWLf#GsU8D zYD>7A9*11$4d>KAJJr47O6rlAg##>=5lCc8c9g;084&?bXQ5m?Eg*lU5Kq~r$8jp1 z#h)5GB8}OqnB;z#!1p$Bf0uHNg{EsV# zsxM~Sh83~mr%DsXn^vnX{>1bs$+tT!rCLcO28vLyci_smaLz?P z2k`Yi5NkU0_?b>CxxY1@_m&k;*zIk3TB#N&)$+WNPCI7i8I}G(Wh5BYqtQS`Rh6Zc mMmOX@m=nT8Kmavexk!_ky&8Io&K?p^p6h)y3?pNwUlD3<~Ahmu_ z4^lm-6cqK;gE#S@77r4O{S(?_j~;sVDuQpircuOUdHeRw`+dyE&YYXI7_cqM9uW3p zZ+$1OF#`)SlL3s6lRx3=2Lj(HhvRSp#vuVGAq}Ts60%T$3Y>xSZ~-pDEL?#(a3C5- zFHwjQ0)R~Wgl!OZ?@ND&{6lS0q{cSriEvwlJ0jer7{1LGYOcK1ce6PWeO~gspQ!mI|+G7(-4Xpnrfy%W0IN#8C3$0 z#ks-1Q@%-3a&>5G=#+zTXmwac8@P3-4%KvI9r$C3asDjEra?%=BZ&hC2O=*i-zBL& zsTRP%MuJ8Eae5UQ*@SC=>PL}n~ ztwmq2=nmmi$sIz(R{kG^1_A-QwgSrZq8%=E{mw))Ox*OtcCgS&SQ#syu@Z*U4H{m; zE#~q?r&PD=^?adR_N^($af>CZP%L|HcB+25jqYid38;=xl9(VOV87cZi-NlgTC7??M2-uJ$5-kX^@a!SE~ zKQnk4;kB#pS2+>O)WKm6Aoz0fC!GC&z~?GJ5h`HA65N1gxCyu5HZ5nqNY%^Mxd35zzyWNhS zym0OMr-=JqzVY^Wqj#{e{%R6d9&uM5E}d%{vj@H(^03U|*Jz9tFZP#@O&=u__A&y` z6?QdHb0~)q2na-`a}b1(z%#uL{@X7S4tEiuXh9^K2yZ*KK(auiGFf*`)rux&0keMK z8Ou1s0QOLmHPo8whZrl06iPWnK@Nb3S-}q+Ww8wGAu_q+B3%^VG9ytVr38XQsUt6y zL90jEAu~RZ4&*+I32C|TG&mHV0YzUypQO!@Z4}roO|c@Q8B#hd zOF7Fd8dWKMenE0PiY%4Yk`qsNvSgd{b;~yC0Y#hgpIpUCIWH~B14v{DaZ34r5EWoK zu-Qf6r{`3AzHZhgO1AfsX;-aE*{g(0d&Hw-a literal 0 HcmV?d00001 diff --git a/tests/queries/0_stateless/data_minio/dv_puffin_warehouse/default/dv_puffin_complex/data/id_bucket=2/00000-16-03e76970-c0de-4110-9928-e74531d8b125-0-00004.parquet b/tests/queries/0_stateless/data_minio/dv_puffin_warehouse/default/dv_puffin_complex/data/id_bucket=2/00000-16-03e76970-c0de-4110-9928-e74531d8b125-0-00004.parquet new file mode 100644 index 0000000000000000000000000000000000000000..cb037bcb4efabf66e7d228d53eee43670eaaf802 GIT binary patch literal 1043 zcmah}&ui2`6n>LrOTeWlI70$KmO#TUrfbu>KUx;li!4>K6c+^XkYs0TV7qOzyZ)f` zZ%~SO=)q$VJgOH3>CIIX#G{CM5vzz)D5#*~n~B?2iWtJj%$xUp^S$>%rjAW}6zCeE z0ljnk`;U}G4czNg2gu$#^b^jy1U7o`1fIcbcmwa?J$!^u@EN|qCM3_EGYY7L03h)) zy+-JVtB;vhpDFLSiObe(XP>l{!hkkwaZ8Ke= z3f1}^%K$3owD={FCGsr#p!fKpG@~i zG>uEG?sxnlC}@_2LNb(U`T=iZY(U1)pJi4Yu&8DZfo=*mO*4aS5VlMly|P`iB(R^P;L!(I{Ql`Of5NB0&UMiMsqy#i6nkv^|x;RP=t;w09LstDW8elI{)MD!z zEdu+cGQ-i?y1HMq-588Q9`JZ3-wrSES>Jd{7VXZJd2UbG?M~bcJGN^lN5fIwo)~fM zM#P)-wmorf(H0TUwOjtI$j6;)$9$n1#T*Zi&E39MbFMXGFS!Y2bc-Tp_iqv!h<4D% zwrFy`|L2`w0%LOCj~BYUGdvq-F7mh?&9yRK-Yew2Og897&3Yy*jTTBlx#2e&h2r=) n_r`)CES0@tX}lhej5Q9IL(dx-^>bzgzp&(m>k7aW{tkWvOij}0 literal 0 HcmV?d00001 diff --git a/tests/queries/0_stateless/data_minio/dv_puffin_warehouse/default/dv_puffin_complex/data/id_bucket=2/00000-23-292ac3e8-88a4-482a-9a88-e08ad8bdd2e6-0-00001.parquet b/tests/queries/0_stateless/data_minio/dv_puffin_warehouse/default/dv_puffin_complex/data/id_bucket=2/00000-23-292ac3e8-88a4-482a-9a88-e08ad8bdd2e6-0-00001.parquet new file mode 100644 index 0000000000000000000000000000000000000000..690316c0bced6e4729a5e503715c5549ab0386c0 GIT binary patch literal 1197 zcma)6U1$_n6h3ozc6K9C8k&1MBh$E!&9IAWy0dY!o3Ka>t!&e@##B+X(ag@yuA`g3 z-HpX0VwRR*rKqiKS||zW4?g6jLb3iLF%g?qv3-!@LnPrg<-dzfpKLj|YNzk5!&mCZzd3vT*L~mK-oNIzFXPJW7y9a%(KC(1 zjbCj}ojku)b>&`KQw|Q%_&~e%q{(s|0G)cufj>j|(*1j>B`A0vJ`&1b9C%|;BVF3~ zmbFKBW{r3A(rNDI^7q~rsyE#ZI){6Y*8Kgouc>vcrTNumEywqLJo|e8kw5>EHcW&c z@_X(&`bXZp`oj*-kFU(O`8RA{8+E<_zKM-(&xyQ{pZny+$(g`jbZ=bynV-rVqa`OC z-`79y32%Q5F^oK3fmQtP^uq_GQH(HO6e$AW0DunjodOdP2k@!i|9ONM5q`0pZ2$t@ zxe^E`0yO|q8InYIB@MENx(^B-U?dTcNK8^8R@lY)BoR%1w>06PlO>T^mJ8ZNL0T*m zH$H|4>noEXgr}G3hs)0>viaB$7JZ)luvaCcRvJ%@6;8N=H2|cG*LWq}T z+Q<$$(|X)Y)0kk#C`MU|svu)t3W4CJD6Ir8B{A}dC&!TLjxMlTpi00y&vhY6q0y^A zAwkqP#~sLJ7bfJ0fSZ5~DV-Z)1GC5c;viE<9YWC>@KS+9&bO!y=`D%~X=i#!^^gwwjn_;)7JJu^3xOLDLx94eWIkHs5g-NE-aHl7mfhA zSwdo}n+yo$Onq^MWted@ld8%Zd(C!T{E6#RatFIirB2D_GCf94@hgSYIxC^n1^i0V zGSi8yQn#y5VG+%*bm;ADKVA8ijM?2|WlWk#>glZMFE&eecBGU;{z6S~zJ{IQQ>rKg zYf)YGe4TyJ&rsNU2*k!NJ=5J|=GL`mJO|87*6Qr=XjNLMO7nQ*Jyts5F~Y%6I37vr t$z-Uew${|D<8dP#(Q3lA2_sOQTpuwsEfCawQk34c!V4n^pqc)t9s`uDBd7oX literal 0 HcmV?d00001 diff --git a/tests/queries/0_stateless/data_minio/dv_puffin_warehouse/default/dv_puffin_complex/data/id_bucket=2/00000-4-fe8ff9da-4256-45aa-9ed1-48200445069f-0-00004.parquet b/tests/queries/0_stateless/data_minio/dv_puffin_warehouse/default/dv_puffin_complex/data/id_bucket=2/00000-4-fe8ff9da-4256-45aa-9ed1-48200445069f-0-00004.parquet new file mode 100644 index 0000000000000000000000000000000000000000..56882346e595e3067aaa68451a744b5903dab4f8 GIT binary patch literal 806 zcmZ`&&ui2`6n>MWHPEG{)EN>8vIKY8Vj4HwZubX!XhB2^V$m%Wv}iWT)DD*arhJ2t$y7B%FdQIUHu;C(g#af-OoM=%IyWIVk$`6R?!EQL zPd1)CfBAZI>&@Gp_UhVg9&fh>VM5AF?Ys(X4-Z$x3!iF8BnT#fjy zUQ8Q$)%OCop%pWr^`hs2K`FJw)&|7Nyp@$cU_Kd??!Q#TazG8GBnIi~q<^Mq_ zz~`{J6;P(?cDU5^qLEq{zv+bye{mshnO4@c;;Bm058Swu%VcwveATX2v*U$=XN^`W iPA+ea=L)Wq9<7e$9m`5*>|xcw!SUIB!B1!!|Bzp-n5bX? literal 0 HcmV?d00001 diff --git a/tests/queries/0_stateless/data_minio/dv_puffin_warehouse/default/dv_puffin_complex/data/id_bucket=3/00000-11-499dd379-3429-4dc8-9457-9b0d5a6a6e9b-0-00002.parquet b/tests/queries/0_stateless/data_minio/dv_puffin_warehouse/default/dv_puffin_complex/data/id_bucket=3/00000-11-499dd379-3429-4dc8-9457-9b0d5a6a6e9b-0-00002.parquet new file mode 100644 index 0000000000000000000000000000000000000000..f9661fe1b8f3dc6051661f59daaa1111584c3f3b GIT binary patch literal 800 zcmZ{jT}TvB6vxlKJIV+m1>M_B46$5rS!>*8v~^WB(1)a?K%y)N!TGqW1M8}@yH+?# z^-%OGLeNt|4`mW4q(VfdB_ZK!PklT{g}(Y&Y@bGzCvm;}hp z2$@9W>%G~}RT?owLMAdm{Kbi1aAgaD8{v?HB86&PZK!X46eWptp^5}~3M>4eatx@@scux+w{={kz7p>z|(#r710|7b!x zb|wIHNg_w-&j>MqZd6bsCex|Np>W?yjpuNi`+bSL)6WMiWfG0D!i^!$T3No}7o7sr zS$V16b6KmQvy8{HZl1MX9c5w0)LG6N2>sY^^AeiIb`|6MP8^M@S`K#&wGQpsF^U62{RhEi(b}^I;nv&8Ad~>llrhlg*B48Vy#Rs*8gQ&+do#g}TJu_zjbYzTf}= literal 0 HcmV?d00001 diff --git a/tests/queries/0_stateless/data_minio/dv_puffin_warehouse/default/dv_puffin_complex/data/id_bucket=3/00000-16-03e76970-c0de-4110-9928-e74531d8b125-0-00002.parquet b/tests/queries/0_stateless/data_minio/dv_puffin_warehouse/default/dv_puffin_complex/data/id_bucket=3/00000-16-03e76970-c0de-4110-9928-e74531d8b125-0-00002.parquet new file mode 100644 index 0000000000000000000000000000000000000000..c9baf49eb9c946293095888191bed65921c0f8e1 GIT binary patch literal 1002 zcmah|O=#3W6n>MWC5L5E=o=CUvIH8oC0(1Y`^zGTpi)FyQ4mBS$BI@K z!GIW0lu(jTlW<j)QJWHD-90KVy8&(Hn>D9QMc|f6(M%lzh^Cks)YIq)h|g(r}EwP;nMy zg-lu@F_qMa`37?atWkzMn>SeTNQxwbY!~>hhG+O!6$`^88re+SNV2r}6Xt9y&LhQ2 zZZQG!C~q`bgXz-fG;)>gub}JI?X4kIYUbSV!bM215DEYK=?O3ez zqgc=Zsf9bt)n4i>*z0a8nK_Wi$^BbI2BIUf;(;ug?*Dan*1?!v_T!bl=uIugx%(pS zMoXQXSMW*&FP9JcQM;K7D>J1^P;L3GR;gU839lFgVWsMoE45}gU2L7JhMqS)<4>3k P`o7W^9xDKI^q2nu>F&sB literal 0 HcmV?d00001 diff --git a/tests/queries/0_stateless/data_minio/dv_puffin_warehouse/default/dv_puffin_complex/data/id_bucket=3/00000-21-55307a4a-f583-48b6-983a-c2993f261fec-0-00001.parquet b/tests/queries/0_stateless/data_minio/dv_puffin_warehouse/default/dv_puffin_complex/data/id_bucket=3/00000-21-55307a4a-f583-48b6-983a-c2993f261fec-0-00001.parquet new file mode 100644 index 0000000000000000000000000000000000000000..4a5ea28ead0cff6cee3f80ddb56c71f42cda3ca8 GIT binary patch literal 1148 zcma)6T}TvB6h8B_(?XkNcrP{>efn6%|Gj1eWwOJ3Cub>#DQ6R%Bp3 z1!ZaY5J3e|LO%~-ND;lHB@|*k6b1QEQBO%k81)i%@2snW5DmlGbI-ZoIp25Au+1BG z>Il$d4h=Er$;|cdoQ%RGARsS5V0hghXi2l6Y+NSYJ5q8C9oTwNxpJ}9%8zuF`%7P# zH*YjCGr@eYxAEG9n!VMeT-f}nS>!XryQkhb&-Sk>15Py0>0|Te(ozP1G#FvgCkCAz zx_yRS)KTN$^wZ^XUu;xtf8Y0L&E4R1|KTf*MKNzR zodJe2;ZX@v(XYwh)4n)j5nogt1>gW6V;wEzLH2S%>`Xh&ZqNAjDm_%C`vZx*olg4ANKGh`h^CBGDin^zEWI|7 kFe6bt9El~(>e^IY)YSFr8lysrlW(o?!c`Wanf&kk1t0AZ0RR91 literal 0 HcmV?d00001 diff --git a/tests/queries/0_stateless/data_minio/dv_puffin_warehouse/default/dv_puffin_complex/data/id_bucket=3/00000-4-fe8ff9da-4256-45aa-9ed1-48200445069f-0-00002.parquet b/tests/queries/0_stateless/data_minio/dv_puffin_warehouse/default/dv_puffin_complex/data/id_bucket=3/00000-4-fe8ff9da-4256-45aa-9ed1-48200445069f-0-00002.parquet new file mode 100644 index 0000000000000000000000000000000000000000..82ba5ea76d3794b3f1e8b936bca6642e3d813d41 GIT binary patch literal 772 zcmZ`%&1(}u6n`^W>ssO`onaRiSr!vg*u-p_Zz5Jtr63eh5Jc!^cT$&pwApL~DS>+R zpa(w=B2usyp;Qlgm*Tnqgr5gb;zQUFWzd$UD@{+DzBB`AMikBNz55RyJ$bhI{KYuhQ58D>mP{$e z@_hc|5iHYBQe9K0u~aT}_k}R3bI`eoJ1!z^U8B1@f8`CXvMs_T(%{>K?MYwfMZl`Y z!lB4*b_7#^DJjrs;9sS{lISYF8@VYcDTaYizo?LV6E z;_nFnzpJp*{9S_Ofj?HbnO6BXIwS9`9Sc0s6yZv4;N28mS9?g-t?2feuuDM}J$UfYU*HEgldVZX9F}L_edm4N=b4Y4y>Q9GfWJz( zkMP>hFQ1MWnDT^*Re;p?+23&WGlDa61AMpzE$Bc9cVH2Ea2M{ueRv3~@B-H16}*Nm zcmvz81JS^jC6EyU;N5;8yo2!bZy%SB{nNfL!3SC5oZSaQeAHiFdGhqx>e~9tjkg2N z2Svc7fGGhd1YiL~z^S8(bp9QhRE!%3uXM=s`f*9u)CMmQgNGwul1@M;ls{1sQglt) z9pJ-xgQ$3bh+rCI58-_g>!b#}s(NuEw4Ely3ScDxn&f*|B)~BqDxMwM3CJsk!Kqxs z&}kA3tmp|)xr*=M7@_2`Oa>^aR>!8sP6?z&R>xUxgKr~OhiW>q4rEioglx&U20|hh zNemqfMK&d}kCJ<+T7VQ^o`#e|O-Suv2Y|d+@M-c(##tbP2||-9*^p0)b!$_;%N!Oo zG9C9OYuVZg>Y3r7Igf%cHGKX z`HU4$IbFZ$#ob~qUvx@!yI#*1%4KHFIF4H^S%qTRbF(w`b0ycZvN=1gQ+{yb?7s6W IG|T^uKV)>MN&o-= literal 0 HcmV?d00001 diff --git a/tests/queries/0_stateless/data_minio/dv_puffin_warehouse/default/dv_puffin_complex/data/id_bucket=4/00000-16-03e76970-c0de-4110-9928-e74531d8b125-0-00003.parquet b/tests/queries/0_stateless/data_minio/dv_puffin_warehouse/default/dv_puffin_complex/data/id_bucket=4/00000-16-03e76970-c0de-4110-9928-e74531d8b125-0-00003.parquet new file mode 100644 index 0000000000000000000000000000000000000000..4d8d2c59eee2d2bf3da40564de2e5cd68abc29d3 GIT binary patch literal 1028 zcmah|O=#3m5T2K0C6%Qp>^u?(vIH8oknT3E`_m;J3JOv@6j3UQha{V=f$g@6KM zWqfz%`R4%xbF^1c1t>fz{enwb2HRWk2HwIByoV3)5kA2e_zvkaa6tl@F#sYp@D^@8 z__}-akj7mGlmaCIRRMJYrvyw2I4$6efGGjf0%izM!9ld}$IkAztcwyVeN#06nR43U z4Ur}C41D?Y=e>+5tsJy}jM8$Hq#u`bP2HlMcxc)0041G+&U5spNvx!6Z01l{reeA# z@UJ6-tGLRz&@}iO!ws=cv<$u?<5aS9fjH;RX1bo=3h@Z}9Rf{?8W$zNEG?CW*Y$>= ztQZCfC812q^TQU!jwCViXXvg41}gdxsJdWdmgo33$rt=y)N15|>R7qz*P33lS*}b>gwD9{2i2NW fsZKP4(edWVTHrXNW8Sdt(ifJ#a9^TdjsE}Nu!qfx literal 0 HcmV?d00001 diff --git a/tests/queries/0_stateless/data_minio/dv_puffin_warehouse/default/dv_puffin_complex/data/id_bucket=4/00000-22-cbc4c1d2-88dd-4d8a-a31a-6db198e852d4-0-00001.parquet b/tests/queries/0_stateless/data_minio/dv_puffin_warehouse/default/dv_puffin_complex/data/id_bucket=4/00000-22-cbc4c1d2-88dd-4d8a-a31a-6db198e852d4-0-00001.parquet new file mode 100644 index 0000000000000000000000000000000000000000..9fcb8cee529f660beb1a0695fc96e23ebfe95a93 GIT binary patch literal 1176 zcma)6Z)h837=P})%QbefEVb{;T{&yH(OZgXnro9ZDN#0&Aa%0NDGbI&FUh65E$!0e z(lWPUP+?<=Xw@2!O}2p_vJWk*;)Eh?$2#dI^^^2Nl_FHOFQY=IVZ}f1U6!_NFg%Xu zectEq@A*B)_3nOM#ek1&r zpMRgu{5UpqcJy%LvGnX@Brw!EQv3GIuIce_Z~by~a`8%jpX&bOr-_d;6Fuj9F7ECB zOqu<6Q4uGswtHm()zH1%jL;(L%P#k=EjRn%a2VMlniaQ~?QdVi0&I zMQJ3li^Rx1j+{e^GrBp|=BNZb8?$a&DKxrw3JIdE4`&DRX$>aiR|nn$Ruk95CN?m; z%{)skA!QUrL*StV2~7KzGQ_Ns2&7bln@EYr9o0gdg|;$|UFj`SVw8eI`I?h#K5an) z9*bLB2hmG6%29k2q;*6|sLD5xA8N2fmUtWna`h<^Q=H^y2VJcw33Wz7kL%ffUoLS( zAJBva=A>+%$mnvnoHMiegeiOFimS~?%H4jioHF!uGADN*JSkfi)hiEb16Dk}dF8A= zlsB?Eb)>X(PWM)erH>Ex%cs4SoQ{ngYllzBq7aNNd7T@1_M-ns;-S+Z^klT`P+m9N z2C~h=dNyYqA8b~AYS5=Pd*XQ`oor5Ybp*TO;gpt21w)aDu6D-biLS63>WU;2{?1e& YoKRJNht?`a>9OV literal 0 HcmV?d00001 diff --git a/tests/queries/0_stateless/data_minio/dv_puffin_warehouse/default/dv_puffin_complex/data/id_bucket=4/00000-4-fe8ff9da-4256-45aa-9ed1-48200445069f-0-00003.parquet b/tests/queries/0_stateless/data_minio/dv_puffin_warehouse/default/dv_puffin_complex/data/id_bucket=4/00000-4-fe8ff9da-4256-45aa-9ed1-48200445069f-0-00003.parquet new file mode 100644 index 0000000000000000000000000000000000000000..50b6c3ed85e6464c819efd6129ded6d9fe2fca95 GIT binary patch literal 766 zcmZ`%TWb?R6h1RqWho&dbcS77$}*6U!Zv0Tb0tW{2c^DLQSgG|W-oOimv*x$NGJ&X z10sDYf(Uu=ArDe06n&`V7x?H85G_9V4;0U2ZL5gGa`v1v-*?V8XJ+-vEt>&*Mp?{u zzi#hk3}#}iG8G`dfAI(0jR|-(1M|S(0tm3743}XUuEI6A0XN|u+=oYytaG<1qzD0o zvID}N6Sn{H(@W)_lAj5TDZ-#H%}|<6X+{*ybo<55tJmWRhjl#fYHZJmlb zRmvfMBK0(jGH_zp9a@Wtbl^uR#`$;3RzS8$CoN_QW=*~kJ|(#cX;mSQ$J0pvk0QMO zdji0Jrq}{MNUp&X5X}c8Av3lmdQMm3F|tJDZw$l8#|>or)^V*)zqcl~tb|fJp~zAG zpM(NJNqe{j+RC~UZ4CW*sT-M_el!UCJ<~4Om4a>N+rzNqnQpCAskQ5Y69koNqv6}- kcH6Dh?P{&zxy5pDsqWf#vE(f37G9jZyCZx;tN8MN0nc`u;s5{u literal 0 HcmV?d00001 diff --git a/tests/queries/0_stateless/data_minio/dv_puffin_warehouse/default/dv_puffin_complex/metadata/0c03abfe-66a8-43ab-a765-a9e8072b59ed-m0.avro b/tests/queries/0_stateless/data_minio/dv_puffin_warehouse/default/dv_puffin_complex/metadata/0c03abfe-66a8-43ab-a765-a9e8072b59ed-m0.avro new file mode 100644 index 0000000000000000000000000000000000000000..50388b8a2ebb6cfce380f516f8023dddb1a5cdd1 GIT binary patch literal 8221 zcmcIpdyExV83$G?lopMm#kiW|Sha2L);lx%?!tlsEfMLK-8Lq$Ue29+?(QseXKrWa zvUj^Bn1<9xYB2?*NnL_Aqz};I);6_K(?}abTPY>czWl>Sqo$Bh7PbaN;&;wG&bf2% z%-+4M$%cfPGvDL)J7RK9Fhuz8J0ODZBKdMDB(9b?Bay4 zzyzIsH@N*%c$;nA>{;@BZ)l@UG|Q%XeH#kc)bqq~GsJZ5F?cRm>mPAk?3NsO<-4hy z)JJyCbCFsZoHcRL8MfRE^UeEUlVp|*13qCc4W^_V$>WmYO5OCiC`lG20WQSUNX9G` zvcwM6*Dp;Kq~&#kWSlD&*(!*nk;qWFp=q%o+e!eKmj*sqnq|5Gd-%QNA)B)_NRF?0 zFCAp%kvmKf)zppB%cIvp}S+jvp_2gldOu?GeC;i z9p>8o+=2Ko1J+v_W(w}yxDJi#?l4;0k61oYk9qjzbXq`#$<14K0SkUbB(DWmkaq@A zD7q2N)fBR9jsAm**oN3wCYhWTlLJn;%=^&#chl~5TmJqf-P(A3@Vn-Daa(vDCOXyLi7ee*vz2~ctbLFqLK8g z3=8TU0;tFoz}mecIoA}iHO-i7iqcT7F{GGq$U(0w#Ali;wqz5Va}Ji;31F#R4P={I zRRl@drgk+57M&Sry+U3ghFaJjy$ViDHdSmMhBZp;OcpSQAueb*SJ2axcA94$4M3-~ z#}{gi&a8AagdOu2F@F<5QrZM-O2y*bf<(KS+DJE3#6}S*#u^~0jkPo8Z#^K1*$!PK zQy?z(XZXm_;Tad+H)~AI9xJi|48B9bcPMLE*~GZ0VmES%Fb`&|eBL2${n;DbkT6*m zU_kZ9N0ncO{Zy=rvoObT+41d26-S$3NCk>83yZwbxYD3@ZqS-w^3|*Y3|YDZ_c6P) z(7M%BOM-1;8atK^XxTZ)jy?o)o`p;@N}nft*D4}!lRU90r^;oj$xG1DD2t^OGEBs^aM_L_ zhA$wRLq@<0Zm1>(H`11iy)l@Mp)mAf7h$;LeU<{Vs&7LwV&cH+(DMP|^$1 z&ALY5njG@C1n;+olQ`r+9h(f;Z3MJGSP}4T21ww`3|ct!wy4~HI2FqR{%OJ@MCB^0 z$b4M_T0;ShsaCH9>XK5qj|sK_{8Q2SYEW79goTjGjq=k1a-g=bxnB{4VM+6FGHCqe zK#Uy`N}seW;2QYqi)jeg1oi1yTj$z<>*HYxvp*KunZpF6KTIBRdBCl&Ok9@qbFd}g z{_yON><73BV&(`pRJJK$6QBPIba(?`cYuWVmaCv>g=?HT89w)+EA)ZMXoc z9~4Nrx$z;HdWTNQJjZLX>{)uIK*h&-2E}=$o>g(;&Lcz;t&7reZQ_+5kcSHP7N<2bp#8sC9JiFu4ON6K&c+I4W=q_- zW}c6MX1bCWIa=KY%$BPG`+~pbaJ*#X>f!nDQ?Gyfh=ZbQW7DBSFO;G8)6=H5L8w;8 zD%l$GhL*@LYp|VK;ybmFa`t(r&i-*=@4I~?>o@KkxmIa1ufF`@Kb&0t^(J}QN8aDO z@Zf>ldRKkt_z%{NzwxKfG_U^9t)HCOwR!j3kA7vey=!{+Li5258*ct>-{S1Vi+65r zcF#9;ez&}Dh}=9X1&4wnc2`R(b| z{T=q}Tkxj;zIEt769-zBl)iBGxxe2({iD8D_MCXG_{7eZcmDC`!*9L$%a_}qEpK1+ z&RqxZFYKGT`q1Q)M+Vp5eebU)mmT?*+w#njUravptEZp6cGX{pKPQz(4t%+3_3i~X zz1XyJ-V5c27QgWLPunKW>`%{*5>7Ke_ayukJX$=BX9(iN161 z-m&7?(ckxUO`Q2s&vj3~q84FJ5)e8$D|V)=!;&aO{W2_kCjj<b>20_0fMjM;E;|eR}HQC(1AP_k8-v9j~2OX$ncIWIRd%MT( zo?Qn*lh6bKif}+E6n=m}B5FyBApB`4B|iX(ZxF32R9Zk$DT1mX_`weZ;+>h@nYq2& z_4(pTR;29Cz8}x~G4J!r%2_qaeLU3Gyf zc8dj& zt3~7o7X1g+aR9MzPBOiuCI_5|nD@f^Z0z+?d-NTF1(6xNBAg3(Z5 zPGPqp%QW#)aY7$r z;dK28$HMMb-p*^d4vjQH;~HG2SaT1NW24B2KZ+0ohZdp}ZU>99yIW}}Z4(dh!ay|) zP#L9epc+n_C2@^2PJzncncp2$ET4;zNtWBN;i5wH20+-e(MG&6A3N1Z`c;_)bpZi1 zbP8bmUXh<`hT59u%r(R4D%TiNOgQ8q?+V$OrmHR4!`^~}Wex#Y=5QC;X0{YTUbdOT zU4lht23oI>SBRk|+9R*v++;J==HdF&Bq&z^a~R^1hI0}<%^0M4wx$c{jKS8Z(NM?8KjQQIMNNTo27b(|BgyR`L zHFR*}#C>zQX9Y8LHh{r*DESUmT)&bV7gOy8*;fFfl1~LgqUX&@Tiq zrrNp^=txTQJ|@`$$WK-0TR~;fQx-y|G%8LD$bt647Jfw_!cvmqWYEORfebrRls<1+ zAT`L<7tu_x#^~o@a?9W7Y?l1xA7s;b852W?YiOZ6H0k#CvADR8B{eUz< z&Aj9q=lh)eeSuUWlaW_QYE#gOs?5!S)G{QXOzjM${3bL-cql6`YA_P0LC!~AxW}XDsM*IL;OuW%cPDp`G z5p6^8qv$8N!?=91C+zH(tt`#!G7OFH5FV>$ekY~M;H0Qr zme3gSSZYUdW@tx%ZvZHwXLge&MDs zR;+q#?LCj(ebKI|e~;gH&c)xZ|L>Cj9q;{g^3}6G>Kk4%uw>woT|JHEk8c<@{ZB@B z{`S(nkG_e%^T_YN ze$Cqb*5)IBzUu02??3$6^Upc^pSga=)77D0^$slmTi^F@7oUFp$>VoDIQuMF+jHt! zy|3KA*gpS9zgYLX#pmuTd^L1nY~TH_|MBPtXRe+8%_Wz7_eHq=;NPEk_vMEQg*yu0 z+;Q*i->+Z()O&~DJoM?N#~-zO&RDwtz>T{r!~4H} zar4o)9=veTxeuJT=j_=ddoCEpp BrX2tP literal 0 HcmV?d00001 diff --git a/tests/queries/0_stateless/data_minio/dv_puffin_warehouse/default/dv_puffin_complex/metadata/321000ac-b08e-430f-9c79-e40d5e4aa42d-m0.avro b/tests/queries/0_stateless/data_minio/dv_puffin_warehouse/default/dv_puffin_complex/metadata/321000ac-b08e-430f-9c79-e40d5e4aa42d-m0.avro new file mode 100644 index 0000000000000000000000000000000000000000..0cce769e50f047c4764f93674602e98fe0811c84 GIT binary patch literal 8145 zcmcIpdyL$~87Co1iV_h7FUnfv$y8F_MYv% z>xG;INIWD!YLvFL=^qHBAkwBS;*UN+0t6)mnyLsSD%3%j-E=(a zBRj`kq*f+pJzNi~eq=J=_y=Q}*Kl3q{4L8z8=yO$4NtFb+5YHl}*Qk|A zAk=p*%}}IO9G95kd5i2*MAAr@RIV^B6XZZo0OM&8gJt<%1h9+mHIA(9TaX+#@s}a; zD=4ZGL^X||;ufm5fJ$5He&|Gw?;)GGBqCt}>07`*>^jkG@+{HH!c;O;>lq+*995)#QK^5%XSHpN)N~*dBeyHp>lblIck$Dvd*Ko)*F9@wo)4 z+Y)OK4GxMLb(a)4R2Y?UNNfagqCpgPqp(h_6pXq8BQI0a@*8S?umVc7?qqdA(}N!l zk;9eH5Tr!6mV&dNn2u+bXCZY2h`d4*1$+DmyDaZe2*;`TSBRk|+9R*v++;J<=HdD?BrwZ>ISg@0!#R(hrVr3O8|(r)eIUC~+jM4q zuq*7Czliyp3zFWiSktK2mlq`3&5UlknXWd9R57*z$>^?~F@HM&NzHcXB4&+5IG*8C zLkBlb+&9}iE10db0SvxF$#o5yump?vXxwbjfHY`ZGWlA54TdZyg!`CXT4?*tR7;L+ zY8r=r0B8j^ZqVbiieewwogF{H{CGT)JaJ|TcH z)z+0jM^c*iG07G{eyTd(3Mz}9vJf()QE^&84zw4x{3`+xmZA(NgC<@MWZ03S^m)qy zsX?y3mBF$_fytFzR4VAq2M|OVW*Z-LNV(taAM(x0cjE6O>OA!61^}Wkutn6Aav*N#iC?Y_@(y? zzf!z(i9YEAnLg=&UE)*O@i6;YfIaAyN*1jgl}1YQW!a|~qm`Z7V<;ulz4Erd^CY-7;s?-T;*DN%LJDk( zXd8kj&67Ep;&M`0`yjTp4iZsX$D8;f#^sYeVP~IgWocfQVQ74Z@K`nTJ1JELCq?D5 zgvN-+Qahq_Qk#0^C*+}ty=7@l4QT%_7AGy0b3+SZ%d)WvnWYjpsafV@qM5GbRgTuN z0ZZj-z`o#bqZlt4x%%k}_^CI(ebhlQ3a}T_p%==K|MWE0KMvIz`i;OMd}vAivIg6! zi({vN+OZ${F8JEO4ts~aZ{wNGKbl{C^6j6m{^zPw2cO^hQt^`GW52xd^785aUr%4U z=~ruR|KaA_H?RKw!zUMx?Ys5RZC`)k!lyo6<*qAy{Ks{dU3l#U#`+gd+_ho*+RZ1% zpWOeuQ~x;g=5I#5?|t&6C%*HyL*s8edG6uYZ@BKpJM0h6o#>f+=8?np?;m}n=jcrz z6}I$Fxv%f)8Qb*5C#t=B-+1*0wti@Lqvs1p-n#n1zrMd=&HT1|_xW3M`MkN$n+ z+E1p}@3*b_xo7`%#p(C9p4@QxzyEAM;B8u%I(~!ojmgKZs&9ShuKRc0bN%}(&MZ7M zb>-8qp{M>Z`25~irjERJ@ahlyUpY1Z@##-LJ2!vyxzz^_+ea48l7&@kuiJFG`j>^i X!(Vy%Pv(8=&9gsyYx|kCX$b!Zz!tjv literal 0 HcmV?d00001 diff --git a/tests/queries/0_stateless/data_minio/dv_puffin_warehouse/default/dv_puffin_complex/metadata/321000ac-b08e-430f-9c79-e40d5e4aa42d-m1.avro b/tests/queries/0_stateless/data_minio/dv_puffin_warehouse/default/dv_puffin_complex/metadata/321000ac-b08e-430f-9c79-e40d5e4aa42d-m1.avro new file mode 100644 index 0000000000000000000000000000000000000000..c9facfde4c040fac3af8044e19b6889af039d5e6 GIT binary patch literal 8132 zcmcIp4Tv1o9rw;s^ei!cOHJW922r~wJ2RWh-O@BkZB8xid6>jh!!dn3`*v@<*_qAE z>|HM(+O(#EC=#V2O7LiDrM8Gf6K#Uu)Ng`{AP8Emii%J_3M~Xm`+x7vy!Uo?XKrtE zg(HxeH~){{|Kt6C|GlPi{PXKv7pr) z;#NbgynUBOpXde6!wq^K{#VcfvZLh$1YXpz8-1w355FI zrJ0Jfn&T2HJYkU~MI?=cMdb?9GC>ZE1TdZkF<7?mMF6|_UgOACeH)VF3jQ)geho#_ z1W`?8R9r{Z7LdNK>4#3__#Ubfmqa8iAY&c)hg~O{Po5=OS(tQFwVnae#L=|W9_J3k zhZ*o=-^NVA)@!dub51mk#y2CDPt;=>e!W;!P+@Xwe$c>*Un$As;0oer5QU$9=fi`~(8s#c|C&oDiyM5S@a&C??IJhdl5YBI3~ z(cqw{)pSXLLxoWVhop)iPBe(ZZWK0&or2L&VB}?L*?vpS4^}{l)(KV@G(Gs?5II~4 z4M9qDXDK*4iRE}!Wgb#TfXFK}QLx32u*>oeg>amTPmZh6SF_6Y-B!c1=y54EsaO#r z90ZuQ8YdzRC!XP`ZPhbFsEmxB1Yas>&SJME%QW#)aY7$u;dK4F zV`Fy*ve-t4G4lP6{oB@lnyIXNIZ4(dh@<25PP#ML3 zpc+e?C2@^2PJzncncp8&ET0RINw(Xn!bOGX4S=v$MH6@;A3N1Z`jy3kx{LrCIt8$M zugK3eLv2lq=9*#jm1_(sCLD5*cZKXs)76&jVQ<;NGDiU{bF7bSGdqePFWbzqKEa|h z1FcucE5uL}?U7e-ZnBwb^Kkt+5?B?$9EP~0;haEEGe&5ht?dIkVB(uME#{BICBsJTii&za3;dq8m4ISJ# zao=q6>|nmh1~B*zCEuZn>sNB)VyfLJY{EQf`L$X|qTaJNxglY)Y`}mTkB_Fj3^!A; z5w5@-CuL{1BU2r1k|7l+!V)avqj9@IBhsKv$>eMM4H&YV5bk4kX`!osNVVkHrlxV| z2Y^;kg4J<^;gD0)MWn20@Xyiil2@a81H-P(nuz z9EMIE<~$#HWR5=1_pYQOACMXeD5t7go#Z9yXq3fL3MmqC94^~=#P9_~tH@1w!6lu< z;8NOhF&~5J7z#r#ju3`BK4vK}tHw6uBc=|lF-?lI7`{stPCI5>V9e$@sm{Y_ZW>%? zGk7cx0j&*KO=RarseQI(Y>f@KoujjWuhSZrcy(Y;Y*0@E0CzZ(23Gh${#8@+fmy?P zx}3-8FmX1c`nF#s_6l&u#SvM>>DBNo2|^%=gQg~^$1tTHP|}0NRgO*InyklLf-g&A z5{Dee_eHaryFu9V8N(H_70D>sP?Bas~DYra6o(F+q2DZ>jBLU-a`EJ{{|U&b2YSBjS| z(I;ah(P{H(w=q!hBO+ zV2}?gm&-6;69(`eoJ;DY8hLM}`CEn_(H&hUq9>lKJK_W`)coSd5xO}oF>@3MvmgaRChQ@aYk5xOrlTu}HQdBNWXpDF) zwI@m^wW(KrLLS=KTb9<;fcF1lane#bH*^rTEF0U9St@apnq@vFn(0bj%`k9Qt;7#fG~FC!QS`9r*3)^_NUOc;PEE_4i)e zb^Mh1n|AHK@2S7t{N(KS?%chy{LsZ$-BFm@`cC@Pod?9I>5z2H~( z9DJ+%*&`cf{&?5KAI{o*-)#@w|Co1ZgzWt81*?W{A3Efo_V4;V+wMN{%+Y=O5B_Dt zFJ1|)Pgn0P{^%e=*#^3nh^Vd(^7CrpZ$Jf2_**_X*ANbjK%DXpx^7|`}Jw1>{ F?SG;LvHAc2 literal 0 HcmV?d00001 diff --git a/tests/queries/0_stateless/data_minio/dv_puffin_warehouse/default/dv_puffin_complex/metadata/39ec1ceb-2dbe-4ab7-ab4e-0d18bed2d2da-m0.avro b/tests/queries/0_stateless/data_minio/dv_puffin_warehouse/default/dv_puffin_complex/metadata/39ec1ceb-2dbe-4ab7-ab4e-0d18bed2d2da-m0.avro new file mode 100644 index 0000000000000000000000000000000000000000..2f84bfe01b743ad7f90816345f78231c4dd7fcf5 GIT binary patch literal 8230 zcmcIpe~cVe9Y1P8C3sc^dO(jp$6qw=!OqOy^=_p|fu1zT(c06f zd)MRAXpr`bhGIx#nu3K0XR$>RsJR$uY6Zjy{z@?gi50*@!rnv z%w%G!i)~H#994WLpUU^U}ZvOS4QDU=P2SJY)-&2FY=~ z_tHUD5xGMIQB7HtUPaZWki4pFIl8M`CMpnvxWq{zWfk~`4c#3Jo&{Q2m}FJ7o&i$E z?hx1R=MKb&8Sqt0!%V@~zqS^Q>h2Kg-GEp=QIC1}<#bv=g~=^ib_ol9MI`rvE66*8 zC=^*lb2Wu5TciJ=GPWW1)k!9&#pHk!F7rOLJ|25HT_1hDkjq!JVWuaNs5B0Vd71{F z2et%AtqZI{G&snulns*dpu#AR9a2CLCmO`iZse4S7J*R_VD!o~r&$#-KUe_;TKiaC z(Dd*QJ4pAG&=5pK*Or2QJIU#0E4l0(^d&qBNCj3@IiYa?tAv@tG!zE!o88f`g@Y16XQL6WONL z6hTt9sXa}CMP~+DuaH-Wp%%7BuYwbkO%mM3VmB`8Hn1(*8m+4#q)46$ zmrj;&O~P?dLPrklIQk&Wc@{FsD1DynT}DLSCPiXXPF2X&l9!;PQ5H)nWSEG(aM_L_ zhA$vmKt{j|uBasjD`?Bb-WW{BP#Ai#i!j{rK1+dF)wdxTF>zq^X_BAC@Li;Esxj+_ zG26>YaUMpaL*P1_!F_QEXsyj^B0e`V)w3;QYizi!9i2n?YOOJdISA~D4eCJv;0|Zf zzzPekB(Lt#Pl=PzX zGF>BZO%D27g4s+si9-%Fu*rbkMnL<66#?I7fCRqGpoK$ki|YM{Q?)GMpC&9qRIajy z%r_(;6AEBVwRR=Ykd*3uOt1yupNh`cg36*NEQC~Ul%E!m1NDV1{E8q9OPYt1LE|q6 zV(f@e`lMw6*T7d_OhdRPXiP_MgKGn>kB2GD{#az^4ik|6FnPq~0k^(7aaqzYz?Ojf z!?QoKAK)g4nHOB+-aaRJU%-{{Wb`T|vMFdpRqW<~Yv~T5jO+}!{3Wteq* zP$1O%yH&jRq|8A;7f9L6i3W*fsg$kiN zdoSiBtD#>?XXsaim(J5Cr7PAa1+WWz%IhX(KMSx29bCzrl_SzfZay#j2xH`u8!8zk zzGPIPWSTc$BzU3uD!;%WA4Dz}W4;s`;Pv1{QpeTsdn?M{V)TUF(a;^bH?QVNaBcVx zpvCwbz2F2FSQXJa1dp4?b1=f?xUl*`tZN-4BDcP=?-5Pa8W2pjsWP zVr#@3S|Y!!!FHvLkD+y7{Wf_GLFLUb=cs`^R27dENCde*WDnr=Dv4bL);HzkOx>>vukR zSND6%7f-ZGt;fIX{O)f{_V#`7lhQGtIu^VnR)qVckC|ho!GWmeOHTCa+mBtldi4CZ z<@OazuW4D-vTb7WqVy$;Pb*KMu$e{s)|)YX%VKc2cknf}e8^?&`* zi4_lCcmH6|@i+f)>phzo6`;*U>n=$n1`-~Yhlk6oDAdG*-Nd)_I0=JH!kKBJGn zHM8q04{V+I?B;c+Rv3R-w|e=#d$<2^(;oNP>E~bDH8tbz7<>M}?7u#`XY2dlDjwN0 Z^WRqww4M4@;f>u>Guh|f`A{@8{{z{c>aYL+ literal 0 HcmV?d00001 diff --git a/tests/queries/0_stateless/data_minio/dv_puffin_warehouse/default/dv_puffin_complex/metadata/41caec64-2cad-46f5-a966-dd75e3635abc-m0.avro b/tests/queries/0_stateless/data_minio/dv_puffin_warehouse/default/dv_puffin_complex/metadata/41caec64-2cad-46f5-a966-dd75e3635abc-m0.avro new file mode 100644 index 0000000000000000000000000000000000000000..3f7df28d60e9e941028adf7999c41026606465e0 GIT binary patch literal 8135 zcmcIpS&S4#7+%&FV-$D*JlJT_i3H=U&UEj>?kdr!U_?a}G%Av|rn_dh*`Ds9yJy!G zHyU+~2jLAQCivnN6z}G9oA$+!c5hF zeD(cR|M$-|<;`>EpB!2vq>3LsVXGER)QC2ug;7wqBCVkDZ+abD8!8pFibGsG)P^>0 z)#6V)T&3&be+4ZdTk1|gU}*)rAt@xrT;ChkCJTIzUeMw_HjZ%4E(pO<_8DTIi_64q zW8^T(@Mkev1U;;i@(u|jc5svC1d==y4coZDA zj$u=z=_PADTnk71$Yh}K4I4GD?$V$c7Mlv1O)9wVM%oblT(wkEEdij!vxw={t7Q@h z_w8FV6l)d7C1!ZsCi@hV^axFASA>=ca-b)GasI?$S-uwm>|(#hk+ppbSam7?GDLm_ zMI!`JPeZ79Un;gn-?7bd-5Obp(jKLK6oY{0O^@cPNDA)O@mBJ$;o`mhaZ9o=KNWt4YO* zBf>#|=~d%IqzVNgK2P6GqovXcbF5SvNU~|Do>4+=WZX%xQ$ceKyLDNnNtQ|yr2%%F zu0QNp*xkV2fhw+PlavOm9InPSiZ#y=IW~%X_@f9haA+Yq;V49ueY+J0(lPM^o*k$L z0V=)N2~>mWuq3WA;}ocL&iu}xVtg(DlPtGx!wZM#3xKd^qqTT#K6Yx5beG8vbru2C zOBBG?cSSzeboFYQmTS7+5!V<}%s6n6-wN4GE2&qqhrL+`%NPK#jKL1rW;7K+9@~t; z4#A>51Fcu!6=JA~>ycM*F4+wA@^Jle5}0Mc9J;^c!a0sRO*^yS%w+Tv!` zS9F9O%NMbHb3xMkm1yd<+U$ZvZ!@El+e}w4ic~SS0Lkd=I%D~^1Cq*ixJAq=iE!Nf zrt~ddxt-v2|=|M;&?joX{B|MX` z9GuXW1BanA41J!DJTgu{&(E$;MLr-E5>QQLo6X`SxzVWnrP@;vaWy=)6Nupph}y_a zbiuM_VQ>z;axrg%X&VYnFOCp~H$GM=(5uEVQ^| z<^$8u?sVCY(Pm-+llqo#6KgIcCsE22b6SovB$9pJd?xm zl;Gz+(TM{GnnobShS>lefxst;>`z7nats0%)oqRqq(6B~g8iAuPJc`w`UQE^;em|4k+_WXvoIx)`AGJs z<^wVWm3hfC&IN4#yg)jUWaJf+niRC-Dsyrmy$lE_Q!@i;zX45gJXDnzG-!!zRY1z$q7BB_$(cq^4X!_0K3_^iyJIFC28CQU5TFn|{EE7L;$&M*rDh?J3q0Y`WISxhTd zMP7Qp$Sb8wXPA?|JToU9uuF0(J04~~3$PD*rITqZN0pH>d|CD>#V9SeRWf>Z$rwt> zbRNDc@I?5Ae87MYs+P+LUlRfFJ2+R=NjLJ{O6#`_KjL92HT$a%o@z`oxl1_S4pZr8TG^n>Muc-#@|59->QrR~&3AU^l8<3f)aFd=T9}~}X zB(G|;wgZ@{Rs;0~e?6kTWa4Vq9QchmetpzNF$%C3(xw;6kpJ|wv40a(Yv|Vli}0o; z^~)Mer{=~ zzh&t~uP>RJy8qUv4~<>2%l1Be;V;8@tn85L;d}n<-uc${MK?TRFJ64jmcPFFYTn#K zpMH7di@!hk<&Tfg?7BnSv+urFcRu&i&7XdMZ13m$e}5yq@71%dQ{Jxkjp(=7@2|PC z|MYiv?A1r3XD)dD=t~E8JmI~(b#Pt(Ll3Swy!r2^&fWLqp2K+2Yd73<^ugW();s$b zEIMiFp$ivYJHNQ9^x};V-223lW#z-0cc1v$n!AhJ4*c5F^ILG*@S*^wjLO+tWSi?%8#X zi;>_F6ctqPfOvsoLJ|@YQR9V*F$e|`1pT2X1`!nTg2eMxRd-cQ&vftX>U&l1dvkT^)&)z>46HFy#k&@6)5575(T21j^y^lr+#LtD0K z(I+mh()IAaoaU3Qb=xPfw1S<0BRQ>1D!A^1+7SI*wNy|o0ieXSi0Rg=CE^SB z?OQVxYZcoeW-w=yJ&H+sgeJ8sLQ4hN*JHpqeDC-0{5LZRQBC=>?bY#OR(oKPDncM|MW(44?dU6yH_rIJKpfE}mf zjoKD=Ht~0$ifh_5r2#94t8k5C%`-%H8HFDFQG^&cv=HrJ93o1;-SPv;nD_zD4^)Ey zm7ebes=;Jf64!`v3RF60erHfIKIec*mQydo3y0_nfUsLeMO@6rP7RXoGTEWdBY=8= z0@(Vl$mW`^UQIJ{P1if(8bgX12M+REA)RRj^-6ZJJMUl_0|1sW*a6#&rXt8b2VZf<$jKqm$cAS1*c0F}487=&<~nkr4o>^efGvLBy^TlXi%f}(U6bf zMrt<1CFtX%?ey!&P+Ob0Nd<{8i->q@+!#=w3}`s+d@ZjEO_m+Nd(0j!v|%mvk`bFq zld*rU+rdB`P`^z-cOdQ{|nQXxLoRAsYSyu>#emA_Pb3L>t9$94)ad;!ria$;R@ zezP#RfL^(nx52axg{BvW2*Vp6sTAl{qZqOgQybPuCPgfUeTl+p#B39^*_@MVKa3{F zAa&M*N9GX7TA$TKIyZV6dt0W~SaaLlIt%!kqj88k3hIe9>hS@<8_ujj6dp_s)KIg5 z>1B7i?8j&`v5ZN5%PSLWF(l*QkgVkLs(XU?0f@vdLle@Yn@SHT>8|`r+amBxjz&|0 zpL;|n4jgD2f#e%z19SuepTx319udef2w0F)5L!6&wV=CTloIc17#7H_850ya327Gd zZ5ikh-kDPE&`RSVC!PYx0Mz80L1m|>Op**4R74BlKxIhl7#_BH}@=N6ssaH zy;tOw(517?N$*R|NeArWoJzKf+0O#(gKp_$#>!D;qzqq{eL^ux%Waj6o?bGBQZk)~ zuL?X7z9Anl;Df5=Qo`3n0Q?Tl6m`;#e7BPNEya(x9UVKMbMr=>1kXnF0bDWBL@!<; z4K|!;3xOxYlN?NFIcco*A-0STh$y4um-r09Ws5x#XOFzflDaM>(C7`}XVs|hq*Zx1 z2`QImG(tSG+Lok~-qa^Q77q>TEzN79LHoZ{oQzcV4NZbAEyf09W-HvJXUWIdGabpR z8m;XBW~b^yI(~|gQ z4W?6zqohFEu^;+Q+I_HIdhEL`uN*nKfA>X?-9OTGY~cqFy;*!OzwGWEFD^cGWUqJD z>X+^--2U@ckbW3=lH6L?N96+{ra*KT@M^Oef9b)k1jgzwjH~E|LglZ9$53! z(_fi4KYj7r#pmvOv+&lBE50ed`t10$`OC{+Y<}Zc_qX>K9{>9IkL!1=`sBdRmz}!% zqit?@bokpFKi?YM@Z0mR?CbvN_e;NgZS3$52NwT(X!rPuC(k{meYoLa^vwBZzp(Yp zdluc5e|*Q0PnSLVvHzL=&?9T#S+?t}-gExR-T6mxzuWiJ{Z}kM@V9ni>7N6qJZKCL zUcdT`@xO+b-eQxz`iixhvE;h(Ys#f1dlp@L+Ri5qZa1E~>5tn74xQn=yYKA|}){keRCg z`0D$s{_mev#XDx6caCq4lM3EHYnv8KREajI`GHq611+QRZ+fhfHkiw3WsBG)UmM)G zO$$G9aD}di|7A3f+*z|c0!z!-_DLo(X1mUqwmrl5=ou~CQ^EmG*##k3%05HvvvHBw zZEW1M{UW@@bC>WGUb#*r=MmHOXux4073^4L;s*ud1l|OEuA~_*@dF&xeE2HF$8sn* zY#qZUOVf+iI=JeOyFr10!Z&QxoSIF8W>{>>XeCm{H9OD->F27YoN5UGC5}l7POVZT zo^ap3HAAshwro=HPupZ(F-ecGK<$dq5<&L#2r$l{5G>Pm0)SoY*Eq5z*92DG#J}{B zTSmb+LDbVA$}XdBGbp#L>iSk-xeh83n*_wqAblC+hixmEh(3$Fata%2^bC+H4#uVT zkaTc9EPz{F6Eg=J)~-NxD;P(^>k#7;%~&3PE~^rN*)6+X1uJ>QEDu8}h_68uia|uV znnA8-(tl7DdvNxRB+F%0a=?m!WiO&n#y*#AjlNYX6l>;Y<|jr}dJd^P%|gzbwns>< zj-o;Ia8OXI+9boF!l;OSQbKS}^bked$gdJJ2BWT!ifYZa$JmrJWr z#fl@so`>mGV+EuF1tC6<-%Xoyxuu0Ixm8Cd)L+Qb{7$ z&yLe}$1D@u>-jrS!BuTLr2#945nQEM^9+$yLV*i^6d?u1uC60zcZ*9pEJND)2@}^g+ufOK-ejvQ9PQCof;(FRbYoY zjR5L73SjHIBAsixdNobTHC^wBYYZu795~2tg=D7X)GOJ+&a{JN^aEJNKnH9ynu;Ke zZN@-{V9}m|)+_J|G1SEM$SXLNY=(My*lwM8g(6@M-CuO!oW`A|_feiL?EpHxFS$@# z+|2sYj<93-B9?C|NP4dlO|4p;UXbW*W^{6!>FPxhE5;Te8J%5cEZ=rOQrQl-NTEUk z95(;4rh^A3-kTxE^d_pT0fT)g+J}m^TTDF{L%ogsD)fT|w_Nr~&^~*k7ZN(l3N)x; z`)J6=a2+)p;3D*K(suH7WT>r8)TDw$m_kInHEs;3PX;s;b-t!sfhNoH;XP)L7Fw~K zdP#{*rLpgNfRZ;UK2lxe)!Vpid7!pDx8Bl&kVxD`Ks!r# zCSf@^p)CjYeQON*JQq2nPCrl2F0Uf*kuve9rYaSh#Y=RfQTa=?ry$}mJhl^v;R}eC zkR9oQ#m&N?n_jt?x52axg{Bt=2*Vp6subu|!x+*LQybP$CWS19eX+u6#B4pZ*_@MV zKaA?*kUHzZLvsjZt;cF2nH%}W-j=B~*4#F?&H}#XXl&w)fqG(%dUOErhBIprg$q*y zHPm!qdfA;W`!U)~%wtmDbW6mX3(43xAPcy>>e)g(A4Fo8p$X~HL!}3lbXRtPWfFKM z$HFPWFL}|40|%N$AlZi503CtACz0%rMg(#U0v6;Hgcc5cE$A*7r9`_Lh6QqK$^=DD zLYl>VTL$vNJ5#D1T4@~QL{k76fSPn?5g)P9zz5g~TQW?YK&u97r$y0?OFTK-zCWQydRfC!3Yr1vG}qyu(QPDRVX>}LV?L63AYY2~OgQidL!b+i6{{?o5d zuKa%C$*sTq`RgAqPwe>Y-yd(7G4s?JxBk_mFB<%$K0N=&XV-l9&w=`!eM9ECuh#n) z=XZVo_Suh|d&ygej%n_OBU`V!%=_t?m9UWR4-Szohhi|<0>kIXdzI@33=8YF#t8QIz?{7PRL&lpJ*w%|DzwMk?Cjgy;bv!+ znb|w`$ngguqOFQeBeca>XbA>LgzJR_-3H$UFn z*`2w)?U`JfboagQkI(nV`+mOr?aZBvmM^u{JT54|{M0>yQ!ZkmORycQq&h-EaKCBO zvO-rXA>=fyXKkTt`#pm9iJ=t8dH7#Ku<%_a&BAarr|33L_`-D47#1cH?j129cz3dj zqr}_-6EyCg6x`F54A$$oI5^Rwj9TUz*M;BS7&vEP)wGDgo*@Or&~n&zWNbKA89wLS z^p4n$;*@Oo%EL!92src{#U@GI^UoSe(axKWOhIukY!{4@PK>5l3?_su&M76`5xU6d zyrdK_2>`{0ie;lz$Y3i_U%xbola|wTEZY|?GR=u3A(V;Sz_dt^Ezt+e%^wde)ifM{ zJ-9Ep$YxCyvZ}}Z(ne+uIeCnTrY@B1CTbHX)m=1g&CyH)WwDMOY$uS|4gM**=9K-< ze64N@OT6_IkfP$`nRYMfAU@Q9ep6Mbg70qWLt~ngN4*1x<`eOlg+G<#6M)LinN~sJ z{0d3#1y_)J1`#N_5Xsd9GA)(-gNlj;v9CMnsSe+pis~7`L`FnU zf}0$gql#W)W$Js$Ig#q1;ndAxO;z-*?mJLWioyg*16mHZDn){|>mkyz$T8uMAVk3- zg{avhU{Q2;OLl~9as#|DP;~-SVzLpaI>Tl$T)m9rpc388ZwxA$&k4vRRWD`Xg+t^8 zK#Gw?o0ZM+*zrb^TQUvm0s<(e2!OSFMSQM_d~2FB*F>?QT%$-);gExFSBTEE6yK5! z#aM8#qz(W}>TDp}q^cr_%QmUAL9ocoKkXHBVCTv(7uHM*Xb^BtF}ri^v7+DBkcN8aiC#g7;>lp<3l49l+oo^6#OHZf0WRBJthGF2X!0 zo4K5go%*xae<5MAEWm*3jgJz03)6!aOW;N8?I^+L=Kc z{mEA~3ovA9HoV95(L#OSBwAu@AlRopR{jG-{}Dh^WMjrUjz%&MLZ@rdyQ zt4EW(EQWib!l}e;D~#E0PV)0G8q0(0bO!gtA)vJut%>N|NLS9bl&#U>wt939;H$Pq z$Hp+QCpxJ60f0B0N&_oQ*c$Mr#sjmK_H;Inkzt~lQhn9TVzmjJ(G>@`y7a1T6k9fs z#Kn>jP>(i3J)oo)CtEcY!!tSTZ3!-g`R2%)&%wG=&f^Y!1S>&1=$~o?A&1j z(jO!bzdT^pS0*k^`UTh$Fn?I~hxP-^1U~cpXWS*Q@%sX%ge9X}A)!q{J*px%2TV&x z0A*-rz~on;8H9(h@`L~*kvo;7*3tivH7wnf(239c3Us&wV3&`C`z=#J&3|>4gM|6yb#gp}Tl5<|M0uU*g)puMjVtr%z&gq)#GX=lhh=42AwIKtE_> zO6IH_o<=hBS=omeBa>WL$%xS)a+j zhjH;_FR(MsT3MLaMHuS6A#SWH`5lwWf)k>0Q9?b$J*jn3I;M?(@_q79!QP^@h6c3% z7mH(-vbmv(utnKefy{h~8`I44(br5?@;pbY+kp9UHDF)x*BXqMlw9py1iuk`uMa;c zI+kMCWax!5bbor3_U1v-;i7c0V!l z%nhwyzvB33-keEodSyfNWf$Hkw>Dijdi2C&@`hD+Oh5isQs?J*6dr?Pu}{#Z+`mr>OIfxAM89;oOtlc-tIHIuRYLSIRE)07xuoE z|H=3@ll{3bmEN5k%&ffnb1&Zh+{#0^{oFUw`IAqbd;6UOQ#-etGiUqKx9D%I{`b#& zUf2O_{rQuDSG7%S(^0*!{yJ&n{Xrbji>=PslGlyL=`!n*Z7r zFP5Lbap|J_r%wKU*^f`369&F247_@D=%YmUzwW(x!~45$Yr%K5Yabmew`8W)FZ=k) z+{?$B_n*6JpQh%frpx)3C1=O8Us-keroO2)e_J*A=PxW9yDIm+Kjt5Lf77he`sI(?%RxGOLsl+k;lw|@4|n-0t2$o&r>T?L!~ literal 0 HcmV?d00001 diff --git a/tests/queries/0_stateless/data_minio/dv_puffin_warehouse/default/dv_puffin_complex/metadata/5b66c606-c296-4fe4-a5f8-d8be592f6b96-m2.avro b/tests/queries/0_stateless/data_minio/dv_puffin_warehouse/default/dv_puffin_complex/metadata/5b66c606-c296-4fe4-a5f8-d8be592f6b96-m2.avro new file mode 100644 index 0000000000000000000000000000000000000000..97a79ed3f6d9c64cf7b4d06301d69181ff4a8f9c GIT binary patch literal 8298 zcmcIpe~c8>9ao_UcwA_KtACuA*J-&Nx|p4ryW?&fJScFHsEB~ImRvS(XW!n9H#@U8 zGkcfQUfQ0ewNwS6RvV4Ut@V$ZnApU&2qleC($LcHd-LPH zo!yz+Tdv84gx&YPKR(|d@B8`g{kcbOT-xMlL!^jbn)jIGj+cm(ksQ}9Yp#@%yl+

q2tvs}y(J5*o4G*ysR&<&Dxu3KcMAd-eKOXY^9#e!@r0bpMK_+V+4=>qKG zeaS;MZ)uQK8@w+aWEGG*L=e@KLFu(rZ3@Y2OO~U%x@Dp~F^Ef?6jIiLf7sC7@!+#S ztCzy6Xgvd@gxw*o-A_7*4>RCiOT$dTgWEQvG2IM|$>o)4R>%A!LPHP{U0n+Lqa>@F+1xm!P6(n`p@D)OmWvIRcgTU`RD5t; zjJ}vvnq`!WW|ki3QiF;OLxgP`(^jLqqzDBOeje>j!?L_4J1WaPfj1S^Gfb$An4SbT z1vE#oQRZbDcqup`_popp)}XFoV~6(+6mdzKplQI$;XYiVSbH8KJ&#-q{wP8W99oFF zGYl5Rceiv;)Fv;$a|2Z`K&7M`fvPuZ7RS}kH~}if%lyWmV)>kcOwx>U9$q*^UjT&7 zJlcY{Bx5HUNpEFYQ0EXpMWz7O?iI1K-9C?dsJ10=Pv zcEr91E*vquYU-bl41*e0g2W7&X~orhemMPP!tkPQUe;Vew6a29kVs^}EEaY46% zZQ0gXO${MM@>IBVvV><6j)M|9a$v{N2Vu^$kV(eq=gHo6ipblfKy1pX^4V(g5_B|5 ze<}A26R{s2+i}G31w`}62zWt9H8EI4TQ2s-U^<4v(2HG!;f?oM3e2j$4ata!1FKJy z{49ohk;19OYzK_lUQUYhFd7>I*Vzp2i$g$bZB`TUxzSlU+cLJshTH1VIfSp;8iSaF zz@FHk9s~g1a3&3`uwZK-nwktu7whSK9;3s=az^zvD^Ij$aK^wcY4zyU>QQ1lKoW0K zrI328rqlyU`lfWNt`T@92mLL<<<4*tha9L{fut*T19Sxfp9Hi&SP}3w2uR>t5L!6& zwV=0Pl@i>lSQhZB2^$o?38^OYbqVMUwKJyLpp?o*POt^w28hmAgUX^OY?4%Nl%E!m z1GReN=`4Lxx?_D(0K34aoNi+FvjF>GHCHlY z<%l$ro6pNW!Wg;ax=KcgFBw%Rna0f*30`Qv${#Su2a(Igm@kC}cs)3g)NwWZ-HP(J z7(HQkG<1jV%`15lJRAN8&|>_JUT}g7tcYk0g2&C{IT+z`Tv+WO*0c^1kz40A@fnOu zCVQcsoxGJrd0mX5{u|=Os*>MvsXRClDi4;Ab!PHSX9`+u=G zZYiG|st8-0jTOkumbh`vJRbwibR{oxw7LzLEms5f1%IvKc*)4s^Ebk8#Qy6e4vMag zO@|J>P=?-5PdmE?pjsWPY-_|DS|Y!!!FH^H3Mc=%e&`Q`(6H}&Sf+_nC| z;-4)zbFT0H6YE+Ry#I+UZ(KNX?y2;z4?pyimFb>ar$^)mrl0P<`PeN>c1dF^MsI8S z&Gfv-r{&j=_w8Twi|?NP<<$NW`&Vmocl>YX25kSBZai&NU3kBt+13(C{IBMY@tCt6=99@UPX z%E(9CULJkpqYv8j-H*3Sz3W^#U@dv=z4EDbts8$!-Z|F)gUb&t(T{I)ckR3V<*r-q zc=Ytajw9bbwC#&uecr_U@{ z`egn}$36dTzu7syVRGO6$usA6cMdFV+IZL1Go`EVpZ(SkZ+&sgm+m>(vuCT^K7aA_ zlh;1@RQblTuYBff&#(A;^R9nOf8H?l-1)mZo?G{YiMA>G-i~LlHD}zFCl4)s^W@c6 zw{|bLH{G>6^Kt8)&g?HZ~+Lf8_hWyRH1U_Rk&J`=5vUqEY%Eh~w|w literal 0 HcmV?d00001 diff --git a/tests/queries/0_stateless/data_minio/dv_puffin_warehouse/default/dv_puffin_complex/metadata/5b66c606-c296-4fe4-a5f8-d8be592f6b96-m3.avro b/tests/queries/0_stateless/data_minio/dv_puffin_warehouse/default/dv_puffin_complex/metadata/5b66c606-c296-4fe4-a5f8-d8be592f6b96-m3.avro new file mode 100644 index 0000000000000000000000000000000000000000..e96733a33279ffda09104a288feb1bd9755ad7ff GIT binary patch literal 8297 zcmcIpdyG_9880;jQLNS!vPBY3FA|r{Y%_E3>@vH}Vq0Wg1eV8cO$uGdGk4DHT=ver z<9)Ch>59ZZT5Gj2m{{vZs;s!NP2(m?Qd>zW)!GI$#TwCQLTc%QXiE$UE&u3u&V8IS zcjn%m9oA$+!pu3}d&O4l1B8RE%om^DEXflxI)jv|5B1gzT#>Yfukj?+awhT(@kSsno48lm>>R@;P2CI58n|kgO()Ml@h3|FT{Gc3kZQjwIft2^LUvSk4rYDl9 zG!BV*ng*Z8rUImP1=b)M9OSrFouoXdFe+f16cNOU1~If7*;S%OV3dXA^2#)?ny#21 ztbhWo!>lf7dU%Izq?bx?3^w^f;FqRBRX`Y+0DL8qFaUD2VX$Xm6UxWH#mxWHPzHo4V?mAXG+7PlB5Q zng_7%@-hv)6r9N9SU7caTvM?=>b(OMT$QG18nANMfvXg2&qJgYkz>LiMTmhz3sJKt zz@qr>md-_O@&dd#Q1t;+a=IO;`l4oWT>Xp_ppw1JZx1S#&nd_xRd6#g@nnn0t2c)J}Ue%9HC+zT!1-_%Z_hHia6Q?Ln=^&1z6;b#`OmEa)Y)6ldo!4 zV93&Jc#qkmg|>c-$04=Ksx!jDv1al!92u9#6OsjAfbR??Ciru)RS-`d| zbF!(1kRo|196DLTGYQ8*2^~4GZENE&=b6YLll1ds@3JED7AX;na;jp!k-P*QjnZGr zJ;OvCgvYjq7`}jL5$ORh=xihg8)(bL-WW{BP#Ai#gD|}DK1+dF)wdxTF>zq^X_BAC za4%9g^_Y#qnC<1HI1i)AGPurWa96`foN(nFg>iN^LdO86Sp#|ubM@o-VDy@*dc2?dewb^ST>Nv8x<*} z9^I6BKuO=2UZbf5p2=~4OK>?GPU4UQ4J(jz-EM%cK;V;r_6I8hz6Jpad<#MghrSl{ z7OYZ&TXo9Gzp=MNK*{xEsO zsbRQ4&6{7E&Nxe z3H@E57YZa&j28-o?)tr$m#l_<$vvT85nj4LpXA7m)ugx$nhnk2qn|L`69s!%~$vX2KgXzxft`M&;YLoCz3j@hQC`;{uZMr z?2fu-)4h2;Pl9K|{{UKyztIa$aDjCZZ9?$4c{~RrT#gHC9>k{BK_YVNye2-6ami#a zv@^?FS(Mks80xWTuI*WJeB0Ps^ZuRNhCVX0 z^z6C0FPy%sW7*|5ax+g19erZ+Thre^p-n7ZcFU>VTi&X^JXR_H?Zr2LvT5_$=U@Ku zv2Q%|(6)Q`-~H6Lau4V3y!7UU3tzVW{JT&7Z1|ar?;g53RlWL8l3>iD z{nsD4vb_3_imV@5HM#HP`0FcQI zj%UfUkG*@_mv%k**KYObFYg=6?)l=n!>9i{eV_3s`~2bZi@iN(p5LE7IaBCaJ-h3! z$DbbA`}0e)Gb`Ph-4nOGv-g>YosplCJ-4qs{==ayyKA*m>;8Rt?*m_(s@dv6_oG7} zobJ2f)ZB`rWW)V$;7yNzx^pYqTO81`FARM5y+g`#zga$c&o^f;-?M7{J95`{GCI_8 z`0szw-@o$oiU(ilTxK5`boB47{NRq>OTQi5sb$X{m%G+KBEN#JeSYe;lgroL)OBR= zPIX=9e~A9tqu<_it~@&Z%FwosyDwgO>FtY$j$J!HG(G;{+^eqs+D#j>T|3C=_Kvf& iudn?WdB1DL=r4-M6}`SdK^Y;6w=7Y8DeRHf?h4HVCuDz_NukDCbXsh_vYiho!!~n zTdv84gx&Z4AHVnezxV(BcUg49*r}5ZF(D;|7sqYp%=VPT)o_NXr$v(saMm}aa*V49 z1-Q5@DKUeqS-+XHKT(AwUJw5ZaJsZHE$b32jSGq)1sr3FrZ#gO0c#H*;Ospy!4&fB zf)He8U)bHJ2vJEXU}IB9iO{NRbFCEKyw6Qemqbm+0o#Ej1yznqh8dAmQ*VdQnKbPs zhAEh71HQ8HkyQ*1Sx2x5;`E%gs*o}gni(OWSR2-JYFfcT6D*nnTuh1!X~pDf@aL?h z5Ninll~hrRsOe->(p~q3TMIMR;<6$|jGH!D#+bxM7{PYk(0oDGc?U3yKQ>sRrkVh| zyPva=jcFpVYLWG&fwVX>6B5FnYEW=Kb{jyU`6Oj0CMG6IO75BUj-Y_>a} zIbJOahuP=}ASuC2Q13SB;Cx5`^_nOU2df`mirQo|ftIX9giknQdiio+%yfIk^JUUQS15|vl7^tefVNqOd#xYRw7W0dPitsrAOcIrJ3|=^d zUjQViG1MqD=Esf=67PzTLmfo``49#$_g#^nYdm{34a+s2FN$jfDPkNrXx$3_ObfAB zvMQ*f4pz7dzzSCv!M1Q#5#(cAxVlKNaL<726?laRYVP%DRd8Oih1ttP(b^8>dMHi7IXJ;BhhP|TGxT{HQl&QhdH(FmSmbpnF6mfP z#Uk0_#ktY2{KeYSCE^lzY}*mR7Z8mh#nA=#WDA3P@s%rBZ7^;_q3IP&B)}VQs}$%} z?HKYS#x|_BOtM)F`#gn{iPvn@`t{V-}vK)nL zb#~#)jz*ExW>8P0QFjgi-f&_KqR?P!z=oP1m~wKb(|!y$6Vr**7qyrqPJv_;!IWlN z@~X5|(hU%aw}d%YdX!@60VRD)aHcFu@Ju$_Q-V*++)f-gkTn7cX3Pfg2n0THWWO^a zpkolQK&K$MaPVt^cflyd*_AOYpj-1yQ0OEiTg(?^pv?77lxmSyG6y-%6o3Z6CZ7!| zIXz~Q6sAGhv;Yp|Ha7Yr0{2*g^f(DL_He)_4v(bIH!PqY=+P4o?wFu3A4>`x8&H4r zm|XVzB0Kysf#`S1!wwH<^qIsZq#uPT0nLZ9-!mVeA+XGIo^eaS=ARc(CzOmLCByrdOqeN|;=^YJo*RCcK48EHR?GQ>&$$6u zcW_=&N8QkO%d6jf{J6KHA{%&ao~e`I*|0x=E5@GaIV-5aj1$cv@Mw6HgB~qMjpaVX zoY4UhX>`^lK1^`=#hx2y8NJH9y3Qw1`wg+qDpTK4tMqU@QqIq)jks;KAW28Pu}{7u z9x~LMpI1+V_Wx3GG*a3(WC=FE7&DL=sc@s7DIXorcqGqiw1NW|sa6B^1%ET$_L7LJ z?y>M2vHkk6jiRXws)3tcC`0R~r}gDcP_2fR)TN@l#>c%<*EEiLP>PJH$?Bi`Elz;l1CT)gSn z>}USHY2Jk6-~asMlgh>g*U#TSe%GsYL+4|$%XPb6JyZ3Q{-NW-OS?Aj{!Y0XzYs|8 z862D)I&kgQ3-$4M;=;vC{O%vD;2ZD$*@AD@8!fv(yHfvR_r6c|R;(F3`Ip}E^4lA> z4?dbc*YFSYt4n=nhL*oSXZ*P_iMm%NJ~igf+2iis-}&IgvHf3rc*ok74ON?#tN+rL zeY|vIk88Mp37UFO8xD3Qr$VD^S)WUR=xQ0$@v%Nj2mCuS+jrgZEDq_t>+u= z?;kw;(0>o_{_;21SHHI~(c676@Ih(l7bW7A)6f31`yrL zKdwv8SoELXR|cw2>{xgB>FY1w=pDGUb3mB&x^k+dc3#QF=z>n+%4>J+er(aS>E$KH zlXYm}`Fp;7@eS>4Yv@E**AL&>qFw#^lgAF8sv7KkKf1l8@7g!|j!b(iaq164$GRWc z@x)`3f4=S7)csP)+PD6^@W8&-g)g6Yd}6#Oq0KOAK0n$&cioqFao?KLH)ZIV#?sc7 z($?gx{yX|U+&NHtbkDLKwRaqUcf;W|`}RgpKi4aN)+OIsKj~5Pr}MU68+f{N+MSn_ sLmzy%ZN z?w*Ykxhg{W_#5R%0thf)K?H3IirOC$rF^_IvpX}l zce_4cT*-=*yP5anndhB(pZB(v_FlI1iohBpRs6`p16nv$BU(WVLceZ>T2ABNth38n zLCgh_D~ue_qEB30rR(8;In5{g>b6f{X$3n0$;HMT&mGmKb9|4H)1p0P9O8^! z5Q44jGsQj!mx$BG#y!*D#}mHy6Q07aH;UwZVtGCdI0~eSUAsbput?m{pMuYoG@~U! zh{JjSUq$%XE(M3JW7y=dvTwwIasEVLS)LmL>|(#h zku7@`uxbgp~9$y15!qCPV^8(+$g9KD*>aSkc*dT(emmlKUe|9 zUWZv-P&WmZpF@0z_V+v4fpnh#kf|6u@$7K3=Y#zRD`g zbLv&MNS8~iam9)w!oH8`Rbz*w3I!oPPu@-Ay53uy(DlAJn}+HcC)7sDodi1-G$*iA zmt`7fsU)HIvEy{SQQN}KPW}#5aZQ`1G+^a$JFZczd4|X?qtJstiVy>b7NQ-DLqzGf zTfQ$D6F=bjfvO*%GV+~3)t?MY;uucleKW*8lDjUmO10|)u7kj^w+y^>w*&O2CUAAn`{ zcfdBYsR**zX7+ao7VQ~my#lWgLrq+dyn-{yW~!Hm<4qF3SOUzU`-?A}OSsdF4U}iS z9YAMnNH5eDH?z^(5q2zJ#PZDq$yl#MQ?J$L7bJR{nVsBbhI&yXim?SqW@pzK%eNhn zRJOw{Qmm2?N6mkt>EOYM_h!(w{HYpiz+fMW_o0&Gl`_x8RBxl82K`{st5gCKw$I-9 zg@n$s3Jq%1KAQ3|+(FHTxCDKiw4Ht(nQChjH>n^I<`5BYjT-~nAOjkVJ73GILX%|& z@E)^A3vJmg-)yB-yJpHzrXHC4ITEMDRpjmlrDJp~bm z;IW-T3|~OBjGR~(tZfzs*V8K(^EQ~aq0sc=5Mg-ZBb5TZY7|2@Vrs)0$)t$IurEnjS?jZ!NasdRV{gmU8f$KwTW0}Zb2JWdM?pQYMm;_N zc*B`Bh{A)Zff{NyFzeZ!F8eXsOe|wk-}1`DS^~*9I3(R%UagxTegGnIfvE}Uv5ra) zDCq_HZrdX8OpZoVg3o$HCk`BF8iC{+W&?Bt0-waPKOPatF$h?YQxIA>^tGV7V3ZQ? zY8V#Ctr-&(ISFYN^KBXE5#E_n?a)f&ASa#z$N<#jn?Yr#r%aMe8B|0I;6Q6*^FJaG z$C8)F$)Jgb11WJNBz@MfKzfiz&pe1RL3=)i+8i55fAW|F`%{sf{g^=X3-YML0~viI zaT)37VM-wLk?c>*2V@8;^WtZm3)t*=fpj9t$SWif}Ir=@U>UW(LxJ1DfJ^ zs4CBC&=T>kMDJz)p<`G!DPbL7^cDEv4PYTgf`2PrP_)Dtr%i<~bLba2FuPKVN(b%b z0EVc-Y>|Tjl$)O%vZc33N|zjOrLyOknE@4_mKhZ1rDoQoiA5SF&?0(eTFBoyW?=%6 zQnE1N=q^2rS;eZz%UCb+O6bx#=45P0&B*}l;+#shi`mZt?1OdE$*h&5%19Z$Ec=9F zl$P5n86&-9Or>Nx4__5{B79RmV891e%cX>`i2(Q=oGI$08~JV}^;?P`aXUJ8KqBfA9S~7Q$1m|&g3A_rBF-Lpl_hmu zN}$mj!q2Kv-$|?Ta1v53&1i&pWVJ0xC%vgpek>ju)LWX@M1%JKQgJd;**7!^wzL=< zkeRD+lb$6XW6yLXuWGcm1DLB;1N8-e-J-o@;_Ad@@EdXT`lyX!=wmmaO)r!o|LJLD z{T`^+z^nTf;Y~~8mo=D9Es2r>X~%x(>pJ_8sITk0@7;Iw;Tvj4ez9O* z*U)=taqxpvzr1kq;!pQ4-T&WvcfUGtaiZ{>V=LY|JovqXpWJrw>Epe- zMt=Y0>0ADBDt}x4ug1}XGk0|@>i%eE-}2WNk3WCa+Rck^-TK+y6Zq4WU)=P;zYg4d z{O9NY`T4gCKfCgcbKYZbfBieXuw(kwuF0#un)&=MtB!2>_)q?+f#ue+r?20ZKeBeo z1H;Qc&`&PxUis3Q|Gdz>{IMUsvwGl*cb_`!J-O!Y3orj&A9&)^KVMyaa?L##rrzIl J_QoVu-vC$Bxe)*W literal 0 HcmV?d00001 diff --git a/tests/queries/0_stateless/data_minio/dv_puffin_warehouse/default/dv_puffin_complex/metadata/b68e784a-6b9c-4894-827d-c56978599ecd-m0.avro b/tests/queries/0_stateless/data_minio/dv_puffin_warehouse/default/dv_puffin_complex/metadata/b68e784a-6b9c-4894-827d-c56978599ecd-m0.avro new file mode 100644 index 0000000000000000000000000000000000000000..2dbdc71cc3cd474bd49c68c44ff4c06be986f429 GIT binary patch literal 8301 zcmcIpdyHIF8Sj?TLTNTjQ#XQ3J-JH2I7?>ko$Wr_wL;xZDza_53o5(o?3p`fW-fc@ z-r0Na>~>q15d5R98wnUNgvz5e#X_~IQDa-wV%nlu358{tF}wdbdg9slI>U})sbS7`%Sx+ zk$Ms_DXS4ZV@o~5+a>Q40~hFd_+L!2$V5rA2pr8~-6k<#m~I+5X(s00kzzW%-dWSj;;@!vxsV1)L_q$0yeZPv7Hn#9BUdr7u@uY z*ba6|HhksbqZt$&c8*~ar|$V@4P3PIrjufzxEF>cqoh-#85W~4DMPZjq&rd%{alok z5G4Vi#864fC>7Gg3e?vxO%bGJHJzmFD;Aj)MA8tZsNBG`NRTbr2h7bM4=mL*9DqHz zFS*EOOckrDBere#!#ZSXxf^inFh)bojAmfA-Na)!@A~7`=9w* z-4s?t>lq+L?Buz2FX zAFP0UtpltsXnMGZZKSzMXb3{0D@#G!MpBxQN>4-T1R%N<>MIyB9jvpwLpB_z;{D@d z^u?@FO}$hwQuH{N>Q`(KB5YZhwi?YL1t^H%^Kfq(Pb4~1lZix^?@d|tj1wv&q9?&k z0nJIQmw1`_UJ6bmx>z`MGpDIoA9CM;0xn83G!0le+=Yu2Yu7`hWsqaSA4Q0PLkm%} z$HAiL?iTL~+vEm#VW8>;sN{GfP<4mR;<$PlCqN~;nco;xET3bLNvdATzzc`y3xKeZ zK?8W8Hg=+s^j3-mbpZjC6BNMey`pxm$zp4oGuLFfp#O9%!Q^ZQ80dtuC{DyM{ zJxyLi^Q^N0=;Srgg<7RE%bg8j$NWXi-d@lO;Tpa2%A-kptVdmV-IZL!7ZA-L-RA|jRuY5H(3XqcF_?~_F!W*vVR+*`mIAY?XG3ko#DUeL zNnRGiy-?wlV>Se1wwsgUJdCFD;5wVZJ#h$Vt;K30IyaK#vn^w5Y`Co)odftPt`hBIkkg$Y{&(bU?&bg-Vz=P^1=tYTDOH8VtQ24{5akT#cIwNDbu29mf~ zkpk+`PN@f!^x}A%rV@B2bKaKVQZkstAqOf}An~%@09}E=CqC`>R|I?w0uuNZgcc5c zE$A&+rTDkXmIeH3jSULlgjACGx&$Nx?To25D5ZRn<8J}D0iyGjpt9%*nooz zi_jBvM_se&-n^VA!L#9g04>Jb==mqOz_N%|A$Z(8o`WGS$Awi7VpZ!P5xI446Q9Gl z+GH=VGs#<7nAb%Z>b)UutjhTvm&$_^qH<9}J;Xh!bx}I5O?>iw@=(U!qO^tvwEq{2 zCSr5qqzXI4C+6 zHf%cdLK(V0Jq>q^Lbcjv$x?|sw1j?HgY8ta=hQWnvoHGk?zQ*+{G%0HmJHo;X0!44 z7nc8E=__ya-g*4Cxp%kJ5p zuGvQinn#`14+gus{&4d4lbg;hE4DoMjj=PY&nC~Ee(u!|E*;x`?qcV)cKqjG z2X?GnG`3{padYtepH{#Bi)RLJ-rfGQvw!*U_ya5ZuWEhxdk;^J-q`Z{<&VFTpIzEC zp=XB+Z`w2G+lHS!^tGwp_fA}V{iV$5J*Bn!kJjfpj=iP5cIIU?anpet&pz?!V?%rU z_vO3$I`W_U^6w6O@vhST^nFh}ziM{d?R%z`@%+G^=l}NAzPtCn_s7c*y>-+2*c=%(F>^!-@>6)=cV{bg0dhsVM$;5|$efiV>zVPvS9adHsfhnkI(l^5e-9qaHLEA?{p7mXEkC`#DIBQ(0XO{l A-~a#s literal 0 HcmV?d00001 diff --git a/tests/queries/0_stateless/data_minio/dv_puffin_warehouse/default/dv_puffin_complex/metadata/c21c4572-6006-4649-85b1-47e82e73a396-m0.avro b/tests/queries/0_stateless/data_minio/dv_puffin_warehouse/default/dv_puffin_complex/metadata/c21c4572-6006-4649-85b1-47e82e73a396-m0.avro new file mode 100644 index 0000000000000000000000000000000000000000..4f57b9fff1fe963049330c0f05a33da882383112 GIT binary patch literal 8157 zcmcIpe~28_9q)yn2hk>%oC#u%@|Z}Cc5~gGxx3t*xk$~ACMKRXxmqP$Hg9L&-c4_I zW;-)`Kbl4(5|h%ZEwvC^4T5O}QArD>wfKh;MM|haYy%C7Pz8VgJp2(bwBPsM%zJNV zcjopsS2zNhdGr18`Tls{&v$Pw{HU$-GT)paW&FgF16nX!A=;4U2VT_-w2a2zthb8V zP&T8LEMgaZZD{WSjXud`GzXXIdH7#O^T>~@mPg=43EMu&M8a&>8P^VG_&2)vricTa z@)ek1(eK6%uEtZI+rusS+x5amkC?7U^@cW-v165pALNM>c(d?au{J#72RNwu@G5lE za;T5&9Cwjg8J%@-#h-A4JoAlzuvc@cwhf;!m&P($k(6-N4zwZqT$PklB>^tPF-hL3 zmJ7ra>YJBlDAG!nP4fOli|kcI(n#c~Twz)w$etbn#?v4K%XFOpU>DzO9ND64LUL^8 zFMZ^eP%uFd)ii{%!&Ges<%TP+Zv~d?pdztJK>Q5ShrvH=TfuDfEYixtlrvQ886XuL zOi1lv?m&E)0lQoiGX+1`y#q~K!35g22eEvj9?S6OvRMTcCb#5zWvuuWle`UFLHrD& zPz)iOs~O~aCjAFhum`cPO|o28O%6B_Fz24r&;iM z>|lh{Es-^d1_uSzicK;cDvS!)Cq)EtqCpgPBfmn-7>v3CBQMju=~mVJUZlDC|EVHQ$HfXFK}QZVWU*k*Z$d^k?UN5|FZt662bcD3x} z>2WDFs#p;s?0J~B8Y>`WCzITgo+t6cY|P$h$&vrsdR@>|kfn!7>H_EMu^RY%}VL zAT8UB!4|=yGXt$x$ScH96YY^#aB8v{YV)w&Y2xJzfH@3tQNy{2o~HNHJloI$bb5bs zp*HBu`i7RUWBwxMZz@Q7pJGk5QdwM(Xg4!j>1MjxC}PFf03@TecEX_bag$-cv9g4m~1=}s8#>G&(kzav%Fz=R1J_(x7-spye$+8Rs zYB)X`@-o~*#Rj+lbDWf&+>Q)&w26jPpa=`Fh>ym#2K7sWwnme$>6T&0vV6FY*`4($8ZILvu2a>z7&p6*?*io8ck#G{<5n6D=kxfmt=QAssPwU=3+fn8om2tZ-^E8-+2O z=cGChqv;87oz38(I0UrTV>OYS8@;u&En{nJxUCX$Yr-v41(2Vr&ewy=qNgl`3~5xD7LWstg)RPyK!hbL!^xltmjelQ z#3+5*vOsE(t1qTOtO=Uav8~Cqfz&6%B(gsd+4;i+q+cYDx;&89*CsAY`bF3hNPlGZ z$Myr#1U2)bYn<XoSA9P5$ z#qlAXdJCs=GRGUS>;-zJL&Ya~2E}=?p4D+;;f4We5#CJ==HUd5>4072Q^9gD`&ocJSTB{#TRAF?l;+E_k1<9mxv7%TlS{@> zN~U%5Re~qXH{=Bd`Ji&S1oJgv0Pn%6q)w`l_g0+0CFl{|(YAcLH?QSMaBYMSpv8n6 zz37A#SQF6(1W%eLb1=r`q_D<8Y-k-MqO^`T@p+6(Cws!qUfIgxye`4e@DAazs^xc5 zstity$|VU65f7y{Md_qA^~#UPLk)XN(i$7k{$DIkS}Ny;I>MG@V+}G3C2mr)%*RMG zUCFB)t!V=m%GH2JX4R!n{Hp7^$N%-y71ur6e%nCDnui|g>cHma!aL7zU%6%f zOE*05mvbMTzwp_w+h%4i{PoTQcbt3ww$(Eo6AwPN@|xohZhU3y(UGoIPrmf&jXR4c z`mY@O;h#o#ukD^Vb?>edXSY{Q-91XK{oa4RweInOTW_zN-ubk>=I}3Oe{<~m@$VmB z{j+B#*Q{`su3Wl#ZFV5Ti842TzG6SWsdVF@S)eRU`;07{og>JNM4)tGoQ%nLFn@ z_dDl&=i9{%O@qfyWLL@3EQJffFuR^wL#goyq`Z_2IY|ysg_OvKwclY@yRGAkW(eHr7p*S@X$Ev&}g8HTNL5x>JV4e@?=s0Q- zAE3N0nr?5S{7M9J zeuRt@=S3A~?L#lY{7Mj2U;wBRh{?Q~oUg+YmXkyk>fa|=um-CzUMfHe0yPAINv#*q zWdT-Y3jtFWfC5a(oN-$q6xbYvA_x6qG~57XlDCZ#U*Z%nPJvhmiZ)OXV->�#LiT zUQ~l6#$~|@^N63E@M@$OY7$@`HepdEtT#=9DzQN6srVI1M`@LBid;R30L!qiaK2%kgQRw5+}0+wvunO zLSW#ym3q%)%Quy1S&F6PYUA0UKs%O>0mj<&n1KOxCy@ zlLuo;S5RV95GW)&iK%5}nT?~@XxPI@>+thrag<%O$LwjFCTcps@}h!@_MQRlUtQa@sE2Ir28XiFaI8U&O6{5+aQ)=lz0;Y$y--$9YR}KWqifJ$d zFvvy)mGWvi*@ePmNkKs4FZ&zNL=ho1S)~ju#`sH2=>_^%K>rE^B^j{j+WKs21Z4)c zE{JF(av#9@SNQ`oEh}3JDKu?iBn^qB;%UJb^F_m97Ab~y707ZSLTA?d&{lPS5yt&t zsIuBVz`j6;-WxVk1Du zbp@6Ht0;UUl6eV;Fs40MwEa7$va+fYXt&6rq7E7nJ3;p&oD_iEXjFWGRpCT)9U!g? z9E_vb;dZ)8OPp?ZiPPaIeMHZ^2tlIv!nIK(xX z=OEgejS1uP%42x8kza{&+U7SbkzQWf=64eIB-I)vjE{^qB`>3e>P4X-9{N0tkTy{U z3X_V0El>s=n|x=qLn?%4|7|g6WF-RFe zZG`#qFiZrs1f!<6O1U8CXq_-9U5r7~nWkJIeL0vYg6B-av)iW(c-BTqwNoTGsT;Nt zdB?}L!s^%v1Syi~DDp-y9Y$IJhD?%C5KRQJz0AOmt@%>S7EYca#m4;P9Uk+etFvKf z2?8mvNl)GYCOzrm(?kZa&S=()@DgMN1K_pxd?uEV56p;d``7hpzyFPG7pIi<-E6wHTlCJ{_+Hz&3t(19 zTgBt<5wC8%P!2qvk*%W_T>!`KD7`(~-30|tt)BMzqd#@mUp(%(Qdia0x1i+giSPSr zPR`z3zwFuRJO1EiAKr33xO~oraN)wvcUyxyqaU_jy&e2ySLgDso&XGo1QpDuirjO=^VRZ@P?t>)VQO1$#?CG53b$odH&;(ymR=r z6%X&2?fxP9@+P0cojFq1wjH{g0w0a(K7XKcUF+BNGvDu;zI3po0}NaG!P#XUw`vY{ zU0EA<-uv4hYnxo&`fUKT~m{;>o$2dp(-^epFSWOIZ24-&v|U}!+S<9bkty#C8oTp$^Iz5b ygU*|G&PQU``kO9YJ+k{j$L*v0{u}pOS^L}WMSt$7=$q8sKIZkiTW^d?F30~qK+=l< literal 0 HcmV?d00001 diff --git a/tests/queries/0_stateless/data_minio/dv_puffin_warehouse/default/dv_puffin_complex/metadata/snap-2216958009875447676-1-b68e784a-6b9c-4894-827d-c56978599ecd.avro b/tests/queries/0_stateless/data_minio/dv_puffin_warehouse/default/dv_puffin_complex/metadata/snap-2216958009875447676-1-b68e784a-6b9c-4894-827d-c56978599ecd.avro new file mode 100644 index 0000000000000000000000000000000000000000..619989a8a3283c08923351d067a48609d3202a8e GIT binary patch literal 5072 zcmbVPYitx%6o!WIhzPboiw(icg2a#(cend2pg{_Ofbv)hBn6hcvv<1#voq_=?A8Su zgb+}{lt(~OqNo8SJVJtsm;fTQJOolu3@9iMDIlSiD5*pd@67DnJF}0r%g>#?bG~!G zbIy0ZnH-qk`Kdw}7lq*x6GjoZ+_K0bC!)wwZV?e9h(Sw-inS~aMd=jD&j_(>l^U(^0&pS41kjHeII=nnA<=8BB;!t32c*rYW0NP_2y7%@ zYlUF@aVylGX)E7!qD7%5EmsrI8U@O+ObpnJO-~qTHz;8sYg}WobOPE9s}qnlsV z@)dc|rnJh!$2iV{WhXhcP!wST|BZ(|g0+r3PZdYeY4zwmZPG+V2O!H!cu1ANdfb2@ za0v4FfhLO6#TZtr5P*C_ldrHclz4bJ4v;)S9CEQ#(eWBOkb3PIU{ zjSC_cjXnmj`d041w%R$hB^Eqw5iAYKrQ)e27xTp;5r`E-xe9_zn9|wieXK@xUw{(+ z2$G?)4@lo+V{$Cdh>57WRq=a3e-8vWAbQa&yG=a38X3t^op`lJiZ#> zDjb&Z{JgQNiOUnKoT~RT%0w}lWKt=6VOl)7(k1#?IF}-l{IVht-eH~z~ zhuZBnm)GgBTD=~()8TNt+^+s=>ZKSHALwfHS_fe2BTz1lBT>S;J+Ew_zK}|_J(%Yj zz%mH0&FZA{d228kZfsa$jI#PQPo&n@YBD@Yf0FKvlFmoOtCAJ*MlHvo5SjWMoRBh7 z8j39)g?OMWI41qh>WL_Xoj?$+F}R^DAVx!z*&Pu8o0|YoU(|uHx%eas<|JTx&8b7m z0%{`6mxE)Xs4jw<*(-Gfxr5$`fLhB4XeQUx5lCMLTolD~q~MwTQx-g9qon&O8l2XR z_=ue26JKGpY!rePNw*X^Bj^?*BY=%gl2#B!1&P1R!jGx>(%cqF-bRZ}_{lju;YXEc z&CyZ>T3(%=oB?!tGR3!286Y~NcrVIJ2yzmUOQOdd_SN`U%DN>F47Q9Yx8dVdQ&BJJ zTxyauM@G?2TCKaK_RLZyhejFoWPnw@8AUbSnV7WpU^93$ii($2@}u2s$FhR5Jhm%} zQBc3oJL1bI7~M~13zjojb}wdfTQsgxFKolKq}%XZG*{XMDfWs6O(}<&nKpeEfw5Em zQ!G_B7?T9}9@|=Bo`&GbQaq`_Fb%QW>}7Vh!|8Q7+-{e{ZufR_Zv56UINChEX$Jc5 z<@{;edfw`K{@kHeeMa}~)^hRL6?-oIdSyw==8KmvKA1Y|;@|$|5AJQOIu&B#(oczH z%ih>fzqET{KnlaPCtqpY^6`c5me%)JwxzyucySElqdv@*F%3X+WJCgrp>HTeGN4_qe`Q-hoBPVv2 z?6}vmZqv0Uv5{dzpEO^1f5Go3x*UVdla3zur+u0cop6dW7!!|TiY^j;WDELl}_C(m|FWF~eQ zv^sC#LhH1?^Blv5mQ{skFa3b));Q0)WYNvjXX+ldKJ2tG|6!G*?#H1g$8_C3=7~-F zx?h^rqe1M_xYu_6=@oA=EDH`_M`n?68uPp?rqE9xMgmz)f~#hL()D8mkUT5kSwH{R$A(o449|C^U%;T$i~Tx zrX%t=qai(QO9Q+2)tx9noFgZyfoyDPMwa|en<|MjY^o4zrU4$`xfPT&oCA*}fP>DW z_GoEoAS%n1?)um|GO}WH-4tSvm>BASCJ1b8F9MnYQe}*6FWqWjMF?Dg7S?D!NxOj5 zBtd40YX)XnC&-^*YaA49lm^%YB}>D#`G$nBs%c>4ykBKO4EC`5rwz;uz&QjwY6pP4 zF%2>72Fk`X(otRJN}m8>qo<3=P*Kt-=2OVxdHX0yR8Il-6e#78;R6LFXakr|BYJbE zs4h!NRKUt>grAo1hRqUb65t*7U`Zv!o7O<}3Q$feN!x%0J7;2FLHrl1o}g91If&gm zG(nNWbttq`?4)WIgDN#hc>$!5RvCE51dim05E4D3*_d>?GayZ)4pmP!2`rNDwnBRL zxN)avn&n$hv}Km+a`o}-Qb3P2Vn7j_o-r_MP{u%DT(MXl0kekn2ndWS){9L(l?Qi9 zcS}Z5*Cir5)v1M+1Q zm>iCTd;5ErJE`{&OmbjBzr2z#pNF_-99cG5?eoH`y?xZa76n^LpLLLDHX5F7e7j1ZkD7e%Q@VHPL=$0y%` zmWV@y7zC3V!3+g}h<>KAIbr}x9|PdFsD-fY*dz`XWnf;-xj_m5^%0h6!!U8wUWVG( zDz$>#qIF`Ra+rZOGEJ>OCR#9294}IX=eJJ*cw(c}+bJHL*NxeTw&OEfAv!h=!HeWM zinb9vhtUu~;gjSRgo+@umjV3vny=1mvE&tAY{pO9;Tb=UI=hCJBk=Ni^t27&(bFit z%w>S-47FaImkiWxCYMZ)bz-Z@v6^km9tf_CDYx)#>Z)iz>s;=UTtmjueOm3ArP;hv zBZI~njWvLEtQkl3-I;l`*%G^RrQJyv~^j2&c2jlz6@4(sy z3$|iDv&G}mdJz}qCf&qx@mzTq)VM1iw5A+>W}4b81{31^*Ep&m7@q_rZrE87p4O1b zqLx(jOR;b?6pi*qBXX=S5>^zsuWwH0KgVYVzu9}G^ZkQMC*NLjVZk5s&mDVr$LI3$ zL;sE5{?^OaSDZIe@lU3veqZ*rdHvYfi)Zg1?^u$`st=_GPs-slKg?CGEZ(wk;gN}h zk;QMApIt1>oG|`=Vz_(%&@^0CS+$~P(YEZ-MU~Di@eXC;($Ue)gI_!tKQglOmg&^l z)7A%9r#D=EeSOESpVwWONySFHrq*9NpV|N2OSr3R^5ad1^&Oe9EmyxC`?2HN>dmJn zwm)z}drvceI{y6`^lS2~&2x@yOje#eaNy$UPha?O?&{|{_8xn!^Uihhnv?$?zW2+8 zmwy?tUs-zJu5If#CG2yzKA$@!y&1l?{m;rx-CK?1U9Vog|DE6d`e@GDXJh*On-A1+ F@joZigB$<= literal 0 HcmV?d00001 diff --git a/tests/queries/0_stateless/data_minio/dv_puffin_warehouse/default/dv_puffin_complex/metadata/snap-243132775900853210-1-0c03abfe-66a8-43ab-a765-a9e8072b59ed.avro b/tests/queries/0_stateless/data_minio/dv_puffin_warehouse/default/dv_puffin_complex/metadata/snap-243132775900853210-1-0c03abfe-66a8-43ab-a765-a9e8072b59ed.avro new file mode 100644 index 0000000000000000000000000000000000000000..151ec4194d68cbbe4dfe21440da196da973bd96e GIT binary patch literal 4966 zcmbVPYitx%6fTbvq@-5CrlDwV*9HjLmYIFETfjirDoZM;i;;(Qy*qoiJ9Tzuotb4_ zu>Rtq5G80NCTb-m5y1p#{Gk{j0V8U7$ScI4!4L}x(MAm+il}&JX6N3S-JR_&N&j@_ z&iT&$&N<)tc15gzTtg#VCrj-LmqaPJ(Y3*)=F-U3<&qI6$#Iw4CBl^Ue+r6x0x4=V z!3zjKPVh*GE3U4!8;VY(p_;@e@_`gp=EXkze_Y~loF0jTDlduP>8HXV%d1K7Xawl! zBx)NUcXjz0X24w>TR>u;HM$^KVy~1`C;%-%pz!Mv;6)J2sYo%?%?D;Jfzwb{c`cu) zl15wae%LKNP zZ?r;u?6@JlXUgSUOtdUzi*mK`Y*3&bE5!h7ZF<7Mm_Z2xj&ZHUG6@(ntVw`lRBOG+ z`{MAzw*tDDm(J0w8&Ubx_C@icZkdffP&+ZNHPt#Oxgl6pAG< z<0t_$g6itha{vyYK0p_~Z zYJi}Yc1o2XSLmDwDB~rdr94w5kdX>J6vguu;Mv`i1D>@}iro|qPU}WoMAh+$ ztFRh23PFox8j7kBOoLGpfTfdU6hzZN;w~Ndu{B?j*CNTYwAh57s>2h0ba6KPEJdK@ zHR-7uz@(>Ce1^&Z(HYHoQC<>IP>5U-Jr=O5#>WblEqP!uWkk6xKTQo44U*2K7Rm5q z6y2uPrdJxvDwXnQl+n%-u(~s&sJ1&3i#8T)35P~e@v_SQWVf3!M^IkGW<|3J`tS2f zTp0zU`^j#=ss?j*Vm7Zu<7)N7Cd^2>jN_uY(k>`4S2SorIqb}oby)<)L-{W-R8BB9 z35eXavBEsH;>pq+sl_hM>R`RBFTi+%ey=YWV3{D}s|ok_w_iT8=-O}m#dr67_1v|| zJ8Ms!J-qx&rlIEf+53h*y7y>v(|rC?tG7*&Din5xz0_m*H1k;_3c$n4Ry^+ zW(OwC-MHwBn)@eAz<<+Xi$1$@piVi|eEReA;r~?cTQ^V4Ts>ve+-?2f`!zd{H~)fq zTOzCWZcSb6>=}+N*f)L7$%FgenOOVeU?czS$ebA$!f!U7ZJav%lY3duuG1Hex!Z3n z-u#36vDzExICpv64lsHX>>7L}^81F~DJ_$J%yxH#!pe~;yJ0R6m^?gsee~$5k${;yBn zIWX(H+w-I?E4FW*_4pmICnAAFNnp52^xMaKQJiw#OG^yTfr%BRpqh~apx6ZzbR0E{ z@tJaM6jeJoHZ}{nN_}*e*TtT|OEe4Q1cAuRg@ECJr$&OJmTn}lSr<4&3lbye;}%2E z>qf{pafXv{)-LoCloo=p2z@}}LAAh0(fL|*VZA&jLFM}h3;JLq)t4gR{Xhyppi!F( zs9`=@q6Y)>Fdt-LOy=es9fSyHg(2sKZY~`3K!N0Kpu`n91=LfZIsiEXD5$1IK>K`9 zzPVCVl_j%HhZSTHKdr(m=c`eZ0PU~_iz=bNX%v)Q2c@Lq7CAa326)L(5aY!nC*-O~ z0a{Y;&~Vb!^5%w=Lu%hY&cqUoVpGqU`lA?olF|$^p-$r zE)J#-`spw$nR4Zvj6&hirNE-`7u+5+QA9{JR!yoFWBhTN^aAB8pnL`VyZ~r)ZDlrP zf)WGk7eqJ|dIDhOtK@;rwbN?{o6xicku*e>ilY`^%oPp>X`~qPRUqvR5IUQ^52aW4 zDp=zVLW!350q&bbOcrrop(dnkRrDRuz5^bX_cRM>*0+zJ5zu-^v!fSX45^3$xuPT& zkE;aexT3&3phc0XL^96<4pz(0MS1@g6c!d00{IrLsHlZP#HK5L#LN4jHx(71r6t%y zTL*~iepaWGvgTyl?G$CRnVn|J`GS&p7J@|YJ6N)*48(kpma=9;K}4%P#oAljM_K9` zOv%Y$ybx{8%82n<-WSg{@+)Dly!lm2q?DI3_??J7iM2)%<0GI=$p~nn=Auv#4}B6w zNS-Ja#T<)*El>g+gM25nLW3yvwxl}0lxeY;?M|D6qMVK#TXuGiJ;&Z=Y2HK&Q^@~a zTvak~^}R7AsmD8>I+0(MUXb4Aud|njuCiY0d$-H`c};nF3px$zwfCTT!TGfZo6g;y z{qnCp3knvmD;mD2hi7@(vnxKko-%jafJschnO(=H?Htv;@Zk5QN7GAd9nbkYEbVyv z;+~vmHgEajjqx8o2%l}}kh%Lz*7eI}m$&WTcIeaXn+EmG=<#aN4%eyOhlWg=cuD#^ zZ_g%o-I{4PIv3Z>xRW>epS1N`m!*os{yhEddG>5C>DJVf-%Rb3S+cY&eJ%4_@sybr z=C_xHW?Hu|Ib1j5T7Ai}YnSPJkBWzXw}1b>h9i@!^S9^kJ)U`>YqzBbN><$M{?&`V zxsN*cznwC^%oP;cAKlsh%FX(xjtw7FH=$pq%h7Jwv|II)svh2WbZ+N`d3`qYzGHDr z$xgkJ`ATgU&x#J+D%0Mvr|h7gK6&ux`m*_d^!;h$x~q@f4YhlBJ-+z+`eO_GuN?c* zny;5Nu3A@mBBR^lp47mK3)0E^M>d@4)W6YIvT5uurHze^_ru=G2m6k`_vPpj8?0mI zyk*T_Iok81Wlfup>)Kr`UwF8@b9sIFxQ}ut&ObeC#tE+B#_LTx|MgWpKK#k^+4-Y4 z9H^@Z?i+WtDQk;!z@T)ndC17Ewz7v054es58yNcsV|w%4&GSR&biMn-yBlXMdi>_? ZU5?eM&Xo%?y4goH)mAk8*EYKJ{{za={7V1; literal 0 HcmV?d00001 diff --git a/tests/queries/0_stateless/data_minio/dv_puffin_warehouse/default/dv_puffin_complex/metadata/snap-7131460395349610904-1-321000ac-b08e-430f-9c79-e40d5e4aa42d.avro b/tests/queries/0_stateless/data_minio/dv_puffin_warehouse/default/dv_puffin_complex/metadata/snap-7131460395349610904-1-321000ac-b08e-430f-9c79-e40d5e4aa42d.avro new file mode 100644 index 0000000000000000000000000000000000000000..929e6598ea9f2d611ee647b86a9ecfa9f9aad704 GIT binary patch literal 4908 zcmbVPYitx%6sApMi6*rr4UvS9%b><+OK0}g-5P?0)`-@MU5rMk>)n~VyQ8x+%gk)I zD=7gZXh=|uqJkzu6aCSk7{p+L!AFS31T+L<@DURM5flXsN_=3vGqZE=o!y=7EkJFK;=@Qq)sW5B(&O%jAA;V0h zWCh{JDH$n}!S$_NQ*j~(%?v(K3S^kNtfujANfU9LAx(s)tf}CEk!7GDn;CFt9N6e2 zYPT$LePN!T=&FpZAT`ZKS2QN}3eAKHuo47@`~(8B3X(+=8E(3zzycFE2X#}n@<|vu zBqlI2NnBP3jyn!TOYM`LYW_i8=CakEbCbHizvA_mf>ilT{Ed|UB0xq?E zfL|m*6LtY%k%Safk-5;zKp1Fs9;qTqsQGLX=`?Q-CGpZJ;G6=53{pLypa2a3B?(zK zw~OkqB!n5Ptc>}o3vWm(;3fg?VHXxxg1u=ORD^-DQ%M*q%o!QY^c2K~aE~z@@gXK88kY?dce*_wHKPuePSyy_lJB&F zG=1CwyJu?UTS>I86)JM|@a#}v9jnHGz&1T$VA`OB0pGZ6v0MVC4eJu%8|oV-F2Wl~V^`0I%zo+v7$E~}iQ#RPx3E4{#e3)pXgl%@lSuWiq!MUX!*yCCwp z+*JVEZ#51qOpH!+ad_IYSQ<)8B|1Sa7SCt15G#gt6$Hf$rL*RJcvjsXN5w=Inb6t? zq;LF~98^WUn6tMkeh;|sfuy1(YeEWa`=n$YuXj8!2C>9a;7$3U9PQ4pr;sY%)elBJ{3(f8^q#Jm(HwWjr`>5MlgU!LYvWW23OqYz$ zrUhiSv0urH*5-FCkzHQi<9AB-q|zFdjE{~trL5zH+J{3SGV~1?A#0)>6rmD@SRfx9 zk9_-DA_@^85JYMiGvoup`kCtHhyVz^1c2J2I>I($lPFl2fVnlN2FVB1Ls+~4!$eUd z1hu+Vst37F>qJ2LAOWprn(Bd!*I}Y4UZ@PuYoC1Z*hZoC;Zsz>=;^#K+EgW(=dQb zPqp|wl>wqNmi3~%q@ZFDxg>h5U|Wrkm2F${fN^C+xtVWMM@3_#bE!vi3>iiDXtisW zrt?bG3>sxLS_Rg&W)#(PXX4SOgRNrGC@Nl7rEhkx8}kK~RqR$QpJ0EW*JH~l7~M}^ z2i7o{zZLVCEgILV7j|Jz(lsm>&6RdRnY*Gv%gW(prl!pzFagScnWOT9@kl`Ys+|?) z=|((RmL+BVQXmit2crTXj)kIpFcgf$A~SaPe|aK4-g#yIvk(6C%Pk{4*X(FHeB`a+ zgLkyIUO4-vyzp}S2OsbG_L0v=!yETyQu~jtGOkSSUz7vwnU-1oe6*!!-HswtGiSai z9vK<_bmr#SC%N0+z57gl)joOq){9M%y>r{bZyY~8w)dW?$uG_>@03;>U8#&X*8e!Z>vyS4>qg6j_) zKe%D%(#z?|k6s>0ezJAdrO9nW#TO4P7~Xm8_D*i&`qLX0KQsK?y5jfu7V;C9&EMCa lAD^*u@x1h@zs|q5yLD8J{?^m_%5_K2_CCEi_If!*{{vBBm+Sxl literal 0 HcmV?d00001 diff --git a/tests/queries/0_stateless/data_minio/dv_puffin_warehouse/default/dv_puffin_complex/metadata/snap-8352556540874311077-1-39ec1ceb-2dbe-4ab7-ab4e-0d18bed2d2da.avro b/tests/queries/0_stateless/data_minio/dv_puffin_warehouse/default/dv_puffin_complex/metadata/snap-8352556540874311077-1-39ec1ceb-2dbe-4ab7-ab4e-0d18bed2d2da.avro new file mode 100644 index 0000000000000000000000000000000000000000..070f738c8a08409fb27780970531ae9a96f0d6ed GIT binary patch literal 4800 zcmbVPU1%It6gK-3rL{jcl%j3Dj-^FNnwe}i*(@y@x~Wtm6*m?ngw5TVySo!-XErml z*{&tfmR5>=s7i&jN>GCMASm^(;EM!N`~ji*P@fe1p(4Hriho!6(;E$nx@Fb8Lf{1u?o|39)A^7aNEsh#dWCf^-w*YA$htbnAf?A#fSnu1@pGI%T3} z0huMPn=Z>b1An3;X;g8DhFl9(ZQX6mcSH!QTBb|9_v^c+w@6SxZ3@W; zHSFR+q#V?UL7FmG#svt6ELMoAlB`)N<%rGm4pEY+pF;jAR4o!S1d6KIL0HoWy}4IZ zpCu(BV3l;>rz^Z^s|rnmg2MqUR6@LI1JsxR<)xB!Ok8$~mK!Qa_+s@Fv?^{9yZ#;8 zpvdti6xu0HR<$Zom71iyASk3=L%}hDBY9K^NsjA|3!Ux_NZY7W^^@M7&agvGOAcFHu+Q@{3)H4 z%!*-1AUpM`#kP%W@HY&53ABzqZxly6At!^L4rwCQ0j!%23@Hs9#LZcjL9huAd{LaS zE1*_+gw2{SUqw2UFg%8VSe|GJ8 zg(4}WV8Hm>*&Iw17t(-L$=70r|5zZs$a@QUZ=t+pBMjH}W|I;W4J|&Wpsf0l>=v4?-i=583_mM?)e~Q$yCE{Yb4_MzsF*#+b zcCGAfRd^2t@1dMw<=R3jV*BKE8`e9_jwree(h~)KMVA)OEFd`UDX@jGUN=2oM z-*-e_b>)!Vsk@VAXH@q+HY`frhJBiXP=w8V4?c(vl#ik`qc8^)ffG{iNJqpWMi>NB z8o>)ifQWXcwKrk_$~XhyuBeN!y|^R}mSA8(&AC5{01XkA>A^2?)G>zI+9`E|+@*73 zpmLmnw(?BfKxVq|P#kZx0Wa*HBJjjUX?9aQIIkOX5k1Feu0k|y9D)}qFcdu_1O}re zfWjxqF9_8@<}M@n2{m7n*J8;lyx5GNp2IVKJaP8@EJxtw4e03^AfTsJe3{Dt(;4c# zI4^l@I7}{?9vk4Q;aJ16We)^X#*|z5H1$<H*f3je|X>XpZ$5~aBcXz#F?)aVrM^@{b8Z+jYocZ>AljfQ~Pf0 zGG-SBCeFogT+2*-c`AM3(y_x&J~wyl>eQ>#?|<~n=aUy_=4b!@aQNy6Zx%OSK5$c8 z+5f;%`>q2op8jj&$1ks~Y`s9<-m>TG-9M&Q=dXYB?|t)w8?D!`|MuU+gZHklT=_!2 TvU={1JJz<|b9uS1@f!aFKTrwl|NPy!H`)D{4G zxR+JfOu!8HLLSCs&g5AjWH?KOf(KRzYOoSYBySrf?#L-%oC4th6l|a%%*ufEdZBi6 zy{HCDOs)kh$RU0@gg0Icqb337VG|Zr!g|vvs9_c;J(UVsU_)|1RBQ#YUo3J$tBMj} z72^(VpwKxnC|ak;6&_JdRH;IZ7k~;WRRi;wg(KQ&2?^(LvVuBYACNYq7DrCD5!gz; z(F)%7;}+>X(^kH*L`z~gCRZEJ1_j!&cnmPsrpFAl8x%8;Fs`*&CIRh+H3>)<)mkq+ z`IDN)lU*{zk(dL|TWRM~kDBOXr(CZPP?e2Ut#!(U59^&A64K z$U{~@4-8S9BB@BNe1H|I4fzUaLy3lm=K;R0zOd!EV{Nnn;Joh zfvpQd4TbIlSbr;dV5VkzO(un=Er_HcvQ+YG@WtF}Fvuds(5?bmPk_+b=6z_Zx>v#K ziXc>2Z6DyiNyKEa;E}3B`c_5n0rNdj$%~b3LNeC&@o^Gb?`U?cqKhGQQ6N`TwBiYV zfR5`5ECN=RIX{wl5eP7>J(sopTUb<7Tm-aRbfBUR8WEeO`w@@mg`OU$_&lq?KIS?= zTxaCxI+;A$K|2|j(?#bnxp~fXJ@qVvi4HvIa5DW6^+8t2nhhlx?e^51C(VVFW$eLJ zS3k}J(c1J!oKM{VJlx2zs2*+g8=gq7FKshC5q}cvjUvuRLaUOK&_*plp&*|6B%F{o zQU(eWi-J8+0vwxuC-g)F!hs>M)>zz70uZaAiSLdu0F#FSh%f3u*kpVX0n5c;X3dF1 zN&spj%$og9(g<@Yhu*7n?#`ESbn+F2PntH(&UFE+h&CCmBw*^l4lTc^HO zyQ}}E9giI!$vwPm>Bc51d%>~VGvyPzUi|Xa&V_}?&YrxPz5Z|E8T!qIZ{BTrz21Lg zbL(?CgLUMm}6<6h~+I@F_&cmtxjivIT!ub#UrG9s}%TYd%=hkayOOBN78*~(uzA*B@>Xf>5a-*7_ewYId>rbD*oe?VEHgD+8J$vRiwf@m` z>A#x|_xe7Rx%{g|{l|^%Rl-v%FRa|RbOy|Ox~BKfD~Gr?bbCL`ud08xUT?YZYs;1) z6H?Y+?HkUn|C&o{>D}#xRM*bpgZazYw37ax94p$taLTc>Yv#e+`n=~{HGf{8aqrya zLn9hDG%q_b>7r-(q6IxiKj^6ImHrXEcrp7#dPe5~b6=TRKJ^5@;=sh)JtjW+&zwtV gyxZ&L_qy$Ob-UWK@Wi@VZ=6f1FDt8hJ*E-<2cc;QMF0Q* literal 0 HcmV?d00001 diff --git a/tests/queries/0_stateless/data_minio/dv_puffin_warehouse/default/dv_puffin_complex/metadata/snap-9221420601809522049-1-41caec64-2cad-46f5-a966-dd75e3635abc.avro b/tests/queries/0_stateless/data_minio/dv_puffin_warehouse/default/dv_puffin_complex/metadata/snap-9221420601809522049-1-41caec64-2cad-46f5-a966-dd75e3635abc.avro new file mode 100644 index 0000000000000000000000000000000000000000..09ad8a80111b87a5f580454e112992741352fa25 GIT binary patch literal 5192 zcmbVPYj6`)6y_mR28NajouVDQrVOGq&L(Noqykf*MT-Rrt+fIYH_1)2bh8_FH!T6H zFuaQc<70+ez<0%QL>w(vs34$I)WXOcg@I8(9FS4)NfoVkceD5I=Fv9%%;xU-&i&3g z-}!c?dqLl{47yg}9}qo$7@${}mYAe)5Sj{10`%~L*OX)8=z#n?Ky!=_ijvF6un;}= zF_86&rh>UkVon5U$&XG%0*R8qaMkFym-nDJrM!oh7@h-<&MpCUjN}JX%YlN9qna77 zsUVM{>T_aavyiLSM`w9m>+guPQqE2qd!4O5eSLU3nU)Y35*n-uT>Y;!*dc;-jA@L4>m>pQv|#ZNPY-3Y6}20 z!AnbY4zNt{LKenkZpqg{h;U&Davtd7LIF1vNZtlY$|I+MdJ5F}A!h&ub+ia*uNTTU zSBk2#WXaQE1sKFnukcFwI@BaUJFLN?O6YGI1!dPkDXF+bjt+``UNRKKc(KR{xhj&M zmef0RgF@M2P~=V#T^>G^s8W+;UH~ej5C+;Y9Y<8IE+kyUh!W~_Wk9-(S{ymqO<+Cw zYAbkqj$5SkOn3Rl5-sp`F}WIeRwT!>{&NOmGqOA7)WMt`GW4&!fdr$fF!uPa8B*)&ZK~L^PywU@fkj z=UGT|=z%JV6+#H9l@HKdSd}loJd|j7SQg+sK`qUOqD9B4>p%jggf`yEq+veM6DZBa z!Sq5O9bzR@ft-_3C>**JSTz2E%Z(Y38 zF|d9?go43a09M{g9@tXhZ$07Xn7ytzDdMnG3OD&L1n9=_ki{uaI?I-TS$w(eSD07);pRVz35^{MHI*tCAoN9 zH9*G|1?B-QicB?AaK(>!crWw}Ma5@n31(>P z0C7FWX|-B%t(2XzI4EbH)k@_$GnLfM2ooI`Xw9Xv5cL6CGS@;uM7zD8ZIrf<%<3NO z=g49_5UtJXi1TU4#>0&aOUNUye$^8x^`#7kC*n_Hy-~#Z2xwI@0@|nrC=|p~pM(>V zM@mJp#G+silmN$|-w8btfv{o-tTj3}lmJ9;XyUsg3}DH}0K^ydB5X1~iGbx{Fsp|@-Vf)q)!6iFj!79%czg-nuK5LpGWzf8c7q4{Fm7EazmijDb6Iy~k_ zk!RJ>5(H9Sjh>_dG=B_L=9lR~LTnsh!xm zv7mj?@8i~3qhag(H(xEAmG(zj+B-MwZ5{6w&$*T{eg6xEnO|(^ zx_PW__qUb(KdG?)R%xpl_e<4;6Zfuf-8FKN(;L3D`>S!LtimehQwhDSRMh-GnPOgPLY#`56k9 ztZ066$X%ah-~M=JN%O+xS5{H@%sf!+5ALkpHNv~$NXuWR zQvNPpHhkT(0~PMKRROKAhi5r5zdl;{AOOHa=_F@pYA8Uo*(PzyF9qjZ1$vSCus%U9zyV^Q#Tp9)p7y zU%YC*;#|3Q@}_0mCw;$n?B0s|*EheubnSJ|fHCiH&06=edB~?-Uw*aqku$FCyPx^7 zYBqm&`jxIT=3iI8ej7z~$9Tr$!x#!X<+qU+;yLW89 ze7QL_KmX)*Zl0rJ>f7UweB5^Qz8P=ycO6@k(l~5C>!|q7F?-@$S?BvK8arj*m;oO& zEeJ8cUFh6)ruC!LpZ}RMcE%k8C*D)E;N6GWjys;eZSbK11E-bVRx$0)#_N@HTbhK< u8C~tC|Gux_V z$c=GZqo%$JSsA#{uJqQFmqANCC(6l&hZwqw_?v8W5powXbP=%svcZ|p{EC4ykN%bo zPF(6!44gRROEx&NiH8_CvhbU1aA0B=F>qk`|Lg7P^7Z%j?0qxowC>h3LZ`LYu9HVcu{jf6 z|GP=XhFo+Jw7qh{iBmS@f&=5<%jfNW-U&W$^KsARb5<|s1fR2bSabQT+0B^XvnCg9 zE}tGAaPM1ps?5$ zpdqYk95x8!5q63(XuvH+hyrZ{MkvrS1{+P7BN)V-!Hc||OdKfWJskaMAWiaOKb z(}kA11?WI6Oi+9b>T~b^i6%SbULK8Sh8`Uzaiek=Bn;1UE8Ms%W#E=)w z-%Z|p`4HC(yg)9QW50wYcy-D}6_lPc-wK>&AAIEYd|FoLd^mkm3C2gF1*g`j{PUON znhZShc(ol?F%V4!wb86O_AQgO>zO`)Vtv<3QTNc-ZH=%EE#59Fp^$<`8I`OLYVzd{ z6H(h%;Hnz1LUM1dOv!8#KzwP!nl3k;j literal 0 HcmV?d00001 diff --git a/tests/queries/0_stateless/data_minio/dv_puffin_warehouse/default/dv_puffin_source/metadata/0e8edb12-fdf0-4ce1-b6ab-c9c2a05d6d57-m0.avro b/tests/queries/0_stateless/data_minio/dv_puffin_warehouse/default/dv_puffin_source/metadata/0e8edb12-fdf0-4ce1-b6ab-c9c2a05d6d57-m0.avro new file mode 100644 index 0000000000000000000000000000000000000000..1f1d617f028d0c90fdbfcc904146a541b0f902d6 GIT binary patch literal 7802 zcmcIpZHN_B7&a};J{F6xFss8rrE$0J%NkD2he|{fMxB1bzPKJ!fXlIWzapc=zrG%fil_ zc|V@_^Evxq>4^&#Ugn#lq>A@nFsTF+HKOz@e&E&3Kq)BUn@+2&^cM?C#UgguSNex0 z75YRiC=RaD^YB+e@yO=7#M!M0Bdu`t_pMwF@HsYQ6qb8ihLzIP|r323oDs05>jVHh|6U1^ExRTJ`JV8lkK76hMhT*t`cZKORa-#nikjOJ(K=IHS9s`lS!r)<>Y`9 z0rQ?)pN+j*OrSqJ6+8G)ECf+ZQBbehq!4tFkQP|bqso3NR11PBa1x`(+*(;qzZY=pQl^OCRJT-jHzmG>`fx+HxVi$qm{sy z5}IS!u8Zo5y_B3#d!fb){o_H z0*kVnQn5E}QyAdcfvOLn(u(as)t5F);2LF|1eI2t0aPrX3y?{sT`xnAK(uiq?3B@3 zyfz;@*+}}8!Gb!A0BR})Fj3-s8s+DjCU=zc=9;Fp1t3F;35OgEyFzxRsd6WEurupm z>Ae7!-q%L9=?z7Ymu-4qn_$sVPwN%(3Nh4pdkiZ$H`#Q#dD!ka@r)8+4m(ubaAwfc zv>uvg%iDlX>&Y(ECY@PZ-WGPuU&Q>)1xf3ctf|*(vkMaKW_ml_Op_Z$su-Jqq_@}3 zn7^%nBxgHx5u-{1Ot*xFOj1JQ_`F%=nBGKMiGT|)f$W_rLcjJoX0ZH=Q z@unI$vtgU_U_duZ7?W@ul+cv}`@S^-d!CCNGESf8d)Fl+?~w}eC>1IjjhrLyXp}_K z?VbBM0K;|yF?<2hGP2`!ere-Azl^qA9InB14TYr_2MEKAkH`h=s*w%(h{+3U#BQQ2 zhVQc2Aj51utl42s%KI=H9|hOh4jzd^z(ze*6WO`ZmE3I^$zscG=E6Q#@{rp&saz*-Y-7V2{L2fdJFw;gP2i(Y|_ zaEa@Wmy+LYMLxv_|V6FU~&bY6bf4N0R&N&*u)173Tt+J$fwWTDOFT*Gp0OE z&orpoEYG08X6ji3ujOv&kQUK=X+mMA>4gr7l;MRAp_{oE=OvunFRh#VmExAu^hxW< z^hpEkVxLNugV~Kdd(bJAoVRjhDk#hsWuIbwLUKzbqh*(jE|pCC=F23Fo3D!@1oZ%}q-sK5!nl0G#_jA9tt`#Wgy4js(F_S=mE>qbst8Vsj%5i=OySmO znb0PW{FneF7+99p)ROgoF)?AO*c%#1SC)+lWTs2Lgl3VCv1WR$F7vFG4VW(90tN*? zOZbY&2-1Z^!yS<&z^T|j#^$^@rSDv`zx(@L8xO48cci{)?xA^yAKdrA=f!I}cCY*B z%cr;Abg=u|1&deSbKNh0zI;r%`h}Cvcfa@8oW+aYez!7j&BhLI{+0HD6IHyf5y1 zbM?+Me}6OoXXn(iKR&(T^pO5z#SKoramS9kHe5DwuJ-FY_WXsCcAoGy6+RZ@=`B oUDrOe=fuW~7Jq&3Wgi zR|=|P;~L!$|EY>g-e{OEfe%$|c|?tbS&qG`Y!|nm#T%|O97z1)MXt*whU3!np{yFV z%_{Ny60v=E3)UsA;TF%we#3)Lp}D3_ZDsocQ@7PikvPxBb#Kk_OU%aLzy-x_SQZ>% zrWIACLaMl7`N|Ysmn9WsN#Gx`4N|fjwK8$J`u3&ilC-L6k&?G>k)x8x3ife{%H^h| zg6wJ$U_m%TU=7Fi0d{^~36QNg1_XIJ_~;?0iu^T#sHQ2DKSkB5sBo(8c&2YUHmVSd z_{39@b_)E%mg#RrYmrtKrh+b8&j6`oe@$qIG&CW1AU@21dB?y^!TDEa(WdFIp_ygG z@`-vZ!e7YeB~+N)s^iwMCv3;7uO#qG$!mm?vFVv77m-6Cq>Ax35FkqQEMDfV0)^6JD$c&|y`2h~+FoQ51$ zmiLj?MV8Jqs=;OtnSm0DTtakn2Agk@l4+O9TM%*Xdr(-Bf)&Td7E2f8!FDP>+Aar8 zP6orV8a2B_w+pFJp>brf>tfm=OrO*skNJACrK}eU$W(D~U{7{4#IJACDZyhX3 zZ%X-zq)kD94+g49fJ)1E1Jz{GEP-p7aS~KoejiY=d{!Zo469Lr9)W1%M%b>PMZB1e zoopmMRboLsKmfG@1u$0P{5HzYHBIg)kIgkr>k2@I6cY|P7<7g7Oe@Hp)W-IKgQZUZ zSo&la*`_xYK~}culU;&EuXkJxv>< zc{bh!blO;Yp|1LYTC=$ik0wle=cEXv!odFpl3hXKlmXsiIvWkFhEszf z%k>)GCo%E-H5>TwgO{zkdyK}j5gQ6bvA>C;t;S=m(@giZj8ofTSl_jaN9gObNJ#Bq}Eu(UIo&_ z2K6WaaECK#V1)zRTQ)Ttn3Jrhi+PL=6VEV)Za5WU90q4B?32NO73DUF>jAgx(-lrV za+G>NN%!RkO@qKSxf+uDM>?Ox+j~fWr-dZ57mo?>6d~RcXyMS?BL4gk=lJ4it9OX- zur5?KaqzYTjB){tU^X-PwxnoWid;zIsO)?*s4RNYdyFoO3ey5|ptZ1rXZAcSc@a(q zO~~m}>_{;2ERHWUh?gg(fwTMebj-BL_JuwXCZ7GN$YzA;0GPh8Nn~(>E@0#A~@5I;2H-zqFvR9eSZdBBgktL+JMJ#bXI4_e(p;{Yr4l4t>(bQhm|@ zyU3@qX=C;x&lYk*$zv-=rh>wJQT7SeCnUF3GFp1c=u*jaZ@x^@xcR!cgdiVeK9*v> z!VL(XvondAP$TZ1BnL~;!@Hwpdi2RU&UN702p6Emgm*%;LkNsTv;|2M=7}6k@G&8* zbr4%x2Z<=Gi&HfrFJ@deVdHj=idL57W>E+8S4;~&XF6(n%ap1w|-2G#p{&c1|_tf0Lv~hD~&hU(D+6Nb>zkTO><=XVQ yM}32r`=*Q2FXaFHwOTHJ^4$+T%EzaV_4f~4Ir{qBy+?kVIQoA7XMKm7y!;OsL?f;M literal 0 HcmV?d00001 diff --git a/tests/queries/0_stateless/data_minio/dv_puffin_warehouse/default/dv_puffin_source/metadata/snap-1455293623190430422-1-c34940c4-6b84-4560-85a3-35e17802d85d.avro b/tests/queries/0_stateless/data_minio/dv_puffin_warehouse/default/dv_puffin_source/metadata/snap-1455293623190430422-1-c34940c4-6b84-4560-85a3-35e17802d85d.avro new file mode 100644 index 0000000000000000000000000000000000000000..a968682b674d42b94dc791d6e4aeb73973fcd00d GIT binary patch literal 4789 zcmbVPU1%It6t2;hRx$LUM$y{dPDLL~WM+TTWT9xxrnHb^+l?ZWu)RBTH#<5%>&$G{ zHRMH#4JmoDh!0ZwAZP_2>Vrk9G1XuMi$uYQR?s5qLkmhpMexCMXLjzrv%9lDecahQ z=R5a1=X~dzrwY&AwtWE4I`(6(S|TPsDa}but|TL*wTx{_3}mC zAn>@T6GL;Q%<;LV6J_j`;6y!;>p8kL1HUy}g*cP8iap)7(9_2bp{nkc(19HC(OJ|X zU6V3tS)Lzkj%^TYMvQLQLhLEq!v>-WB3FNjAl*WRnn&C)-Fjd}2wcXFr_+4$Zkecg zKxT>SmdCQr!Jp_*7FArLA*3$Bb4Oor%-STRZGN*fTAjP5!N(9Z|)a0 zU`a^}Sf&pAbcHu*SD{HzcsPWGN{BaYfEpH{{8aL;h0AWq_96v|UaWqCR>dn}FStV+ z6gk<1LOaFHt9B)>Qe%`C1ch{JC_E-`B&URs5NyGNKoqB( z3aC{PVXGF%SBVZK43A+TmM5CUMx{}7MuiUKV1DT6oum+Rx)Uh2T3~9Vh%1ICWoS-O zp%@WTFkt+hd;unk3u(xz6lgKSKN3nW^4~)KTc~I|2*b7g*`x%;1B(lyQZC;Du>V%? zz{>pe{GbHWW`Z=-m&(XIyI8Jbniv#=UWH(_#OZ8%A6ZoQ=SeMZ5)adT!1^YR$q7q! zYGr?`!h0xu4;2i%&=yh=+oz~Iu-;*IMA2oCz9{f3dbD`f41(jn0^10?u08`YZzGFT z>2sIv-?40VB8%uPx~OPE24Z*kenho3qV_}a4eXJDa2){G4=bs3Ix(6YP9&94IhB-C ziNt+=>O+7D2m0ilK)i{)p;_X%u-W^PyTWxe6l}phWv8wZSei3+-?O-vjW*boib@xM z;E4R{$`QL$cPGuxsP20lSd_X0`!oZg2%GsHd=MR|07Yp=VGbw;C!*f5j)+4f7z9%q z!3)KJh<2v6H(~(FJ_f*DQ5Ru*aY-C3&A`H%bAJ>A8X+v#gJ0sP!wj{xQ|bn}OXtKu zc^!!v$-aSr?}N8sfR>FF6Dq^DJUnacpv8S1<^ zFGXy)OfH!o8{n$pSi`bq4+K-jlw0^T4ODcLbuPC^fgj`O5v>lrQYWj_%AavYM_R!8 z&Wxi*?#wJ&C)gGajibV{s(-Ra%~&j`Zh={$Ho^Zs@5YsJFutFn2CQeWcqbO|T0AbT z7nrah={Amw=gPaF!Cdj64dsY3)7E7%m;~p)!BEA)L?j@0%fnDFWAAWT9_dhH5Z@H}BOz+*@ GnEU^9TW6L4 literal 0 HcmV?d00001 diff --git a/tests/queries/0_stateless/data_minio/dv_puffin_warehouse/default/dv_puffin_source/metadata/snap-8278841256334127309-1-0e8edb12-fdf0-4ce1-b6ab-c9c2a05d6d57.avro b/tests/queries/0_stateless/data_minio/dv_puffin_warehouse/default/dv_puffin_source/metadata/snap-8278841256334127309-1-0e8edb12-fdf0-4ce1-b6ab-c9c2a05d6d57.avro new file mode 100644 index 0000000000000000000000000000000000000000..ef9e9c5fc2f370ef040371b7984987eee8580041 GIT binary patch literal 4870 zcmbVPU5pb|6c$7ib%ToQh7h9H;VWlh<#w;Z0xig)6@3ft6%Tv?o zo%5ahopZkP^@-e$Id^s8QQKPWs0C8QyQQ6yJ60l6TC$01*_zZXnYc*57qO}5iQ{JT zxrRxgTYwYQK#ptc<}iHMEEVExvQ+HqmWiHzY9p%XZUH@-K^{7b z+M{byIw8vw-Sx2zVh)SZ4NHi<*>bUgXoASm#|hF+lpAx26Qo-WtO$Wi*miZAPu3|B zH4Dfraou!T)*1K_Z5%>nhiJ&PP{r2W+I(X|Sk*FJ;=NyGK@9eY|D%JfJaP*J1=OaI zJfLA0cOzv$BL=CaoPW(X8ju!FFs5qfj4 zs6I8F zXo4cg>riN?I9b&yM^$Q&@`9j{_81C|2^`72LP+w0?zqtD-hecXx}kcqNnnwDzZJCE z;|_T}(=6Y5qHU{EmurY;p8|TU5d(_Y^o)U7gE9spYjEh9QCMRHqi(HXehYVc3hHb?kYqINAw074&pS6R8eh-E?3`Y2YAk&aw=GO?cpo z;%2)HYL!RW9P{O?K!*~B$1o7f6OCe{Tq`>LLI-j%FLd}$Qh>Q_Hc)Ifz|=?{mkn1+ z)11OWkrGlcVEpZD4kn5VX~3%FYca!rGLT;6y@kBDP~Nf;hHHDXNePMu78gXhRJs9R z@2&QMm5FT=-4aY&5u~BIR8kY{VwrNWh(R&vRR~rKoX)2AkwtZXo{VLS#Km+Uu)c|6 za)YVbVjy?=B zZy}RZ=yQke-@&1w4MT|DqJ@gaWFWTF^CPOI5p_Nk-@q>E3f2K|y*w35rc%91EYTN_ z!$&eM_b>EP?*U9WaBEDK7X$T0?DmWj+kxFaH-2BRkb3+*I5)OfR|%}m;i~i5vxE&d z7?!e1SHJIxy!y%^!&CJq_1>uJd~8^ix(ypO4WS5|`Zk;p9Vs70sYhWRC;}&>-;th( zL&O*a(;C4IMSzHgrm;I>07{Ypa9`9y*misp2TL%ppyu2mMSz9~%e3K`IBFk5ZS0j= zL2l7IF;F?qKpVNHRvc7+-R?;mxfZG%O7 zv5?#1acRB4hWSZ1@mxGt-UT)GiU+MJN0^zWK8wM`IR7=4Dhehf0ht@NR>0Fzm@Mi^ zMZ=`@CK9p!cwa28^vk_*xi=QOY3;=NZ~y!4;>@DepC8|~ao;!d&fPq9MtS^%yzrLo z_oPdmUFwR@GL`%Ff)2Z>jdygrD$LFhB*McKo-~ZzF z(Tl4motFv^;gd7h3ug!aI`?kRt7{f%1IsT=96UU=skEf<{{BM`_r1O9w!|mP4rHbt z{OE%Z2iEQSb>VZ*?(UrW`TO(NF8zD@#K_?E)zYOuCTCWxee9KI-n_T`!^oZ=Cw)%Ac3NyL$cdmF;(|`{~N5J8SR$KkU$n8vp SparkSession: + return ( + SparkSession.builder.appName("generate_iceberg_dv_fixture") + .master("local[1]") + .config("spark.jars.packages", ICEBERG_PACKAGE) + .config( + "spark.sql.extensions", + "org.apache.iceberg.spark.extensions.IcebergSparkSessionExtensions", + ) + .config("spark.sql.catalog.spark_catalog", "org.apache.iceberg.spark.SparkSessionCatalog") + .config("spark.sql.catalog.spark_catalog.type", "hadoop") + .config("spark.sql.catalog.spark_catalog.warehouse", str(warehouse)) + .config("spark.ui.enabled", "false") + .getOrCreate() + ) + + +def generate_simple_fixture(spark: SparkSession) -> None: + spark.sql( + f""" + CREATE TABLE default.{SIMPLE_TABLE_NAME} (id BIGINT) + USING iceberg + TBLPROPERTIES ( + 'format-version' = '3', + 'write.delete.mode' = 'merge-on-read', + 'write.update.mode' = 'merge-on-read', + 'write.merge.mode' = 'merge-on-read' + ) + """ + ) + spark.sql(f"INSERT INTO default.{SIMPLE_TABLE_NAME} SELECT id FROM range(0, 200)") + spark.sql( + f"DELETE FROM default.{SIMPLE_TABLE_NAME} " + f"WHERE id IN ({', '.join(str(x) for x in SIMPLE_DELETED_IDS)})" + ) + + +def generate_complex_fixture(spark: SparkSession) -> None: + spark.sql( + f""" + CREATE TABLE default.{COMPLEX_TABLE_NAME} (id BIGINT, data STRING) + USING iceberg + PARTITIONED BY (bucket(5, id)) + TBLPROPERTIES ( + 'format-version' = '3', + 'write.delete.mode' = 'merge-on-read', + 'write.update.mode' = 'merge-on-read', + 'write.merge.mode' = 'merge-on-read' + ) + """ + ) + spark.sql( + f"INSERT INTO default.{COMPLEX_TABLE_NAME} " + f"SELECT id, char(id + ascii('a')) FROM range(10, 100)" + ) + spark.sql(f"DELETE FROM default.{COMPLEX_TABLE_NAME} WHERE id < 20") + spark.sql(f"DELETE FROM default.{COMPLEX_TABLE_NAME} WHERE id >= 90") + spark.sql( + f"INSERT INTO default.{COMPLEX_TABLE_NAME} " + f"SELECT id, char(id + ascii('a')) FROM range(100, 200)" + ) + spark.sql(f"DELETE FROM default.{COMPLEX_TABLE_NAME} WHERE id >= 150") + spark.sql(f"ALTER TABLE default.{COMPLEX_TABLE_NAME} ADD COLUMNS (label STRING)") + spark.sql( + f""" + INSERT INTO default.{COMPLEX_TABLE_NAME} + SELECT id, char(id + ascii('a')), 'new' + FROM range(200, 250) + """ + ) + spark.sql( + f"DELETE FROM default.{COMPLEX_TABLE_NAME} " + f"WHERE id IN ({', '.join(str(x) for x in COMPLEX_DELETED_IDS)})" + ) + spark.sql(f"UPDATE default.{COMPLEX_TABLE_NAME} SET label = 'updated' WHERE id = 25") + spark.sql(f"CALL system.rewrite_data_files(table => 'default.{COMPLEX_TABLE_NAME}')") + + +def cleanup_crc_files(table_dir: Path) -> None: + for crc_file in table_dir.rglob("*.crc"): + crc_file.unlink() + + +def generate_fixture(warehouse: Path, *, simple: bool, complex_table: bool) -> None: + spark = build_spark_session(warehouse) + try: + if simple: + generate_simple_fixture(spark) + if complex_table: + generate_complex_fixture(spark) + finally: + spark.stop() + + if simple: + simple_dir = warehouse / "default" / SIMPLE_TABLE_NAME + if not simple_dir.exists(): + raise RuntimeError(f"Expected table directory at {simple_dir}") + cleanup_crc_files(simple_dir) + + if complex_table: + complex_dir = warehouse / "default" / COMPLEX_TABLE_NAME + if not complex_dir.exists(): + raise RuntimeError(f"Expected table directory at {complex_dir}") + cleanup_crc_files(complex_dir) + + +def main() -> None: + args = sys.argv[1:] + complex_only = False + if args and args[0] == "--complex-only": + complex_only = True + args = args[1:] + + warehouse = Path(args[0]).resolve() if args else DEFAULT_WAREHOUSE.resolve() + if warehouse.exists(): + shutil.rmtree(warehouse) + warehouse.mkdir(parents=True) + + generate_fixture(warehouse, simple=not complex_only, complex_table=True) + + if complex_only: + print(f"Wrote complex Iceberg v3 deletion vector fixture to {warehouse / 'default' / COMPLEX_TABLE_NAME}") + else: + print(f"Wrote Iceberg v3 deletion vector fixtures to {warehouse / 'default'}") + + +if __name__ == "__main__": + main() diff --git a/tests/queries/0_stateless/data_puffin/dv_sequence_number_not_minus_one.puffin b/tests/queries/0_stateless/data_puffin/dv_sequence_number_not_minus_one.puffin new file mode 100644 index 0000000000000000000000000000000000000000..9b260e2de1d5e61086507cd4be0ae3ffa7a9200c GIT binary patch literal 297 zcmbV|Jr2S!423yL<(be{0tvCPFn32ClE#5ZAqk0Fsj7NXE`w8X0zCP9vY&st+qK>O z9UXTspj=QHF4cVjnj>-s8Xb}iUQ9SpNMdSXUxBgc^j6t3WiMlX1`<|ti)f34?-Big z!jCV7iRFgXU`$40I5If&4|+13&P*|dh>0u~*98(9OvOM8ElCPv-L_6$s3xf#)=?%B fO^E}iJdet1gG}0b=`79c1P!cfzkYs?DQA8Gj#Nq~ literal 0 HcmV?d00001 diff --git a/tests/queries/0_stateless/data_puffin/dv_snapshot_id_not_minus_one.puffin b/tests/queries/0_stateless/data_puffin/dv_snapshot_id_not_minus_one.puffin new file mode 100644 index 0000000000000000000000000000000000000000..7216285d651e5c67f97df45d8cb734d331b8a526 GIT binary patch literal 297 zcmbV|Jr2S!423yL<(be{0tvCPFn32ClE#5ZAqk0Fsj7NXE`w8X0zCP9vY&st+qK>O z9UXTspj=QHF4cVjnj>-s8Xb}iUQ9SpNMdSXUxBgc^j6t3WiMlX1`<|tOH&klkLU*! zetaoRtcsT9U`$40Ix;x)4|+13&P*|dh>0u~*98(9OvOP9ElCPv-L_6$s3xf#)=?%B fO^E}iJdnz2gG}0b=`79c1P!cfzkYs?DQA8Gk%>wt literal 0 HcmV?d00001 diff --git a/tests/queries/0_stateless/data_puffin/generate_puffin_fixtures.py b/tests/queries/0_stateless/data_puffin/generate_puffin_fixtures.py index dac6eb811559..120643cf21ad 100644 --- a/tests/queries/0_stateless/data_puffin/generate_puffin_fixtures.py +++ b/tests/queries/0_stateless/data_puffin/generate_puffin_fixtures.py @@ -5,24 +5,11 @@ import json import struct -import subprocess -import sys import zlib from pathlib import Path - -try: - import lz4.frame - import xxhash -except ImportError: - subprocess.check_call([sys.executable, "-m", "pip", "install", "lz4", "xxhash", "-q"]) - import lz4.frame - import xxhash - -try: - import pyroaring -except ImportError: - subprocess.check_call([sys.executable, "-m", "pip", "install", "pyroaring", "-q"]) - import pyroaring +import lz4.frame +import xxhash +import pyroaring OUTPUT_DIR = Path(__file__).parent PUFFIN_MAGIC = b"PFA1" @@ -345,6 +332,21 @@ def generate_missing_required_fields() -> None: ), ) + # Puffin v1 DV blobs must keep snapshot-id / sequence-number at -1. + for name, field, value in ( + ("dv_snapshot_id_not_minus_one.puffin", "snapshot-id", 0), + ("dv_sequence_number_not_minus_one.puffin", "sequence-number", 0), + ): + case_payload = json.loads(footer_json.decode("utf-8")) + case_payload["blobs"][0][field] = value + write_fixture( + name, + build_puffin_file( + BLOB_PLACEHOLDER, + json.dumps(case_payload, separators=(", ", ": ")).encode("utf-8"), + ), + ) + def generate_invalid_property_value_types() -> None: """Property maps must have string values; non-strings must fail with BAD_ARGUMENTS.""" @@ -618,6 +620,48 @@ def generate_invalid_file_metadata_properties() -> None: ) +def generate_mixed_blob_types() -> None: + """DV plus a non-DV blob: footer parse and Puffin SQL must tolerate the sketch entry.""" + ok = OUTPUT_DIR / "file_properties_ok.puffin" + if not ok.exists(): + raise SystemExit("file_properties_ok.puffin required to build mixed_blob_types.puffin") + + sketch = b"\x00" * 16 + puffin = ok.read_bytes() + blob_end = puffin.index(PUFFIN_MAGIC, 4) + dv = puffin[4:blob_end] + + footer = { + "blobs": [ + { + "type": "apache-datasketches-theta-v1", + "fields": [], + "snapshot-id": -1, + "sequence-number": -1, + "offset": 4, + "length": len(sketch), + "properties": {}, + }, + { + "type": "deletion-vector-v1", + "fields": [], + "snapshot-id": -1, + "sequence-number": -1, + "offset": 4 + len(sketch), + "length": len(dv), + "properties": default_dv_properties(cardinality="2"), + }, + ] + } + write_fixture( + "mixed_blob_types.puffin", + build_puffin_file_from_blobs( + [sketch, dv], + json.dumps(footer, separators=(", ", ": ")).encode("utf-8"), + ), + ) + + def generate_unparseable_footer_json() -> None: """Malformed JSON / oversize integers must fail with BAD_ARGUMENTS, not STD_EXCEPTION.""" write_raw_footer_fixture("malformed_footer_json.puffin", b"{") @@ -653,6 +697,7 @@ def main() -> None: generate_missing_footer_leading_magic() generate_invalid_file_metadata_properties() generate_unparseable_footer_json() + generate_mixed_blob_types() generate_cardinality_mismatch_large_bitmap() generate_dense_range_100k() diff --git a/tests/queries/0_stateless/data_puffin/mixed_blob_types.puffin b/tests/queries/0_stateless/data_puffin/mixed_blob_types.puffin new file mode 100644 index 0000000000000000000000000000000000000000..90c0f6dccb26114012aedd2dca32d565687e98a6 GIT binary patch literal 444 zcmb_ZF-`+95Ok2}dBVw}JaczZghVhvgu@4V(LjBV^qr0C?g@D`qcDESQQ14K|- zwwf7vt(n=)-EGHC=ko3SMlFu?b<5ZnU$6_(&)7MuKVDupv_J9Q`abb>bw6=D1R%s^ zki!T0!p#(cwKRAz^Zv?ODs7V(&$kbMa!Zq5 Date: Tue, 15 Sep 2026 21:49:10 +0200 Subject: [PATCH 2/5] Resolve conflicts in cherry-pick of #2183 Kept the antalya-26.8 side for everything outside the source PR's scope (point-in-polygon / paimon cache entries, lazy-materialization `rows_to_read`, identity partition columns, `_headers` count-cache guard, 26.8 path resolution without secondary storages, existing manifest-count and snapshot-summary comments) and applied the PR's deletion-vector changes on top of it. Also moved the PR's `use_puffin_files_cache` settings-history row into the branch's own (empty) `26.8.1.20001.altinityantalya` block: the three-way merge had spliced the PR's `26.6.2.20001.altinityantalya` block into the middle of the `26.8` row list, breaking the file. Applied the PR's 6-line `icebergCluster` deletion-vector docs section to `docs/reference/functions/table-functions/icebergCluster.mdx`, the current location of that page on antalya-26.8, instead of re-adding the whole pre-restructure `docs/en/sql-reference/table-functions/icebergCluster.md` file that the cherry-pick resurrected. Adapted: Parquet `Reader::prefilterAndInitRowGroups` lazy-materialization slice and the trailing `rows_to_read` bound now read `global_offsets[...]` / `global_offsets.back()`, because the PR replaces the running `total_rows` counter with `buildRowGroupGlobalOffsets()` Adapted: `InterpreterSystemQuery::getRequiredAccessForDDLOnCluster` emplaces `SYSTEM_DROP_PUFFIN_FILES_CACHE` in its own case, because antalya-26.8 replaced the grouped fall-through with one `required_access.emplace_back` per cache type Adapted: the PR's delete-file classification (deletion vectors vs parquet position deletes) is applied in `IcebergIterator::decodeDeleteManifests()`, because antalya-26.8 moved delete-manifest decoding out of the constructor into that lazy method Adapted: deletion-vector loading uses `object_storage` + `persistent_components.path_resolver.resolve(...)` instead of `resolveObjectStorageForPath(..., *secondary_storages, ...)`, which does not exist on antalya-26.8 Adapted: `ManifestFileEntriesHandle::getRowsCountInAllFilesExcludingDeleted` keeps antalya-26.8's `std::optional` signature while delegating to the PR's shared `getRecordCountInAllFilesExcludingDeleted` helper Adapted: `IcebergMetadata::totalRows` keeps antalya-26.8's delete-file bail-out, comment and mismatch warning, and only adds the PR's `common::addOverflow` guard on the cross-manifest total Adapted: `canUseCountFromFilesCache` replaces antalya-26.8's `hasAttachedDeletes` gate (same purpose, both call sites), which is what the PR's `effective_need_only_count` refinement supersedes; antalya-26.8's `rows_to_read` and `_headers` guards are preserved Adapted: added `ObjectInfo::clone()` / `IcebergDataObjectInfo::clone()` (two lines, verbatim from the source branch) as the minimum scaffolding for the PR's new `gtest_iceberg_data_object_info_clone` test; that virtual came to antalya-26.6 via Altinity#2154 (upstream ClickHouse#90740) and is not on antalya-26.8 Adapted: `#include ` in AvroForIcebergDeserializer.cpp, which antalya-26.8 does not include but the PR's `Poco::toLower(file_format)` needs Source-PR: #2183 (https://github.com/Altinity/ClickHouse/pull/2183) --- .../table-functions/icebergCluster.md | 136 ------------------ .../table-functions/icebergCluster.mdx | 6 + docs/reference/statements/system.mdx | 4 +- src/Access/Common/AccessType.h | 3 - .../AggregateFunctionGroupBitmapData.h | 7 +- src/Common/ProfileEvents.cpp | 3 - src/Core/Settings.cpp | 4 +- src/Core/SettingsChangesHistory.cpp | 4 +- src/Interpreters/InterpreterSystemQuery.cpp | 9 +- src/Parsers/ASTSystemQuery.cpp | 4 - src/Parsers/ASTSystemQuery.h | 3 - src/Parsers/ParserSystemQuery.cpp | 3 - .../Formats/Impl/Parquet/Reader.cpp | 21 +-- .../Common/AvroForIcebergDeserializer.cpp | 4 - .../DataLakes/Iceberg/IcebergDataObjectInfo.h | 2 + .../DataLakes/Iceberg/IcebergIterator.cpp | 83 ++--------- .../DataLakes/Iceberg/IcebergIterator.h | 4 - .../DataLakes/Iceberg/IcebergMetadata.cpp | 44 ------ .../Iceberg/ManifestFileIterator.cpp | 11 -- .../DataLakes/Iceberg/Mutations.cpp | 4 - .../DataLakes/Iceberg/Snapshot.h | 11 -- src/Storages/ObjectStorage/IObjectIterator.h | 4 +- .../StorageObjectStorageSource.cpp | 79 ++-------- .../tests/gtest_rendezvous_hashing.cpp | 5 - .../01271_show_privileges.reference | 3 - ...117_parser_system_query_variants.reference | 3 - .../04117_parser_system_query_variants.sql | 3 - 27 files changed, 49 insertions(+), 418 deletions(-) delete mode 100644 docs/en/sql-reference/table-functions/icebergCluster.md diff --git a/docs/en/sql-reference/table-functions/icebergCluster.md b/docs/en/sql-reference/table-functions/icebergCluster.md deleted file mode 100644 index 312e5b30ec0b..000000000000 --- a/docs/en/sql-reference/table-functions/icebergCluster.md +++ /dev/null @@ -1,136 +0,0 @@ ---- -description: 'An extension to the iceberg table function which allows processing files - from Apache Iceberg in parallel from many nodes in a specified cluster.' -sidebar_label: 'icebergCluster' -sidebar_position: 91 -slug: /sql-reference/table-functions/icebergCluster -title: 'icebergCluster' -doc_type: 'reference' ---- - -This is an extension to the [iceberg](/sql-reference/table-functions/iceberg.md) table function. - -Allows processing files from Apache [Iceberg](https://iceberg.apache.org/) in parallel from many nodes in a specified cluster. On initiator it creates a connection to all nodes in the cluster and dispatches each file dynamically. On the worker node it asks the initiator about the next task to process and processes it. This is repeated until all tasks are finished. - -## Deletion vectors on cluster reads {#deletion-vectors-cluster} - -Iceberg v3 [deletion vectors](https://iceberg.apache.org/spec/#deletion-vectors) are loaded on the **initiator** while it distributes tasks: for each data file the initiator reads the Puffin blob, validates it, materializes deleted row positions, and attaches the bitmap to the task sent to workers. Workers apply that bitmap when reading Parquet; they do not fetch or parse the Puffin file again for that path. See [Processing of tables with deleted rows](/sql-reference/table-functions/iceberg.md#deleted-rows) for format limits and caching. - -On wide tables with many deletion vectors, initiator-side decode and per-task bitmap serialization can become a bottleneck even when Parquet reads are well parallelized across the cluster. - -## Syntax {#syntax} - -```sql -icebergS3Cluster(cluster_name, url [, NOSIGN | access_key_id, secret_access_key, [session_token]] [,format] [,compression_method] [,extra_credentials]) -icebergS3Cluster(cluster_name, named_collection[, option=value [,..]]) - -icebergAzureCluster(cluster_name, connection_string|storage_account_url, container_name, blobpath, [,account_name], [,account_key] [,format] [,compression_method]) -icebergAzureCluster(cluster_name, named_collection[, option=value [,..]]) - -icebergHDFSCluster(cluster_name, path_to_table, [,format] [,compression_method]) -icebergHDFSCluster(cluster_name, named_collection[, option=value [,..]]) -``` - -## Arguments {#arguments} - -- `cluster_name` — Name of a cluster that is used to build a set of addresses and connection parameters to remote and local servers. -- Description of all other arguments coincides with description of arguments in equivalent [iceberg](/sql-reference/table-functions/iceberg.md) table function. -- An optional `extra_credentials` parameter can be used to pass a `role_arn` for role-based access in ClickHouse Cloud. See [Secure S3](/cloud/data-sources/secure-s3) for configuration steps. - -**Returned value** - -A table with the specified structure for reading data from cluster in the specified Iceberg table. - -**Examples** - -```sql -SELECT * FROM icebergS3Cluster('cluster_simple', 'http://test.s3.amazonaws.com/clickhouse-bucket/test_table', 'test', 'test') -``` - -## Virtual Columns {#virtual-columns} - -- `_path` — Path to the file. Type: `LowCardinality(String)`. -- `_file` — Name of the file. Type: `LowCardinality(String)`. -- `_size` — Size of the file in bytes. Type: `Nullable(UInt64)`. If the file size is unknown, the value is `NULL`. -- `_time` — Last modified time of the file. Type: `Nullable(DateTime)`. If the time is unknown, the value is `NULL`. -- `_etag` — The etag of the file. Type: `LowCardinality(String)`. If the etag is unknown, the value is `NULL`. - -## Altinity Antalya branch - -### `icebergLocalCluster` table function - -Only in the Altinity Antalya branch, `icebergLocalCluster` designed to make distributed cluster queries when Iceberg data is stored on shared network storage mounted with a local path. The path must be identical on all replicas. - -```sql -icebergLocalCluster(cluster_name, path_to_table, [,format] [,compression_method]) -``` - -### Specify storage type in function arguments - -Only in the Altinity Antalya branch, the `icebergCluster` table function supports all storage backends. The storage backend can be specified using the named argument `storage_type`. Valid values include `s3`, `azure`, `hdfs`, and `local`. - -```sql -icebergCluster(storage_type='s3', cluster_name, url [, NOSIGN | access_key_id, secret_access_key, [session_token]] [,format] [,compression_method]) - -icebergCluster(storage_type='azure', cluster_name, connection_string|storage_account_url, container_name, blobpath, [,account_name], [,account_key] [,format] [,compression_method]) - -icebergCluster(storage_type='hdfs', cluster_name, path_to_table, [,format] [,compression_method]) - -icebergCluster(storage_type='local', cluster_name, path_to_table, [,format] [,compression_method]) -``` - -### Specify storage type in a named collection - -Only in the Altinity Antalya branch, `storage_type` can be part of a named collection. - -```xml - - - - http://test.s3.amazonaws.com/clickhouse-bucket/ - test - test - auto - auto - s3 - - - -``` - -```sql -icebergCluster(iceberg_conf[, option=value [,..]]) -``` - -The default value for `storage_type` is `s3`. - -### `object_storage_cluster` setting. - -Only in the Altinity Antalya branch, an alternative syntax for `icebergCluster` table function is available. This allows the `iceberg` function to be used with the non-empty `object_storage_cluster` setting, specifying a cluster name. This enables distributed queries over Iceberg table across a ClickHouse cluster. - -```sql -icebergS3(url [, NOSIGN | access_key_id, secret_access_key, [session_token]] [,format] [,compression_method]) SETTINGS object_storage_cluster='cluster_name' - -icebergAzure(connection_string|storage_account_url, container_name, blobpath, [,account_name], [,account_key] [,format] [,compression_method]) SETTINGS object_storage_cluster='cluster_name' - -icebergHDFS(path_to_table, [,format] [,compression_method]) SETTINGS object_storage_cluster='cluster_name' - -icebergLocal(path_to_table, [,format] [,compression_method]) SETTINGS object_storage_cluster='cluster_name' - -icebergS3(option=value [,..]) SETTINGS object_storage_cluster='cluster_name' - -iceberg(storage_type='s3', url [, NOSIGN | access_key_id, secret_access_key, [session_token]] [,format] [,compression_method]) SETTINGS object_storage_cluster='cluster_name' - -iceberg(storage_type='azure', connection_string|storage_account_url, container_name, blobpath, [,account_name], [,account_key] [,format] [,compression_method]) SETTINGS object_storage_cluster='cluster_name' - -iceberg(storage_type='hdfs', path_to_table, [,format] [,compression_method]) SETTINGS object_storage_cluster='cluster_name' - -iceberg(storage_type='local', path_to_table, [,format] [,compression_method]) SETTINGS object_storage_cluster='cluster_name' - -iceberg(iceberg_conf[, option=value [,..]]) SETTINGS object_storage_cluster='cluster_name' -``` - -**See Also** - -- [Iceberg engine](/engines/table-engines/integrations/iceberg.md) -- [Iceberg table function](/sql-reference/table-functions/iceberg.md) diff --git a/docs/reference/functions/table-functions/icebergCluster.mdx b/docs/reference/functions/table-functions/icebergCluster.mdx index d1e6837943b1..72be387d8ce4 100644 --- a/docs/reference/functions/table-functions/icebergCluster.mdx +++ b/docs/reference/functions/table-functions/icebergCluster.mdx @@ -12,6 +12,12 @@ This is an extension to the [iceberg](/reference/functions/table-functions/icebe Allows processing files from Apache [Iceberg](https://iceberg.apache.org/) in parallel from many nodes in a specified cluster. On initiator it creates a connection to all nodes in the cluster and dispatches each file dynamically. On the worker node it asks the initiator about the next task to process and processes it. This is repeated until all tasks are finished. +## Deletion vectors on cluster reads {#deletion-vectors-cluster} + +Iceberg v3 [deletion vectors](https://iceberg.apache.org/spec/#deletion-vectors) are loaded on the **initiator** while it distributes tasks: for each data file the initiator reads the Puffin blob, validates it, materializes deleted row positions, and attaches the bitmap to the task sent to workers. Workers apply that bitmap when reading Parquet; they do not fetch or parse the Puffin file again for that path. See [Processing of tables with deleted rows](/sql-reference/table-functions/iceberg.md#deleted-rows) for format limits and caching. + +On wide tables with many deletion vectors, initiator-side decode and per-task bitmap serialization can become a bottleneck even when Parquet reads are well parallelized across the cluster. + ## Syntax {#syntax} ```sql diff --git a/docs/reference/statements/system.mdx b/docs/reference/statements/system.mdx index 3a99b8b5b007..e2f4fde13d7c 100644 --- a/docs/reference/statements/system.mdx +++ b/docs/reference/statements/system.mdx @@ -134,18 +134,16 @@ Clears the per-URL Confluent Schema Registry caches used by the `AvroConfluent` Clears the parquet metadata cache. -<<<<<<< HEAD:docs/reference/statements/system.mdx ## SYSTEM CLEAR|DROP PAIMON METADATA CACHE {#drop-paimon-metadata-cache} Clears the in-memory cache of parsed Paimon metadata files (manifest lists and manifests). ## SYSTEM CLEAR|DROP POINT IN POLYGON CACHE {#drop-point-in-polygon-cache} Clears the cache of preprocessed constant polygons used by the function [`pointInPolygon`](/reference/functions/regular-functions/geo/coordinates#pointinpolygon). The configured size limit (the `point_in_polygon_cache_size` server setting) is left unchanged, so the cache keeps accepting entries afterwards. To disable the cache instead, set `point_in_polygon_cache_size` to `0`. -======= + ## SYSTEM DROP PUFFIN FILES CACHE {#drop-puffin-files-cache} Clears the Puffin files cache used for parsed Iceberg puffin file content such as deletion vectors. ->>>>>>> 4b7cecaa3cf (Merge pull request #2183 from Altinity/feature/antalya-26.6/iceberg-puffin-deletion-vectors-read-2):docs/en/sql-reference/statements/system.md ## SYSTEM CLEAR|DROP TEXT INDEX CACHES {#drop-text-index-caches} diff --git a/src/Access/Common/AccessType.h b/src/Access/Common/AccessType.h index 94abce275135..56748e427c75 100644 --- a/src/Access/Common/AccessType.h +++ b/src/Access/Common/AccessType.h @@ -329,11 +329,8 @@ enum class AccessType : uint8_t M(SYSTEM_DROP_PAIMON_METADATA_CACHE, "SYSTEM CLEAR PAIMON_METADATA_CACHE, SYSTEM DROP PAIMON_METADATA_CACHE", GLOBAL, SYSTEM_DROP_CACHE) \ M(SYSTEM_DROP_AVRO_SCHEMA_CACHE, "SYSTEM CLEAR AVRO SCHEMA CACHE, SYSTEM DROP AVRO SCHEMA CACHE, DROP AVRO SCHEMA CACHE", GLOBAL, SYSTEM_DROP_CACHE) \ M(SYSTEM_DROP_PARQUET_METADATA_CACHE, "SYSTEM DROP PARQUET_METADATA_CACHE", GLOBAL, SYSTEM_DROP_CACHE) \ -<<<<<<< HEAD M(SYSTEM_DROP_POINT_IN_POLYGON_CACHE, "SYSTEM CLEAR POINT IN POLYGON CACHE, SYSTEM DROP POINT IN POLYGON CACHE, DROP POINT IN POLYGON CACHE", GLOBAL, SYSTEM_DROP_CACHE) \ -======= M(SYSTEM_DROP_PUFFIN_FILES_CACHE, "SYSTEM DROP PUFFIN_FILES_CACHE", GLOBAL, SYSTEM_DROP_CACHE) \ ->>>>>>> 4b7cecaa3cf (Merge pull request #2183 from Altinity/feature/antalya-26.6/iceberg-puffin-deletion-vectors-read-2) M(SYSTEM_PREWARM_PRIMARY_INDEX_CACHE, "SYSTEM PREWARM PRIMARY INDEX, PREWARM PRIMARY INDEX CACHE, PREWARM PRIMARY INDEX", GLOBAL, SYSTEM_DROP_CACHE) \ M(SYSTEM_DROP_PRIMARY_INDEX_CACHE, "SYSTEM CLEAR PRIMARY INDEX CACHE, SYSTEM DROP PRIMARY INDEX, DROP PRIMARY INDEX CACHE, DROP PRIMARY INDEX", GLOBAL, SYSTEM_DROP_CACHE) \ M(SYSTEM_DROP_UNCOMPRESSED_CACHE, "SYSTEM CLEAR UNCOMPRESSED CACHE, SYSTEM DROP UNCOMPRESSED, DROP UNCOMPRESSED CACHE, DROP UNCOMPRESSED", GLOBAL, SYSTEM_DROP_CACHE) \ diff --git a/src/AggregateFunctions/AggregateFunctionGroupBitmapData.h b/src/AggregateFunctions/AggregateFunctionGroupBitmapData.h index 805cd32392f5..42f6294de6da 100644 --- a/src/AggregateFunctions/AggregateFunctionGroupBitmapData.h +++ b/src/AggregateFunctions/AggregateFunctionGroupBitmapData.h @@ -626,9 +626,6 @@ class RoaringBitmapWithSmallSet : private boost::noncopyable } /** -<<<<<<< HEAD - * Return new set of the smallest `limit` values (as UnsignedT) which are no less than `range_start`. -======= * Count set bits in `[range_start, range_end)` without allocating a result bitmap. * Used by need-only-count DV filtering to avoid an O(N) dense Filter over file rows. * Implemented via roaring `rank` so repeated per-row-group queries stay O(containers), @@ -674,9 +671,7 @@ class RoaringBitmapWithSmallSet : private boost::noncopyable } /** - * Return new set of the smallest `limit` values in set which is no less than `range_start`. - * It's used in subset and currently only support UInt32 ->>>>>>> 4b7cecaa3cf (Merge pull request #2183 from Altinity/feature/antalya-26.6/iceberg-puffin-deletion-vectors-read-2) + * Return new set of the smallest `limit` values (as UnsignedT) which are no less than `range_start`. */ UInt64 rb_limit(UInt64 range_start, UInt64 limit, RoaringBitmapWithSmallSet & r1) const /// NOLINT { diff --git a/src/Common/ProfileEvents.cpp b/src/Common/ProfileEvents.cpp index 6cf52db6b2a5..3b38488c5f79 100644 --- a/src/Common/ProfileEvents.cpp +++ b/src/Common/ProfileEvents.cpp @@ -120,18 +120,15 @@ M(IcebergMetadataReturnedObjectInfos, "Total number of returned object infos from iceberg iterator.", ValueType::Number) \ M(IcebergMinMaxNonPrunedDeleteFiles, "Total number of accepted data files-position delete file pairs by minmax analysis from pairs suitable by partitioning and sequence number.", ValueType::Number) \ M(IcebergMinMaxPrunedDeleteFiles, "Total number of accepted data files-position delete file pairs by minmax analysis from pairs suitable by partitioning and sequence number.", ValueType::Number) \ -<<<<<<< HEAD M(OneLakeAccessTokenRequests, "Number of access token requests issued by the OneLake catalog via the Entra ID refresh token grant.", ValueType::Number) \ M(OneLakeAccessTokenRequestFailures, "Number of failed access token requests issued by the OneLake catalog via the Entra ID refresh token grant (including expired or revoked refresh tokens).", ValueType::Number) \ M(OneLakeAccessTokenRequestMicroseconds, "Total time spent requesting access tokens via the Entra ID refresh token grant in the OneLake catalog.", ValueType::Microseconds) \ M(OneLakeAccessTokenExpirations, "Number of times the cached OneLake access token was found expired and renewed transparently.", ValueType::Number) \ -======= M(PuffinFilesRead, "Number of Puffin files read (footer or deletion vector blob).", ValueType::Number) \ M(PuffinFileReadMicroseconds, "Total time spent reading Puffin files.", ValueType::Microseconds) \ M(PuffinFilesCacheHits, "Number of times parsed Puffin file content has been found in the cache.", ValueType::Number) \ M(PuffinFilesCacheMisses, "Number of times parsed Puffin file content has not been found in the cache and had to be read from disk.", ValueType::Number) \ M(PuffinFilesCacheWeightLost, "Approximate number of bytes evicted from the Puffin files cache.", ValueType::Number) \ ->>>>>>> 4b7cecaa3cf (Merge pull request #2183 from Altinity/feature/antalya-26.6/iceberg-puffin-deletion-vectors-read-2) M(VectorSimilarityIndexCacheHits, "Number of times an index granule has been found in the vector index cache.", ValueType::Number) \ M(VectorSimilarityIndexCacheMisses, "Number of times an index granule has not been found in the vector index cache and had to be read from disk.", ValueType::Number) \ M(VectorSimilarityIndexCacheWeightLost, "Approximate number of bytes evicted from the vector index cache.", ValueType::Number) \ diff --git a/src/Core/Settings.cpp b/src/Core/Settings.cpp index f51912d4d104..2b4e4af8426e 100644 --- a/src/Core/Settings.cpp +++ b/src/Core/Settings.cpp @@ -6030,7 +6030,6 @@ Minimum time of delay between 2 background compaction operations. DECLARE(Seconds, iceberg_compaction_data_cleanup, 60 * 60 * 3, R"( The time after which the data will be deleted. )", 0) \ -<<<<<<< HEAD DECLARE(UInt64, iceberg_compaction_commit_batch_size, 100, R"( Number of merged data files that background Iceberg compaction accumulates before publishing them in a new snapshot. @@ -6038,7 +6037,7 @@ Compaction results are published in any case once there are no candidates left t how long already merged files stay unpublished while compaction keeps finding new work. `0` restores the old behaviour of publishing only when compaction runs out of candidates - a state that is never reached while the table keeps receiving new data files, so every merged output is then written to object storage and never referenced by a snapshot. -======= +)", 0) \ DECLARE(Bool, use_puffin_files_cache, true, R"( If turned on, Iceberg reads may utilize the Puffin files cache for parsed puffin file content such as deletion vectors. @@ -6046,7 +6045,6 @@ Possible values: - 0 - Disabled - 1 - Enabled ->>>>>>> 4b7cecaa3cf (Merge pull request #2183 from Altinity/feature/antalya-26.6/iceberg-puffin-deletion-vectors-read-2) )", 0) \ DECLARE(Bool, use_query_cache, false, R"( If turned on, `SELECT` queries may utilize the [query cache](/concepts/features/performance/caches/query-cache). Parameters [enable_reads_from_query_cache](#enable_reads_from_query_cache) diff --git a/src/Core/SettingsChangesHistory.cpp b/src/Core/SettingsChangesHistory.cpp index 24a230c7a01d..f9cc37b180e2 100644 --- a/src/Core/SettingsChangesHistory.cpp +++ b/src/Core/SettingsChangesHistory.cpp @@ -43,6 +43,7 @@ const VersionToSettingsChangesMap & getSettingsChangesHistory() /// Note: please check if the key already exists to prevent duplicate entries. addSettingsChanges(settings_changes_history, "26.8.1.20001.altinityantalya", { + {"use_puffin_files_cache", false, true, "Enables cache of parsed Puffin file content such as deletion vectors."}, }); addSettingsChanges(settings_changes_history, "26.8", { @@ -215,9 +216,6 @@ const VersionToSettingsChangesMap & getSettingsChangesHistory() {"show_remote_databases_in_system_tables", true, true, "New setting to control whether `MySQL` and `PostgreSQL` databases are shown in `system.tables`, `system.columns` and `system.completions`."}, {"use_constant_folding_in_index_analysis", false, false, "New setting to fold partition-level constants into the filter predicate per part during MergeTree index analysis, improving pruning for filters whose branches depend on partition values."}, {"join_runtime_filter_size_from_hash_table_stats", false, true, "Use hash table size statistics collected from previous executions to size the JOIN runtime filter. When disabled, fall back to the fixed `join_runtime_bloom_filter_bytes`."}, - addSettingsChanges(settings_changes_history, "26.6.2.20001.altinityantalya", - { - {"use_puffin_files_cache", false, true, "Enables cache of parsed Puffin file content such as deletion vectors."}, }); addSettingsChanges(settings_changes_history, "26.6", diff --git a/src/Interpreters/InterpreterSystemQuery.cpp b/src/Interpreters/InterpreterSystemQuery.cpp index 8cac9a6d7f7a..89f8e0409994 100644 --- a/src/Interpreters/InterpreterSystemQuery.cpp +++ b/src/Interpreters/InterpreterSystemQuery.cpp @@ -504,15 +504,13 @@ BlockIO InterpreterSystemQuery::execute() #else throw Exception(ErrorCodes::SUPPORT_IS_DISABLED, "The server was compiled without the support for Parquet"); #endif -<<<<<<< HEAD case Type::CLEAR_POINT_IN_POLYGON_CACHE: getContext()->checkAccess(AccessType::SYSTEM_DROP_POINT_IN_POLYGON_CACHE); clearPointInPolygonCache(); -======= + break; case Type::CLEAR_PUFFIN_FILES_CACHE: getContext()->checkAccess(AccessType::SYSTEM_DROP_PUFFIN_FILES_CACHE); system_context->clearPuffinFilesCache(); ->>>>>>> 4b7cecaa3cf (Merge pull request #2183 from Altinity/feature/antalya-26.6/iceberg-puffin-deletion-vectors-read-2) break; case Type::CLEAR_PRIMARY_INDEX_CACHE: getContext()->checkAccess(AccessType::SYSTEM_DROP_PRIMARY_INDEX_CACHE); @@ -2808,15 +2806,14 @@ AccessRightsElements InterpreterSystemQuery::getRequiredAccessForDDLOnCluster() required_access.emplace_back(AccessType::SYSTEM_DROP_AVRO_SCHEMA_CACHE); break; case Type::CLEAR_PARQUET_METADATA_CACHE: -<<<<<<< HEAD required_access.emplace_back(AccessType::SYSTEM_DROP_PARQUET_METADATA_CACHE); break; case Type::CLEAR_POINT_IN_POLYGON_CACHE: required_access.emplace_back(AccessType::SYSTEM_DROP_POINT_IN_POLYGON_CACHE); break; -======= case Type::CLEAR_PUFFIN_FILES_CACHE: ->>>>>>> 4b7cecaa3cf (Merge pull request #2183 from Altinity/feature/antalya-26.6/iceberg-puffin-deletion-vectors-read-2) + required_access.emplace_back(AccessType::SYSTEM_DROP_PUFFIN_FILES_CACHE); + break; case Type::CLEAR_PRIMARY_INDEX_CACHE: required_access.emplace_back(AccessType::SYSTEM_DROP_PRIMARY_INDEX_CACHE); break; diff --git a/src/Parsers/ASTSystemQuery.cpp b/src/Parsers/ASTSystemQuery.cpp index 3da44cdf3f6b..673a6b071ddb 100644 --- a/src/Parsers/ASTSystemQuery.cpp +++ b/src/Parsers/ASTSystemQuery.cpp @@ -635,11 +635,7 @@ void ASTSystemQuery::formatImpl(WriteBuffer & ostr, const FormatSettings & setti case Type::CLEAR_PARQUET_METADATA_CACHE: case Type::CLEAR_POINT_IN_POLYGON_CACHE: case Type::CLEAR_AVRO_SCHEMA_CACHE: -<<<<<<< HEAD -======= case Type::CLEAR_PUFFIN_FILES_CACHE: - case Type::DROP_OBJECT_STORAGE_LIST_OBJECTS_CACHE: ->>>>>>> 4b7cecaa3cf (Merge pull request #2183 from Altinity/feature/antalya-26.6/iceberg-puffin-deletion-vectors-read-2) case Type::RESET_COVERAGE: case Type::RESTART_REPLICAS: case Type::JEMALLOC_PURGE: diff --git a/src/Parsers/ASTSystemQuery.h b/src/Parsers/ASTSystemQuery.h index 65ab87adf720..3831eefb861e 100644 --- a/src/Parsers/ASTSystemQuery.h +++ b/src/Parsers/ASTSystemQuery.h @@ -49,11 +49,8 @@ class ASTSystemQuery : public IAST, public ASTQueryWithOnCluster CLEAR_ICEBERG_METADATA_CACHE, CLEAR_PAIMON_METADATA_CACHE, CLEAR_PARQUET_METADATA_CACHE, -<<<<<<< HEAD CLEAR_POINT_IN_POLYGON_CACHE, -======= CLEAR_PUFFIN_FILES_CACHE, ->>>>>>> 4b7cecaa3cf (Merge pull request #2183 from Altinity/feature/antalya-26.6/iceberg-puffin-deletion-vectors-read-2) CLEAR_FILESYSTEM_CACHE, CLEAR_DISTRIBUTED_CACHE, CLEAR_DISK_METADATA_CACHE, diff --git a/src/Parsers/ParserSystemQuery.cpp b/src/Parsers/ParserSystemQuery.cpp index 6393b38d4e94..4ac8beaf354b 100644 --- a/src/Parsers/ParserSystemQuery.cpp +++ b/src/Parsers/ParserSystemQuery.cpp @@ -322,12 +322,9 @@ bool ParserSystemQuery::parseImpl(IParser::Pos & pos, ASTPtr & node, Expected & {"DROP ICEBERG METADATA CACHE", Type::CLEAR_ICEBERG_METADATA_CACHE}, {"DROP PAIMON METADATA CACHE", Type::CLEAR_PAIMON_METADATA_CACHE}, {"DROP PARQUET METADATA CACHE", Type::CLEAR_PARQUET_METADATA_CACHE}, -<<<<<<< HEAD {"DROP POINT IN POLYGON CACHE", Type::CLEAR_POINT_IN_POLYGON_CACHE}, -======= {"DROP PUFFIN FILES CACHE", Type::CLEAR_PUFFIN_FILES_CACHE}, {"DROP PUFFIN_FILES_CACHE", Type::CLEAR_PUFFIN_FILES_CACHE}, ->>>>>>> 4b7cecaa3cf (Merge pull request #2183 from Altinity/feature/antalya-26.6/iceberg-puffin-deletion-vectors-read-2) {"DROP FILESYSTEM CACHE", Type::CLEAR_FILESYSTEM_CACHE}, {"DROP DISTRIBUTED CACHE", Type::CLEAR_DISTRIBUTED_CACHE}, {"DROP DISK METADATA CACHE", Type::CLEAR_DISK_METADATA_CACHE}, diff --git a/src/Processors/Formats/Impl/Parquet/Reader.cpp b/src/Processors/Formats/Impl/Parquet/Reader.cpp index 8461292e6891..4c5e176286da 100644 --- a/src/Processors/Formats/Impl/Parquet/Reader.cpp +++ b/src/Processors/Formats/Impl/Parquet/Reader.cpp @@ -13,11 +13,8 @@ #include #include #include -<<<<<<< HEAD #include -======= #include ->>>>>>> 4b7cecaa3cf (Merge pull request #2183 from Altinity/feature/antalya-26.6/iceberg-puffin-deletion-vectors-read-2) #include #include #include @@ -364,7 +361,6 @@ void Reader::getHyperrectangleForRowGroup(const parq::RowGroup * meta, Hyperrect } } -<<<<<<< HEAD bool Reader::spatialBboxStatsHaveNoNulls(const parq::RowGroup & meta, size_t spatial_key_condition_idx) const { for (size_t bbox_pc_idx : spatial_key_condition_bbox_col_indices.at(spatial_key_condition_idx)) @@ -376,7 +372,8 @@ bool Reader::spatialBboxStatsHaveNoNulls(const parq::RowGroup & meta, size_t spa return false; } return true; -======= +} + std::vector buildRowGroupGlobalOffsets(const parq::FileMetaData & file_metadata) { if (file_metadata.num_rows < 0) @@ -411,7 +408,6 @@ std::vector buildRowGroupGlobalOffsets(const parq::FileMetaData & file_m /// mismatches would break previously readable files. return global_offsets; ->>>>>>> 4b7cecaa3cf (Merge pull request #2183 from Altinity/feature/antalya-26.6/iceberg-puffin-deletion-vectors-read-2) } void Reader::prefilterAndInitRowGroups(const std::optional> & row_groups_to_read) @@ -747,23 +743,18 @@ void Reader::prefilterAndInitRowGroups(const std::optionalcolumns.size() != total_primitive_columns_in_file) throw Exception(ErrorCodes::INCORRECT_DATA, "Row group {} has unexpected number of columns: {} != {}", row_group_idx, meta->columns.size(), total_primitive_columns_in_file); -<<<<<<< HEAD - total_rows += size_t(meta->num_rows); // before potentially skipping the row group - /// Lazy materialization: skip row groups that contain none of the requested rows. std::pair requested_rows_slice {0, 0}; if (rows_to_read) { - size_t group_start_row = total_rows - size_t(meta->num_rows); + size_t group_start_row = global_offsets[row_group_idx]; const auto * begin_it = std::lower_bound(rows_to_read->begin(), rows_to_read->end(), group_start_row); - const auto * end_it = std::lower_bound(begin_it, rows_to_read->end(), total_rows); + const auto * end_it = std::lower_bound(begin_it, rows_to_read->end(), global_offsets[row_group_idx + 1]); if (begin_it == end_it) continue; requested_rows_slice = {size_t(begin_it - rows_to_read->begin()), size_t(end_it - rows_to_read->begin())}; } -======= ->>>>>>> 4b7cecaa3cf (Merge pull request #2183 from Altinity/feature/antalya-26.6/iceberg-puffin-deletion-vectors-read-2) Hyperrectangle hyperrectangle(extended_sample_block.columns(), Range::createWholeUniverse()); if ((options.format.parquet.filter_push_down && format_filter_info->key_condition) || !spatial_key_conditions.empty()) @@ -839,9 +830,9 @@ void Reader::prefilterAndInitRowGroups(const std::optionalempty() && rows_to_read->back() >= total_rows) + if (rows_to_read && !rows_to_read->empty() && rows_to_read->back() >= global_offsets.back()) throw Exception(ErrorCodes::LOGICAL_ERROR, - "Requested to read row {} of a parquet file that has only {} rows", rows_to_read->back(), total_rows); + "Requested to read row {} of a parquet file that has only {} rows", rows_to_read->back(), global_offsets.back()); if (row_groups.empty()) return; // all row groups were skipped diff --git a/src/Storages/ObjectStorage/DataLakes/Common/AvroForIcebergDeserializer.cpp b/src/Storages/ObjectStorage/DataLakes/Common/AvroForIcebergDeserializer.cpp index fd912f27c012..9d3445adb9b3 100644 --- a/src/Storages/ObjectStorage/DataLakes/Common/AvroForIcebergDeserializer.cpp +++ b/src/Storages/ObjectStorage/DataLakes/Common/AvroForIcebergDeserializer.cpp @@ -17,11 +17,7 @@ #include #include #include -<<<<<<< HEAD -======= #include -#include ->>>>>>> 4b7cecaa3cf (Merge pull request #2183 from Altinity/feature/antalya-26.6/iceberg-puffin-deletion-vectors-read-2) namespace DB::ErrorCodes { diff --git a/src/Storages/ObjectStorage/DataLakes/Iceberg/IcebergDataObjectInfo.h b/src/Storages/ObjectStorage/DataLakes/Iceberg/IcebergDataObjectInfo.h index f8da4f7da7f2..c5062f74ecc5 100644 --- a/src/Storages/ObjectStorage/DataLakes/Iceberg/IcebergDataObjectInfo.h +++ b/src/Storages/ObjectStorage/DataLakes/Iceberg/IcebergDataObjectInfo.h @@ -98,6 +98,8 @@ struct IcebergDataObjectInfo : public ObjectInfo, std::enable_shared_from_this clone() const override { return std::make_shared(*this); } + Iceberg::IcebergObjectSerializableInfo info; }; diff --git a/src/Storages/ObjectStorage/DataLakes/Iceberg/IcebergIterator.cpp b/src/Storages/ObjectStorage/DataLakes/Iceberg/IcebergIterator.cpp index 569899006f85..256117981885 100644 --- a/src/Storages/ObjectStorage/DataLakes/Iceberg/IcebergIterator.cpp +++ b/src/Storages/ObjectStorage/DataLakes/Iceberg/IcebergIterator.cpp @@ -316,39 +316,10 @@ IcebergIterator::IcebergIterator( , blocking_queue(100) , callback(std::move(callback_)) { -<<<<<<< HEAD /// Decoding any manifest reads settings from the context, so a missing one is fatal either way. if (!local_context) throw DB::Exception(DB::ErrorCodes::LOGICAL_ERROR, "Context is required to construct IcebergIterator"); -======= - auto delete_file = deletes_iterator.next(); - while (delete_file.has_value()) - { - if (delete_file.value()->parsed_entry->equality_ids.has_value()) - { - equality_deletes_files.emplace_back(std::move(delete_file.value())); - } - else if (delete_file.value()->parsed_entry->isDeletionVector()) - { - deletion_vector_files.emplace_back(std::move(delete_file.value())); - } - else - { - parquet_position_deletes_files.emplace_back(std::move(delete_file.value())); - } - delete_file = deletes_iterator.next(); - } - LOG_DEBUG( - logger, - "Taken {} deletion vector files, {} parquet position delete files and {} equality delete files in iceberg iterator", - deletion_vector_files.size(), - parquet_position_deletes_files.size(), - equality_deletes_files.size()); - std::sort(equality_deletes_files.begin(), equality_deletes_files.end()); - std::sort(deletion_vector_files.begin(), deletion_vector_files.end()); - std::sort(parquet_position_deletes_files.begin(), parquet_position_deletes_files.end()); ->>>>>>> 4b7cecaa3cf (Merge pull request #2183 from Altinity/feature/antalya-26.6/iceberg-puffin-deletion-vectors-read-2) producer_task = std::make_unique( [this, thread_group = CurrentThread::getGroup()]() { @@ -479,8 +450,10 @@ void IcebergIterator::decodeDeleteManifests() { if (delete_file->parsed_entry->equality_ids.has_value()) equality_deletes_files.emplace_back(std::move(delete_file)); + else if (delete_file->parsed_entry->isDeletionVector()) + deletion_vector_files.emplace_back(std::move(delete_file)); else - position_deletes_files.emplace_back(std::move(delete_file)); + parquet_position_deletes_files.emplace_back(std::move(delete_file)); } } chassert(in_flight.empty()); @@ -488,9 +461,15 @@ void IcebergIterator::decodeDeleteManifests() /// Sort objects by common_partition_specification, partition_key_value and added_sequence_number. /// This is needed to efficiently match delete and data manifests in defineDeletesSpan(). - LOG_DEBUG(logger, "Taken {} position deletes file and {} equality deletes files in iceberg iterator", position_deletes_files.size(), equality_deletes_files.size()); + LOG_DEBUG( + logger, + "Taken {} deletion vector files, {} parquet position delete files and {} equality delete files in iceberg iterator", + deletion_vector_files.size(), + parquet_position_deletes_files.size(), + equality_deletes_files.size()); std::sort(equality_deletes_files.begin(), equality_deletes_files.end()); - std::sort(position_deletes_files.begin(), position_deletes_files.end()); + std::sort(deletion_vector_files.begin(), deletion_vector_files.end()); + std::sort(parquet_position_deletes_files.begin(), parquet_position_deletes_files.end()); } ObjectInfoPtr IcebergIterator::next(size_t) @@ -500,33 +479,18 @@ ObjectInfoPtr IcebergIterator::next(size_t) Iceberg::ProcessedManifestFileEntryPtr manifest_file_entry; if (blocking_queue.pop(manifest_file_entry)) { -<<<<<<< HEAD IcebergDataObjectInfoPtr object_info = std::make_shared( manifest_file_entry, persistent_components.path_resolver.resolve(manifest_file_entry->parsed_entry->file_path_key), table_state_snapshot->schema_id, Iceberg::getIdentityPartitionColumnValues(*manifest_file_entry, *persistent_components.schema_processor)); - for (const auto & position_delete : - defineDeletesSpan(manifest_file_entry, position_deletes_files, /* is_equality_delete */ false, logger)) -======= - const auto & raw_metadata_path = manifest_file_entry->parsed_entry->file_path_key.serialize(); - auto [storage_to_use, resolved_key] = resolveObjectStorageForPath( - persistent_components.table_location, raw_metadata_path, - object_storage, *secondary_storages, local_context, - persistent_components.path_resolver); - - IcebergDataObjectInfoPtr object_info = std::make_shared( - manifest_file_entry, raw_metadata_path, table_state_snapshot->schema_id, storage_to_use, resolved_key); - - object_info->info.requires_external_storage = (storage_to_use != object_storage); const auto & data_file_path = object_info->info.data_object_file_path_key; bool has_deletion_vector = false; for (const auto & deletion_vector : defineDeletesSpan(manifest_file_entry, deletion_vector_files, /* is_equality_delete */ false, logger)) ->>>>>>> 4b7cecaa3cf (Merge pull request #2183 from Altinity/feature/antalya-26.6/iceberg-puffin-deletion-vectors-read-2) { const auto & referenced_data_file = deletion_vector->parsed_entry->lower_reference_data_file_path; if (!referenced_data_file.has_value() || referenced_data_file.value() != data_file_path) @@ -544,19 +508,6 @@ ObjectInfoPtr IcebergIterator::next(size_t) if (!object_info->info.record_count.has_value()) { -<<<<<<< HEAD - ProfileEvents::increment(ProfileEvents::IcebergMinMaxNonPrunedDeleteFiles); - LOG_TEST( - logger, - "Processing position delete file `{}` for data file `{}` with reference data file bounds: " - "(lower bound: `{}`, upper bound: `{}`)", - position_delete->parsed_entry->file_path_key, - data_file_path, - lower.has_value() ? lower->serialize() : "[no lower bound]", - upper.has_value() ? upper->serialize() : "[no upper bound]"); - object_info->addPositionDeleteObject( - position_delete, persistent_components.path_resolver.resolve(position_delete->parsed_entry->file_path_key)); -======= throw Exception( ErrorCodes::ICEBERG_SPECIFICATION_VIOLATION, "Data file '{}' is missing record_count required to validate deletion vector positions", @@ -564,18 +515,13 @@ ObjectInfoPtr IcebergIterator::next(size_t) } const auto & parsed_entry = deletion_vector->parsed_entry; - const auto puffin_metadata_path = parsed_entry->file_path_key.serialize(); - auto [puffin_storage, puffin_key] = resolveObjectStorageForPath( - persistent_components.table_location, puffin_metadata_path, - object_storage, *secondary_storages, local_context, - persistent_components.path_resolver); /// For icebergCluster, next() runs on the initiator's task-distribution path: DV /// I/O / CRC / roaring materialization happen here, then excluded_rows is sent on /// the wire per task. Workers apply the bitmap and do not re-read the Puffin blob. auto excluded_rows = Iceberg::loadDeletionVector( - puffin_storage, - puffin_key, + object_storage, + persistent_components.path_resolver.resolve(parsed_entry->file_path_key), parsed_entry->content_offset.value(), parsed_entry->content_size_in_bytes.value(), data_file_path, @@ -632,9 +578,8 @@ ObjectInfoPtr IcebergIterator::next(size_t) lower.has_value() ? lower->serialize() : "[no lower bound]", upper.has_value() ? upper->serialize() : "[no upper bound]"); object_info->addPositionDeleteObject( - position_delete, position_delete->parsed_entry->file_path_key.serialize()); + position_delete, persistent_components.path_resolver.resolve(position_delete->parsed_entry->file_path_key)); } ->>>>>>> 4b7cecaa3cf (Merge pull request #2183 from Altinity/feature/antalya-26.6/iceberg-puffin-deletion-vectors-read-2) } } diff --git a/src/Storages/ObjectStorage/DataLakes/Iceberg/IcebergIterator.h b/src/Storages/ObjectStorage/DataLakes/Iceberg/IcebergIterator.h index caebff5b865d..78f6d222c37d 100644 --- a/src/Storages/ObjectStorage/DataLakes/Iceberg/IcebergIterator.h +++ b/src/Storages/ObjectStorage/DataLakes/Iceberg/IcebergIterator.h @@ -110,14 +110,10 @@ class IcebergIterator : public IObjectIterator ConcurrentBoundedQueue blocking_queue; std::unique_ptr producer_task; IDataLakeMetadata::FileProgressCallback callback; -<<<<<<< HEAD /// Filled once under `deletes_mutex` and never mutated afterwards, so `next` may read them /// unguarded once it has gone through `ensureDeletesReady`. - std::vector position_deletes_files; -======= std::vector deletion_vector_files; std::vector parquet_position_deletes_files; ->>>>>>> 4b7cecaa3cf (Merge pull request #2183 from Altinity/feature/antalya-26.6/iceberg-puffin-deletion-vectors-read-2) std::vector equality_deletes_files; std::mutex deletes_mutex; bool deletes_ready TSA_GUARDED_BY(deletes_mutex) = false; diff --git a/src/Storages/ObjectStorage/DataLakes/Iceberg/IcebergMetadata.cpp b/src/Storages/ObjectStorage/DataLakes/Iceberg/IcebergMetadata.cpp index e3c1f6f6a12f..35a45bde43fd 100644 --- a/src/Storages/ObjectStorage/DataLakes/Iceberg/IcebergMetadata.cpp +++ b/src/Storages/ObjectStorage/DataLakes/Iceberg/IcebergMetadata.cpp @@ -10,11 +10,7 @@ #include #include #include -<<<<<<< HEAD -======= -#include #include ->>>>>>> 4b7cecaa3cf (Merge pull request #2183 from Altinity/feature/antalya-26.6/iceberg-puffin-deletion-vectors-read-2) #include #include #include @@ -1247,7 +1243,6 @@ std::optional IcebergMetadata::totalRows(ContextPtr local_context) const && *actual_data_snapshot->total_equality_delete_rows > 0) return {}; -<<<<<<< HEAD /// Row counts stored in the metadata layers above the manifest files are not used as /// data sources, because writers derive them instead of measuring them against the data: /// - the snapshot summary's `total-records` is maintained incrementally (parent total @@ -1263,22 +1258,10 @@ std::optional IcebergMetadata::totalRows(ContextPtr local_context) const /// required field in every format version, so summing it over the live data files is /// exact, at the cost of opening the manifest files (served from the Iceberg metadata /// cache on repeated queries). -======= - /// Do not trust snapshot-summary `total-records` for the answer. Those totals are optional, - /// writer-maintained incrementally, and a single bad commit can poison every later snapshot. - /// Sum required per-data-file `record_count` from manifests when there are no live delete - /// files; otherwise fail closed to a real scan. Summary is compared only for a mismatch warning. - /// - /// Manifest-list `added_rows_count`/`existing_rows_count` are not used (some writers stamp them - /// from snapshot summary and can report 0 after compaction). Subtracting live position-delete / - /// deletion-vector `record_count` from data-file totals is also unsafe (duplicates, stale - /// references, DV supersession of parquet position deletes). ->>>>>>> 4b7cecaa3cf (Merge pull request #2183 from Altinity/feature/antalya-26.6/iceberg-puffin-deletion-vectors-read-2) UInt64 result = 0; for (const auto & manifest_list_entry : actual_data_snapshot->manifest_list_entries) { auto manifest_file_ptr = getManifestFileEntriesHandle( -<<<<<<< HEAD object_storage, persistent_components, local_context, log, manifest_list_entry, actual_table_state_snapshot.schema_id); /// Live delete files make an exact metadata-only count impossible: @@ -1297,19 +1280,6 @@ std::optional IcebergMetadata::totalRows(ContextPtr local_context) const /// nullopt means a corrupted manifest file with a negative `record_count`: fail /// closed to a real scan instead of returning a wrong count. auto manifest_rows = manifest_file_ptr.getRowsCountInAllFilesExcludingDeleted(FileContentType::DATA); - if (!manifest_rows.has_value()) - return {}; - result += *manifest_rows; -======= - object_storage, persistent_components, local_context, log, manifest_list_entry, actual_table_state_snapshot.schema_id, *secondary_storages); - - if (!manifest_file_ptr.getFilesWithoutDeleted(FileContentType::EQUALITY_DELETE).empty() - || !manifest_file_ptr.getFilesWithoutDeleted(FileContentType::POSITION_DELETE).empty()) - return {}; - /// nullopt means a negative / overflowing per-file `record_count`: fail closed to a - /// real scan instead of returning a wrong count. Do not use optional column - /// `value_counts` here — nested fields can report element counts larger than rows. - auto manifest_rows = manifest_file_ptr.getRowsCountInAllFilesExcludingDeleted(FileContentType::DATA); if (!manifest_rows.has_value()) return {}; /// Per-manifest sums are capped at Int64::max; still guard the cross-manifest total. @@ -1317,20 +1287,6 @@ std::optional IcebergMetadata::totalRows(ContextPtr local_context) const return {}; } - if (auto summary_total_rows = actual_data_snapshot->getTotalRows(); - summary_total_rows.has_value() && *summary_total_rows != result) - { - LOG_WARNING( - log, - "Iceberg snapshot summary of table {} claims {} total rows, but its manifest files describe {} rows. " - "The snapshot summary is inconsistent with the table data (possibly a corrupted commit in the table " - "history), using the row count from the manifest files", - persistent_components.table_location, - *summary_total_rows, - result); ->>>>>>> 4b7cecaa3cf (Merge pull request #2183 from Altinity/feature/antalya-26.6/iceberg-puffin-deletion-vectors-read-2) - } - const auto summary_total_rows = actual_data_snapshot->getTotalRows(); if (summary_total_rows.has_value() && *summary_total_rows != static_cast(result)) LOG_WARNING( diff --git a/src/Storages/ObjectStorage/DataLakes/Iceberg/ManifestFileIterator.cpp b/src/Storages/ObjectStorage/DataLakes/Iceberg/ManifestFileIterator.cpp index f50307841899..2f163e0f1df3 100644 --- a/src/Storages/ObjectStorage/DataLakes/Iceberg/ManifestFileIterator.cpp +++ b/src/Storages/ObjectStorage/DataLakes/Iceberg/ManifestFileIterator.cpp @@ -169,7 +169,6 @@ bool ManifestFileIterator::ManifestFileEntriesHandle::areAllDataFilesSortedBySor bool ManifestFileIterator::ManifestFileEntriesHandle::areAllDataFilesEligibleForLazyMaterialization(Int32 table_schema_id) const { -<<<<<<< HEAD /// Equality deletes force reading all physical columns of the data files they apply to /// (see IcebergMetadata::getInitialSchemaByPath), so the pruned main read is impossible. if (!equality_delete_files->empty()) @@ -192,7 +191,6 @@ bool ManifestFileIterator::ManifestFileEntriesHandle::areAllDataFilesEligibleFor std::optional ManifestFileIterator::ManifestFileEntriesHandle::getRowsCountInAllFilesExcludingDeleted(FileContentType content) const { - UInt64 result = 0; /// `record_count` is a required file-level field in all format versions, so the sum is /// exact: no fallback to optional per-column statistics is needed. The field is parsed /// as a raw Int64 though, so a corrupted manifest file may carry a negative value; it @@ -200,16 +198,7 @@ std::optional ManifestFileIterator::ManifestFileEntriesHandle::getRowsCo /// silently produce a wrong -- or, after the conversion to size_t, absurdly huge -- /// count) and rather than rejected (the count is only an optimization, a malformed /// value must not make the table unreadable). - for (const auto & file : getFilesWithoutDeleted(content)) - { - if (file->parsed_entry->record_count < 0) - return std::nullopt; - result += static_cast(file->parsed_entry->record_count); - } - return result; -======= return getRecordCountInAllFilesExcludingDeleted(getFilesWithoutDeleted(content)); ->>>>>>> 4b7cecaa3cf (Merge pull request #2183 from Altinity/feature/antalya-26.6/iceberg-puffin-deletion-vectors-read-2) } std::optional ManifestFileIterator::ManifestFileEntriesHandle::getBytesCountInAllDataFilesExcludingDeleted() const diff --git a/src/Storages/ObjectStorage/DataLakes/Iceberg/Mutations.cpp b/src/Storages/ObjectStorage/DataLakes/Iceberg/Mutations.cpp index b224c27e11f3..109f6a9affad 100644 --- a/src/Storages/ObjectStorage/DataLakes/Iceberg/Mutations.cpp +++ b/src/Storages/ObjectStorage/DataLakes/Iceberg/Mutations.cpp @@ -45,12 +45,8 @@ namespace DB::ErrorCodes extern const int BAD_ARGUMENTS; extern const int LOGICAL_ERROR; extern const int LIMIT_EXCEEDED; -<<<<<<< HEAD extern const int NOT_IMPLEMENTED; -======= extern const int SUPPORT_IS_DISABLED; -extern const int QUERY_WAS_CANCELLED; ->>>>>>> 4b7cecaa3cf (Merge pull request #2183 from Altinity/feature/antalya-26.6/iceberg-puffin-deletion-vectors-read-2) } namespace DB::DataLakeStorageSetting diff --git a/src/Storages/ObjectStorage/DataLakes/Iceberg/Snapshot.h b/src/Storages/ObjectStorage/DataLakes/Iceberg/Snapshot.h index c204a8215bf6..113bd4982f01 100644 --- a/src/Storages/ObjectStorage/DataLakes/Iceberg/Snapshot.h +++ b/src/Storages/ObjectStorage/DataLakes/Iceberg/Snapshot.h @@ -16,7 +16,6 @@ struct IcebergDataSnapshot DB::ManifestFileCacheKeys manifest_list_entries; Int64 snapshot_id; Int64 schema_id_on_snapshot_commit; -<<<<<<< HEAD /// Row-count hint from the snapshot summary (`total-records`). Only used to log a /// warning when it disagrees with the row count derived from the manifest files; never /// used as a data source, because the summary is maintained incrementally by writers @@ -24,19 +23,9 @@ struct IcebergDataSnapshot std::optional total_rows; std::optional total_bytes; std::optional total_position_delete_rows; -======= - /// From snapshot summary (`total-records`). Compared to the manifest-derived count for a - /// mismatch warning only — never used as the trivial COUNT answer. Summary totals are - /// maintained incrementally by writers and can be poisoned by a bad commit in table history. - std::optional total_rows; - std::optional total_bytes; - std::optional total_position_delete_rows; /// Rows in equality-delete files (snapshot summary). Not a count of deleted data rows; /// used only to fail closed early when present and > 0. std::optional total_equality_delete_rows; - std::optional partition_key; - std::optional sorting_key; ->>>>>>> 4b7cecaa3cf (Merge pull request #2183 from Altinity/feature/antalya-26.6/iceberg-puffin-deletion-vectors-read-2) std::optional getTotalRows() const { diff --git a/src/Storages/ObjectStorage/IObjectIterator.h b/src/Storages/ObjectStorage/IObjectIterator.h index aa0029552216..467474e9d8e3 100644 --- a/src/Storages/ObjectStorage/IObjectIterator.h +++ b/src/Storages/ObjectStorage/IObjectIterator.h @@ -55,15 +55,13 @@ struct ObjectInfo FileBucketInfoPtr file_bucket_info; -<<<<<<< HEAD /// Lazy materialization: if set, read only these rows of the file. /// Sorted absolute row indexes within the file, see FormatFilterInfo::rows_to_read. std::shared_ptr> rows_to_read; -======= + /// Polymorphic copy: preserves the dynamic type (e.g. `IcebergDataObjectInfo` with equality / /// position deletes, resolved storage, and metadata path) where a plain copy would slice it. virtual std::shared_ptr clone() const { return std::make_shared(*this); } ->>>>>>> 4b7cecaa3cf (Merge pull request #2183 from Altinity/feature/antalya-26.6/iceberg-puffin-deletion-vectors-read-2) String getIdentifier(bool include_file_bucket_info = true) const; String getIdentifierForPath(const String & path, bool include_file_bucket_info = true) const; diff --git a/src/Storages/ObjectStorage/StorageObjectStorageSource.cpp b/src/Storages/ObjectStorage/StorageObjectStorageSource.cpp index 0de0832349a5..420639170f2f 100644 --- a/src/Storages/ObjectStorage/StorageObjectStorageSource.cpp +++ b/src/Storages/ObjectStorage/StorageObjectStorageSource.cpp @@ -222,25 +222,19 @@ static void logIcebergFileStats(const ObjectInfoPtr & object_info, const LoggerP #endif } -<<<<<<< HEAD -/// Whether reading this object goes through row-level delete transformers (Iceberg -/// position/equality deletes, Delta Lake deletion vectors). The count-from-files cache -/// is keyed only by the file path and its modification time, but delete files change the -/// number of rows the file contributes WITHOUT touching the file itself, so both -/// directions are unsafe: a count cached before a delete resurfaces deleted rows, and a -/// count cached after it goes stale once the deletes are compacted away. Such files must -/// neither use nor populate the cache. -static bool hasAttachedDeletes(const ObjectInfo & object_info) +/// Count-from-files cache key is data-file identity only. Skip when row filtering can change +/// independently (DVs / selection vectors, Iceberg eq/pos deletes) or when the task is a bucket subset. +/// Cache must stay fail-closed even when need_only_count is allowed for position deletes / DVs: +/// the key is path + mtime only, and deletes change the contributed row count without touching the file. +static bool canUseCountFromFilesCache(const ObjectInfoPtr & object_info) { + if (hasNonEmptyExcludedRows(object_info->data_lake_metadata) || object_info->file_bucket_info) + return false; #if USE_AVRO - if (const auto * iceberg_object = dynamic_cast(&object_info)) - { - if (!iceberg_object->info.position_deletes_objects.empty() || !iceberg_object->info.equality_deletes_objects.empty()) - return true; - } + if (hasIcebergEqualityDeletes(object_info) || hasIcebergPositionDeletes(object_info)) + return false; #endif - return object_info.data_lake_metadata && object_info.data_lake_metadata->excluded_rows - && object_info.data_lake_metadata->excluded_rows->size() > 0; + return true; } static bool readsIdentityPartitionColumn( @@ -305,21 +299,6 @@ static std::optional buildIdentityPartitionColumnsDag( outputs.push_back(&dag.materializeNode(constant)); } return dag; -======= -/// Count-from-files cache key is data-file identity only. Skip when row filtering can change -/// independently (DVs / selection vectors, Iceberg eq/pos deletes) or when the task is a bucket subset. -/// Cache must stay fail-closed even when need_only_count is allowed for position deletes / DVs: -/// the key is path + mtime only, and deletes change the contributed row count without touching the file. -static bool canUseCountFromFilesCache(const ObjectInfoPtr & object_info) -{ - if (hasNonEmptyExcludedRows(object_info->data_lake_metadata) || object_info->file_bucket_info) - return false; -#if USE_AVRO - if (hasIcebergEqualityDeletes(object_info) || hasIcebergPositionDeletes(object_info)) - return false; -#endif - return true; ->>>>>>> 4b7cecaa3cf (Merge pull request #2183 from Altinity/feature/antalya-26.6/iceberg-puffin-deletion-vectors-read-2) } StorageObjectStorageSource::StorageObjectStorageSource( @@ -898,17 +877,10 @@ Chunk StorageObjectStorageSource::generate() /// Do not cache filtered cardinality: filter DAG, PREWHERE, and row policies all /// reduce rows seen by generate(), while the cache key is file identity only. if (reader.getInputFormat() && read_context->getSettingsRef()[Setting::use_cache_for_count_from_files] -<<<<<<< HEAD - && !format_filter_info->filter_actions_dag - && !hasAttachedDeletes(*reader.getObjectInfo()) + && format_filter_info && !format_filter_info->hasFilter() + && canUseCountFromFilesCache(reader.getObjectInfo()) && !reader.getObjectInfo()->rows_to_read) addNumRowsToCache(*reader.getObjectInfo(), total_rows_in_file); - -======= - && format_filter_info && !format_filter_info->hasFilter() - && canUseCountFromFilesCache(reader.getObjectInfo())) - addNumRowsToCache(reader.getObjectInfo(), total_rows_in_file); ->>>>>>> 4b7cecaa3cf (Merge pull request #2183 from Altinity/feature/antalya-26.6/iceberg-puffin-deletion-vectors-read-2) total_rows_in_file = 0; chassert(reader_future.valid()); @@ -1081,28 +1053,12 @@ StorageObjectStorageSource::ReaderHolder StorageObjectStorageSource::createReade return schema_cache->tryGetNumRows(cache_key, get_last_mod_time); }; - /// Row-level delete transformers need real row values: an equality-delete FilterTransform - /// evaluates its predicate against column values, but the count-only fast path - /// (`input_format->needOnlyCount()`) makes the format emit synthetic chunks filled with - /// default values, so the predicate would filter the wrong rows and count() would come - /// back wrong. Position deletes and deletion vectors filter by row index, which synthetic - /// chunks do preserve, but they would still build the huge synthetic chunks only to drop - /// rows from them, so the fast path is disabled for any attached deletes. This also - /// covers the count-from-cache shortcut below: a cached per-file row count is keyed only - /// by path + mtime, both untouched by delete files, so it must not be used either. - need_only_count = need_only_count && !hasAttachedDeletes(*object_info); - /// The count-from-cache shortcut builds a `ConstChunkGenerator` without opening the read buffer, so a /// requested `_headers` virtual column (the HTTP response headers of the data `GET`) would have to fall /// back to the metadata-probe headers (usually a `HEAD`), which can differ from the actual `GET` /// response. Skip the shortcut when `_headers` is requested so the real `GET` headers are used. const bool headers_requested = read_from_format_info.requested_virtual_columns.contains("_headers"); -<<<<<<< HEAD - std::optional num_rows_from_cache - = need_only_count && !headers_requested && context_->getSettingsRef()[Setting::use_cache_for_count_from_files] - ? try_get_num_rows_from_cache() : std::nullopt; -======= /// Equality-delete FilterTransform evaluates predicates against column values, but need_only_count /// emits default-filled chunks — so disable the fast path for equality deletes only. /// Position deletes and deletion vectors filter by row index (preserved on synthetic chunks); @@ -1115,12 +1071,11 @@ StorageObjectStorageSource::ReaderHolder StorageObjectStorageSource::createReade const bool effective_need_only_count = need_only_count; #endif - const bool can_use_count_cache = effective_need_only_count + const bool can_use_count_cache = effective_need_only_count && !headers_requested && context_->getSettingsRef()[Setting::use_cache_for_count_from_files] && canUseCountFromFilesCache(object_info); std::optional num_rows_from_cache = can_use_count_cache ? try_get_num_rows_from_cache() : std::nullopt; ->>>>>>> 4b7cecaa3cf (Merge pull request #2183 from Altinity/feature/antalya-26.6/iceberg-puffin-deletion-vectors-read-2) if (num_rows_from_cache) { @@ -1404,7 +1359,6 @@ StorageObjectStorageSource::ReaderHolder StorageObjectStorageSource::createReade builder.init(Pipe(input_format)); -<<<<<<< HEAD if (!identity_partition_columns.empty()) { if (auto dag = buildIdentityPartitionColumnsDag(builder.getHeader(), identity_partition_columns)) @@ -1417,18 +1371,11 @@ StorageObjectStorageSource::ReaderHolder StorageObjectStorageSource::createReade } } - configuration->addDeleteTransformers(object_info, builder, format_settings, parser_shared_resources, context_); - - if (object_info->data_lake_metadata - && object_info->data_lake_metadata->excluded_rows - && object_info->data_lake_metadata->excluded_rows->size() > 0) -======= /// Deletion vectors (and selection vectors) address absolute file row numbers via /// `ChunkInfoRowNumbers`. Iceberg equality deletes use a plain `FilterTransform` that /// shrinks the chunk without maintaining `applied_filter`, so DV must run first — /// otherwise later DV filtering maps dense post-equality indices to the wrong file rows. if (hasNonEmptyExcludedRows(object_info->data_lake_metadata)) ->>>>>>> 4b7cecaa3cf (Merge pull request #2183 from Altinity/feature/antalya-26.6/iceberg-puffin-deletion-vectors-read-2) { builder.addSimpleTransform([&](const SharedHeader & header) { diff --git a/src/Storages/ObjectStorage/tests/gtest_rendezvous_hashing.cpp b/src/Storages/ObjectStorage/tests/gtest_rendezvous_hashing.cpp index 65c6a02783cf..b10d1825048e 100644 --- a/src/Storages/ObjectStorage/tests/gtest_rendezvous_hashing.cpp +++ b/src/Storages/ObjectStorage/tests/gtest_rendezvous_hashing.cpp @@ -1,13 +1,8 @@ #include -<<<<<<< HEAD #include #include #include #include -#include -======= - ->>>>>>> 4b7cecaa3cf (Merge pull request #2183 from Altinity/feature/antalya-26.6/iceberg-puffin-deletion-vectors-read-2) #include #include diff --git a/tests/queries/0_stateless/01271_show_privileges.reference b/tests/queries/0_stateless/01271_show_privileges.reference index e874c73a0b39..20d91d0da6fd 100644 --- a/tests/queries/0_stateless/01271_show_privileges.reference +++ b/tests/queries/0_stateless/01271_show_privileges.reference @@ -135,11 +135,8 @@ SYSTEM DROP ICEBERG METADATA CACHE ['SYSTEM CLEAR ICEBERG_METADATA_CACHE','SYSTE SYSTEM DROP PAIMON METADATA CACHE ['SYSTEM CLEAR PAIMON_METADATA_CACHE','SYSTEM DROP PAIMON_METADATA_CACHE'] GLOBAL SYSTEM DROP CACHE SYSTEM DROP AVRO SCHEMA CACHE ['SYSTEM CLEAR AVRO SCHEMA CACHE','SYSTEM DROP AVRO SCHEMA CACHE','DROP AVRO SCHEMA CACHE'] GLOBAL SYSTEM DROP CACHE SYSTEM DROP PARQUET METADATA CACHE ['SYSTEM DROP PARQUET_METADATA_CACHE'] GLOBAL SYSTEM DROP CACHE -<<<<<<< HEAD SYSTEM DROP POINT IN POLYGON CACHE ['SYSTEM CLEAR POINT IN POLYGON CACHE','SYSTEM DROP POINT IN POLYGON CACHE','DROP POINT IN POLYGON CACHE'] GLOBAL SYSTEM DROP CACHE -======= SYSTEM DROP PUFFIN FILES CACHE ['SYSTEM DROP PUFFIN_FILES_CACHE'] GLOBAL SYSTEM DROP CACHE ->>>>>>> 4b7cecaa3cf (Merge pull request #2183 from Altinity/feature/antalya-26.6/iceberg-puffin-deletion-vectors-read-2) SYSTEM PREWARM PRIMARY INDEX CACHE ['SYSTEM PREWARM PRIMARY INDEX','PREWARM PRIMARY INDEX CACHE','PREWARM PRIMARY INDEX'] GLOBAL SYSTEM DROP CACHE SYSTEM DROP PRIMARY INDEX CACHE ['SYSTEM CLEAR PRIMARY INDEX CACHE','SYSTEM DROP PRIMARY INDEX','DROP PRIMARY INDEX CACHE','DROP PRIMARY INDEX'] GLOBAL SYSTEM DROP CACHE SYSTEM DROP UNCOMPRESSED CACHE ['SYSTEM CLEAR UNCOMPRESSED CACHE','SYSTEM DROP UNCOMPRESSED','DROP UNCOMPRESSED CACHE','DROP UNCOMPRESSED'] GLOBAL SYSTEM DROP CACHE diff --git a/tests/queries/0_stateless/04117_parser_system_query_variants.reference b/tests/queries/0_stateless/04117_parser_system_query_variants.reference index 7a4a135737ac..197276c6dac6 100644 --- a/tests/queries/0_stateless/04117_parser_system_query_variants.reference +++ b/tests/queries/0_stateless/04117_parser_system_query_variants.reference @@ -16,12 +16,9 @@ SYSTEM CLEAR ENCRYPTION HEADERS CACHE SYSTEM CLEAR COMPILED EXPRESSION CACHE SYSTEM CLEAR ICEBERG METADATA CACHE SYSTEM CLEAR PARQUET METADATA CACHE -<<<<<<< HEAD SYSTEM CLEAR POINT IN POLYGON CACHE -======= SYSTEM CLEAR PUFFIN FILES CACHE SYSTEM CLEAR PUFFIN FILES CACHE ->>>>>>> 4b7cecaa3cf (Merge pull request #2183 from Altinity/feature/antalya-26.6/iceberg-puffin-deletion-vectors-read-2) SYSTEM CLEAR FILESYSTEM CACHE SYSTEM CLEAR DISTRIBUTED CACHE SYSTEM CLEAR DISTRIBUTED CACHE CONNECTIONS ON CLUSTER cluster diff --git a/tests/queries/0_stateless/04117_parser_system_query_variants.sql b/tests/queries/0_stateless/04117_parser_system_query_variants.sql index 344cdcb4a900..9093c599264c 100644 --- a/tests/queries/0_stateless/04117_parser_system_query_variants.sql +++ b/tests/queries/0_stateless/04117_parser_system_query_variants.sql @@ -26,12 +26,9 @@ EXPLAIN SYNTAX SYSTEM DROP ENCRYPTION HEADERS CACHE; EXPLAIN SYNTAX SYSTEM DROP COMPILED EXPRESSION CACHE; EXPLAIN SYNTAX SYSTEM DROP ICEBERG METADATA CACHE; EXPLAIN SYNTAX SYSTEM DROP PARQUET METADATA CACHE; -<<<<<<< HEAD EXPLAIN SYNTAX SYSTEM DROP POINT IN POLYGON CACHE; -======= EXPLAIN SYNTAX SYSTEM DROP PUFFIN FILES CACHE; EXPLAIN SYNTAX SYSTEM DROP PUFFIN_FILES_CACHE; ->>>>>>> 4b7cecaa3cf (Merge pull request #2183 from Altinity/feature/antalya-26.6/iceberg-puffin-deletion-vectors-read-2) EXPLAIN SYNTAX SYSTEM DROP FILESYSTEM CACHE; EXPLAIN SYNTAX SYSTEM DROP DISTRIBUTED CACHE; EXPLAIN SYNTAX SYSTEM DROP DISTRIBUTED CACHE CONNECTIONS ON CLUSTER cluster; From 8c609f0e81a311713114fe064ba957f9dcae9a06 Mon Sep 17 00:00:00 2001 From: Andrey Zvonov <32552679+zvonand@users.noreply.github.com> Date: Fri, 4 Sep 2026 15:04:14 +0200 Subject: [PATCH 3/5] Cherry-pick of https://github.com/Altinity/ClickHouse/pull/2271 with unresolved conflict markers (resolution in next commit) --- Original cherry-pick message follows: Merge pull request #2271 from Altinity/iceberg-delta-bin-deletion-vectors Support for Databricks deletion vectors # Conflicts: # docs/en/sql-reference/table-functions/icebergCluster.md --- .../table-functions/icebergCluster.md | 136 +++++++++ .../table-engines/integrations/iceberg.mdx | 8 +- .../functions/table-functions/iceberg.mdx | 8 +- .../Common/AvroForIcebergDeserializer.cpp | 11 +- .../Iceberg/IcebergDeletionVector.cpp | 57 ++-- .../DataLakes/Iceberg/IcebergIterator.cpp | 5 +- .../DataLakes/Iceberg/ManifestFile.cpp | 2 +- .../DataLakes/Iceberg/ManifestFile.h | 13 +- .../DataLakes/PuffinDeletionVectorReader.cpp | 113 ++++++- .../DataLakes/PuffinDeletionVectorReader.h | 31 +- .../tests/gtest_delta_bin_deletion_vector.cpp | 186 ++++++++++++ .../gtest_puffin_dv_referenced_data_file.cpp | 44 +++ .../altinity_dv_puffin_repro_v3/.gitignore | 2 + .../altinity_dv_puffin_repro_v3/README.md | 13 + ...84-47b8-ae93-31b30e5d8359.gz.metadata.json | Bin 0 -> 655 bytes ...98-46cf-9c64-8d0cdf419c13.gz.metadata.json | Bin 0 -> 1001 bytes ...46-4a90-bf5a-80cf5e93be12.gz.metadata.json | Bin 0 -> 1167 bytes ...c1a95b-41b1-4c5c-a585-cd0480cc6739-m0.avro | Bin 0 -> 7998 bytes ...531858-556b-45f9-a989-a7ba0b0ae967-m0.avro | Bin 0 -> 8007 bytes ...-7fc1a95b-41b1-4c5c-a585-cd0480cc6739.avro | Bin 0 -> 4816 bytes ...-e5531858-556b-45f9-a989-a7ba0b0ae967.avro | Bin 0 -> 4895 bytes ...r_3d169d0e-b938-4854-8cca-d9d38949f1b0.bin | Bin 0 -> 252 bytes ...1-4bc7-b94d-2e5639a05e68.c000.zstd.parquet | Bin 0 -> 50617 bytes .../test_deletion_vectors.py | 281 ++++++++++++++++++ ...est_deletion_vectors_databricks_uniform.py | 180 +++++++++++ 25 files changed, 1045 insertions(+), 45 deletions(-) create mode 100644 docs/en/sql-reference/table-functions/icebergCluster.md create mode 100644 src/Storages/ObjectStorage/DataLakes/tests/gtest_delta_bin_deletion_vector.cpp create mode 100644 tests/integration/test_storage_iceberg_with_spark/data/altinity_dv_puffin_repro_v3/.gitignore create mode 100644 tests/integration/test_storage_iceberg_with_spark/data/altinity_dv_puffin_repro_v3/README.md create mode 100644 tests/integration/test_storage_iceberg_with_spark/data/altinity_dv_puffin_repro_v3/_iceberg/metadata/00000-102ad1cf-eb84-47b8-ae93-31b30e5d8359.gz.metadata.json create mode 100644 tests/integration/test_storage_iceberg_with_spark/data/altinity_dv_puffin_repro_v3/_iceberg/metadata/00001-364c19a9-7e98-46cf-9c64-8d0cdf419c13.gz.metadata.json create mode 100644 tests/integration/test_storage_iceberg_with_spark/data/altinity_dv_puffin_repro_v3/_iceberg/metadata/00002-8de40d94-9846-4a90-bf5a-80cf5e93be12.gz.metadata.json create mode 100644 tests/integration/test_storage_iceberg_with_spark/data/altinity_dv_puffin_repro_v3/_iceberg/metadata/7fc1a95b-41b1-4c5c-a585-cd0480cc6739-m0.avro create mode 100644 tests/integration/test_storage_iceberg_with_spark/data/altinity_dv_puffin_repro_v3/_iceberg/metadata/e5531858-556b-45f9-a989-a7ba0b0ae967-m0.avro create mode 100644 tests/integration/test_storage_iceberg_with_spark/data/altinity_dv_puffin_repro_v3/_iceberg/metadata/snap-4017675268503509610-1-7fc1a95b-41b1-4c5c-a585-cd0480cc6739.avro create mode 100644 tests/integration/test_storage_iceberg_with_spark/data/altinity_dv_puffin_repro_v3/_iceberg/metadata/snap-6006002030350834282-1-e5531858-556b-45f9-a989-a7ba0b0ae967.avro create mode 100644 tests/integration/test_storage_iceberg_with_spark/data/altinity_dv_puffin_repro_v3/deletion_vector_3d169d0e-b938-4854-8cca-d9d38949f1b0.bin create mode 100644 tests/integration/test_storage_iceberg_with_spark/data/altinity_dv_puffin_repro_v3/jk/part-00000-c0585bde-5e01-4bc7-b94d-2e5639a05e68.c000.zstd.parquet create mode 100644 tests/integration/test_storage_iceberg_with_spark/test_deletion_vectors_databricks_uniform.py diff --git a/docs/en/sql-reference/table-functions/icebergCluster.md b/docs/en/sql-reference/table-functions/icebergCluster.md new file mode 100644 index 000000000000..72577f140e71 --- /dev/null +++ b/docs/en/sql-reference/table-functions/icebergCluster.md @@ -0,0 +1,136 @@ +--- +description: 'An extension to the iceberg table function which allows processing files + from Apache Iceberg in parallel from many nodes in a specified cluster.' +sidebar_label: 'icebergCluster' +sidebar_position: 91 +slug: /sql-reference/table-functions/icebergCluster +title: 'icebergCluster' +doc_type: 'reference' +--- + +This is an extension to the [iceberg](/sql-reference/table-functions/iceberg.md) table function. + +Allows processing files from Apache [Iceberg](https://iceberg.apache.org/) in parallel from many nodes in a specified cluster. On initiator it creates a connection to all nodes in the cluster and dispatches each file dynamically. On the worker node it asks the initiator about the next task to process and processes it. This is repeated until all tasks are finished. + +## Deletion vectors on cluster reads {#deletion-vectors-cluster} + +Iceberg v3 [deletion vectors](https://iceberg.apache.org/spec/#deletion-vectors) are loaded on the **initiator** while it distributes tasks: for each data file the initiator reads the Puffin or Delta `.bin` object, validates the `deletion-vector-v1` envelope, materializes deleted row positions, and attaches the bitmap to the task sent to workers. Workers apply that bitmap when reading Parquet; they do not fetch or parse the deletion-vector file again for that path. See [Processing of tables with deleted rows](/sql-reference/table-functions/iceberg.md#deleted-rows) for format limits and caching. + +On wide tables with many deletion vectors, initiator-side decode and per-task bitmap serialization can become a bottleneck even when Parquet reads are well parallelized across the cluster. + +## Syntax {#syntax} + +```sql +icebergS3Cluster(cluster_name, url [, NOSIGN | access_key_id, secret_access_key, [session_token]] [,format] [,compression_method] [,extra_credentials]) +icebergS3Cluster(cluster_name, named_collection[, option=value [,..]]) + +icebergAzureCluster(cluster_name, connection_string|storage_account_url, container_name, blobpath, [,account_name], [,account_key] [,format] [,compression_method]) +icebergAzureCluster(cluster_name, named_collection[, option=value [,..]]) + +icebergHDFSCluster(cluster_name, path_to_table, [,format] [,compression_method]) +icebergHDFSCluster(cluster_name, named_collection[, option=value [,..]]) +``` + +## Arguments {#arguments} + +- `cluster_name` — Name of a cluster that is used to build a set of addresses and connection parameters to remote and local servers. +- Description of all other arguments coincides with description of arguments in equivalent [iceberg](/sql-reference/table-functions/iceberg.md) table function. +- An optional `extra_credentials` parameter can be used to pass a `role_arn` for role-based access in ClickHouse Cloud. See [Secure S3](/cloud/data-sources/secure-s3) for configuration steps. + +**Returned value** + +A table with the specified structure for reading data from cluster in the specified Iceberg table. + +**Examples** + +```sql +SELECT * FROM icebergS3Cluster('cluster_simple', 'http://test.s3.amazonaws.com/clickhouse-bucket/test_table', 'test', 'test') +``` + +## Virtual Columns {#virtual-columns} + +- `_path` — Path to the file. Type: `LowCardinality(String)`. +- `_file` — Name of the file. Type: `LowCardinality(String)`. +- `_size` — Size of the file in bytes. Type: `Nullable(UInt64)`. If the file size is unknown, the value is `NULL`. +- `_time` — Last modified time of the file. Type: `Nullable(DateTime)`. If the time is unknown, the value is `NULL`. +- `_etag` — The etag of the file. Type: `LowCardinality(String)`. If the etag is unknown, the value is `NULL`. + +## Altinity Antalya branch + +### `icebergLocalCluster` table function + +Only in the Altinity Antalya branch, `icebergLocalCluster` designed to make distributed cluster queries when Iceberg data is stored on shared network storage mounted with a local path. The path must be identical on all replicas. + +```sql +icebergLocalCluster(cluster_name, path_to_table, [,format] [,compression_method]) +``` + +### Specify storage type in function arguments + +Only in the Altinity Antalya branch, the `icebergCluster` table function supports all storage backends. The storage backend can be specified using the named argument `storage_type`. Valid values include `s3`, `azure`, `hdfs`, and `local`. + +```sql +icebergCluster(storage_type='s3', cluster_name, url [, NOSIGN | access_key_id, secret_access_key, [session_token]] [,format] [,compression_method]) + +icebergCluster(storage_type='azure', cluster_name, connection_string|storage_account_url, container_name, blobpath, [,account_name], [,account_key] [,format] [,compression_method]) + +icebergCluster(storage_type='hdfs', cluster_name, path_to_table, [,format] [,compression_method]) + +icebergCluster(storage_type='local', cluster_name, path_to_table, [,format] [,compression_method]) +``` + +### Specify storage type in a named collection + +Only in the Altinity Antalya branch, `storage_type` can be part of a named collection. + +```xml + + + + http://test.s3.amazonaws.com/clickhouse-bucket/ + test + test + auto + auto + s3 + + + +``` + +```sql +icebergCluster(iceberg_conf[, option=value [,..]]) +``` + +The default value for `storage_type` is `s3`. + +### `object_storage_cluster` setting. + +Only in the Altinity Antalya branch, an alternative syntax for `icebergCluster` table function is available. This allows the `iceberg` function to be used with the non-empty `object_storage_cluster` setting, specifying a cluster name. This enables distributed queries over Iceberg table across a ClickHouse cluster. + +```sql +icebergS3(url [, NOSIGN | access_key_id, secret_access_key, [session_token]] [,format] [,compression_method]) SETTINGS object_storage_cluster='cluster_name' + +icebergAzure(connection_string|storage_account_url, container_name, blobpath, [,account_name], [,account_key] [,format] [,compression_method]) SETTINGS object_storage_cluster='cluster_name' + +icebergHDFS(path_to_table, [,format] [,compression_method]) SETTINGS object_storage_cluster='cluster_name' + +icebergLocal(path_to_table, [,format] [,compression_method]) SETTINGS object_storage_cluster='cluster_name' + +icebergS3(option=value [,..]) SETTINGS object_storage_cluster='cluster_name' + +iceberg(storage_type='s3', url [, NOSIGN | access_key_id, secret_access_key, [session_token]] [,format] [,compression_method]) SETTINGS object_storage_cluster='cluster_name' + +iceberg(storage_type='azure', connection_string|storage_account_url, container_name, blobpath, [,account_name], [,account_key] [,format] [,compression_method]) SETTINGS object_storage_cluster='cluster_name' + +iceberg(storage_type='hdfs', path_to_table, [,format] [,compression_method]) SETTINGS object_storage_cluster='cluster_name' + +iceberg(storage_type='local', path_to_table, [,format] [,compression_method]) SETTINGS object_storage_cluster='cluster_name' + +iceberg(iceberg_conf[, option=value [,..]]) SETTINGS object_storage_cluster='cluster_name' +``` + +**See Also** + +- [Iceberg engine](/engines/table-engines/integrations/iceberg.md) +- [Iceberg table function](/sql-reference/table-functions/iceberg.md) diff --git a/docs/reference/engines/table-engines/integrations/iceberg.mdx b/docs/reference/engines/table-engines/integrations/iceberg.mdx index 5472415d2a99..74ebe2fee57b 100644 --- a/docs/reference/engines/table-engines/integrations/iceberg.mdx +++ b/docs/reference/engines/table-engines/integrations/iceberg.mdx @@ -149,19 +149,19 @@ ClickHouse supports reading Iceberg tables that use the following deletion metho - [Position deletes](https://iceberg.apache.org/spec/#position-delete-files) - [Equality deletes](https://iceberg.apache.org/spec/#equality-delete-files) (supported from version 25.8+) -- [Deletion vectors](https://iceberg.apache.org/spec/#deletion-vectors) stored in Puffin files (Iceberg v3, read-only) +- [Deletion vectors](https://iceberg.apache.org/spec/#deletion-vectors) (Iceberg v3, read-only), stored either in Puffin files or in Delta-style `deletion_vector_*.bin` files using the same `deletion-vector-v1` envelope at the manifest `content_offset` / `content_size_in_bytes` The following limitations apply to deletion vectors: -- Only `deletion-vector-v1` Puffin blobs are supported +- Only `deletion-vector-v1` blobs are supported (Puffin container or Delta `.bin` slice) - Data files must be in Parquet format - Column-scoped deletion vectors (user column ids in puffin `fields`) are not supported. Writers may set `fields` to `[]` or to the Iceberg reserved `_pos` id (`2147483645`) for file-scoped deletion vectors. - Writing deletion vectors is not supported - `DELETE` / `UPDATE` mutations on Iceberg format version 3+ tables are rejected (writers must not add position-delete files) -Parsed deletion vectors can be cached in memory when `use_puffin_files_cache` is enabled and the puffin file has a non-empty `etag`. Empty deletion vectors are cached as well, so repeated reads do not re-fetch the puffin file. Parsed footers for coalesced multi-DV Puffin files are memoized with that cache (same identity: storage, path, `etag`) so slices share one footer parse; the memo shares `puffin_files_cache_size` / max-entry limits and is dropped when the cache is disabled (`puffin_files_cache_size=0`) or cleared. The cache can be cleared with `SYSTEM DROP PUFFIN FILES CACHE`. +Parsed deletion vectors can be cached in memory when `use_puffin_files_cache` is enabled and the deletion-vector object has a non-empty `etag`. Empty deletion vectors are cached as well, so repeated reads do not re-fetch the object. Parsed footers for coalesced multi-DV Puffin files are memoized with that cache (same identity: storage, path, `etag`) so slices share one footer parse; Delta `.bin` files have no Puffin footer and skip that memo. The memo shares `puffin_files_cache_size` / max-entry limits and is dropped when the cache is disabled (`puffin_files_cache_size=0`) or cleared. The cache can be cleared with `SYSTEM DROP PUFFIN FILES CACHE`. -For [`icebergCluster`](/sql-reference/table-functions/icebergCluster.md) (and `object_storage_cluster`), the initiator loads and materializes each data file's deletion vector while distributing tasks, then sends the resulting row bitmap to workers with the task. Workers apply the bitmap; they do not re-read the Puffin blob for that path. On wide v3 tables this can make the initiator a serialization point for deletion-vector I/O and decode. +For [`icebergCluster`](/sql-reference/table-functions/icebergCluster.md) (and `object_storage_cluster`), the initiator loads and materializes each data file's deletion vector while distributing tasks, then sends the resulting row bitmap to workers with the task. Workers apply the bitmap; they do not re-read the Puffin or `.bin` object for that path. On wide v3 tables this can make the initiator a serialization point for deletion-vector I/O and decode. ### Basic usage {#basic-usage} ```sql diff --git a/docs/reference/functions/table-functions/iceberg.mdx b/docs/reference/functions/table-functions/iceberg.mdx index 07fef0d65761..83384c1cedbe 100644 --- a/docs/reference/functions/table-functions/iceberg.mdx +++ b/docs/reference/functions/table-functions/iceberg.mdx @@ -126,19 +126,19 @@ ClickHouse supports reading Iceberg tables that use the following deletion metho - [Position deletes](https://iceberg.apache.org/spec/#position-delete-files) - [Equality deletes](https://iceberg.apache.org/spec/#equality-delete-files) (supported from version 25.8+) -- [Deletion vectors](https://iceberg.apache.org/spec/#deletion-vectors) stored in Puffin files (Iceberg v3, read-only) +- [Deletion vectors](https://iceberg.apache.org/spec/#deletion-vectors) (Iceberg v3, read-only), stored either in Puffin files or in Delta-style `deletion_vector_*.bin` files using the same `deletion-vector-v1` envelope at the manifest `content_offset` / `content_size_in_bytes` The following limitations apply to deletion vectors: -- Only `deletion-vector-v1` Puffin blobs are supported +- Only `deletion-vector-v1` blobs are supported (Puffin container or Delta `.bin` slice) - Data files must be in Parquet format - Column-scoped deletion vectors (user column ids in puffin `fields`) are not supported. Writers may set `fields` to `[]` or to the Iceberg reserved `_pos` id (`2147483645`) for file-scoped deletion vectors. - Writing deletion vectors is not supported - `DELETE` / `UPDATE` mutations on Iceberg format version 3+ tables are rejected (writers must not add position-delete files) -Parsed deletion vectors can be cached in memory when `use_puffin_files_cache` is enabled and the puffin file has a non-empty `etag`. Empty deletion vectors are cached as well, so repeated reads do not re-fetch the puffin file. Parsed footers for coalesced multi-DV Puffin files are memoized with that cache (same identity: storage, path, `etag`) so slices share one footer parse; the memo shares `puffin_files_cache_size` / max-entry limits and is dropped when the cache is disabled (`puffin_files_cache_size=0`) or cleared. The cache can be cleared with `SYSTEM DROP PUFFIN FILES CACHE`. +Parsed deletion vectors can be cached in memory when `use_puffin_files_cache` is enabled and the deletion-vector object has a non-empty `etag`. Empty deletion vectors are cached as well, so repeated reads do not re-fetch the object. Parsed footers for coalesced multi-DV Puffin files are memoized with that cache (same identity: storage, path, `etag`) so slices share one footer parse; Delta `.bin` files have no Puffin footer and skip that memo. The memo shares `puffin_files_cache_size` / max-entry limits and is dropped when the cache is disabled (`puffin_files_cache_size=0`) or cleared. The cache can be cleared with `SYSTEM DROP PUFFIN FILES CACHE`. -For [`icebergCluster`](/sql-reference/table-functions/icebergCluster.md) (and `object_storage_cluster`), the initiator loads and materializes each data file's deletion vector while distributing tasks, then sends the resulting row bitmap to workers with the task. Workers apply the bitmap; they do not re-read the Puffin blob for that path. On wide v3 tables this can make the initiator a serialization point for deletion-vector I/O and decode. +For [`icebergCluster`](/sql-reference/table-functions/icebergCluster.md) (and `object_storage_cluster`), the initiator loads and materializes each data file's deletion vector while distributing tasks, then sends the resulting row bitmap to workers with the task. Workers apply the bitmap; they do not re-read the Puffin or `.bin` object for that path. On wide v3 tables this can make the initiator a serialization point for deletion-vector I/O and decode. ### Basic usage {#basic-usage} diff --git a/src/Storages/ObjectStorage/DataLakes/Common/AvroForIcebergDeserializer.cpp b/src/Storages/ObjectStorage/DataLakes/Common/AvroForIcebergDeserializer.cpp index 9d3445adb9b3..fe983f88a825 100644 --- a/src/Storages/ObjectStorage/DataLakes/Common/AvroForIcebergDeserializer.cpp +++ b/src/Storages/ObjectStorage/DataLakes/Common/AvroForIcebergDeserializer.cpp @@ -322,6 +322,7 @@ ParsedManifestFileEntryPtr AvroForIcebergDeserializer::createParsedManifestFileE case FileContentType::POSITION_DELETE: { /// reference_file_path can be absent in schema for some reason, though it is present in specification: https://iceberg.apache.org/spec/#manifests const bool is_puffin = Poco::toLower(file_format) == "puffin"; + const bool has_dv_offsets = content_offset.has_value() && content_size_in_bytes.has_value(); std::optional lower_reference_data_file_path; std::optional upper_reference_data_file_path; bool bounds_set_by_referenced_data_file = false; @@ -337,9 +338,9 @@ ParsedManifestFileEntryPtr AvroForIcebergDeserializer::createParsedManifestFileE bounds_set_by_referenced_data_file = true; } } - /// Parquet position deletes may fall back to file-path column bounds. Puffin deletion + /// Parquet position deletes may fall back to file-path column bounds. Deletion /// vectors must use the dedicated referenced_data_file field only. - if (!bounds_set_by_referenced_data_file && !is_puffin) + if (!bounds_set_by_referenced_data_file && !is_puffin && !has_dv_offsets) { if (auto it = value_for_bounds.find(IcebergPositionDeleteTransform::data_file_path_column_field_id); it != value_for_bounds.end()) @@ -354,13 +355,17 @@ ParsedManifestFileEntryPtr AvroForIcebergDeserializer::createParsedManifestFileE if (is_puffin) { - if (!content_offset.has_value() || !content_size_in_bytes.has_value()) + if (!has_dv_offsets) { throw Exception( DB::ErrorCodes::ICEBERG_SPECIFICATION_VIOLATION, "Puffin deletion vector entry in manifest file '{}' is missing content_offset or content_size_in_bytes", manifest_file_path); } + } + + if (is_puffin || has_dv_offsets) + { requireDirectReferencedDataFileForPuffinDeletionVector( bounds_set_by_referenced_data_file, lower_reference_data_file_path, manifest_file_path); } diff --git a/src/Storages/ObjectStorage/DataLakes/Iceberg/IcebergDeletionVector.cpp b/src/Storages/ObjectStorage/DataLakes/Iceberg/IcebergDeletionVector.cpp index 91d40f6df66f..7f77e3f40e3c 100644 --- a/src/Storages/ObjectStorage/DataLakes/Iceberg/IcebergDeletionVector.cpp +++ b/src/Storages/ObjectStorage/DataLakes/Iceberg/IcebergDeletionVector.cpp @@ -18,6 +18,7 @@ #include #include +#include namespace DB { @@ -131,7 +132,9 @@ DataLakeObjectMetadata::ExcludedRowsPtr loadDeletionVectorUncached( ContextPtr context, LoggerPtr log, bool disable_filesystem_cache, - FooterBlobsPtr preloaded_footer) + FooterBlobsPtr preloaded_footer, + PuffinFilesCache * footer_cache = nullptr, + const std::optional & footer_key = {}) { RelativePathWithMetadata puffin_object{puffin_path}; auto read_settings = context->getReadSettings(); @@ -148,16 +151,35 @@ DataLakeObjectMetadata::ExcludedRowsPtr loadDeletionVectorUncached( if (!file_size) throw Exception(ErrorCodes::BAD_ARGUMENTS, "Cannot determine Puffin file size for '{}'", puffin_path); - FooterBlobsPtr footer_owner = preloaded_footer; - if (!footer_owner) - footer_owner = std::make_shared>(readPuffinFooterBlobsFromSeekable(*seekable, *file_size)); + const auto container = detectIcebergDeletionVectorContainer( + *seekable, *file_size, content_offset, content_size_in_bytes, puffin_path); - bindDeletionVectorBlob( - *footer_owner, - content_offset, - content_size_in_bytes, - expected_data_file.serialize(), - expected_cardinality); + if (container == IcebergDeletionVectorContainer::Puffin) + { + FooterBlobsPtr footer_owner = preloaded_footer; + if (!footer_owner) + { + if (footer_cache && footer_key.has_value()) + { + footer_owner = footer_cache->getOrSetFooter(*footer_key, [&]() + { + return readFooterBlobs(object_storage, puffin_path, context, log, /*disable_filesystem_cache=*/ true); + }); + } + else + { + footer_owner = std::make_shared>( + readPuffinFooterBlobsFromSeekable(*seekable, *file_size)); + } + } + + bindDeletionVectorBlob( + *footer_owner, + content_offset, + content_size_in_bytes, + expected_data_file.serialize(), + expected_cardinality); + } auto deleted_positions = readDeletionVectorFromPuffin( *read_buffer, content_offset, content_size_in_bytes, expected_cardinality); @@ -173,8 +195,9 @@ DataLakeObjectMetadata::ExcludedRowsPtr loadDeletionVectorUncached( LOG_DEBUG( log, - "Loaded deletion vector from puffin file '{}' for data file '{}': {} deleted rows", + "Loaded deletion vector from file '{}' ({}) for data file '{}': {} deleted rows", puffin_path, + container == IcebergDeletionVectorContainer::Puffin ? "Puffin" : "Delta .bin", expected_data_file.serialize(), deleted_positions.size()); @@ -367,14 +390,10 @@ DataLakeObjectMetadata::ExcludedRowsPtr loadDeletionVector( } /// Footer is keyed by file identity only, so N DV slices in one coalesced Puffin share one parse. - /// Resolve the footer only on a deletion-vector cache miss (nested memo lookup). + /// Resolve the footer only on a Puffin deletion-vector cache miss (nested memo lookup). + /// Delta `.bin` objects have no Puffin footer; detection inside the uncached loader skips memo. return cache->getOrSetDeletionVector(*cache_key, [&]() { - auto footer = cache->getOrSetFooter(*footer_key, [&]() - { - return readFooterBlobs(object_storage, puffin_path, context, log, /*disable_filesystem_cache=*/ true); - }); - return loadDeletionVectorUncached( object_storage, puffin_path, @@ -386,7 +405,9 @@ DataLakeObjectMetadata::ExcludedRowsPtr loadDeletionVector( context, log, true, - footer); + nullptr, + cache.get(), + footer_key); }); } diff --git a/src/Storages/ObjectStorage/DataLakes/Iceberg/IcebergIterator.cpp b/src/Storages/ObjectStorage/DataLakes/Iceberg/IcebergIterator.cpp index 256117981885..baa575fdc03a 100644 --- a/src/Storages/ObjectStorage/DataLakes/Iceberg/IcebergIterator.cpp +++ b/src/Storages/ObjectStorage/DataLakes/Iceberg/IcebergIterator.cpp @@ -518,7 +518,8 @@ ObjectInfoPtr IcebergIterator::next(size_t) /// For icebergCluster, next() runs on the initiator's task-distribution path: DV /// I/O / CRC / roaring materialization happen here, then excluded_rows is sent on - /// the wire per task. Workers apply the bitmap and do not re-read the Puffin blob. + /// the wire per task. Workers apply the bitmap and do not re-read the deletion-vector + /// object (Puffin or Delta `.bin`). auto excluded_rows = Iceberg::loadDeletionVector( object_storage, persistent_components.path_resolver.resolve(parsed_entry->file_path_key), @@ -537,7 +538,7 @@ ObjectInfoPtr IcebergIterator::next(size_t) has_deletion_vector = true; LOG_DEBUG( logger, - "Attached deletion vector from puffin file `{}` to data file `{}`", + "Attached deletion vector from file `{}` to data file `{}`", parsed_entry->file_path_key, data_file_path); } diff --git a/src/Storages/ObjectStorage/DataLakes/Iceberg/ManifestFile.cpp b/src/Storages/ObjectStorage/DataLakes/Iceberg/ManifestFile.cpp index bffd91213282..201fdcd3a7eb 100644 --- a/src/Storages/ObjectStorage/DataLakes/Iceberg/ManifestFile.cpp +++ b/src/Storages/ObjectStorage/DataLakes/Iceberg/ManifestFile.cpp @@ -92,7 +92,7 @@ void requireDirectReferencedDataFileForPuffinDeletionVector( { throw DB::Exception( DB::ErrorCodes::ICEBERG_SPECIFICATION_VIOLATION, - "Puffin deletion vector entry in manifest file '{}' is missing referenced_data_file", + "Deletion vector entry in manifest file '{}' is missing referenced_data_file", manifest_file_path); } } diff --git a/src/Storages/ObjectStorage/DataLakes/Iceberg/ManifestFile.h b/src/Storages/ObjectStorage/DataLakes/Iceberg/ManifestFile.h index 96b4897818e6..2d9f2389bf90 100644 --- a/src/Storages/ObjectStorage/DataLakes/Iceberg/ManifestFile.h +++ b/src/Storages/ObjectStorage/DataLakes/Iceberg/ManifestFile.h @@ -15,7 +15,6 @@ #include #include -#include namespace DB::Iceberg { @@ -97,15 +96,14 @@ struct ParsedManifestFileEntry : boost::noncopyable Int64 record_count; Int64 file_size_in_bytes; - /// Iceberg v3 deletion vector metadata (position delete entries with puffin format) + /// Iceberg v3 deletion vector metadata (`content_offset` / `content_size_in_bytes`). + /// Present for Puffin containers and for Delta-style `.bin` files that store the same envelope. std::optional content_offset; std::optional content_size_in_bytes; bool isDeletionVector() const { - return Poco::toLower(file_format) == "puffin" - && content_offset.has_value() - && content_size_in_bytes.has_value(); + return content_offset.has_value() && content_size_in_bytes.has_value(); } ParsedManifestFileEntry( @@ -178,8 +176,9 @@ std::optional getRecordCountInAllFilesExcludingDeleted( std::optional getBytesSizeInAllDataFilesExcludingDeleted( const std::vector & files); -/// Puffin deletion vectors must identify the data file via the dedicated `referenced_data_file` -/// manifest field (non-empty). Position-delete lower/upper bounds must not be used as a fallback. +/// Deletion vectors (Puffin or Delta `.bin`) must identify the data file via the dedicated +/// `referenced_data_file` manifest field (non-empty). Position-delete lower/upper bounds +/// must not be used as a fallback. void requireDirectReferencedDataFileForPuffinDeletionVector( bool set_from_referenced_data_file_field, const std::optional & referenced_path, diff --git a/src/Storages/ObjectStorage/DataLakes/PuffinDeletionVectorReader.cpp b/src/Storages/ObjectStorage/DataLakes/PuffinDeletionVectorReader.cpp index b91a87fc6269..92a11a6457d0 100644 --- a/src/Storages/ObjectStorage/DataLakes/PuffinDeletionVectorReader.cpp +++ b/src/Storages/ObjectStorage/DataLakes/PuffinDeletionVectorReader.cpp @@ -8,6 +8,8 @@ #include #include #include +#include +#include #include #include @@ -107,25 +109,126 @@ void checkDeletionVectorBlobReadLimits(Int64 length, std::optional expec throw Exception(ErrorCodes::BAD_ARGUMENTS, "Deletion vector blob is too small"); } -void validateDeletionVectorEnvelope(const UInt8 * header, Int64 length) +bool isPuffinFileMagic(const UInt8 * header) +{ + return std::memcmp(header, PUFFIN_FILE_MAGIC, sizeof(PUFFIN_FILE_MAGIC)) == 0; +} + +bool isDeletionVectorV1Envelope(const UInt8 * header, Int64 length) { + if (length < 12) + return false; + const UInt32 combined_length = readBigEndianUInt32(header); if (std::memcmp(header + sizeof(UInt32), DELETION_VECTOR_MAGIC, sizeof(DELETION_VECTOR_MAGIC)) != 0) - throw Exception(ErrorCodes::BAD_ARGUMENTS, "Invalid deletion vector magic"); + return false; if (combined_length < sizeof(DELETION_VECTOR_MAGIC)) - throw Exception(ErrorCodes::BAD_ARGUMENTS, "Invalid deletion vector combined length: {}", combined_length); + return false; UInt64 expected_blob_size = 0; if (common::addOverflow(static_cast(combined_length), UInt64{8}, expected_blob_size)) - throw Exception(ErrorCodes::BAD_ARGUMENTS, "Invalid deletion vector combined length: {}", combined_length); + return false; + + return static_cast(length) == expected_blob_size; +} + +void validateDeletionVectorEnvelope(const UInt8 * header, Int64 length) +{ + if (!isDeletionVectorV1Envelope(header, length)) + { + const UInt32 combined_length = readBigEndianUInt32(header); + if (std::memcmp(header + sizeof(UInt32), DELETION_VECTOR_MAGIC, sizeof(DELETION_VECTOR_MAGIC)) != 0) + throw Exception(ErrorCodes::BAD_ARGUMENTS, "Invalid deletion vector magic"); + + if (combined_length < sizeof(DELETION_VECTOR_MAGIC)) + throw Exception(ErrorCodes::BAD_ARGUMENTS, "Invalid deletion vector combined length: {}", combined_length); + + UInt64 expected_blob_size = 0; + if (common::addOverflow(static_cast(combined_length), UInt64{8}, expected_blob_size)) + throw Exception(ErrorCodes::BAD_ARGUMENTS, "Invalid deletion vector combined length: {}", combined_length); - if (static_cast(length) != expected_blob_size) throw Exception( ErrorCodes::BAD_ARGUMENTS, "Deletion vector blob size {} does not match combined length {}", length, combined_length); + } +} + +namespace +{ + +String formatFourBytesHex(const UInt8 * bytes) +{ + return fmt::format( + "{:02X} {:02X} {:02X} {:02X}", + static_cast(bytes[0]), + static_cast(bytes[1]), + static_cast(bytes[2]), + static_cast(bytes[3])); +} + +[[noreturn]] void throwUnknownDeletionVectorContainer( + std::string_view path, const UInt8 * header, Int64 content_offset) +{ + throw Exception( + ErrorCodes::BAD_ARGUMENTS, + "Deletion vector file '{}' is neither a Puffin container (header magic {}, expected PFA1) " + "nor a deletion-vector-v1 / Delta .bin envelope at offset {}", + path, + formatFourBytesHex(header), + content_offset); +} + +} + +IcebergDeletionVectorContainer detectIcebergDeletionVectorContainer( + SeekableReadBuffer & file, + size_t file_size, + Int64 content_offset, + Int64 content_size_in_bytes, + std::string_view path) +{ + checkDeletionVectorBlobReadLimits(content_size_in_bytes, /*expected_cardinality=*/std::nullopt); + validatePuffinBlobBounds(content_offset, content_size_in_bytes, file_size, "Deletion vector"); + + if (file_size < PUFFIN_MAGIC_SIZE) + { + throw Exception( + ErrorCodes::BAD_ARGUMENTS, + "Deletion vector file '{}' is neither a Puffin container (file size {} is smaller than PFA1) " + "nor a deletion-vector-v1 / Delta .bin envelope at offset {}", + path, + file_size, + content_offset); + } + + file.seek(0, SEEK_SET); + UInt8 header[PUFFIN_MAGIC_SIZE]; + file.readStrict(reinterpret_cast(header), sizeof(header)); + + if (isPuffinFileMagic(header)) + return IcebergDeletionVectorContainer::Puffin; + + Int64 envelope_end = 0; + const bool envelope_header_in_bounds + = !common::addOverflow(content_offset, static_cast(8), envelope_end) + && envelope_end >= 0 + && static_cast(envelope_end) <= file_size + && content_offset >= 0; + + if (envelope_header_in_bounds) + { + file.seek(content_offset, SEEK_SET); + UInt8 envelope[8]; + file.readStrict(reinterpret_cast(envelope), sizeof(envelope)); + if (isDeletionVectorV1Envelope(envelope, content_size_in_bytes)) + return IcebergDeletionVectorContainer::SliceOnly; + } + + throwUnknownDeletionVectorContainer(path, header, content_offset); + UNREACHABLE(); } std::vector deserializeDeletionVectorV1Blob(std::string_view blob_bytes, std::optional expected_cardinality) diff --git a/src/Storages/ObjectStorage/DataLakes/PuffinDeletionVectorReader.h b/src/Storages/ObjectStorage/DataLakes/PuffinDeletionVectorReader.h index b78f57041d02..4412015160d1 100644 --- a/src/Storages/ObjectStorage/DataLakes/PuffinDeletionVectorReader.h +++ b/src/Storages/ObjectStorage/DataLakes/PuffinDeletionVectorReader.h @@ -4,10 +4,11 @@ #include #include +#include #include +#include #include #include -#include namespace DB { @@ -44,6 +45,34 @@ void validatePuffinBlobBounds(Int64 offset, Int64 length, size_t file_size, std: /// Iceberg deletion-vector-v1 envelope magic (`0xD1D33964`), shared with SQL `Puffin` decode. inline constexpr UInt8 DELETION_VECTOR_MAGIC[4] = {0xD1, 0xD3, 0x39, 0x64}; +/// Puffin file magic (`PFA1`), shared with SQL `Puffin` decode. +inline constexpr UInt8 PUFFIN_FILE_MAGIC[4] = {0x50, 0x46, 0x41, 0x31}; + +enum class IcebergDeletionVectorContainer : uint8_t +{ + /// Classic Iceberg v3 Puffin (`PFA1` + JSON footer). Footer identity bind is required. + Puffin, + /// Spec-aligned `deletion-vector-v1` envelope at `content_offset` without a Puffin container + /// (Databricks / Delta `deletion_vector_*.bin`). Identity comes from the Iceberg manifest. + SliceOnly, +}; + +/// True when `header` is the 4-byte Puffin file magic `PFA1`. +bool isPuffinFileMagic(const UInt8 * header); + +/// True when `header` (first 8 bytes of a blob) is a `deletion-vector-v1` envelope for `length`. +/// Does not throw; used to distinguish Delta `.bin` slices from unknown files without catch-as-control-flow. +bool isDeletionVectorV1Envelope(const UInt8 * header, Int64 length); + +/// Peek the object start: `PFA1` => Puffin; otherwise require a valid `deletion-vector-v1` envelope +/// at (`content_offset`, `content_size_in_bytes`). Throws `BAD_ARGUMENTS` if neither matches. +IcebergDeletionVectorContainer detectIcebergDeletionVectorContainer( + SeekableReadBuffer & file, + size_t file_size, + Int64 content_offset, + Int64 content_size_in_bytes, + std::string_view path); + /// Fail closed before envelope peek / full allocate. Shared by SQL `Puffin` and Iceberg loaders. /// Order: cardinality ceiling, then length bounds (`length < 0`, absolute blob cap, min envelope). void checkDeletionVectorBlobReadLimits(Int64 length, std::optional expected_cardinality); diff --git a/src/Storages/ObjectStorage/DataLakes/tests/gtest_delta_bin_deletion_vector.cpp b/src/Storages/ObjectStorage/DataLakes/tests/gtest_delta_bin_deletion_vector.cpp new file mode 100644 index 000000000000..62f3db885bb4 --- /dev/null +++ b/src/Storages/ObjectStorage/DataLakes/tests/gtest_delta_bin_deletion_vector.cpp @@ -0,0 +1,186 @@ +#include + +#include +#include +#include + +#include +#include +#include +#include + +using namespace DB; + +namespace DB +{ +namespace ErrorCodes +{ +extern const int BAD_ARGUMENTS; +} +} + +namespace +{ + +/// deletion-vector-v1 blob for positions {2, 5} (cardinality 2). Same bytes as +/// gtest_puffin_deletion_vector_envelope.cpp. +constexpr UInt8 two_position_dv_blob[] = { + 0x00, 0x00, 0x00, 0x24, 0xD1, 0xD3, 0x39, 0x64, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x3A, 0x30, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0x00, + 0x10, 0x00, 0x00, 0x00, 0x02, 0x00, 0x05, 0x00, 0x2C, 0xDB, 0x9F, 0xC1, +}; + +/// deletion-vector-v1 blob for positions {7, 9} (cardinality 2). +constexpr UInt8 second_two_position_dv_blob[] = { + 0x00, 0x00, 0x00, 0x24, 0xD1, 0xD3, 0x39, 0x64, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x3A, 0x30, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0x00, + 0x10, 0x00, 0x00, 0x00, 0x07, 0x00, 0x09, 0x00, 0xB7, 0xB0, 0x20, 0xFF, +}; + +/// Coalesced Puffin with two equal-cardinality DVs (same fixture as footer-bind tests). +constexpr UInt8 two_equal_cardinality_dvs_puffin[] = { + 0x50, 0x46, 0x41, 0x31, 0x00, 0x00, 0x00, 0x24, 0xD1, 0xD3, 0x39, 0x64, 0x01, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x3A, 0x30, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x01, 0x00, 0x10, 0x00, 0x00, 0x00, 0x02, 0x00, 0x05, 0x00, 0x2C, 0xDB, 0x9F, 0xC1, + 0x00, 0x00, 0x00, 0x24, 0xD1, 0xD3, 0x39, 0x64, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x3A, 0x30, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0x00, + 0x10, 0x00, 0x00, 0x00, 0x07, 0x00, 0x09, 0x00, 0xB7, 0xB0, 0x20, 0xFF, 0x50, 0x46, 0x41, 0x31, + 0x7B, 0x22, 0x62, 0x6C, 0x6F, 0x62, 0x73, 0x22, 0x3A, 0x20, 0x5B, 0x7B, 0x22, 0x74, 0x79, 0x70, + 0x65, 0x22, 0x3A, 0x20, 0x22, 0x64, 0x65, 0x6C, 0x65, 0x74, 0x69, 0x6F, 0x6E, 0x2D, 0x76, 0x65, + 0x63, 0x74, 0x6F, 0x72, 0x2D, 0x76, 0x31, 0x22, 0x2C, 0x20, 0x22, 0x66, 0x69, 0x65, 0x6C, 0x64, + 0x73, 0x22, 0x3A, 0x20, 0x5B, 0x5D, 0x2C, 0x20, 0x22, 0x73, 0x6E, 0x61, 0x70, 0x73, 0x68, 0x6F, + 0x74, 0x2D, 0x69, 0x64, 0x22, 0x3A, 0x20, 0x2D, 0x31, 0x2C, 0x20, 0x22, 0x73, 0x65, 0x71, 0x75, + 0x65, 0x6E, 0x63, 0x65, 0x2D, 0x6E, 0x75, 0x6D, 0x62, 0x65, 0x72, 0x22, 0x3A, 0x20, 0x2D, 0x31, + 0x2C, 0x20, 0x22, 0x6F, 0x66, 0x66, 0x73, 0x65, 0x74, 0x22, 0x3A, 0x20, 0x34, 0x2C, 0x20, 0x22, + 0x6C, 0x65, 0x6E, 0x67, 0x74, 0x68, 0x22, 0x3A, 0x20, 0x34, 0x34, 0x2C, 0x20, 0x22, 0x70, 0x72, + 0x6F, 0x70, 0x65, 0x72, 0x74, 0x69, 0x65, 0x73, 0x22, 0x3A, 0x20, 0x7B, 0x22, 0x72, 0x65, 0x66, + 0x65, 0x72, 0x65, 0x6E, 0x63, 0x65, 0x64, 0x2D, 0x64, 0x61, 0x74, 0x61, 0x2D, 0x66, 0x69, 0x6C, + 0x65, 0x22, 0x3A, 0x20, 0x22, 0x2F, 0x64, 0x61, 0x74, 0x61, 0x2F, 0x66, 0x69, 0x6C, 0x65, 0x5F, + 0x61, 0x2E, 0x70, 0x61, 0x72, 0x71, 0x75, 0x65, 0x74, 0x22, 0x2C, 0x20, 0x22, 0x63, 0x61, 0x72, + 0x64, 0x69, 0x6E, 0x61, 0x6C, 0x69, 0x74, 0x79, 0x22, 0x3A, 0x20, 0x22, 0x32, 0x22, 0x7D, 0x7D, + 0x2C, 0x20, 0x7B, 0x22, 0x74, 0x79, 0x70, 0x65, 0x22, 0x3A, 0x20, 0x22, 0x64, 0x65, 0x6C, 0x65, + 0x74, 0x69, 0x6F, 0x6E, 0x2D, 0x76, 0x65, 0x63, 0x74, 0x6F, 0x72, 0x2D, 0x76, 0x31, 0x22, 0x2C, + 0x20, 0x22, 0x66, 0x69, 0x65, 0x6C, 0x64, 0x73, 0x22, 0x3A, 0x20, 0x5B, 0x5D, 0x2C, 0x20, 0x22, + 0x73, 0x6E, 0x61, 0x70, 0x73, 0x68, 0x6F, 0x74, 0x2D, 0x69, 0x64, 0x22, 0x3A, 0x20, 0x2D, 0x31, + 0x2C, 0x20, 0x22, 0x73, 0x65, 0x71, 0x75, 0x65, 0x6E, 0x63, 0x65, 0x2D, 0x6E, 0x75, 0x6D, 0x62, + 0x65, 0x72, 0x22, 0x3A, 0x20, 0x2D, 0x31, 0x2C, 0x20, 0x22, 0x6F, 0x66, 0x66, 0x73, 0x65, 0x74, + 0x22, 0x3A, 0x20, 0x34, 0x38, 0x2C, 0x20, 0x22, 0x6C, 0x65, 0x6E, 0x67, 0x74, 0x68, 0x22, 0x3A, + 0x20, 0x34, 0x34, 0x2C, 0x20, 0x22, 0x70, 0x72, 0x6F, 0x70, 0x65, 0x72, 0x74, 0x69, 0x65, 0x73, + 0x22, 0x3A, 0x20, 0x7B, 0x22, 0x72, 0x65, 0x66, 0x65, 0x72, 0x65, 0x6E, 0x63, 0x65, 0x64, 0x2D, + 0x64, 0x61, 0x74, 0x61, 0x2D, 0x66, 0x69, 0x6C, 0x65, 0x22, 0x3A, 0x20, 0x22, 0x2F, 0x64, 0x61, + 0x74, 0x61, 0x2F, 0x66, 0x69, 0x6C, 0x65, 0x5F, 0x62, 0x2E, 0x70, 0x61, 0x72, 0x71, 0x75, 0x65, + 0x74, 0x22, 0x2C, 0x20, 0x22, 0x63, 0x61, 0x72, 0x64, 0x69, 0x6E, 0x61, 0x6C, 0x69, 0x74, 0x79, + 0x22, 0x3A, 0x20, 0x22, 0x32, 0x22, 0x7D, 0x7D, 0x5D, 0x7D, 0x9A, 0x01, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x50, 0x46, 0x41, 0x31, +}; + +std::vector makeDeltaBin(std::initializer_list blobs) +{ + std::vector file; + file.push_back(0x01); + for (std::string_view blob : blobs) + file.insert(file.end(), blob.begin(), blob.end()); + return file; +} + +} + +TEST(DeltaBinDeletionVector, DetectsSliceOnlyAndDecodesSingleBlob) +{ + const auto file_bytes = makeDeltaBin({std::string_view( + reinterpret_cast(two_position_dv_blob), sizeof(two_position_dv_blob))}); + ReadBufferFromMemory file(file_bytes.data(), file_bytes.size()); + + EXPECT_EQ( + detectIcebergDeletionVectorContainer( + file, file_bytes.size(), /*content_offset=*/1, static_cast(sizeof(two_position_dv_blob)), "deletion_vector.bin"), + IcebergDeletionVectorContainer::SliceOnly); + + const auto positions = readDeletionVectorFromPuffin( + file, /*offset=*/1, static_cast(sizeof(two_position_dv_blob)), /*expected_cardinality=*/2); + ASSERT_EQ(positions.size(), 2u); + EXPECT_EQ(positions[0], 2u); + EXPECT_EQ(positions[1], 5u); +} + +TEST(DeltaBinDeletionVector, DecodesTwoSlicesInOneFile) +{ + const auto file_bytes = makeDeltaBin( + {std::string_view(reinterpret_cast(two_position_dv_blob), sizeof(two_position_dv_blob)), + std::string_view(reinterpret_cast(second_two_position_dv_blob), sizeof(second_two_position_dv_blob))}); + ReadBufferFromMemory file(file_bytes.data(), file_bytes.size()); + + const Int64 first_offset = 1; + const Int64 second_offset = 1 + static_cast(sizeof(two_position_dv_blob)); + + EXPECT_EQ( + detectIcebergDeletionVectorContainer( + file, file_bytes.size(), second_offset, static_cast(sizeof(second_two_position_dv_blob)), "deletion_vector.bin"), + IcebergDeletionVectorContainer::SliceOnly); + + const auto first = readDeletionVectorFromPuffin( + file, first_offset, static_cast(sizeof(two_position_dv_blob)), /*expected_cardinality=*/2); + ASSERT_EQ(first.size(), 2u); + EXPECT_EQ(first[0], 2u); + EXPECT_EQ(first[1], 5u); + + const auto second = readDeletionVectorFromPuffin( + file, second_offset, static_cast(sizeof(second_two_position_dv_blob)), /*expected_cardinality=*/2); + ASSERT_EQ(second.size(), 2u); + EXPECT_EQ(second[0], 7u); + EXPECT_EQ(second[1], 9u); +} + +TEST(DeltaBinDeletionVector, RejectsGarbageHeaderAndInvalidEnvelope) +{ + /// Parquet-like magic; slice at offset 1 is not a deletion-vector-v1 envelope. + /// File must be large enough that offset+length is in bounds so detection reaches magic checks. + constexpr UInt8 garbage[16] = {0x50, 0x41, 0x52, 0x31, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00}; + ReadBufferFromMemory file(garbage, sizeof(garbage)); + + try + { + detectIcebergDeletionVectorContainer( + file, sizeof(garbage), /*content_offset=*/1, /*content_size_in_bytes=*/12, "not-a-dv.bin"); + FAIL() << "Expected exception"; + } + catch (const Exception & e) + { + EXPECT_EQ(e.code(), ErrorCodes::BAD_ARGUMENTS); + EXPECT_NE(e.message().find("neither a Puffin container"), std::string::npos); + EXPECT_NE(e.message().find("50 41 52 31"), std::string::npos); + EXPECT_NE(e.message().find("not-a-dv.bin"), std::string::npos); + EXPECT_NE(e.message().find("offset 1"), std::string::npos); + } +} + +TEST(DeltaBinDeletionVector, PuffinHeaderStillTakesFooterBindPath) +{ + ReadBufferFromMemory file(two_equal_cardinality_dvs_puffin, sizeof(two_equal_cardinality_dvs_puffin)); + EXPECT_EQ( + detectIcebergDeletionVectorContainer( + file, sizeof(two_equal_cardinality_dvs_puffin), /*content_offset=*/4, /*content_size_in_bytes=*/44, "dv.puffin"), + IcebergDeletionVectorContainer::Puffin); + + const auto blobs = readPuffinFooterBlobsFromSeekable(file, sizeof(two_equal_cardinality_dvs_puffin)); + const auto & blob = bindDeletionVectorBlob(blobs, /*content_offset=*/4, /*content_size_in_bytes=*/44, "/data/file_a.parquet", 2); + EXPECT_EQ(blob.type, "deletion-vector-v1"); + + const auto positions = readDeletionVectorFromPuffin(file, 4, 44, /*expected_cardinality=*/2); + ASSERT_EQ(positions.size(), 2u); + EXPECT_EQ(positions[0], 2u); + EXPECT_EQ(positions[1], 5u); +} + +TEST(DeltaBinDeletionVector, AcceptsBareEnvelopeWithoutVersionByte) +{ + ReadBufferFromMemory file(two_position_dv_blob, sizeof(two_position_dv_blob)); + EXPECT_EQ( + detectIcebergDeletionVectorContainer( + file, + sizeof(two_position_dv_blob), + /*content_offset=*/0, + static_cast(sizeof(two_position_dv_blob)), + "bare-envelope.bin"), + IcebergDeletionVectorContainer::SliceOnly); +} diff --git a/src/Storages/ObjectStorage/DataLakes/tests/gtest_puffin_dv_referenced_data_file.cpp b/src/Storages/ObjectStorage/DataLakes/tests/gtest_puffin_dv_referenced_data_file.cpp index dba4b03f5244..568ca89a0797 100644 --- a/src/Storages/ObjectStorage/DataLakes/tests/gtest_puffin_dv_referenced_data_file.cpp +++ b/src/Storages/ObjectStorage/DataLakes/tests/gtest_puffin_dv_referenced_data_file.cpp @@ -8,6 +8,11 @@ #include #include +#include +#include +#include +#include + using namespace DB; using namespace DB::Iceberg; @@ -68,4 +73,43 @@ TEST(PuffinDeletionVectorReferencedDataFile, RejectsEmptyPath) } } +namespace +{ + +std::shared_ptr makePositionDeleteEntry( + String file_format, std::optional content_offset, std::optional content_size_in_bytes) +{ + return std::make_shared( + FileContentType::POSITION_DELETE, + IcebergPathFromMetadata::deserialize("s3://bucket/deletes/dv.bin"), + /*row_number=*/0, + ManifestEntryStatus::ADDED, + /*written_sequence_number=*/std::nullopt, + /*written_snapshot_id=*/std::nullopt, + DB::Row{}, + std::unordered_map{}, + std::unordered_map>{}, + std::move(file_format), + IcebergPathFromMetadata::deserialize("s3://bucket/data/file.parquet"), + IcebergPathFromMetadata::deserialize("s3://bucket/data/file.parquet"), + /*equality_ids=*/std::nullopt, + /*sort_order_id=*/std::nullopt, + /*record_count=*/2, + /*file_size_in_bytes=*/45, + content_offset, + content_size_in_bytes); +} + +} + +TEST(IcebergDeletionVectorClassification, OffsetsClassifyAsDeletionVectorRegardlessOfFileFormat) +{ + EXPECT_TRUE(makePositionDeleteEntry("puffin", 1, 44)->isDeletionVector()); + EXPECT_TRUE(makePositionDeleteEntry("PUFFIN", 4, 44)->isDeletionVector()); + EXPECT_TRUE(makePositionDeleteEntry("parquet", 1, 44)->isDeletionVector()); + EXPECT_FALSE(makePositionDeleteEntry("puffin", std::nullopt, std::nullopt)->isDeletionVector()); + EXPECT_FALSE(makePositionDeleteEntry("parquet", std::nullopt, std::nullopt)->isDeletionVector()); + EXPECT_FALSE(makePositionDeleteEntry("parquet", 1, std::nullopt)->isDeletionVector()); +} + #endif diff --git a/tests/integration/test_storage_iceberg_with_spark/data/altinity_dv_puffin_repro_v3/.gitignore b/tests/integration/test_storage_iceberg_with_spark/data/altinity_dv_puffin_repro_v3/.gitignore new file mode 100644 index 000000000000..cea42dcdcd26 --- /dev/null +++ b/tests/integration/test_storage_iceberg_with_spark/data/altinity_dv_puffin_repro_v3/.gitignore @@ -0,0 +1,2 @@ +# Root .gitignore ignores `*.bin`. Keep Databricks UniForm deletion-vector objects. +!*.bin diff --git a/tests/integration/test_storage_iceberg_with_spark/data/altinity_dv_puffin_repro_v3/README.md b/tests/integration/test_storage_iceberg_with_spark/data/altinity_dv_puffin_repro_v3/README.md new file mode 100644 index 000000000000..356bdfea265d --- /dev/null +++ b/tests/integration/test_storage_iceberg_with_spark/data/altinity_dv_puffin_repro_v3/README.md @@ -0,0 +1,13 @@ +# Databricks UniForm Iceberg v3 + Delta `.bin` deletion vectors + +Customer-provided table (`altinity_dv_puffin_repro_v3.zip`). Databricks UniForm writes Iceberg +metadata under `_iceberg/metadata/` and data / deletion vectors at the table root. + +- Schema: `Id String`, `Name String`, `UpdatedAt timestamptz` +- 10000 rows (`Id` `"1"` .. `"10000"`) +- `DELETE WHERE cast(Id as int) <= 1000` → 1000 positions in `deletion_vector_*.bin` +- Delete-manifest entry: `file_format = PUFFIN`, `content_offset = 1`, `content_size_in_bytes = 251` +- Object header is Delta version `0x01`, not Puffin `PFA1` + +The integration test copies this tree, flattens `_iceberg/metadata` to `metadata/`, and rewrites +Databricks `s3://` URIs onto the ClickHouse test warehouse path. diff --git a/tests/integration/test_storage_iceberg_with_spark/data/altinity_dv_puffin_repro_v3/_iceberg/metadata/00000-102ad1cf-eb84-47b8-ae93-31b30e5d8359.gz.metadata.json b/tests/integration/test_storage_iceberg_with_spark/data/altinity_dv_puffin_repro_v3/_iceberg/metadata/00000-102ad1cf-eb84-47b8-ae93-31b30e5d8359.gz.metadata.json new file mode 100644 index 0000000000000000000000000000000000000000..287f3d2187499ad5804fe562a6e9a2fda7bce085 GIT binary patch literal 655 zcmV;A0&x8wiwFP!00000|IJiQkDD+M{VzVX5(WZEfJi-5J?*8Js;6qTGWL+T3)r=X z&3-8Vy<-R@TWzZz_XNzxo0<1!cwvokok0Eo&TG@NT^=zMbqjnLv|>A!ZOTS8jpSvX zHhf(|#p}AvIb?OS-PTo8Wd(~^Yox#tfcbovB#4R+y~Za#cn}^F=pQ~I(97N3X!mHG z*h4~TqBVP;$W>mbJTG{?%~QV4%aYe=k?~5&vIN*{3$;zg)+n|pRw`3#4nmfEovvzL zXIsUyJgc*s8itKX?zDs&+&*+i9dHl72IwX5e(1=~cInwMSS1h??>yVBin1s+G}F2$ zva?^BcIbK@=pc6F;2iV>ClAnx(Mmwm7XkcuVfbVr@PKZR$Y$Rh*=T6h6c{Y4i1nfi z@i&TM9W|*PkfRZ;hv*7S_j`7VrmSqq`WhbpotH0p->0EJ<3Ey7cchr;EIzZ-?ddjR z3K}uAG`s~l4`hWy9kd^<(`48FW#AEEWb~2}VyddTLN*S$aSGf*?P8g*UpZfcTW2f~ zG?3+m9i2vqJ3v9Bir5M~yu!%nVEvXrIniT1U^#_BVqv7`emZDc_`bMyLT!J|bJ{|*0m-6qB!KJsO zuz#g(QU5L8_Rrcbk*59$lJ;+E`wEcG_=4w$t`kJtWrJJYX5!$r_1l_Z_DJbz!E>K~q^3Q09tX69zg={NYKt~4wL000=(JiGt^ literal 0 HcmV?d00001 diff --git a/tests/integration/test_storage_iceberg_with_spark/data/altinity_dv_puffin_repro_v3/_iceberg/metadata/00001-364c19a9-7e98-46cf-9c64-8d0cdf419c13.gz.metadata.json b/tests/integration/test_storage_iceberg_with_spark/data/altinity_dv_puffin_repro_v3/_iceberg/metadata/00001-364c19a9-7e98-46cf-9c64-8d0cdf419c13.gz.metadata.json new file mode 100644 index 0000000000000000000000000000000000000000..c52c0f805bea3eeb632176f225ff7e7fcea24d44 GIT binary patch literal 1001 zcmVy)5>$2m(GvHd{&5 zl2o3^f8U{G%TDWUdJGW2&dhK)^Ue1S{gSlS^&*gaaNgJp&?S%Ie|>JRVACY$#}9!I;%vO6C^&r-flzCWExC(B9jNyR&@d~HDvx8$AwFsa|@6pC<&dVmN zdCjteU)gTz2NK~xbkyV=446*sp%=52Zj$9cOz_W35}rngI0QGTkQ{x-$ksrom%wPj zA_mdN_#2I}4vRDn(9w#{!{#iR8FtC*W;vCur~Zr|zkM!W&;7kL{U!Y3a4KY}_oGzu3?oc(NFIy?8T4bK^7L{CRzOw%-HnXL=NIt^}xcC{?I z-#FhUADtZmQv+IFk_Tr3q&))iroF+53}WY>AHK!2T=N(6Ztk+Yv}^dFnfJ}lo*uctk8oJwRoBEc(O zW>sC%yrfOVPy+)z2IB0)9D!v}8YSJ%8t8=?;^+HW4w{!v3~HaeVg)OC7D-=NpTCjy zxI%u|t~{^s_yT6EFY5OAfCvO+h=dke<64f8lGb!l)CpV74is(zzaT$jzu4zU;>~mH zrlb{%&y(7VVTaDs*p!`ya7cKz!RCr^e-JaMZqnxj2o{Na;o8N1@!G}pQv(MxG9kd~ zIHek`U#eLm-a2s#qKmS7>pU!z(>tDg83n#OS2v5}ilvOE^y(Ap0Qq{AF;?IMo--&K z(Uy6ps|qp&3}1LOY0dDjW$jGGUJRziF(TJ}_&5vT=M}n*gd`V;NEqR*VxlP}DVSuW zP^BWGtV^OaEo!P%#q%Z=_s(8;v7J8&jIIQ<5B{LH`1T;n1HxnG7x4BCsebURszFqi z9}K?~$vYa0cOUzICnGO7&(5MF^*;1CBepm@nhDKROF-5IDY&eOfF>t7lR1S_*Lm5b XyXVuc|K?Ev_=Dv?U47e<-U80qY2m*>6+1%Bu zB9$ld-#e6S$=*0|_EMxk0V8&1hQr}E-#6rQP_}(79l3+vnzjixd=)rZR6vHo=wK5> zi6~`RDk3;7Nh}~IMIktWs3_C4$jdy+f>ltpO1d#YVEJafc21L_G47EJ7Nm7+XzqS> zfIn_;N4s@xFSoG9WwL6wcC8rCH0K#9Qcg+Cg&+mZB9dz*1VEBzI$e*YVQiK$9qE{W zRD#5m6{Lt#O(Gr@QGo?RBG_{NSBDi43x5yLC?L&HqdnL}d&kge=|Gd(1{;=% zEK718B}^Rrs;!2)As!AyM-6=s4W?5&sO4y-t6=&M6a4iYxW^7W4z3@R3l6?RWNDz% zQ=qr7RnW-V$6srV?XgJH13F4sSy&w%;5#yzbBG&|lp@ExebfjZec<9k0^r;{D~=L3%!Mnsn-J|@reqs(pZNZV`Z zXJ}{3l>5Yan!M}V4lp&Ktc1$S3POcXq~C{TRYShusWogB@vPXLqf$#I@u*K!?$c(lf#G zVGbsE@KFo_owxsl2+pYWmUEJ(=XKPGG`85 z@r+L?#ZN>z!khoMo^1qL+=@1us~^OrA;7 zKaNkfI?ak*%F--}Qjz$)dCIVkd4wEK*9iae6qiq6`ueZBAg$@G^!f( zF#%kQL_V|aWIwy@`tBgVY1xg!PidWy>oRv zIj&g9Xh^TVpbn6)R}o_|Uf`TToXaxdkxo;H6fnHu(WEuQPsz%WinVM^iETu#TlaYe z!0QpZ_Jky7h)5X8O2uTJ6eMN^Be6;pk%>r%(li!SsWjtxDDV3A%#GdKMPPI$V0&-( zdh^%ox#z`Y)gkp>iTvY2!e~t^U{8MU9;yGinTygfo!{k|Ze#Eh%#%bAf-dLehdt$kXg28rxrD$2z0_ zeTM>Y2#*bZS(MP5z+z_v`_I9^2v1|ha+#A1 ha$KAuAh}9oA~aRHj9HGAoPpuyoO>(`LT%mY9sI}MHYkg4bGMmiarI$_CBzxtm zhgB(fpjNF2LZJ!@KJir*5wR8@U_}K*L9mKq6`vqd5&zI{CdtesyGi%}-)t)HJauV@Z;X>FKD6*wC77xaWkB%*uWkfNK?#59vP@;5SWqe!u}xnY*nX=* zzo-Sp!Bx6`y$NeQa!cLv2>ejNwoeL@G0PcKrVHUNjqe(vn_J$e^L^WDCJu>{h0COrM7*J$Gf8&1Y+Po*pq=kH#M5QN@l`fqf<71l|;Umoko)fYG4t z!%q=ZmP5l}>q3}lp7B}-*ZgreC;?jWhV6<|w{7^Z?E)w}3W`Z8xNZl^0R1joQe{hk zEpZG|a_ZGG@!GJaOV%ouO-lZpO?FEr;Y4tW+U22Tg6wIqrV&`fbpn8$?^i-(P1k?| zyDI$AM{WfL;{;Jp1E@Gi-4>8KSaW?Vuv`b3#3ljp3rHJ;{IG2WQ}MUhD?27tm!oHZ z)Nn8^yh97ns)h4m0bJu6m^s+CWj)$u1>VA=ENv$0o;3G{bN#|dtZjUb9C3hFhR z6v7NKDr29R2;imIb9czE5hInoCS@OXSIKbe^0BhIkG+nta;B#mF7}ZXI-$s=OgC1r zwVRYIr&OMT6X&^yjTJlC>IT?m<$`=zPR+;5M99isnm`(^Q zAXONN{Cj$&OsHyaX;M}D;%vhC8BK@@LTzM>64)uBIf?DM=&m?RDG9X?1|6L%W0ryK zt>GlE;u_4q7>$j%MzIbvM3#vH7yc+h3>?}&mOlXzWe=rdUpl7nfaeFQet=3VwgXjv zIxK-}RB;kiT5%3gv3f2*B^h?zgc*V8;6~Un(Fh*N$4(BC?kcfEoksvQl>(S(aef=+ z*P13zl#|w)rnLniLy8%P8VsjGcBQHEBz3Se?_lYD0G8h0Mz!e;O^{b@dViZ>(W{>J zE7TQYsPXX_c5rU7>GJTf-Ce{hl>u|$Z}GsH!%Wk9Xr1-80iD*9-Kb3_v)0=db}V1S z^34TFTP;OXuhr%kBs$IXcBYvo4~kSXHUUX*Z=A7wTLDR~c9}P2p|b6kbI(PWr;%R+8dq{F6`usHS8sen0wJsduZ&oqE-u4SYBs=S zpkKmv_BhgIY7=uyVD__!C}gzBfOte!cxK$uomJjzayR^{yA?hV3Ho1&_*8{XX z6Kc7c?w~>N0~&?3Kr&%1_{eq9El=Z$<$*}@++9sQaAm`o^HxUJVLgKN;00mdx5gac zlazcUevG&uBcKrh%+9%6J`aynh(`sIS)#j>N+h0hRMt{X&$AzfD|iYq)B$G`*)hq# zypiRvpo18PZAs}2Ba&gI$}Ab}8i&*w zl94$C8r5T+l7n87@iOVfC~qU7&WXk*&KO7`M&IKDfSa9JgD6~h4Uj|42WB;!^n#^P zig6Cp?S^X-qZ5*`LH92YwNlq4@qAEx3v`7Gk}fJppwSl;msG@UvVaxWzXB;ZGbBul`JAbB<{q^uLHnRB%f9XMhmtvhaZ=|0O^;5d{@idFlVolFPPM8fUuuvWezdb+XV&4PRKXZUiKe&!3ytw#^s`)z(+{!24bob{wiG1UYnk! zb{9GH^BkC6p)!So)^Y$tR691wL4(GcpB(Z7V zd(z4*!w;X1w&l}@?4$yMYa{vqBPO~N;uXSR;zXNBHxZtw!IVl9#+n~u)99cOMRZ94 zC+sDH%a?IH&TcWv(uz$OP6Qg=A>mmim7B0C4ksnjvWzBf;kk%45ra$wVp)*TWm&PL zsMr6c&qSnxO*BZctQZ^cnvEnAo<)_$p6L_3tiDd3PUxu&w^kDhUC+0eG1H-9?*RmbRc6Bi9VdCGIWo7LYA&kXFj|Hv6*A9Y=^ zVb$|L9$9(G3)g=0i1xwaMSJhL>igSvf423h2Ui|_Puu&-is`E#{%g8}?7Vr&ve!CK z+}FA3#J=l4o+vF@eDRGBJaFLBWh=h>`}j+5RHxJ%56t}f`=@Um9^U`Zrs1*2Z$Ey+ LhW&poOhfV?MNWOP literal 0 HcmV?d00001 diff --git a/tests/integration/test_storage_iceberg_with_spark/data/altinity_dv_puffin_repro_v3/_iceberg/metadata/e5531858-556b-45f9-a989-a7ba0b0ae967-m0.avro b/tests/integration/test_storage_iceberg_with_spark/data/altinity_dv_puffin_repro_v3/_iceberg/metadata/e5531858-556b-45f9-a989-a7ba0b0ae967-m0.avro new file mode 100644 index 0000000000000000000000000000000000000000..c49e7340201c6e44a92d1b71d9bd76b6261af2ef GIT binary patch literal 8007 zcmcIpX^0#}6gDxC7_U_%hz2d&V^wBlx_h(PU3Nn}vN57-G>NFo+SGK{>~_-AJ?ZYr zvda3y3*!|~F+ZY+x1#x>Q4l;5Aq0hZix-Hf;15we@jww@Rd-ia_e^(YXIBV;^i;j$ zdq=(Rsq zhIWkvA+FWK>y?@DnPqsJ@0}NU^U*5qr%z1Jr?$s_)Uaz;;hsU<(4U6ylE?885E?cD z_$h|UcBvWcTm%#KGdb(xdNApQ2B4K(7*pJa14?r0&DHf^Xh7&4-^t&ucl_deT z#5IZGHfj~(cOXrdq*ZN)7{Q!H_DUilBG{mExoNo|`x=~S2G;c45Mbx`l?YkOGa4<$c~gyIjsb4N@#AwPD4~z;-%z-+7FG6_LT|S#LmX3 zlh<$^`d@;^Ww=hUjyy!Rg+dShP=pvbw0`Vh3s{ujl*;{Co1y^E4^#sHl~(Qqs)4Lo z0@pa>B&f9V9H3(PT!Kt8orVQH0@22euxp`FJX(yMY$Uy9u%ON(fSO7HOqDo)jf!(k zlRL^Gb4}Aa0+1oaghLKST_HcyRJoJ7*qwK<^nL(KALt<4^rj*x$~JwVL$K&mPwN%( z3Nh4pdyFc$Fxhmuc{tt_@r?>#4)a^maOTj{v_6_=D?5Nr>&q|H7M)pJ*%5ZkU&Q<^ z1W8*VS<|T3=NBZ}&Gb&XnI<=iOfj|qN$;$kF@M_uNzQiYB1Vmbm`({znWRL<@%Ltp zYx>i5HaEdNl-xrV$Ey^^MVGr#P=_(j@T%2-gzaZ<@Z>psR&TQD^ypfITa2~;V(1LIf*b^?y zlXUn<{TR=FYygc1z4}$7FE;Wv57Y(LIK-U*D#XV3BmnTTGihLj2ipMI)M8*(uudpjI_A}5c))zc=qQaTM(vs;QzuVk--_}p94b7%c2EK@g_hh5eXG#d1eRDj;h?9 zfY8#<6DG3_5b{%{%psu+~9rX&oe@ur5vD zguIk-#WarF*(+LEmaz%JDMRBoB#Kp>kmS;<5 z^!mT}nXpt$6HQbs&&DRaW<$w@W|5_dX1aox*;m^J%$9`#uYy0z`QXTC)1tBQV`58y zX|exAJM85fZj#`;EQ-KKSJBFS~ZE`U`)4 zaOmR~)|@l8WMt|353XJEXl);U@qvqXEj#-1RY%>wr#5EZy!6erM_%&Lf(y*9lRmrU ztWRI+{$b#Q)z3V9z+J2EXdJ)goAq6bx^?TNg(p9@I6Qmz26ue%6}R3vG_?2X-J|dR zzSbZ=cmMvAcgicTe*MXfyH@|RLP8 literal 0 HcmV?d00001 diff --git a/tests/integration/test_storage_iceberg_with_spark/data/altinity_dv_puffin_repro_v3/_iceberg/metadata/snap-4017675268503509610-1-7fc1a95b-41b1-4c5c-a585-cd0480cc6739.avro b/tests/integration/test_storage_iceberg_with_spark/data/altinity_dv_puffin_repro_v3/_iceberg/metadata/snap-4017675268503509610-1-7fc1a95b-41b1-4c5c-a585-cd0480cc6739.avro new file mode 100644 index 0000000000000000000000000000000000000000..dc9dbead027c07245f7c4bcf2b044958c7ef52fd GIT binary patch literal 4816 zcmbVPU1%It6mFwdMWcCW(^lJZ9j$_F>P(W|O@c*AH_~dg#Z8NCZI`<*jY4{WEOrerPG~`;SZ0l}qzWqX2)iPb;y(?M1axp{&DYDbW~ zRl_chA!VyZ3{sc5k`y2u@^p!qD#@6oLYCM(?+_*F>M7)(LghR$L!hXP9fUQF(3^Wj z^;uHl0#-o>e%iv@WtE{xP;fYag-VDwt$|7iP+lq-$HYY^Z@Hm@gfCV-L962CvFqQV z35q;WheA8W$*5K-s#4o1F9-^0S5R|(qPlq&->HyYF2Zode4&r7l%OKc<2fiqd z*(Fe`9KvSBm#;h>N*Er)KrBx*iH%aN=nM-T$ickO;X6qI=Al-g*ld8QksK}=t~5e( z3Jb-Mkb(i@Z)dVFQCvs^RwZAH8UCI?dXe`Q^4>x@%SITk?ad}7C>mH?5T#=A8i2jG zItNx}4$Z_Qn6?5)Lv^VP&9IB5ON9ak#h_OqSj}@fo8Ctj)qR3gG6mvdx(`_2L@_yP zs&=L5ZB=*=1@ED(VP%^_Dq{QObQ{(?%#J9!4AK(?enpoS&m2c^+*4o+VaL(OLFO%F zk}`en(EYnDl^RVUdW$wH+AjmKt35xWS{hMrgW?<5CB4Bq0Iu&Hl$B&68ShUF#pQvx zJe*MEbzbUyfC&e>Eibtkyl+gWOu6Wq}~}-eUA-`Qnz8BjzB2FX1)U-L2sduC!;t>4| zf+>yQg(5&iJJZ-3F#sjW0Jtk^BWx!wiG#%%SWt8Bk0L-rgrz(1OB^-9P#Zg?c97e2 zP7G8YV4#gWQ#+99Harx^8?3<#yQc^|u~F*X6c5hp##}_l@tLa-4I78xMG6c>#|VMJ zXb7P2N%9LqHITW>2!2A%SLd}@@(M3DnX%t`PGQf0( zIxo&k4jT@WOQy#LxN107vuxP|!IUxO7CudV6&+@s%Po@c$2fXOs{^mp$|^PTXPnWY z2C$wpliHBiG{ot zk4x(XCd^N|iR0qA@-C<`S3GDo&geoqldl&*+->k1s4gw|YlkHuv+B>x^42 zAMbwQ*xZ-N!=KK6Ja@;RqX%}}zCU&I{5QW9FP(X4-_;Xi6E_@vxiI~ry5_`V3zshc zsBQf9-@hl`Sv#Ln5*NQu?=_yf|A)=*{V_Q8*1+83y)T^|er@*cN5;DK`<9ZM)5bsO z+x2G>*K6v^rP%((JC8kocH>sesXeYdHL%3D@)HV&o7_+;_#+@SI$3Bd#(QgiVAUQ literal 0 HcmV?d00001 diff --git a/tests/integration/test_storage_iceberg_with_spark/data/altinity_dv_puffin_repro_v3/_iceberg/metadata/snap-6006002030350834282-1-e5531858-556b-45f9-a989-a7ba0b0ae967.avro b/tests/integration/test_storage_iceberg_with_spark/data/altinity_dv_puffin_repro_v3/_iceberg/metadata/snap-6006002030350834282-1-e5531858-556b-45f9-a989-a7ba0b0ae967.avro new file mode 100644 index 0000000000000000000000000000000000000000..4cd9d9d485fee397ff2395be274269623a3cf028 GIT binary patch literal 4895 zcmbVPU5pb|6ov&92_gi;k0EkX3`DbwGwrt9T_tg4*9d|xbTLta^>%u@9i7gUnd#Q< zPXyElksm>bz=OevQ4G#dkdAZoa}MijhFEqo zqiO^mXH=r4tx(t6;i?mPY-iy_Ign+Wsy+ap(}n_ZRu~GlRYON>Rxd+E)y|^F639Vk zQQOpXs4FT-qpj7kHKGrQ(KSPe-EY`fLo`8TsY3**I!cvnVtMJ71It3-JT`5W=99GY zL`ecNOI+1$mURNYM9X?m!6Ip78>ncic4fX{A*^EPHgVoBvmgdr>V9be^B{?c!{9BoVT)u?JJ1?V&xOGDs~pz z?j7o&NRcWO+9_62F$zJI>ZZIPD5P0J-Z6nAsa*(3UQ#U^I^7wNx>0+}C+h?j$#+{J zJ$Bq4r)TQrTTQfS6svOe@$6DSkJVy87Mq?iFlJE3Kww<4SRMgmhV=*tj4IZPO+J+e zcS_fW^n#{^Kz7Pgi%k=k;AeGjbI%f zxT4r^7C@~s2B6*(1_e3qx|P^#Tg+54EVEL9VBY8QkeZ0?)zLUgEH6uBCOxu5_XpMnRv zA`TH|5KL_ZKNJ8WTAJGahylnQ41hbM2EsPulsH(Ffq6CO9w`9SM_8f>&%{w<47Ijf zY6Q7K_ryS@2m`IjPm)^@s)Nj72Jqu+zAC@Pl9ze089z;jXZ$zobPp$Y8 zmjR|T)O~SYGFY>iTrxe@z*)nwiiOJ_2*!*lxA61SRnbn?x!fkXo{XdWwA%AaW7(xz z9*r{^uL0}0Gmh%JGqY)9!PanT92J&T`DeS|kOhLOIv5se6`bGbjW{z7#`lxogf$Ho z?8bb4i^rw)0wd-oUB`9tTzMB%7%U#Nq8xr^>N+h36XyI^n5rNcp9Cas*<1ln_rYXQ zS1MYjc1iAtbwtClc$Au^&X_EXJHG4s<&Fi*-|sna@$ez#;{1slCZ0L_#PSK<({H=- z{hqHD-G3wc@0@QwpL*`+=u&k0>lKU8)jKz>?O6PodF|wcU&cN9$n2qs*RLFpz4_4O z-c>hm_Kh5SV_DLA$J%seXorC#GoIZ0!_e?s2j}I6zy09hEf;2P-?P5+ubJCdA3Jd5 zmFJIacy{=%?)apYn=j3r`}@vc=bqTP{Cwx${*N|FLm$32b^5NA`wvff;Gb!ykuiHm z%hn~yRVVu|<|Z=5~# z_lZ9FinQdFlWA z&iCGVlRdMuGqZbkZ|-kqa&Ocm)VYaX5&fJ&CrWxiBe2dV1@VdY{--si=V*E@94E;uwJ-!eD{o60gX3St)(V@cr*gM#HkELWl6OzTWrSnG9hOdLRHlraxQN_l+9#k~IDVgy zOo?XN44V7|6XO6IBPeh{Dct131OQs30rYd_;}ZbrwtH}m!(*Dhz40O?6P-|s21yP~ z3?LKH(T)U%r>CnaPee})$c?yihcU{4%*{dc1Tv<5K&5-rK3W+Zo3!*a+;nt29s&ZV z(6qF4QnK0L=xC*hXgNk18Bjr7dT0cha$0&sLBC4Zn(1yV{|kKWDWxL z4NNFO_~bRB^kNibr6;8#=*To_KNs-H&qvd{g(fi(bg@R5K!~#lGZN>b>6HtZ$RZ&? zc?gpxxjckL9#V89C4a@NfS~}PmxnOQ*RsgtGg#BhXVAyM$3Xc4K8Va5y)vXo{-2#b zhH*6cU;nQyjBG7^`UJsYV2zqx?Q$^er^n2VuUYFPNCj9kXMSW}a0i)43t;lU-XQ9j zTcr$1_9FJ`I#kUOp+`KLmJ44U9QH+ zl8he0lb@C>AHp*hOo(B9G(Z?R!d$J!gvaO>vK=g0E@z=${yA9PIz+Nu4M;e~M1&Ed zKFef^kr_E5cd`WJlUxDCgh-oY;o0+ChC8q$r>h7M%mNa(WFp7kYmJCp&Gr$#uOMV> zuxJ>RV;T|K=3|{OrJqHkqfdrNR_4WjG8K4K0#*m7p{I*ycKG@Jd;rPla5{i{LP7)$8e&m2H$I(&tP*`Bin4Sd5FaCu26qc@ z4mXALR?-egH#<6jkxc6s&cxz65*Vk5oG?X?78qbYC`W%tTfmrmC}obnB~cyD6HC(t zH+*DG3f}1eqFpIQ(*g3!#6Byvk_<7?I?>`IOZ1Q`W_AAJ%*f#i@O6 zcLS&PR&zQrocw7H3W7oH4PiCMtJTNt_WBPDWHMtTq@qI*`ggo?kSpC-d5DyI--}$z zlUyoRm6TE-e9s2`q2b9D0UAM==CAH zO8)@D$Vv~#$~cB{1K%iQR|v8@!t}qu;UR|B;Y#9$f%WakE14uq>PHD-957EU0cCu~ zFmN~Hv+G)kI zXuNX9U zhgirjGK|jkEo+sL<|vvJaTRba>Ofd@Y5R2A=_B~(bRddz$n_P9W`>ZV3N~wcMF^i( znqn$LlnIA?VS0iLZ4v+<4=(}lAEg5m!h-<>cnLbwFV7 zKYujrVd8ozNi!HlRR3Fj0{w5M`T_!$9RKqL-l{>xOiaCGI_tsu(BW=M=unPb2T_z+ zA92*Mz#vo%3Kq2!4!3|oSyRAK|Gx6ee&`^v{t5i{IvRw6eeB}kN_HF=4h(AdfA_@7 z|K1#@*U!N!;h{p4nZ5pL{Ci^45KAj5DXFh4d_Xiy)_gNpQ7M=-T0SqCWxw48L;h@I zY2mEPY{Zw4ipFUL-}X|HUsgg=)7S5%CRKijI;7mTS^cm?i^<8a=S_PjHO>2EHz|M| zGLxO3Pn#;A-fsniHVatUb;O`IQU4Zlk6x-LYdZC|6Jh)TJ*|YALXoI)tsk4jU{HOS zxit)Wx>92elYrftt9zT+!9IXtC6@ms`TmoYlB0U?Y6<*c%KwgZtMq>d`%gGsar}kBM2=LQUN{!+P$bs?fDk$p9d( z0-wQqJ@HQg0*S!o+{+Jz9zwjA>FMb>mT*mO`oqLm5NL>(cOI_#kKohcwTxQKXJ$E@ z@F-HqTXp1!A2V_tau7c91UZN#K%P!sJvf+N9iP=7x{gT0Lo07ys%k5K2w zhkM1xqs7MKCddXO2LZ#S=)*^7@#!hw_L`#M1Fpi+!sNn%;c^3SB>=lv@L71YBT|y) zv~WMk0otwJsPJ$(xo|lHfE;NUuvLPd9_}1G0*nvGMUM!_#Y;d-z{SJ!dJmMO2TB3~ z@TKH};r`)rKsZiEtUHpe@2DYQdFSHgn%d00(v`i0k&$@MywKW zJ9f+R@5y%x;jf=Ui@D! zU+yehU(9~KoN*q(bLPS0H1*B*@BS|@{qT|BD+@TMe&T$2c|3>kfiLb_sSTw&hnhy14|$8ixO&FANG-H`~HHxjHz?d^g~j>V+PE`PW0wFQ64o zg+{!fa=eydLYK<1Dgf41eWmC1O;i$;|g|Z&En#wxnJ4~P<3lq1y~fU zk4^_xg9((uv0WT_)C87;y|#wASHah;fag-s|6&(f{SV~-+5$8i{!797QI*l)n?4u4 z0VzER{X1a1c|5+Hv!qNsy{hxKZ-kJPc!?`a`Va|3I%0ZxPDiu&M>*%1ybJmu`tMwl zSDBlV-@akd%fBWm0&odozCMV<%AuEqV9`r)-AF*pFaF82a*+4(-M5{IS5$)43lB#v z;I;Smusi7vkDb1j)8hfBG1ue1+yijtKw>D(MyXfz3V6TWX@1xqwhI2;>9qs@kuwd> zJoJKx_OZiu@K31QF?|1>|D9qVOwk24ynt5!&ynCf^8gG3Wu>R1bMwXsz(MaIvk&@V z2O7${q!i%A{>cT`@$xpJ??rrHFE2p@+^uKje=KmG_HWVuFS$$&NVNh@Bf^34$S5IS+QRcY zf(Yp&@o1Xw;JoyEXdFXp2xUtnw$9^QO7 zMi5SHvt?AzYltx&sY@$fjLyJvxD^nY4nS(g2PnY9>y3tEJiM$QZ#F!9nfTwjTj6Lj z$aJI@a328qY~H_;0`9r>&n4ov7_T^>o<@XC*hD+4&Suf(dbeI3U{O|W(sQ(Gngx}$1`u~wQ z$3GFN6tN6B+$9s?qb#Z!J{lJdB{9CLTz}sM65%TprdG#Ff1TRO-V-g$Ke$fAMqjcm zP8aQm>fmld_HGlwD2&?eI#evurvkqVWbf)0C-L6W?{t?BqvCO(Qo@l-^> zLb0vXB$fs1eUvzTmzsoVqlc+ZM*ZrgYP$tK851Hew~b;nZNM~^w^d$>{?%ge)-vx= z8|zwQzdAI6Pq1P|6l`bN7mU+^gMfSHgYy?*uU0CIQ z4M|~-FV^ojkt7pyMDnou++cH>`pGJJ^5muvR-1ys&v`sQqc&YXbRAX|XPp=-cc8t@ z?TGH8(}X&0JKeXEouofD2R>5yRq)>Hmu=tS_Q$_A4ps4YTp)#ujLZ3>ZwRl37ueBj z(PWf-Yk#!!WIs@>Wk=ukBkm+GVPRRW=w(v0;?oU1oFg2harXGLbV(Dx8f^Wj|2{Fx z<}=0B9D6Q5@0nsP&tq+M!+G!ng>9#RRj$omobuh6j7K9l!2RK|;PLHn>+3+{xWL;l z2d?gx1pL*=&gn`o%TZs!-3pCVorX zSAt!WX2~B_%Jee-bcZRzTH33eK`caO`X4%ljR{O6^m0CH7msSx;x8YoT+9hRr}Gk# zVOW^`dHb?j$2cD|(nLHLfD?AhHZM{@i@|m>7yor*o5uKS|fXEo7?WSNz- zr5?q7zpS4}X@ITOSU1}&>%_2rydT5yV$M(>V}q=cxW$2xs6?Rk98G>C1}39vQq8+e$uqqV$a zBuZ_8p+}n0b_-67mZu;+lO-dfJn?|po2|o+ZDx3-_ZRiovOFmBERovc_%g0JTQVg& z8^s5G0}D-;TWW$fRQP4*r7PpgeGT|?L|ik9f2qf_cla)62(hl$te z@fOke!Z^`B7>Hj2TyR^A;+rvkgm=x#t@DjqgQfw~c-pIP zHWmX@(0pDDzqx>RPc$QhqkAo-?5Y(b*I5Zv;-#`pcxdBqD*Vj6-eh2GYE^0=Z}Xe) zlP6KLoPQUMm3xGAZkO7N1w@`427ztWk?9}ASVu-2+gABgCD41CT6uE$jObe3{>G!# z0pJh))RZ?T_KHbAk1ECHF#PSTc;I?=?IJuF;xp=#nzZHgV|wMvVadR${cz&VYk^@? z(dmSjx44*6ptW}viSboaJUc%dC*NVpF)PHea;AEZ6I+gOdt(3SA8m)NtYowkoW7sxdvEKiT^>Z*$P(QesfZb zA`slr^&a!)%gZ>=GNWC%7lS7#1KZBdXQurgwTQwpqcu#tj9(nbwaECr^>k;Slu@rxpur0BBGP~@4{Wjou zje|UW-k3|oz2<9S)4HQ++03ykJ|h(|D}Bz!S8K_@S`=@tgFnV&cOA_L`r4PX%dFs&Q1vWlT$|Pw9>K~JyY`20-O-Zi zgqd$$)=E^?-hQuH6$vvk)bg=)Hh^^go&UM;)>68Lkvmc^TYllxt}#yR7!Wbl*_$_n z664q4a?egu&y%CAcWM&d;81E-EpRgK48Aq{owrAR9!Q4v(^jSJEYmf6IWI+d|Dt@| zCd0v6x|_)K9ID_4N#w{=@k9)DX)5j$GaO2p1)&P87_0Te))W@%DO+sqFpH5(jG6I7 z%PMI*5ORmA6c5=H7b+x!R@t9oR)0aPKey#1yAmx+E@s!HBWh6&C3G#YYt{x2R?IBs zVIMC2jNT4TxFp~;YtYo{C>TxE2(#V&xm(2@=VNI&#BtDL{yed;{TP9(yxPn2CCvC7 zo8L&6ZiRBnLNvPcdDw~UZDxi+`1&_7f^3F$vmin5A3}x$cHGskg=r~KHGit>lR-11 z(KD-8WtO)#g9wCChQ%$v#=fm-nu-ut@!%bszA^S~q-aTPF6ZzQn=9 zjM6Ct361%Y2Ja(STGT$?*(_EsO6mHLqaj34mOB`}JMM91@@VWUb+Sjq+N`d_eZyi%y6Z2=SVcT`V}k z))9U75G2WPN^qCk=v_h{4sV&a(_f4vb6zt|%WHu({dR~lne`{5VV92Ct39+bqqHYA zFe2>QjOW-Ex6$SW5cNC z+C0d%asl`6ji^^f=RJbU0=R>(5BD(JyxI|Lio;Xi^tL6Eg^pIYRbQ)+Aes>`YH0Q2 zw)L9|mwxTL_z9(z&XLz_lpCk#m$6%Mob{4K4K^v+VHd-9znojJpk5F6V}{Y~>S%8P zvi0XXsyO39SqpPCR*hp5-r6mqX`*K5t85Tv?|WmF$h04PIQ5|ErjlF9$$6eRUIq@* zX*1m@tO+#o5MX?(5JP_6`mgDlbkWF7nVtYs2QtHXc6JiO-Fe=7J8(2~?A7#QP6&^E zbsJ5bru!VnbW&UR=VUc+$DNwBLm}09?hHoBx*3~RSyOc{#U|nd8R{zzhXP`|SJ&d& zJ9pdK@3CkjxX1<`<30!1YY$}S(LBkos(t(dd!zeG>thi{uUn4gHK5VeJ)9LLoQ3hW z*N5_KF0LlD(UXeFh_;R0gT45~~{30^Xz)3UAYQZ+r~5OyG#s5(%3pa@;=yz zUpU*~J+OAyllkNFagQP-2k?tqt{|qSP`rBS<5{+=Ui~S>BVJI9vzDjVz&rCvSP9v; zq+5hRpT_eFcZXRA8G|&IvSPXLK$|MGUR`^Yi6irqT z5$tlE(j3SIWL5ANnVR(l8kA*o)H?j~yHT$d{N%2@gc6^V_Hf+&rW>ZGU`wKLrG&3&5j!48fM2?}l?A@K$rmv2brfcH1T{5YhPVG9F#>hFpi;5K6EVk*P zWfp`}TZWxp@iz5SDJF}n^dwGkk}_$(^f|`zxlB3+ZPPJ9$CXoNpQ zwIGJ});zewtxEytShV&Dg8j^>E@*OCR(2aMl9$XFvdKN6rbaFtz;|K~0`wfEiY7nL zyLQ&5-9H3BW+4TsYNF4y2CI|M>31?~8U&HE=mtE28c4;0KJQe>vT#mIT|we37zI%V z8W;`Un~X#8CTCgV-z>b{RYjLEu@2qARp_qg(D$3$kab69XRI4$tXs|HPi$W*Ou7i z86Y?;^b4)=cl_WPM+cu%6H1&{DhhoDOLS4f_`7l>|23S75@3yKW&a- z;xbZFLI0?Nw_vbRZPWSNr*cR%#c1(IB`D{yK6x^u_7yIRG5Z3Zb5KdTHvK(AqnNht zc1UwGh=$L>F!C9n zsdANDpzIecT@vlt;CX3bHCWvzo0UAfW$BhX`ta}VPD}Rq?FJ63a=pla=7mkf_ibh- zn8vJ?HH@=}y-Z94rP<8{T$mRQayBTfXk^!guUIEid1A64U3~fV;umYU_U#6;X_$LW z?t50K`|h)Lbw@%w8y;~?D~bl`ZCtr58R`yw0yM*0p5aSfl9eKcnaVcM>W_Mb)TwG` z#B7-)#wLF3^W7xXpz%H`3uRWsB1@~^BYh>Xy?|YBjI(cfi5N!2C`jufVUhe+jQ%2K zzUB+Noec;B)2fC#mC`5;H<7Gve%>r+_2m&x8*7^TqKV3wxdn@E>IFZ&d^Pzs-__!u zlOzuD;5trCV@YRPXdRkVUlunQ_$t44O;P7?WbcYqi1kjPhzb$%FEo=2pe7f}+s!z3hpSw^jP&KYxRFAqtp{tSrX)f@v<; zg}ae|yzQrO0SeY(-ui zy2qitt2v;_@@hd?!B|Pxr9w&b;EWjELc3Y>dy65Q)cn8mGFpXbm&%a_)d+beY#par>V zddWAj01)ItzQ_m9gz88aTZo{q^V{MV=R->YLDkWLtW@#*7ySc`;>cfAE5wV{a7wYO z+)W)xX`I!RQ3~l7oHNY5p9Ufybtf)%{GUj5vLMZ!d7tww3pQEp(n6y>J!u7oOz*z= z3O2-{CX+59c4_=kEa@QS`~>`?mNFKDIeD#$kN@|i^y3dFIld(ZE7C#aO&MF8FGY>l zN>glxVE?YDnvqkf6Y`L>_x%A$s~#bVnQrk2xB{_f1VKND2YaLwv$*~1;=kOhBGSqu zx@&D~y9tH7|K$FVXXZ4`ME=;1C=|Hzs_gl+>WyE9-X>89lLM~>zPgSm)|CJ39B*B0 z+T3U^P644;14W=?>mp{rd+f`%aZGCLgOdEbPe^!@tc4sO*_}cP#NvK!Wlz5y`<^~B zt;`oEq@)xC@F7u5n03eNBTJCmYgCAA#xa-ta~>ug0=mtpfTr!DavFBbt-kz8=LJ z;*fW1?1)q-JAY&BzaSly7$lDCIEE2YD{>~wbzm^RUSAdqk860-p zNIRg>HFgkHVv%Mjyy*vCEym87+yDi=o4n#;kL=Nw3~A5f^r$6*XbDnqQ)oMA8R_gg zE+a#$S#d(;0eU+&T-M{)hek1@d=(mpg10Ra8`5u+0xP#RJ$LbAG14ycHzJ#yMp--M z`nKqZ5vr?^mamcnXjykTK?og^*%1Tf0z!Na4ulEAE{mro54FY4?9szvM`B{ zz9g|^q=%KTF*u0U5pkA+{Ae@WawFKK)5^Lr{XEHz(L(ZY=>-Ww;_guHpu^AAspNjv zG4F&UXI4ASZ`gSN4rQMFpSxuD7M8?5055@zqoHzbdQuHyE+D{st+yS*)JI&i{Qx+}IZ%s9i9yTiw9dCR~y zrL9!&d?n;~_7_Y$z(&CsHw`1>iml2gHd+RM z-}Rrf&_TG~{ce!AIa^wMS0sdEA(CWA7yE8|wA(rZyU5@5+Woek1u68El-{u<_2s^i zhKdZ&$m5@(`-`>)s?6`-5n26hH6C%5{_=CwE32?MbLCpmuWB;eA&47}(%#0d{1Hn? zbwV~2cx@M-razT6&wQgo;=4TjfBOOwfJu7?IM?pg~FiF@N=22u`P+9@RCVb4g0#rVH1dsvTe@G<#DTA z7-)ZpIfwl7t?ubWRPYBi<3#ZHO99<89Cdff>HR=mGtt*I){y&|osXWw3DTEL!UAp$QB}iuj?EQo*Vg@acR~0E z^LsMnU@T6lJ5z)2o32q7w6_ME(bQOe+3v&2$Cv~Mf<6i`5gB%&5K{w&=DZi4H0-Cl zMPVdn7}-6*=bPoMr$?s)c)gy|mCWmo5NY_&o{S=s%$zkLlli$ujuBdD)^CEDoT!&)({V1}h=&Y3}Xki~21z0W&= zkoL(BV_xvk0%5ji*}#-WOuygeiJ13I+79^J_iiPO7P{F8lx(8L81B+4ip#q;Fb|=^ z%1P^L6J9R8eLhP1NrWB8^k~c=n3l~PR*RBW)tHt{941Bi?w!1g+%>tgLrKIa+s*xE z7IcMQ0Kea3R$tVOWa#T?CT`1{Oo`WA(AoVYm7!YOSV7*t64GT#-4LUDW%-Uk+7%&J zCLU28=K`dZyD>;(wElhK0p%LJ0_0Tqb>ALgyMoi1$wJ1hY< zV$P;jhFYd1#dhqZWx2lFncR{8J2nN~)Orr*vrN93a*=VR#j7*gVOU(MHBQ}X<`~eC zPfEqOI_yaD<5rD*)nEotF*)7(1eZ2(aF>LEz+EuWJ*=Zw)Uwn?m=}#%hk|d@)u%9 zIE5(jbs$=OFG|#wM_Ax&{PK;JAs5M3(@O!vYGwR!3T^bY^Du|^n|&(Mgkp!BjcZgd zVXnqG93RU0mM-vh>a?E6ih#Z8deaG(wGQO#$akky%)@ULn46fy(el)z)!2z{%s*bm zDwC?@bWaio&EgL`OfRNu66f08gbv^vGEYRIeGt)oS#5n`c+GXUql46p`o`ahq(sPb zc*Me2@b9gH(1qmQAJxV$_9cza{=58p!P+lV@*!5_OZRqCZ2&VIGojl1uH__)wix7* zPrhyCDjE4#0!(t~U8>@I66$uUc%gLWtjM`>pG++iYnGHJ*>7Y7yDQlP_z;@OXBD*W z_vr%Dr)z~ch5aYqJ54zpbxlca4(IfOL__L~oRnx_7skT@g*5BP>ow6Z$~VObv4z_P zUDbUnmwyOebOc6Wb*e&#`C@-R>JK}et}JKN6J-3_$K22IlgxDiKEGbUh~HQ$_gbWC zu${aXGyaJESpAJcd*I_~)g;sA^mr@jcz;`)(g^$Uq6@L@_F|Mwc3hqAH$FXwiP-S( zMAZ1-tvPgFP3(zP`xn=5y}I?W73n?eeulNm=Xo134JvE+d*j1?=fRk8O7@I}V%bk( zr|mjwMeN$33pa}%R-4R)5|TYxVa)T9+;JdGvq394Dx38L1WHlC*uop_(mPPxv^a4! zldXZ#V*|UXK<(@BHCc_R0&DG{DhxfW*r|^KG0dr_W8X!tTk}DVD=F#PStO_(HS+oa zJe;TaM9OGK$hKpDf4S<6odBi%vXfDl-n_1 zXPr*?nX;UbH@h0BTGMPIZ)3Z!X{TMc=@xV)I#%&=SH(c@eBq?<~vp0&~F3`C! z4ShIO3SUc-_WQ-X)$G!S4%#poJ%J{z*|9!g2I$>tjq^poEzcoZ= z+_a3ODno_~@ZEMap@??%!^a@7W^k}pVTgz(3!}ZeEoUsJdVoC7o5$!fiEvF^F-nCT zSN--+n(EgG!Y$CtquRW`|$MN`)fHSB=%z zw)}jzGhaAZXq)L=TOFyWT9%sxQwCFN>}8eJu+Eb*H_3lp%X+@mi7Ef&5@Lb#rjaWb z5A=YfG14dD6|-a3R6jkjl_`d>$MhV}HMJ7kumWbB^4!(ddBr!#T4xA= zL(mV{kf}fMGs_^_?w2`A>Fb%jTXix*%-Sd)w>}Ol%N`&jwjjZ+@fy(~O%qtvX63yq z)_W12Tbo&)5S-=f;X9nQTV>PDZuAkh9ME4aJ=O?$kF?0=dH!>NR_Q>eqmDh$r<_5! zWgeD3UEXuHe*fwp(Iab2=D zZSdr7bfmO>mY$tN{jeWK`&+YIDoLHeTwo?5fKh?oE$z3&G)iqg6|tBBY!&mu{&_)VcBJd@DDDfOe1l_?-0NzPN3T5_N(si8x^1?O}iE`{yX zQ=Wh~Z!4R{0rlJ5Wo9k22@e^36I+Q7zk-rX8`a0XM@P?GTl4YG~HnT0M5UbU5rWI9NYhr20MWPC!CY zhq&%x1sF(VhZvRdMW%`zh+Z3LdyAR8J~qqu_ZyfvYRc}_3enj8>(=*(Y|_IJ!A=qc zwk#b!p6pNqlX-KuzPaNMf4HB9Zb$q&=l4Hw{(-_@cpa3?-*&D!r%1YHWddgneW}sy z82I#JkW;U}i?R0-6tcct$o?%f<=y3dpj!z?`}5}{h_sh6$Yw~eX*yGQmuLUtjdwu$ z-QnUx4Z0-{HqI)goVkqe>j7h0;(#SNJtVYHWL&C{p`NhuPbt4MO=S;Fy0nnrxHimN z90-!1@{rcEj{dwJ9gi@~jOE~6OygoY&r4f87&N?-$&gs2km6%04v(1U`|jc5ib~7# zR-~R8i84lDR&C%dB~KEnyhd zJB!X-v(z|?&wjk#QG3VRZ%4Dvn9Qkp==0X!y>V{J9mm;}Q7ds@>%yDV3iJ*|bhKAG z9zl4J2sX18=Uz*ap~J(~T%vzfQQA{YO;JCC9^fDADviwRWqa=R2 zC6Sog0VIe*y+3$QVWL=_MZ|%0NlKqmS5XuYAZA66Fa^Zwj-u-@l8h`35dq5{c(UAOT6w<^O)-7GEbE$}s7gFq2rVlW>0l(pRODrLL)PBqu4>_Ay z9WTM)=RA_v_0SKP2;(7pxbQbb!-L%CvS&v7CUoh)Ub*Is5-6J=Q=LWzWC?% zZHq9SR+J}|Fy^dhFTXv+4~gOV!bHKn3nXTjhblX5qXzW;N=HMqmIA_r@=4%i-!|>9n4{Sf+mYa z@s6DF*rdL4y?E1GS~N>RWyGk+YUE z-u#jJQazdJ6J_M~mQlPXd^E$w@Xhs46ZymAtLg(K>bEOZ4Ca`-Ex2qU#-FY>Pk+fz zw0c2us2`bBS=Ig^>(;!2VYo{T!tSAA|o)30dAa&lkkX_3Xr`TTn&?)6!E>cC;G_5BB znWl$Z^#S@fs~abfv`vrE0!ZUUbux4Wk*qe9*zV0<=`5xk_Um2)hP+JYMvK_2MuM2z zu)l9ZoV4XHk-OB>gc>krw50V_|GGG2Yv^xIK0LEhhWoQ5#{MR_w%+QrxSNz+Z=4j+ ziH<%O-fRt(GAEcX!Xh~D)#ZRCeaZS+;I=%{h|?p$d0qhIpC{cxQ6D*=c^!B|u9|p| z0a-UcH?NzM=P`+Kv=B^1(MWbX5b+tMNW{SRn!+0L<EpH;MDvK)Ve*?DWsi66)nrZ;-70?T= z?@c`af2ehSclkhZ+s%=~L}*=lt=1SC{5KIYQN zEV3yNuA~7Kguj5lu`K{{vkJbAtFD@iWc5LR1iP2}7=4$na$m zDwkSy=+b?nw9PWldfCI{ioGm#ux{~H*1=!X!6(eUnA@>f z;DH8cbK9paslN6=76>@%Q%`{PCeYv!9eIQ%NaAuCEMKqHA-fUkejidP3PF!0m8mrgu5Z4M)Z);i>Z2_8~FVW!PBL$A`Sai+Sn64)TZWfv!UDlDaN6h=ugn zvu~7%Zl}bX#nVp-w4Now?RahPQc+-IZF%9blbQtEKYFuF=G=-Ur#R$Z5q^eUxkz(6 zNQ;&iE%gW2Xwrv0cv>d}(n|X|yvqjOiBwtGx-u3bp*7_)_v3iaPHF`&qWw%b-k5DP z(Zf5x#3ZwZp7K1ht+tWzld)t@jRK@rmblh^i1qHtK=_JtH;jcm z(_DB9O=?auNc8fd!m4j^#^qN|av9zYNi!)15JXy0v4mmc6g3|0TyD*y=)U9{AR?%sN(kShK~*Y z6N5{XYswdZT-ga{~3EM+GDbS_y=I&pv?o+xvJN7YTE*owOr&aPI@ z%kGNHIj%my`{k3l;=yJ8gr1~0ETM@MEpOv50S%Cpr1ndO5Dbe~U&g%wl{=ynX{<%(giz9VRQ)IJ`?zk*l+mPknjKn#QG{CxHgEA5!>Ac9XdmoWT{tS4p!| z#nUNHx(2ad_mXAry{GeX6AJScTMbdyj+0Zh)+ct?A1K`ToXXi?i*3~f92F76^yXvX53RIgG3-kKC`n3v> z8|1rdjuU9nBX&sr$DV-EWX-;Ec~DQZl0B1Zo>$>~9~G>$y`f`cR^fT|Mo7)?;!{Ps zTZXTmsK#XJUWBfh$^wU1S2;^#yJF3QS^1x+B-LXSerl7nCh;uIYo{V$VK5a&^BAZ8 z#dhZD5qM13#e}$gj$!Xd^KUDyaux_wy_yey)y7xfjZouyceObA{4hmt}ZG#QzXdu6{W-N`S z_T4Op2)1H1U5%cj6OTgSa&eeR%!1{NQq)l1c%-1xyfwe{OkJz*2)2pH1Za0p(2|{j*!dR`;|5ZSTH*tn5>yI=fR5Bn00uTU*cqYj!NMP)*RxjayH7uKTpyEk4V} z?4xd!0xV;hW*V!8eablg7e@BG4Zj(z-(brs`sUiNXZSGos7n5d+h`XGCqe_Ut9U{8 za~k$oxvp!6o=eGa>#4lyhgqp%GfehzwM6^F*v{=mqS2vQ--l{C}rLaHk( z6KMYH$8Ih7Q>U2ZniS7#x5MMG9Orowc~u5!p*G?ed4f{1h6aALNGuD3<1y5!$roHe(aPf33O zxDCRvLBo?Nxps?lYjSH0Q^A!V=*j*nCO9$E4GEQLjuPE}sTaLDc7bt6B6ax}(BTOT zkiXjh(H64h_WRp61!Xb8xK3m}F-SG5*1S#3zTf2{>ADZ7p*@^wJV@Lx814DcCVI9zpbueKM$#V)B8?Ilvb(h^?}BoQKj@~1j`7mGpcLOv!6i`xK_!p*me)FLfAhxc*Y=jX+68#530zH8 zi?Y4GlfFxmcDw+CN2y%2Zk0DT?F#==Pgxvqur6eZ;5eB`n~@MX86O#=>L0!iz5JXx zaI&`!5%@?kK!L+kAkM{0W2^n)de$ce2Sr8pR;Y|X=k&7?G1_Buw!wJ!xzx&(9yY&% zFWQdvPBL0rd~;V!@dnC{4R($!M>%B8a<pp*lWD zcEr91N9$|u7Z~KW4UxjtlUVx%W+%4SOAA}pK|!umGEJgU5)>VfYJ{vIy*whrS+Tf| zsdV!_97iob?u?t2$7UkK1~Z8n>fxpQgWI#My9uW#D&{0Cr5= z8@TE>def+9^QweO43U~CQ6+IIRdSo1_~htcDCVx(HY*!tI*#a-?-4AddS6B+D@Df8 zu@hZ*j9W8T*-q<5?2F|QOjID`&V8&?!WYnlVTdn4j z2ZNlc!;{z2&Tx-ad!MnzZ_uNOPqGUsqrOHwa2bik?JTJ$cI&+C@F=4wEK51{O#R7|9C*^F;< zFR`0%6UjFkdBv0DkdO2HunIA(?eG4sG2;19l?%3-sX=89`s1ReRO?ZwzSr)KikgMZ zi9O|F87+oi_^K65F&|EJ18tj#gk2`myJ&SXTnH3!NbD!PWfv>*MCO9ze?A#*GBPXT>|lR*irMjG%CJ0-Xi~ezKbf z(tm;davQ)V>4EQw;Ls3!%eXK!J~ZfhfS24;O~*L?keZ9pywq^=W5C$fRQFkA%VWNB zl{^~&crNuW4iA{5?jHf)3V-x>k}{-(EGgKHY;nN}9|S~Y1w=5*brNKKUB9vSCK`K8 z1056F#8Va0ZSnB_N@sp7Enk#N2@!E6+KoXqziGDw9^32yh{YHzOq2DI*EF4Kigpr6 z%{;uH+yQaD0-GXCx2r#y^9=|PoEx`OBc)I&>vqq>G%^F+g$a-N1T*285Kl2bGybUM z;Bla&(qH?0=~6#Q8<{aJ$sjwS=eg;1_@HMMcW=)9g3%%NVSLqx*BXZ3xrbRZ5!Axs zjNPqhKL1B%_Vxw@2}yQPlpJsbHd#sQF2FvhS0|`uss9T|L*ePfl@> zD5%3P>qY~*GE|&Id?N1I?x`)S&!6fSADD*Sy|%^sIAxjxy|cwso}xM$*muafqQiX)8x$5=k+ZxLF;Ziuet8_(D#cxn?H7K)Pv&Ci^?HtJ8kWp?+)!G1-TdENU znETjj!ccZV3kIdt3o2IkdRr@EXNx97yoxl@orkXRJw!c2M_DyWN{4w*l67kEb=|vL zxM*^AVpS&^Gnq{-t~`%_RCG~v{rWlOWaG!|@=m%gQBC!LV9jurpg+}QO`|^ioQ3tX zdsYP#E)l2hZZgGB#vh73l7Y$)d!cxnn5diO3?(;aC}IIHr#aaYgOIfH%ZH}$Tq)cgSka% z=_1CmKh>uf=v<-5JGwY_x`#K}#J-xWN6_NsBO%LI%h^V!vmewNTRef-w>`O`;56!H z50guQ6vUBEU$0i4HuN#(Z61T0d(lYjI`-WmQa9~&3Bk; zmmXk2FGI|CGV={u6F@R%ylUFm9>G=QE%MoM-ef?Z2Z1OUU-kr;@@hZxNyc;eaH_hy ze9lqdEzR==cRC%-a4=j=V6Eyx8%>jLj+jIhX^G!Uk=?I^J1NJHL6NKU2u_f~uegNMN)l|HMe6Lk zfYvmN-^$HsN}q>o8V+sgqT+54iaUippdSy?xcq@t#!j!pV^8EBA+FU!)v=?kQ667j z9Yp(A&EmElM73#{HsnXBDi77g@T&9D8VTL<`bs<}O#iVtH-DevFiv0m**@-;TaC|A z!Y~E1)VLCIUWW1PrGg-mssS~&Oz3^o;|Bp(jh*1-sGbZO;LM*#^QWZ_Y&Phkmc@_E zlniIwjBFS}cLtp-#0<3X9~W1#D*B|yRa%j16O!~7;}J`Er$kCl*4;LrYA!gyMk%R4 z!)H`$#!_Vkrx7H%V-awd*qFHpsEb}iC-v9uc!2*9UU|+A)&2-&!PFEN(gpjgzoRsr zk3bI`PhbwWbo%H!nS8dmMo zN=M7Kc7F^Im!VHRP69HMA-0)j35TkdzFa?-Q{8e$x_BofZniJ5LXFDATGvb5V8+-! z*GZ{a&Nm{3qCl#Cm%C^bdA~aLg^RX)3q>apKt)t-__GF2|8Zn!z3qcJ*6P|)r5Y|l zb}AaxxZkf^d!b>$$gTxEcj!4{LqL1bOL5hebzy4Tr%i#QFe063Rs zn0kw8Z0dRXkEFLk<<)XpN^albQxPWxJqz?4P*g%~EZAnu?Y|ajc%`{{v>hpPK5Zyp z&~Dx0quxwUz-G2(C)7}7TUWO#pih=tOiQIk$vLlGg~)whW^^Ntil_z-r{zEcj2*cCGit=E)h2{g|oBp90?%PukYXsE3&6e*mbAYN9XU4$C$t!P1^$ z9ldkt?LM!4ux*4diyLzXbC`eEkGqw(`2eLVY;1CAD8`PeigT}%87RK#TGJ~`^04EW zKKr7~T~bySzO`z3z)4N2l{P=pb?T~u`y~{mrNt?DAJHmFnb!@fQqrRqA!!Ry4c5SO zG(W-4_pWnd7OcjzjY`1k<`}0eLuXQJZr<+<6K9&>n=1@fRh}qgI=p~5t9no~6$J8` z-qXR|+G(pmO!N~nr`bI4bAZc2PREb1A#iXISw0oACXVJ{T9|AV%aXCi?~6+F>7rsy z_F;5gYK+DJBr#|jh6dVW>71#VBB9S@7CjJE0=;@q>$=)7;9%h%pZL8V zUQr18@ee*B0F2(ZS-j)E5iMm5*(rpSnfkQRB&<#y6B!%iPhhGcAD0I$W&ds z8k>s6i^W3tSqOQ78G5doo^H2H8IJ-11zGa|I};8$;QFYob0>zt>We{)Kz*omSiax) z&ayeRJAE{qHt8=+dUYw;mpXPerS0ung*e)DPlwzwO~1goH&t*m2UQspy0~oaC{+c$oQ(|%j5#V5-`xF5LgQ5%fn7X+){|e`FvsGk)lWGm-B;5^B8TO zR5f1P8j?G~$oZx>{~84zu*aIc!!HG2eXGwNj&Lnj1!5mN!2E+4aDeBSM1fOS+rooxp}42CfQRL9k4GtI7O$ zuShAiuNBq@+b%sU75-0oVk||`nl<8^+psdjJM{*7T?KN&%A4%QwD!{XWz&$Vi?Xotd^TIWAL zv+1yOV!P9xYhP0!H3+zxqklbp&cFLi`tyUGqZ=lG+`yTel7_Kh{-p~oeQS-NDWIqe zz5JbK_l|2*SahLMOMfRtg4yt}hLAHWp?5NWJ#V3zj5k;bydoG@Pp+9F_*!ElXj|wf z0Bsk!H2Sdzi+bXyt}=d+CG6q!?Ym2IGzWZz$;GshR8+V4dXMGfMp!r5%K48p@DKC) z;Bc~8{ETg2%it+X#znCGiU)P(!*9{-)c#AcxFZ3Npo^UC- zaZL^Gvp!jf2bRoj4d_9wpR(hvwp$E&V%3u+d`^dTDeESwYpHH%z&0b9ch9{pP$A)s zp7;0(>4lWLlFZ5n7*6)?IrkVvybs6_`D(zlV;8oCXF8f*UutO!c~|x^vHBrpl=Fz0 z$7r?^He2uPanQOBk7Xnt&wSA!<%cD@J09`sBy0@bX|=n7Ewg#LpYkHz0wf}bKPG(S z)L{M(iopDN5u%h_8+nSEF5|ccT3R4$iAJQ}JiMSw?&4nstp3MUaRo{KyeB=^=SBsPo{!F6|T-I9mw>rqkE2? zJCw*QU~t7&?l{te%$K5RA2#c;VO?1V(c>+co3+xuGXy){Pl$^=E)f-!cc2VUpS)+j zW@X}$GAA`5dt8rFxY{8eBVJZGv!Mxp9m|=UZ0D0ee7+yS6wOcc$AmQ!N+>09!h*LB zrrlJn8{x%0cHI7XLiCbSl(VyO?1DAQZVz&6y<70hp^9=OyT{BA*}5LC#)=!qMr6v{ zpv=g!2G1NTO#`CZTiC|_>z0h$JG6GTQFv=jhHJ|Mwc|vrFj3|bXp#tQb$2$9<&jFP z0F`Y8W5P{qQ35Eg^b6yzRMzkQSx^)BpDW9`V6Vh3yw1!%fP{S$T0Bo`qUEK((^t@D zJ=R&@f2p=l0<1(AtxG zL`*jXbRH|bQTG0FCcC+gfRGmNQ315_?2m2oyGGsLAE*t!iqW9$JGU9W4nN)k+1;D3Emvmujv?~d`)PPLjSzj< zKRyC0A-m2L?Xmn$rRXM9hR#_Q?Q>#Qs!zfq(~j<#qsNpBeZD|-A@;{%@pgX`um}}K zL|R5OG(Y9MTPt_ZQc9-*D^)TPY?h0a?{}Yrn95eO9bsI~PBsH3=1nwqw~_-rY`2+S z?^E4zv^<1%MuC5I;_$;OQ;Wi`{md4qR|NyQB!Gu+IW#oQlh*wu9pJiXzr}afDiCgj zJE)$3)Bltxw2@d&VTF78bbCy=m) zm$3z}O5k|!I;!&0d~9R3g->Icp1q5XxeP@ic3%xJIudFash*6NhIKuncZ%>ztT@JK zopf1|H;QJFh`7){5$*IY`X&)S-2kITZ@l{^P>Tq<+1Uw;QBU-UdqeHYiomII)o+>T z)nmbFc1`1yWB!MktY8^}%akeIrf`12*-?g0ev5(!_k|aWOEOvglSI60N^Cy`xnw?z z{oF~D$xINmF(GGKjRi}b(|8b6x(IUq}SgW}fF1bR%HDZl`^peRvH{t+-Bx*~qf{Wn~y+63-cr zAv4_UfoLSDx~t%{tJ>qh%6>U}N3cE}clWxTcD~^VJ2TfLanrhKG5|a*iJ4;gzLsT) zdD)z_cH)7zld3TA+ZJ0Vb${LbY8`bDvFYYT>l1y#Pd@u(HyFpI8fBwHVdH$dZ_YoC zOQBjdQ#LK}FBQYklM@bJehSQ|ion!bGM;M-jqo+i_(AO-CA@%SZHR|jgI zQPbovrf_fSR%v5!JBE}_5#^B=`kH>iSII!%R}us;xH{Gzv96z1=pzMpc>D6S@*KOE zxgrH>`I`)%Cs+O~COtiq4Z|@Pd5^LL;(W9%#7P}Ui{5;-8R$;X`D7&t0p2{TP(5e! z75?XD3N|x316oZu5%W`LeM1z2pH-Oa=JwHa4i|TRE!1NuZJWn+B<>m^6HX@;3vUPR zb^x>JYwhIO5@`FpT5D(n+s%h9vZu@!o-@f^lbGdObC7CZEC|P zvM+uzJ@FPvOy7#79}gS33};|X5zy9oH-BUeDbH`bi1@I>;B4kh{7iEvb2{t1>)~Ky zI+t8MQ&*h3)D=8=MB&fNz|Kezzz5r&v=Z{8ZREHa(~avuND`{00JRPl$5g7S8gdiU zUwhtabNnb7Z@)Zdq4heg+Ba>BLls781 z@@@4@aeBklwy|#>jkB6$0$FBP;ySeSLf{M#a4NM+D83$<9#t48-a}o)Hrf=VA;6zn zL5?zoWwC@9e7_{W7Y&!YsTBB?=BRQcG&qFKu3`AF(Dgo_yAl%K^I9|EIAo}MyXbiK zE4F_uct_O)e4t_z?5es-Tzt>)JP|=?Qu_>>VdzrtN7mKrQ$pl#s#mptE|(%qNDL{4?P_U^ut+@5$0ke$Ak)gWn)yF&ME zeeYFf4dB#vI>V7Pk$uJciWfT}jrt#@4cb8KJpjIiS79@0?C% z-drbto^0>^`~kk;&>X@Xb|t^ch50DCBHS>M^>SUtm=c}~f)C14yUUUwB9Z*)b8l~T z#^oWVK-#`h9P?}6I)NQK$SK7kO9nEYwPT+N$EQ&Y$F`=u6Sg}Pk2-y0mH|szr&5pc z`v4h0zEhIf9hD7qzxEAwN>c8En=?5s4s8U@$}Phk79wJVrs6?$=a17CLD9i=J>z}r zwq6@WlekM`xCp3%@1N*m%hOL|08wK(G!X!!8rNez#|G|CPb_Jo{WR=e2EwgzG-l9B zCgOc4RzCjHL84S#7#meM2J`REAr^(Vz`8Jbr|pmVjvzPzunY?cehEtKzci%K7;-*^ z`--B6+turo?n}W)Ss~au4{;gy7eo8&P`k=eep+OlgNV5vqjAm8<&@}KkdEfB&YD$f zm>Hf3+@HMJ$hW3AlMh*Wt#C%Miq&ITl&UwxhEMvG7^oIh+EEa@4DR9B%cbYMh{`*D zZszbzUePopVjf^xK&vZDK6|~H0-QVTz|XfGC+rA~+!=$Z>!SG##oZrmcI{9}jPrC= zXsR~lt!M3@6|>3&zPC1XCG3>u-nSjJN0C}Ks~fMa|b@i6VVgC zv9w|$r7=pI>ib)O?P<< zi@Rkf4?&a?U~IAYanu+#)#)~ibK_WD;pM2TV+8ay2ZVmSzSZgR`vuZunrA;dE~_$$ zcLzgBz&i6v^7W#@*G-rBGrj?t=OuucCpFuU{n_c3gU~UxAqt!$Txav9>q_RLa(v%z>c?Z>L)c@kH>=Na@)rUE<1`7l zz2wz4y2P&O!W2Wt&`aiYnrFNn+dVG4dZh1rgh{N>2a>(WLQ4$vuc0_9J0N2aY(u;T zhR!Wb2_k{^%OaB3M@&YpBKg>rImWsDA$aK(ZKI}I$~mk@L3Zj6wi%_JItHpeVB z6>!qPM-APRsu}g&f39E&Y+Nt8l?r=u(hhb`9<9SUUY?C=wrNwSEGIT1X&d%1EvV`8 z%YyX`o4_eev?&4ej&Kxy&l*aXe$%l$fXILka9oy#vU*Bw&+mDzvHReBxO5rVQQeT= zjImkbG8%$V;!Eyu>EveDiiD7wa0dG|P6zkI5@iRy!`2(|8D&_urtPG_4KE zUU@mWW7%Q*mS779B;3Guo3Mjuj(?8e1o$@%8R22QN;e_w})y zESL%Nz{Ju}VwG#;K(~lwA4wdHhx+rJSQ<_ZTO;PR=I3t?hcZ4b;sKE5slaInFFM$INsQV00Jxr6WZEIY#Dr8eDh6qmF%t4Xov zEVElxC?SJg3yVlfJ1+GogoNp$VoW6aFS^?f32!+^ThDadGC8e)rv=)rVxJ0tRrq+3Yi+TP(k1{TZQ13-mVOfIj?*30XA zZMaM|MaWiLN891+aqYmxK@-v z3Co2>AfnOK58`o5H39ss+|{Q-!Ze3V%+`_D2z8gqgf1<=6Hw_0WCl%9;T!dTxUh1qLJvXr;8JD)>+2-544@*e(Wu0EAOpGtJuQ$A1?Ifvn; z!Jc)VSS3y9{gdw3yur&GbG`!+qsj*yV}AGc@ZCQ!=V9z^Ty%bO=ON#$zp>}7$3a1? zK)@G6z6fDPk$>Ex=GSN5Oi%jgvUqHr@LHd00B!;C0fH1_*EOA25N}bzEgcAu-(3i- z;}u*QM;0)tGhx8<2&^;lt}_{{HZk3Et>eQpPlrR`9UEs$&Eo%!tsTESlqx{?TrE{^~t;%f$1jPZ)|&>vCD#m8y-BjIt1RO@k<`}K<6x= zI56-Ao^|PxW$DHeW3ev&H`xBS_XtP5IxEX~566uS$g2O1X8bSv*niTe{Pr*d)BSCT z?ce9_k&=@778K1u!i)>RL6L;<7s3WY_U42Y)x3zhzj z4g354{%xBr^w$3-#sWJa)BZ1`xtV`e#gGNZu*3i<_9A}_{e9rC;@=*ogm~p;B8>#< zg@i8&r)&%J)@Tw#wOZUEad{QAMtO1O)U|OKC=$Gbt!gNBU+jJ3Bq)njb1_2VYAi|6 zqTg9aY#VRKev!x;-x4w;$ySspRHVU>5dMG>5~P?Luk^txkBQhwfKiw#PM#tcrUzY} zDi(;uwU?kqmr(rhMdbt9yF8M3@w_i65_uT!@&r)iKjbKWK$j3s9$=478tG(-{*YqG zRy|;8oFbLaltP9RZJ8}hu_{7ZU0D<1Q4{GWicT7AxnMY_k8Vi8m>(=6f*Q^sKpHk>Wv$B>#1cAp)5gfz9 z3KL`v!=mwDhh?HTEE3lRKTQ{bZf~>L^C|pQ%3o6UY7zl}SQOGYDR}!I0@3)^E3%0Kyjv+@5d)BER`zj}bA^uL;aH>yRddSl4NfmMSghDJPR z0tmGxRjM)%cT!8B0ix_lN65vq5?JGnhGKG~6~ptQb8*D$fTx=q%}P2{#acx(FgYAX z8*Z)F{61d{PKuqtXSd1!Zcpi>8u3AiPC9sfL zkV)bW0N2`JRS+kYoDCxjHx|Kt$NKNr?mvdy{Eu&j^S_tX0ZR!mRO4Swho*n!{Kw<` zJ5AYtI_6Ts@7iGhKj!>@RV@B%G3W1E!}&ky)XbZ|3HNW^K9FAhr{)NR=l`qro16VN zIR4+_|D%EPhS&bzG$0}WPZjZ>c1ivZr`fd!KAfmP?X z84MHrwv7f-@}rSqfepyj2TnZ{RAOQh)HegzP>J7eZ|3i@-wG7qAaKPCRGP@9ZdV2Z;hQ_=ABD;0>sPee=8xqo8nF)iWKxP%b2p zX@ndPLu zs%8~$%3FH4S(uG@aN;FbTXNQ@E3-&3vRE_}T&dsKZ#0sucU+mY)^Mo<;`aaC+Wx%z zzufZ<0(}GU|H}q`0h~FHb)Iu|HcPiwkFS4Bk4ND2YX#!*fphxR7JiyvpQsAiP z@9F&))qm=h|7Arf{+)dZxHiWhr~5yRKL-CYG&}hxNqhonL-baE_}jDk@45He`2Feu zxXT_xCTjh5d2_~W-9^9ke=7v+G5DepWDT(5_RD_>?F+{j{~ae9p&3OgZco+_{ua<5 zO*UZnws6Kk^k#oFJ}{SP%$A7#@}JJ)U!3&czkt86^Z)ny|DgQe%dK!un-wZBZN4aIPmF`j3ip@trLm?E`SC!Nl>B%V#LUTgUCoKfgT`H{|^~(mWz|2 zild4Py%{7}z;%1bg2_;$fr99uAP8|1agyM-XXUq2fC3b~C4bBR7C{y*1{47g69atw z7W`lC{_?Wq?e)li+5QE%pT?;U+dP{sXTg|dip{KhfK7T?#HPksbt8! zH)21_AHe>LFp7U!BwoGm|L#BgyZL4j79jnAAmJJr2c1HO0YOH72i;>3V7@$+Z}AnTTi(CZNlc#ldeL> zAfrVN&ziGL_M%(eHaN6G(OXJAxxL4hHZSz&AEg9%mntLoHz+Q7&P>g{FAJEnd^2kG z6ThUsGO!bSTgZU)^|0we7v>yA98BoDJxw5V<*%tMNp4{*9G|y%yi0@SU6VYB+1?b%h`G&oecrd>${`vylaP?g!gW zOjng2QE4_AZf`a6C@JE@E2%Z zw%v$49`fShxHQ;&|75Vezfye=>FMNEH19(H{&mYG7y8J1rqz|+?L9tU=|b!b=QwOH zl-DmeK^@33yyEynKXx;vd+l@s_aA}q7S}6xFRA44~W7ogyMByL?Tj z_mqk_7!u4bhEv(S&8JFG?@>Apb$93j+pVE}i2bJWc&=d$i8N+h* z&gvl_ct&K>RE(|EUL!mmhL%~sfOuT<)S6ez_-}K$G7;hgbxoc8k|27Wvs5k>OwLvO zNKM(hm}8TZIkv{se z9RE^(*sh+-tPMmRkqp`74^m03QIP0sdm~y_y0toS^4V4O?{LEkqDlP4(OBBQz(e$y z%;fWTlVW7BN?V@U_o803KK{5e9Qxrj1fSHV%E>3GE6vP9I}RCpp-L# zgW9t@JM2s+p1GIUaP7T;au|kyFdOharbQh41{*dK?v#cobogrhA9U)5Qw^4vwXZTO z3@;xJ&neeFo-@`5-kBBosWXbM0)AGCPeK50y!s<)kFPe__e331WnX~ZHkql8a zeQzkvh|sTdsXvkI@pvZgr`CGmVK~oU>D5{F`T{5&wLuF^XTO8Xyz70XJ#MNy^By~` zk2OXb>Kswsgt#oCyXw*|sY9RHDDx5DYZBiTFboRc$q&7aU9j=EKUCVQ@Oxw{k0Qa|(QUAC4O4ZG824H(|x5SlX3|F>dr&+8f-oJQa(Qz%TcVLu3BD)`$S7&F{uxWV43s2~&Jt$4Aa&UW8 znFM!*C4A>58kRMvltR&4zNw#lkeBbcF9jF+u-b;=IR;1E-NCZ~=5%FrtoN;vMF!B546z)NI6pP6#(VozZ0M4nsO*ohbLc1S=_&h7ZunO!e z5lwYV)*B4Y*39Rj9*RT=IIL#GTAvKWfAR8q2l4ITo`X}a-`&YyKPWHQ*>%wvHXsC} z`qv@!1S0;M;jY$~1Hl zXPB>I@=&NC3Fbieik?&57stgonAQY4gUl=QBHwW+xpT^VxFjx|(y_LGIkOt3T0GV_SDKVJ*p8lDIenJQ z5@7zdE5@Y%Zr|){w)#8Iz*l^-e9wu9^U>nU5iaW<4`uhx6q(hY*?j2!k)ImrdG0Mw zmTARL`+Qc#Tj$ri_fV&^T($(C0eWlJC69z_?^f`ttY4O1hfg*_@A&t7pIl98Z%SHC zOfTsDcF@sUvUY%HLBj zlJa!sT*#~8gzR*!522|kM$@fmyZ`lH}0$sZc-f5fa0v70q101ljirS(8IjYDyPFaezhjWZdkR>?tN1el!af?}&%hqS$QNO};`hORaO ze3~<;u5!DgX+|r9ZVJC$&B_Q?3T9xl~HBYJc@lY83T zqm}jw{V8-;>!x^j@;(nqb(gX)ruc)Pb#xfYI$ixqX~!o>e{mBme5UqF;>DRAw8dkjUJ#7zFW^w3`~hrjU#R4PrlC} z!_{oqqx#qzj)Eu6hvZ2d4LJ~M zIRMew;VRlZMH)Yu#=)%6%z&DY(I0#(owUM)WBjJ0Bw}3)zs1O#BRxkalhT-OF;BYf z75{oYxSr2nV4ke%e&DQN;c%4MkkIP1RB%QDawlTYUHDU@=W zyh=jN{0Eh!6?22X$)=&wiU^Pli<#c)*fs{jf}^0R^h&pKhPH=1r`n7xchhbzW$tqu zosv`FU#PM)aWYU|=?ZE=It%~RkV#rEUG$F5HjdtdQnl4*Eg+%$dntIk-Qiy0uqr@r zV;WA?$#5n+1r0@z8aaoIU!LNlL0)oi@nPe|ybP9fQdTw?W7buYCU(T+4J9q10xDxx zwy12B5Wh~?Ejz|Jn^FB?fr6Eg5t6o^s32Iz>YAXHqeNWJZQrOy>Ye92GCJct-9Dw4 z182OKM>b1SEZ__)vgXxgHK?H%Y%`l81jw$P{Br zW#^E?YtW~Mf0Cuyf0pcY%aq+nlSAJQ6$dTzKEw{^PigGO$0tuL`#flF&kmyjN0JVq zBI;AU(BRKbE!>~rIYw$>m=pQ>bnwgQtsEndCogqD{ZY# zM2p--RwAh}BXJM(VeA!GU;jG3gDp%7;Jy(?1CJ(H6SwxlLU7Apro#%qTn*{AYGcPY=q&nb~JSf}9-+Sw$I7bdI0 z#`a;uJ(Fv2`KUu8vfWOw%HQ0skbsM8KG1ijK`8~BgCFI&VrD3e=i1RJJE%ldJ3L4FSQV z&#fxnrPMJw?LTL&Hk9b+=x(Z(QWRG=_~ms9t~?r&+51YZP<0S6t{I;^%zzhA8UgRp zm&wNBjq+KG8Qh+20(BdggLEtjNRJvWT;x(QnY2(?YHIMo_MXBWhpj#yD!=$GL!N`= zUY;txwm1qV3D~58pA-sx<3(E97?UMb`h(UQG%(gX&SpuaE7@PL0y(IaW~Agh8IT&V zl$2Jc&4}j6tz$9cIhJZuhG!Z}E~Zx}&4ZT0He$Bqr@HNjHa>VeL;IH zRX=Z)%rg?!G_FBN0bL#a;Ey26b;pX7GN4U>+>QcJ7bR@YSq**GQhm%)*F~~P4|Wgk z76dS>NQ6piKh(~%4WsX6gS{=|W)_6Yd_!}>wIhxS=1m^Odm%;S}glF(J77uQ+)S7di*RwgO7TA0HCEbI(_Wr_{Fr=?pDokgb_;nh%7tuwUy z9HTRn{I2_OC(4YrM8PeQigP0Q$FyUeFnJhYde%HI&k})>`dwTlNzj8u11RMU7c93ecr`MGso=k5x0;`j#-e-9h)Pj^0v?c#M!Tl~J<}KbZ0q z)=gr*n&%xWILWPk6%Jl>n2}4XqN_>G8l99>O`KE&6D%-BQ!m}D^8Kdz;lsM2aFsmQ z1QV*nVW>Xwi=i^8zczXmZJ6wMS)A8Qka_O~0UQLrx06EQQeROHs{UR~!&QM^WATM3 zxw0c2+~X61Qt6inzuj+gdi)^YxXdvVZ?hZlK}gxjaF#PhhMmemS3F|BwXqy$VZD@z z;!Az7!Injtz5$eyM2v<}=NZbBi4M?;ozY=NKp&pMcWbPZ>ZjFasHp`QG7x_m{}3>@ zL6O-iZwcTZH2^F8Y_*B+OdRxFLICoePN)hiU7aQPhr@Syn5$>y4+Ij5@aC$x1co4s z;RS^mMipOu-|Z;+`1k3H6=OdjO0x6qJos{e)+cI$5Btdjc)HOLnwgjd&X-L1pO|Gl< z7$iSmhmZZj`l~!IeBv7+NbAmJ!!XLoDR@{L18tM~P$}${ep}fgxtdp@qlwutoONIT zUKYpEDy?L?7sgmCTaaSTL68@MLO$J`Wnd_mOK--f-DNQZb1{~&rJ4e$oe~7Dn$Tp9 zvU-(g^|Kx7Nb`>YbV$U!?lbdYr&#I}Wc$VM>CO^lgE+!^PiXwIoDb2_ZD>=eXhKB% z=n)Mn83bXx3FLoiCfaH~Hv~AMCBX?5O?*N>*8}e(F;I(=FiT|n>=&HLdPwi<6ZpM9 z(nPi{R0SEkQGdvUW3*9PE1kDANatV^0ac-DN6k%%AV4}h@QziGwerP!zy-ZnOC--` z`UH>7c!d50AAtf%*(09cjfelj-gQJ*#Z0#Yp=-DHI3nZ8(>*T z2fp6+tuMu5-GCX`QDy|M^KlK>) zOMi8y$uy0!`kWn3oOiUz(nqJek8>fM!&(@hAL!@XWms?9oq5R``#dbN5-eyodb=2{ zc&A}$DiQc`fSxyODbKET(+^6qvqM;je9sugRW%u_8H?jr;~VE8S;OZOYn5Dm;OJS` zEH){1>ayH;6$V6+E@ebf_aSPhZ0mI?O|*f}Rr969qt8jz)3Z&$?ZpmWcFHo=Q{9A& zktU`;L@g3J!?p01yyoEr)8ERT%#D|H^=`QDTe+9t?dtvXc`b9$M|uce*9tZ1fw^q^ z_G_xCn=!T+5`ahma7JPfdFENzCP>z$>n}deb@{L^g zjV0b-#`UUCC!gyq-o=;&J*LNY2`%mngt=KPXOsb*7vb{a%=PeN;Ii#)eBZ+K5NWI3 zjv61EUATR6h1ch4m{L=5)7+lDUC3;;p*1Pjn^vF$j@e;jBGd$4GvPse{9skw8i^#X z087BDy4M%*xnaZdmgU^X-fZ$|@{_Lm`45uNDY!gNNHmD_d>W?Sq!s)p4!3R75?$Nl zY|F$)Fb}v51)tC&9oQ2eYmqKc5!eC}>xf8reCkF5U3;@{9Rt5oUDWTf)-@+cu6Rcy zHyKWCtr*<7t-h-@0({ywwb%+qde>NPrk%&?XFxp#uUvjMCsboJMxF^(Q)oxK&!6HF zY+=m?-?leUg^4eH+)P-wuz47K>`UJ+8_YOG+rhcls~W)hCK{aPm-A3w>-4^NG)c|H z`%o}{O7G4wM{T`@%^QUN%u1==#mca{g;8puwJX2986sfQIigokWzV<+7 z*Crxl**!L>&o$T(uhQ6SW+;!|Z;YnyhEMm|Ep@T=2qfcr`@DF?s1m;U=X}Z2`;qZe z-J&CjuZFbaaZZ^JY*$u4* z;q?}n;dg2*Wg{hxg2Si!k13`Rwj-KfT>KAiI?Z-Z@OfQJQXvKA0j^e(fQO2!8-6(T zXEX@{z;S5o82E7t6FGlC*na5_@wn{f*pgnv&;1|86D3bS{o6lZQS=QQTiK8iBAKXz z`{fKC)aPA6-ql2)p9b9CM_zqsVb@0PV^Zk5v#q=U{#s<|!J`0ognJchrvIXHcgcA_ z_eoTaz4OJE@|Qk#!&u_!&|1%!uOt zJTBDpJrfQ*1%P*+xFJZ}?&p%wWW!!D^G_A;5C`3+8V9euet45*+xqK(z4NrRO-QiK z-}zaz;TuHl+=62sIi(zSx?W}t3TwC6<+pvaiGL=-UccthY^cNhWnG?7^sbf-@w`iu zctOme+0~iD1QEu0T7v?a-ba=pgBMao9)5RXkWI%-uR86Cgu>5#Tkv8K*@8y^qn}O` zEF#)U;MrdDbIh zTD0NWPAd?L|LBN>enRGH@ER)@F880$a_xTxE)w@uV8>_5U)TrkV(NUKr_P$LhDfK* zXCnIaKICTNVR-|)MfigPul?#&dy#5)XUHc!DD4Z54fh9nu3wydkgg~HrqCup}sap!smQKGu=Z-QooF%w4F@!@+ zicLlH7MqcB^S~X=@qO{JG`5~#25EdAT;dA$pWCkA9`kZ&VfCif1QP&Ui7Ri^ow@hM8f}_RGUgWDqBWlR*t) z{)y7>*lNrx$~{0i$&sntlLdQjSsVMf zaD|>yA)WsEj<@deLwoMuf1`*QgNxG1WU4`H!gzS4{tfyFPg z<452Yv_EDS`cqQK+As-7MFxjDfh#leyWarbua6Wsi-|WG(bgY(JhLa%>gy_@J)EJw z_B!mse+^ATJ4GU3fD3;#8^_MQL);YCLn9uM%7 zeup+K)Z#g=hh)14K3Yg0Y_F^_&s&N<(z`iv*&PJiEYg=_e7D;>PTIOg(p!8)4{Tkl zriETZheqQz4gKZZlsbJ2op~ovpuy>9G{$%_DDz%z@9H9ZR!nynGRV-t#Wbp^sK$0% zH>+#6u6>~XyApkRF;LLJ5Q|6rhH6hZ;wJn_EI!igpQCoQv+-jchtD8E8k?A&?e}5z z{+T+iU;e+^-a06*E^8lc+&#EE1b24=Aqkd{0706@y^+S<-2%ZK0!eUpcXtccxVvlr zc;A`1U(I}Tt8U%8x2pf>v-WzPv)11GtRsDTKUDrCiyI(Z<#y#aI%l_mDtkU~@u!B( z4z^#DXjGr7hy%`89W5w(pDP+D&)znt6`Y76uMh1GQHPJr$q<*vQ6pG?f?%bwR&M?H z%pNO_zrzT57f0TzIy`w&z#b(PN|3ZI3jH1Wsv~o_{_X2AB-Q*hP8SBxAS>Lr3WG?y z8nZYDYgx{%(bxNNwBd55$N|M&ggy?U{DYmma?73wZu(@4sOe)v`X8l}0t!$<)-k#b zkQptrUD=R1<5}=J(x^U}J}=^o_kS1~Adr*P6k(IzUVEiztBPSV9>mZk4kN+zMS0w* zKo^@CHREg6^o)V`T)*_M3rxG88~H$0bXvXEzwz#(TXBSV%xZ9QN%k$Y-&g2TjR#_H$aZ& zv3eucZ+1W3w0jv(DbN!oJ-x2 zz!`7s(1EJBmif$;VNSqD!P}M{w@NKJde2-dk;BVk+>TKUBbR-m^B|Y(_%u9rpXwb@ z(_UB+3y9oxVLbJXhL6_K2hJPtd)j)x>q~wDH1&x!{kZn%p4BUsDlI&F+u{c|?*j89 zEWUa;cFl1z3vhTeI_WIi`3UXqSz7)b_V_r9f}i`ed_Y8%trIu=nL^0y&B0{0@GwV# z&sbX%y@mFz&yj9aRX`8Shw_5QOO;88SDKpB?9`ob!)Mh7rAjAT?v}dQM}ngUJ!<@> zpcE=;7PIGYeOrMG%qC(hBP9B%$Z!7g1RuG(MkrF%WUQKgcF^OA|4&9=TeIV?+#6a2Mp+h$+NQjl5KocLKIwtmBFXa$ZVe7Dm_kkB zQ-Z#*@a@A~AN8DRUuO5|R?(-drv>_2Z-e#D!s=&N>dW-|E>ot19OG6=??$Sh6{o}m z*3%dwS7LuME5T2%2CBb+sFJ`pJ<@OOlkOx(IB&KIPSoaS?na)N+@+&Bs0}!wLBXMu z)&>J#vr%dMlW5+AKr^ukQSDJnbvr6lN2X1y5Dx0b}mpw(nhGxdP_wK@$HBbK9|l1W{lAgU3XZi`KBni*&}fJ!kXLsG6;A0Y|~4MpQu zplxTMB=C^Fk^4$zboP(p79@kNVc^)wxdK<_+d5!R4RBoo^H}~=j%HGV3mwOfr`2h z9*C9F#1cNxKRTpA)o|@G9WX){49iO6*)Zul@ zc&d4XYT_c$oD$NbpW<%{04!0+VmG0w1sP+;!`e}zZ#dgn+Q$2 zAw4zhnYkG`Uy98Foz|mus?p2Fr{nxa@#YmL%;__903ab`1goHYz<*`@xtB8#2TLuc zh#>&Kn+z=o>DcI^U?+PSFdstYu6sQ&?ihwj^Mx-F9|v357`2s$1&?I(TRQ5YfjO@F zMK+{3{HIh3EuNQXUbYBoraUIV|C$CNCmBF}Ozo}-W$NduA6G!6AxCtJ-$F(y(N0Eq zH5z{}s6%MZ|64Jd+>E4|4-uQD+Tjqe%~$`Z)6#Bn+WrdFsY@~W26?`NAx!U8Kd}wA zj1hmNFnKybj(oe;rCPM@`mazd8b{N#B}q7|A&O1PKRhzmE z?6cqCc9$GVU}~Rw*fEhTbv{LB9!V9=j$0BVeM1Q}at#tt+<2Co-x zqx8w{ZcwFN6y22nN@aou5UP2JDW@w)tBsdHP;SpaJH!@kIfahndl+TQ@v zq5V%p4f!Avqf|F3t|RjND*y|xfzO()Z$e5lT`pcZobu0dw~ari_@=#&w=iUI*QSfc zS|}fbkZFq?&Vq^b`V10hkPw#fdujiYEPxG84Kd%Tq}d%x7?KmcHi{6J81wUrRO_M% zz(5m@4OglhActxI0N^{UWRu&u!9nb%7h3FC2TmdZO*g+{FgB%Ot=0ywj95CqsMB%3 z##+y^s`e|_@Pi|{@C(Y33Zm)1uS0b@Tq&nJ<@0@ad}wL`LvscjBS1 z@0<^d`Da%)k6X<|yl<<7m={VS?3b^yc*6?9G~*c7-t#Z%#v{XB&!!SeF@*%pr%&S1 zAiA4=hBt@k)=Yk=#XLPhh0L(4oaw`cUk+<9`Lel^qFPU+zX#!eRV|oNiIZ zT{v#+xF>`tf6#{opI5!X?_Cu5(LH5>+6|aIKAvw&ZONLt-h-K|n@?izZJA59On~%` zHiQx(bjvdp=C2gV55NE5az27Ua2sJWaUYT7B2NBr?pFkT?M7x0zN&=0Xn2@x7|!@R(|&PMc&sgK|HTFD zi*U)%PbwdcE3m@yuI6S3sQRtdl#;lvQX9*4_mP{e>yYhe-TXE!(aj9(2CPZGO+#A>Y7vmG#QG?Pzr3>yLq zGYdjPeR}QFvk^?6w^XC)i&-ZU#eZS`>cSMlPIE{AUsHGy>F?^tpFu?svTCN-K3W+7Flx zUu_{9jX2^A((Ozlx%;H$XRy-On##0tr7+BmEk#dp0=RiI7O{J@zIt45qp!a0t#jpF z@Uh(cI6kay6k9I5>~vdXR(x)XZ(*sY(-osCIdo|vU&Ht|*V#5;qt$Vn*68(CIz_tO z0iWJ+?ehrsIf)iNVGp&|y!rG+Gj|}yhl{}5<25ITuN4>aM(Di-D)GwD2xe~&R!>T| z7D^5@a=TPKGc+@DdrW-toPm@C9+N^F@yHkwmT?l*$76WU!k5IHf9B%EjnhbXyl}i9 z+`AnEd~$5m1b9=8H{wKQk%u$u9!wF+VAzX0P|YS*@jC-w1gT}q=rpG?h`%tjFFlaJ z6_RqqCmyiVn?^2JVsDo$ItbTu_USeT{Wdf~sWAwTKMIFwlx#Q;(IX;$i$1NZ*z4v5 zWW4uJ1N;VboUf0^hR(eF`-t={{xD_UW(?9E;;p#GYjtP4=I@9|aEjUOqkVEk!;sq&l|oxQ66VWi57P7H*p4ZD-Z_REB?%5 zT$>=NvtyX%t$?3UyxrOyH^#&>u072P@D)>>@?DVoX)ZNO6Nw&+#U^JH49saXU(VJ* zhWOK4MsXX5O7Hq=sP(APdsT1whAG+| zT?Lw@wMnTdb{fylH%6r#LUp%^b%ijqz7x*W-UN7sV-MGT`;4SBB5d3(y0l`ud|zZ@ zcE-7>gQAuFb39bcLGt)ea{Ra+dQI^{uIvlt#stSP|){LPzjv($QCyTJf*%J=vFq?@M$X&8kq>OY; z+Mg9!B0Nl4n=;dnpjY*1ou3_aJX?wH2_7dW+xvP-2*kvIFk)DQ&g|^cpq@EKuQ-my z)FAltAQz<0vZN{;PATpgpC4{%Cq8FDOoYH=?)m~aabhnY*XuWz?s8Bb3yu!9xSXzo zwO_8KPteovT?q!(cMxd(*_~`H#^oW6lMnDz=dP{Du8s{?5A6;?6W8F4lr>Tf{%$9kPNaWcQSImgnE)q;u*bvu z_GW0R^+OZaKI2SV_te2JFSGdAk}i)%rh3d4VHdhO^Qzpn@ms48yxb?(dJGRqbH8_v zZuYgGOr;E-U4cL3u28OMuKFHEO@1u%J?WH!+YAe5g&g9%Qk)AYv4>HZU)LF$9kjY? zJq~;1Hp6R6^abP4#vhX#cbtAUJ&EZi-P1*ADt5Bt-o`tsvTt22k=iL8J*e!v6S^IK zR9O970^ys907p3w+Otrv*lMlp6CMq37kV{ia2*Neeba7)SB>CYA-~eJzFZ4-dd2ho zq3sDI*RHv~O5ev_29fe~A$deGNI%l7!rF?Lyj`x#jPlu-3*iX!nm}lEXd%t8N*cOR z^;qe)6X3q*wLj5dk_*7#OA9IGYdYy=sPk+7K)?jXt^1;5if1k-a7Z1sA{PA%HlBXA zcXv<~u8J=d%hk`=sj|enGqlCAx)!s_{Ed0tXm-LUjsjLYb9Sz<2Rl2tlQ?Z^Yi9Zw zv&=MMVxH=LD(;6x-L#>YBM~`ybJr|;>*PY?l!zf$*K=~3aDu@_t|?;_)w6QGp8~10 z7F)sB{?ZE%6ztwIdd_Gc+R1;e^hPWwi*kPrmG9KN0)EAcI$~FH)TlcD3UwB;#nPj; z(40v#Z55c#0N=mmxG3Vt+=+M$O{5moB2%0qxo1un%o0;+3DLn>;s$AqQQ+_*Ut_eF zhV4CxHD0Nh2=&jM2yJCL=-uNiZN(@*U%H)0FXC10*jo3odR;~2@}xUTqlwjvlIvzH zi)MoY#i?!RsDEW+9=RvPOtQGxti*j$$X|m_;%YS0{osaq)sZ=H1Qz#kkQf)fqC}?K z>TU3}loR8sqsI)9nTam?IrdHa*8$gQoN0g;BldgT2-O>-m^dzs1-QE> z4nbRy1Y!Hn7=>Uw;yX9V(T{n?<^(^+)1^s!Ew%4O%8E0*EJk_7a!Pe*2Nz`-kZV@J zwe{86LRmauD}v^{M+~inc!YM=6FI^-!Pg+xQfU_fPzOlwkhOB0sfO%{3q9X_#SeX-9{si2R#xvMnOP zu(rj6*XFS#U*YftgB|>lt)7SxRIA@WAqcm5M}))@b~B?b`&QHE@6r4t?Ua9-LUqN} zz8csl>?viAjahC%BnIpBI0*0e7OMOlkUMTE-_>b4t=@G3p;!Xsb-Ej~N{P=O-I@YN zeZi@s4}IO}lTm_vKKBQshZOcUP4ha!30+5?77T?2EUNCFweQC&Q`XV0QzgA@NLMRg zd)8Gu;m;B&c{Xb*=9yyHtp49&1>QIJy zqN>^^&IS~(N<1gE2%Z*qG^Xw;AHY+!;!40IiLXY8+~@C+M2{n_$B8<(UnH4Y&pkY* z?4$Ogq05uhSN3f!={*>^ZvK1^fwcyI=v{>BTWuVtgyu|#QuKweNsEIq6Lvof2%!a15zSc+iDmjL~(pjA2? z_?b3FIl=G5wx=gm4Nx{+IjaBZdh(%5&m&6GDYTL4k!+G%y`sajldjcyDOjGzTMX6oVtCv5bN4Z z61ediw~baTelUvm7Lmwbp4LmK>$zP|=n1*{S?whTe^NP?7Z3m}q7V&$|IPe6V{yh9HugWSGUE-95G<#IiAnVqATz1;2 zEFu>X2SR7CR|nz<>uugVJHf@A?;NARh_uU3wgO4ixi=`weun30jp2{K!6uD;OF89@ z^h1z&gG_TVud+iW{%7JoN`NWj44fvvG@ojx|M0HjukFsH1jui=TA%!S0!2;2PUc=Y`VbwMXB`P~$C6|fg7iT<15q!P>} zb2HdA(C=WRv!=XwW{pt*qV(ffPx`9Hkep*ew~urt>8|~Ex-BJ}5xH^ey^+8lyb%+< zR^eX52qlC~G;|TUtGbKOeO1EYpAF(#4!W+D+XeJInOQbqiv|NF%uRigCnzJ^efwuwL7cFtl z?%pU5uj|8v1=!QU*O0%}q|o|CQEorb3}HEWvw$(F7zt6fN$-up{aWM%{W`AZ#$-he zkap(hu&%3P1}3v+U3*j`5L8c4ZWtDy(lL7+(NHh>JjFG+9gqj$NUiPL-Vf9%7MwY< zINPG$v_Iu5*uXr-nmF&Dts zlhV*_zM1x*T7JE+DVCPwglo=hWrFpwCe9}RLO8BsH04Na>oc>hOg^$dvMJ>>Vx%ks zB}PEb24x3&swl*I4^etPEPF#jHX#$Ke;aj&-TeVWC&XWGrpqFcsC!?Q{1-9lhv-g% zoelhj6x69FGn(9ZEr8@C4mLA6k?(qqmED^b+fz~feWLB}Q8l6b=X*>Uojrfv88Q8a zEzz-a$4vE@a;!fa5z7+NR%5N13B&u3G~ibn$4w?EMv8BS{_ZbonDG@{dNc&x8 z3*Nu2<*x6oaN*mr-$aB-v6^=R8{eGJZTquMS--?L{J!S@VyLdnq4ERAs>Ko=$ugTe z{yW*nx8d?NpC)IP&L&;d!rvdtyVW`9=q(bS%-h~DwvavxI+Jk^9>|;Ybco*q`lSb@ zmyQDJsMxIu%+apDoRS_qT01851KnII;<*|f{AtKMeaYdR0rYGe$D?X)IdmneR)O28 zj6tOzqG$G1Z#VH7^GQDs70RWVnj8(aD0e$MXU=gQjXTtL?Hhj)Nj##w<5#&T8GOId zqW@Vzap;_;yWK1si{+kfhHK`#mix;pF+^gIL;`9_rD^EZYRV>%4GLR_T=(#`g}_Eo z%Ilf7xh%capX>t#m1Do1zJ8u8pH(8pxKuD=Uri@RU|p(e3=<v(^m$o?&&A zR+wjHVex~Boh#|X*Ywgr$409H^Zn8xM9V|WPbOr?c-?;zttWpaYWa%yk(q2ck{i$D zG5sE!%eJr5VS3A>`MIPqvCn;LpB22WjYk*JJjQH8Y$a!pL;D3xIjn#SaD9IQw|Dws zOqa@m_iW~Cg4S#9mbFhdyPN^h^pQ5Qr4Pgjh)Oq+)yI5p3{psXmqvzST7~9T0mLlv zW;f|`Np_L7&eb|p#f2ZOQHTDV-1>N#*IwO{wzBgD@F-0fP3am@D#tpWa$88)6|NMm0t9{kuW|h(<}^8?h(lS>nrF+Bh4rv}nVT-) zW5jlqA13*717>SS%JT+T2RS}eA#^g6x;uE~Kg$!#j z=`J`gc+T4DWA2inSk`;MP~0rOvFLkB$<^cDFhR>N>^26c&B;HPKMz#KwyBuun-ROL zZV-Q8_5u8UND2cw)8LaKV!cCNr|{)eh}n#KL;|h%8oPi0W1}I6X^x%Bb3_E>bcCWE zzt)IHe~+KAzFDcWc$5~LG_6s*m$#rbwA6#6QHY&)_kFRTGt}#gfd_gL;9}9KHS}Vr zphR#dY=o07JJUOw(m`wsKVffck3T8@o4~e*zJ*I>QJUrxve;W~(T2~3=sQ)8*WSq@ z_{`@7TEFJWN@6&SUle<;m8}-;EH9f&d$?YK;fkU>X1boG9T4))cAp7&gh=k53sJBU zHcL9TE+p@@B}w%aB|c>BV?SX`UtZwvVtCb4OA2KVsnqQ^U5V}Acu(C0QJG5`DFiCW zq+QflMYmd210sFE4Ej|*)8Mt=gDpoiZYHEOd~GfINyU!0e8bjmptCj~AWklD+$o#CEep&inm_c&-QKGt5-07sy0F(w-BL6cgp^}E3a|ODk`P^i?ZS~g)h~QkzDe4 zo=F=C0z(U+(rOm{Dsc#-1&uiBGXixo^X3Cx-m2+W;Hk_$`YuvKk0hVIbc~w*2zZ2} zykrH=ee9Y=*Y0TWcnN1N>F}%xh$~|`bK_Am%FPt%>jR0O%66~FCV!JiBZ|zlWkij~kKaR=-JUg4+ z5VJ^5N;41P=Bv+~tXiH(1^qq0Wwrawas(@ToVn_@Y)UzujAJHRpj1CHsg7PR-8F3W zr9CP>(MXhDPxFo6%>QtR8I~$5Boi$$TwNY^*7AC@94~oc_vF*lWPmaf;3Y|^L@}sS zBLeLeQ8dabc`crBlY{jgqy5MAYCQv{U&zj3PvKb6^IkH8sEV?h;Lh5H%qHI-^_KU# zwd;*XTh}-2`@yQCMGq{yQ4oQCS-(zLHDPy*np5Vd)}7rcJrj8?eVmXOCiVGQ&E(ys zx#y9|`toCe*_;#fr8k1BvOk(f#ml6|wlWB{RGofqP^MZ;MY)9S(mq)U#{mNT z?G2~T!@>&|3f*TCsXs<8`bZOvdzcg5R*#+HKQEX?Upz=Lr0ZlbSl9(4%`0^ywY_fRx3}BK+94>hrmHc#*a@j`35H>$W>3rJA#xn5XK`u+u%u zq(KXjvWT~e!=$$%F%bmx{Zi&eLLU8gutlLtaLpndvX$Opo7qy*6}yITzUqB-uVZ~k z19YGHL(@KW+yVuA66c6uuKS-|sri_RCIVinZ{Dx5R%ZJYM-I`-Xbr? zfA*@dDVR1`?~B=RAEi0zK4yueo|&>(!))7k@Q*RQM@m34c%AuELifX7wAr;`?}WeX62apWO!TsSjt}4ada}cbneTuX(p)% z4Yp^pp;@B`!W|yJ6EFBh;mGBVA^1(wx^dTgR5?RVCXv=Thg~Yj>8GZb%cP^T6OZgD2#kPu}wS!$Q6Kc8daKDui5*=*Vw#BY3r+3z45by)36 zEpI*sLzdeje0&QfsRKF^kh+X68&8#%@u|@>$G(Kw-V>a_AjwxOK2)=k^su^75)p}& z4M#TX>b0yk#S^5yRkP=t0S&s8!|o$3!VS0 zV3l>lu3n5rYO1-D`Qv+Iwp9n$hOH-5WZN#eRWVDZZ{K$(PLe=>{h`JvMR$pQgU@6A zu}!=eD*; zY~DFGKgIpx1+Jdr4qnpz#D$Y9ru($=w(@Zk6(NA-UJ)(R4*PO=CGsU0tfcdfQA%h3`Y!}wnCX+}SlcH!PdOy|TuPfrcPy@a;4WU(AfyuWJ-n?1i zmvmf>k{T|*VRRn0cOInzJ5d~7xh4*x8JmvAU`v8U-(S_<`|y?8+hKdG)OOwZ?VWdA zpMOWuQ^b0<8~M$)1~%osrUCjl<*)dG_hbuySf`r8!)8kJ1yeTi;LWa?`@3x}uH(;K zed;nbtQKZ()-HW->r!@*gOH^B4%!9^wIjzbH%C$rv?m!OI0l4OwR`VIM{9)6Z-0rl z)rV0M7m4Y$z=){$Z_09>xA$m$Dmdd9Nxxh)%0v_r7W)x z9&k{|kQKUG9n@Tb)joZ9An&as+3a1+zO|mjuEeT~5oBXJk)PSUy$LK@Pu?|0@0Gh= zA#NPythAY5OC~@L>RY`KsW>@+sSzJPOT4ivtedOb4C}FUma5h*>sH+lp~CU8I+2M6 zPszF|s|c1KFs32xj-=}bK{#&e;wL#3r7B#PqK!VB1yX#ujpBhAw(0VWtu3o>kB50r zo3wHF!X4m@X$O1y>v-p2q}wh%tpXPWhb}|PSKrqcEDvC)A7)rx*5K!{t|jeqRca&t zDYHzkva@n#52}H#pb7Pe?1INWpyZ8$9l9AEGV@qyk#G;I*A-Ie6)I>ClA+dgYHx1CF5I58?O;@J;{?a?S2}IzmXe^ zFU>1J8PFqvZ$Vl*%46z#s#(9Vkc;|u$I#XeJvC2>;7WE0R-SNqAQi(Cu#vzfm;=qL zm`-rR*q)GC1YUa}MJr8U&!oibw=4QBE>dp8(1~r#;O;l|+s-dCN$=v|74E-`m%I+L-z%VOb1zLp0L|rrsYwNSOY>ZEUSj-9FqYL$#!j98@gK-k0HJlYCv-uk3$-F>{2k1pV`U~Jo=vvyu=L5cnZ%PjIGQ>7+r?oJOnbh@ za3Y$#u;AuSEXd2kx!$488aVMhnB%pP`;;fL6`nay+}9#Qtbpj82dcarRf4U7 zZ7V{@d7EetdS~%Ef{gtg8mmsOouFugiQ)%|yNyDSh7@AsOs8Y^W z^v#7M|A%j zctW+Y^Q=W#Q$xoAuDSFCpDy35`*NJF0DXm5bLM)Tun)Xyt>~I~)1Y(L!#-7q+(sLbA^Q zRIYvzbtcJ)VaJu?+`r3ScQ_>3b35J7+HE*Mw&$qR$~x^YEQ~ zT@p`TZi=GZvFbKRwMr}$P#^M}PRR2W3XG_y;Qg}ni0~avt4A^dMsTSWfL&oFa6I|2anmRX@w15+dlqZK zc_yNk#BJgmMNRei)8eDocE%m73aWBxMw5z2rRL7~eeqg1p8ufBP5(k`UUR4>~ZhnE9Qs9SW%87zPr4d%`Lq791pQT&4yZnAp| z*do6{%3-qdn$)1IV8g4?J^X=7q1$DMhjc`@g#;{O>~eo1fNhtm6|;%A%ZK#>=0M^ck&2K$hk z$_KECM_0`U(IL8D6s!pJdFPV3(Y|WF=i>{XXNLz{;GM0CtLZ&ysqH=&r0kCH>(J(X zb4vi(#I-j8^|5VlmY>qBU`IBi>~6&FJnUXWd+)KAjWc(b+wpBYU~W{hIEfy0U2fqS z2F}ZSF0773^?B{1`FxQl((Uc+=PiC*pfqH&6PK@M{7el8$J5_n$tX>|cPxpuI5X*J zOjhMT-Q{kqctT{6-29`sb{xtDalR0x-T7!Kc43ROpqRdA?WX~O zV!Jyfuj}4r5-E*HZ5njdXS%r%+c!_%hSRt6Z#O1;nyyN%meu95Zn{5qo zDw3;UkD@%T@Cf;iS5?tOeO4*OI+nk4 zP~X*|pun6vsX*T1g9(FGsD%2?;F>B~3v|PPwYWbG2JyvuP}IH#&Fxvl601R6Dtv-3 zl&%%)#D)$7=O(y^9nhfJyT1?flLMpZNyrBi2xJ@KQsZo$29O1jo7GcHWD)nGHra_P zEfCK5G5;h=Bcn@6yF!0FhZkClic<)`Tlje;|Hi0l*$GZkw2tb*AV9`Oberv)*O^36 zw-Dprl>iyvH-0q_I3GVPalt@szSvWVGVnZQBm>uqc?>9fWD{vS?UM}sIEKgxayFi8 z^ZV<=)D$hFMfdk_%)IFE-fE}^dxrj?7g3mPb4^6%$W%Qsit(Ww##C;neus)l1|SXH zauJ{e%f*^makT=9G^i3N^S-wp=Ulu)OwI`LYbi89B%@6ha(Y5bDYiz$PMYYB3vfqK z5cYgjka!iU^4Z3tiZY(ZoLYDW`Q1tU3SDUTP`4bpcy9Q9o zC#~%_u=5YxXslq<$Kt$#2*91F55pEcBw~X%GEbtBEbb`Ra5adTW-BrY_eNpY`0SoP z%X^@Ft~!eL!@wlu(VDWh;FxWF3AtE6zAsyjaPI)haI3YYC03QHgso{E} zL6rdja6!C`SK7#c7rg;=LJGpB--kUY|IivhBt#`dr4R(Lp#IB;egeU#*7SquVqXKNHGQbiz@&?^g>NaK`PFKNW5<*e?a=%<^Q&0h!Vb&{1qJ&DpBtv5`eUsgc|H%#tE_mz4Qy`-(AFMZDIy8a+3!^ zOf)QPOh^C#64L+W19@rr3br*NL3sl);`*#12Oy#NUrzrZQc}?n5&VD5Lh`>@oJ|}Y zEx@)UNc@~UoZJ8s*8k<Ub2mo|BapSqzw`b#S6dTDh>0<$#b1$lc(K#{ zEBwEsdok0A^8BaqKOz5(_HUq<7yG~E#{2(MZh!Iqw;=vA(Epv=%X`WHZv0Qkf1~{? zH{SoujqiVQ`)f)%LO?cl{~rzAU)29Cj{l7H-!u6uO1}TM>G64a>3IQ2sExojoPSMV zLk9~ZE60BxmX=^cMF+5xosuyL12>N;4~R!lfJ0b_*Mx(|h?}1SB)})YAtC}Y6yX*T zF%jhB0gzDrR~{p4rluF83BPJM!W(d!Y^KyCbm`d0j}IUm830g|`N;Es0JrI+Y5)KL literal 0 HcmV?d00001 diff --git a/tests/integration/test_storage_iceberg_with_spark/test_deletion_vectors.py b/tests/integration/test_storage_iceberg_with_spark/test_deletion_vectors.py index b6b42786c07a..1d5fd3e64d63 100644 --- a/tests/integration/test_storage_iceberg_with_spark/test_deletion_vectors.py +++ b/tests/integration/test_storage_iceberg_with_spark/test_deletion_vectors.py @@ -1,3 +1,5 @@ +import json +import os import uuid import pytest @@ -40,6 +42,223 @@ def _strip_file_uri_scheme(path): return path +PUFFIN_MAGIC = b"PFA1" +PUFFIN_FOOTER_TRAILER_SIZE = 12 +ICEBERG_WAREHOUSE = "/var/lib/clickhouse/user_files/iceberg_data" + + +def _avro_long(value): + if value is None: + return None + if isinstance(value, dict): + if "long" in value: + return int(value["long"]) + if not value: + return None + return int(next(iter(value.values()))) + return int(value) + + +def _read_avro_file(path): + import avro.datafile + import avro.io + + with open(path, "rb") as f: + reader = avro.datafile.DataFileReader(f, avro.io.DatumReader()) + schema = reader.datum_reader.writers_schema + meta = dict(reader.meta) + codec = reader.codec + records = list(reader) + reader.close() + return records, schema, meta, codec + + +def _write_avro_file(path, records, schema, meta, codec): + import avro.datafile + import avro.io + + with open(path, "wb") as f: + writer = avro.datafile.DataFileWriter(f, avro.io.DatumWriter(), schema, codec=codec) + for key, value in meta.items(): + if key not in ("avro.schema", "avro.codec"): + writer.set_meta(key, value) + for record in records: + writer.append(record) + writer.flush() + writer.close() + + +def _resolve_under_table(table_dir, path): + path = _strip_file_uri_scheme(path) + if os.path.isfile(path): + return path + joined = os.path.join(table_dir, path.lstrip("/")) + if os.path.isfile(joined): + return joined + raise RuntimeError(f"Cannot resolve Iceberg path '{path}' under '{table_dir}'") + + +def _current_metadata(table_dir): + metadata_dir = os.path.join(table_dir, "metadata") + hint_path = os.path.join(metadata_dir, "version-hint.text") + if os.path.isfile(hint_path): + with open(hint_path, encoding="utf-8") as f: + version = f.read().strip() + if version.startswith("v"): + version = version[1:] + path = os.path.join(metadata_dir, f"v{version}.metadata.json") + if os.path.isfile(path): + with open(path, encoding="utf-8") as f: + return json.load(f) + + candidates = [ + os.path.join(metadata_dir, name) + for name in os.listdir(metadata_dir) + if name.endswith(".metadata.json") + ] + if not candidates: + raise RuntimeError(f"No metadata JSON under {metadata_dir}") + + best = None + best_ts = -1 + for path in candidates: + with open(path, encoding="utf-8") as f: + meta = json.load(f) + ts = int(meta.get("last-updated-ms") or 0) + if ts >= best_ts: + best = meta + best_ts = ts + return best + + +def _puffin_to_delta_bin(data): + """Extract deletion-vector-v1 blobs from a Puffin file into a Delta `.bin` envelope. + + Databricks UniForm / Delta DV objects are `version=1` plus the same + `deletion-vector-v1` bytes Iceberg stores inside Puffin. Manifest + `content_offset` for a single blob becomes 1. + """ + if len(data) < 16 or data[:4] != PUFFIN_MAGIC or data[-4:] != PUFFIN_MAGIC: + raise ValueError("not a Puffin file") + + footer_length = int.from_bytes(data[-12:-8], "little", signed=True) + flags = data[-8:-4] + if flags[0] & 0x01: + raise ValueError("compressed Puffin footers are not supported by this test helper") + if footer_length <= 0: + raise ValueError(f"invalid Puffin footer length {footer_length}") + + payload_start = len(data) - PUFFIN_FOOTER_TRAILER_SIZE - footer_length + if payload_start < 8 or data[payload_start - 4 : payload_start] != PUFFIN_MAGIC: + raise ValueError("invalid Puffin footer magic") + + footer = json.loads(data[payload_start : payload_start + footer_length]) + out = bytearray([0x01]) + offset_map = {} + for blob in footer.get("blobs", []): + if blob.get("type") != "deletion-vector-v1": + continue + old_offset = int(blob["offset"]) + length = int(blob["length"]) + slice_bytes = data[old_offset : old_offset + length] + if len(slice_bytes) != length: + raise ValueError(f"truncated Puffin blob at offset {old_offset}") + offset_map[old_offset] = len(out) + out.extend(slice_bytes) + + if not offset_map: + raise ValueError("Puffin file has no deletion-vector-v1 blobs") + return bytes(out), offset_map + + +def convert_spark_puffin_dvs_to_delta_bin(table_name): + """Rewrite Spark Iceberg v3 Puffin DVs as Databricks-style `.bin` files. + + Spark `DELETE` on format-version 3 writes `PFA1` Puffin. Databricks UniForm + writes `deletion_vector_.bin` (version byte + envelope) and still + stores `content_offset` / `content_size_in_bytes` on the Iceberg delete + manifest. Overwrite each referenced Puffin in place and retarget offsets + so ClickHouse must take the slice-only path (skip the Puffin footer). + """ + table_dir = os.path.join(ICEBERG_WAREHOUSE, "default", table_name) + metadata = _current_metadata(table_dir) + snapshot_id = metadata.get("current-snapshot-id") + snapshot = next((s for s in metadata.get("snapshots", []) if s.get("snapshot-id") == snapshot_id), None) + if not snapshot: + raise RuntimeError(f"Snapshot {snapshot_id} not found for {table_name}") + + manifest_list_path = _resolve_under_table(table_dir, snapshot["manifest-list"]) + manifest_list_records, _, _, _ = _read_avro_file(manifest_list_path) + manifest_paths = [ + _resolve_under_table(table_dir, record["manifest_path"]) + for record in manifest_list_records + if "manifest_path" in record + ] + + dv_paths = set() + for manifest_path in manifest_paths: + records, _, _, _ = _read_avro_file(manifest_path) + for record in records: + data_file = record.get("data_file") or {} + if _avro_long(data_file.get("content_offset")) is None: + continue + if _avro_long(data_file.get("content_size_in_bytes")) is None: + continue + dv_paths.add(_resolve_under_table(table_dir, data_file["file_path"])) + + if not dv_paths: + raise RuntimeError(f"No Iceberg deletion-vector entries found under {table_dir}") + + conversions = {} + for path in sorted(dv_paths): + with open(path, "rb") as f: + original = f.read() + if original[:4] != PUFFIN_MAGIC: + raise RuntimeError(f"Expected Puffin magic in '{path}', got {original[:4]!r}") + new_bytes, offset_map = _puffin_to_delta_bin(original) + if new_bytes[:4] == PUFFIN_MAGIC: + raise RuntimeError(f"Converted DV '{path}' still starts with Puffin magic") + with open(path, "wb") as f: + f.write(new_bytes) + conversions[path] = (offset_map, len(new_bytes)) + + patched_entries = 0 + for manifest_path in manifest_paths: + records, schema, meta, codec = _read_avro_file(manifest_path) + changed = False + for record in records: + data_file = record.get("data_file") + if not data_file: + continue + content_offset = _avro_long(data_file.get("content_offset")) + content_size = _avro_long(data_file.get("content_size_in_bytes")) + if content_offset is None or content_size is None: + continue + local_path = _resolve_under_table(table_dir, data_file["file_path"]) + if local_path not in conversions: + continue + offset_map, new_size = conversions[local_path] + if content_offset not in offset_map: + raise RuntimeError( + f"content_offset {content_offset} missing from converted Puffin '{local_path}'" + ) + new_offset = offset_map[content_offset] + if new_offset + content_size > new_size: + raise RuntimeError( + f"Delta .bin slice [{new_offset}, {new_offset + content_size}) exceeds file size {new_size}" + ) + data_file["content_offset"] = new_offset + data_file["file_size_in_bytes"] = new_size + changed = True + patched_entries += 1 + if changed: + _write_avro_file(manifest_path, records, schema, meta, codec) + + if patched_entries == 0: + raise RuntimeError(f"Failed to patch deletion-vector manifests for {table_name}") + return len(conversions) + + def add_equality_deletes_by_id(spark, table_name, ids): """Commit an Iceberg equality-delete file for the given `id` values. @@ -143,6 +362,68 @@ def test_deletion_vectors(started_cluster_iceberg_with_spark, storage_type, run_ ] +@pytest.mark.parametrize("run_on_cluster", [False, True]) +@pytest.mark.parametrize("storage_type", ["s3", "azure", "local"]) +def test_deletion_vectors_delta_bin(started_cluster_iceberg_with_spark, storage_type, run_on_cluster): + """Iceberg v3 DVs stored as Delta `.bin` files (Databricks UniForm), not Puffin. + + Spark Iceberg DELETE writes `PFA1` Puffin. Convert those objects to the + Databricks layout (`0x01` + deletion-vector-v1 envelope, `content_offset=1`) + so ClickHouse must skip the Puffin footer and decode the slice in place. + """ + if storage_type == "local" and run_on_cluster: + pytest.skip("Local storage with cluster execution is not supported") + + instance = started_cluster_iceberg_with_spark.instances["node1"] + spark = started_cluster_iceberg_with_spark.spark_session + table_name = "test_deletion_vectors_delta_bin_" + storage_type + "_" + get_uuid_str() + deleted_ids = [2, 5, 7, 100] + + spark.sql( + f""" + CREATE TABLE {table_name} (id bigint) USING iceberg + TBLPROPERTIES ( + 'format-version' = '3', + 'write.delete.mode' = 'merge-on-read', + 'write.update.mode' = 'merge-on-read', + 'write.merge.mode' = 'merge-on-read' + ) + """ + ) + spark.sql(f"INSERT INTO {table_name} SELECT id FROM range(0, 200)") + spark.sql( + f"DELETE FROM {table_name} WHERE id IN ({', '.join(str(x) for x in deleted_ids)})" + ) + + converted = convert_spark_puffin_dvs_to_delta_bin(table_name) + assert converted >= 1 + + upload_table(started_cluster_iceberg_with_spark, storage_type, table_name) + + expression = get_creation_expression( + storage_type, + table_name, + started_cluster_iceberg_with_spark, + run_on_cluster=run_on_cluster, + table_function=True, + ) + + expected = [x for x in range(200) if x not in deleted_ids] + settings = {"use_iceberg_metadata_files_cache": 0, "use_puffin_files_cache": 0} + + assert int(instance.query(f"SELECT count() FROM {expression}", settings=settings)) == len(expected) + assert get_array(instance.query(f"SELECT id FROM {expression}", settings=settings)) == expected + assert ( + int( + instance.query( + f"SELECT count() FROM {expression}", + settings={**settings, "optimize_trivial_count_query": 1}, + ) + ) + == len(expected) + ) + + @pytest.mark.parametrize("run_on_cluster", [False, True]) @pytest.mark.parametrize("storage_type", ["s3", "azure", "local"]) def test_deletion_vectors_aggregates(started_cluster_iceberg_with_spark, storage_type, run_on_cluster): diff --git a/tests/integration/test_storage_iceberg_with_spark/test_deletion_vectors_databricks_uniform.py b/tests/integration/test_storage_iceberg_with_spark/test_deletion_vectors_databricks_uniform.py new file mode 100644 index 000000000000..76baa2d11826 --- /dev/null +++ b/tests/integration/test_storage_iceberg_with_spark/test_deletion_vectors_databricks_uniform.py @@ -0,0 +1,180 @@ +import gzip +import json +import os +import shutil + +import pytest + +from helpers.iceberg_utils import ( + default_upload_directory, + get_creation_expression, + get_uuid_str, +) + + +DATABRICKS_TABLE_PREFIX = ( + "s3://ttd-unity-useast/env=test/__unitystorage/catalogs/" + "c137d337-b630-4388-b072-9dc88ee567d6/tables/c771d2d4-eac8-401b-b26d-232b2b8fae5a" +) + +FIXTURE_DIR = os.path.join( + os.path.dirname(__file__), "data", "altinity_dv_puffin_repro_v3" +) + +DV_BIN_NAME = "deletion_vector_3d169d0e-b938-4854-8cca-d9d38949f1b0.bin" +EXPECTED_LIVE_IDS = list(range(1001, 10001)) + + +def get_int_array(query_result: str): + return [int(x) for x in query_result.strip().split("\n") if x] + + +def upload_table(cluster, storage_type, table_name): + default_upload_directory( + cluster, + storage_type, + f"/iceberg_data/default/{table_name}/", + f"/iceberg_data/default/{table_name}/", + ) + + +def _replace_strings(obj, old, new): + if isinstance(obj, str): + value = obj.replace(old, new) + value = value.replace("/_iceberg/metadata/", "/metadata/") + if value.endswith("/_iceberg"): + return value[: -len("/_iceberg")] + return value + if isinstance(obj, list): + return [_replace_strings(item, old, new) for item in obj] + if isinstance(obj, dict): + return {key: _replace_strings(value, old, new) for key, value in obj.items()} + return obj + + +def _rewrite_avro_paths(path, old, new): + import avro.datafile + import avro.io + + with open(path, "rb") as handle: + reader = avro.datafile.DataFileReader(handle, avro.io.DatumReader()) + schema = reader.datum_reader.writers_schema + meta = dict(reader.meta) + codec = reader.codec + records = [_replace_strings(record, old, new) for record in reader] + reader.close() + + with open(path, "wb") as handle: + writer = avro.datafile.DataFileWriter(handle, avro.io.DatumWriter(), schema, codec=codec) + for key, value in meta.items(): + if key not in ("avro.schema", "avro.codec"): + writer.set_meta(key, value) + for record in records: + writer.append(record) + writer.flush() + writer.close() + + +def prepare_databricks_uniform_table(dest_dir): + """Copy the UniForm fixture and rewrite Databricks URIs onto `dest_dir`. + + Iceberg `location` is `_iceberg` while data files live on the parent table root. + Flatten metadata to `metadata/` and set `location` to the parent so + `IcebergPathResolver` can map both data files and the `.bin` DV. + """ + if os.path.exists(dest_dir): + shutil.rmtree(dest_dir) + os.makedirs(dest_dir) + + shutil.copy2(os.path.join(FIXTURE_DIR, DV_BIN_NAME), os.path.join(dest_dir, DV_BIN_NAME)) + shutil.copytree(os.path.join(FIXTURE_DIR, "jk"), os.path.join(dest_dir, "jk")) + shutil.copytree( + os.path.join(FIXTURE_DIR, "_iceberg", "metadata"), + os.path.join(dest_dir, "metadata"), + ) + + with open(os.path.join(dest_dir, DV_BIN_NAME), "rb") as handle: + header = handle.read(4) + assert header[:1] == b"\x01", header + assert header != b"PFA1", header + + new_prefix = dest_dir.rstrip("/") + metadata_dir = os.path.join(dest_dir, "metadata") + for name in os.listdir(metadata_dir): + path = os.path.join(metadata_dir, name) + if name.endswith(".gz.metadata.json"): + with gzip.open(path, "rt", encoding="utf-8") as handle: + text = handle.read() + text = text.replace(DATABRICKS_TABLE_PREFIX, new_prefix) + text = text.replace("/_iceberg/metadata/", "/metadata/") + metadata = json.loads(text) + location = metadata.get("location", "") + if location.endswith("/_iceberg"): + metadata["location"] = location[: -len("/_iceberg")] + with gzip.open(path, "wt", encoding="utf-8") as handle: + json.dump(metadata, handle, separators=(",", ":")) + elif name.endswith(".avro"): + _rewrite_avro_paths(path, DATABRICKS_TABLE_PREFIX, new_prefix) + + return dest_dir + + +@pytest.mark.parametrize("run_on_cluster", [False, True]) +@pytest.mark.parametrize("storage_type", ["s3", "azure", "local"]) +def test_deletion_vectors_databricks_uniform_bin( + started_cluster_iceberg_with_spark, storage_type, run_on_cluster +): + """Read a Databricks UniForm Iceberg v3 table whose DVs are Delta `.bin` files. + + Spark Iceberg writes Puffin; this fixture is the customer layout: `file_format=PUFFIN`, + `content_offset=1`, object bytes `0x01` + deletion-vector-v1 envelope. + """ + if storage_type == "local" and run_on_cluster: + pytest.skip("Local storage with cluster execution is not supported") + + instance = started_cluster_iceberg_with_spark.instances["node1"] + table_name = "altinity_dv_puffin_repro_v3_" + storage_type + "_" + get_uuid_str() + dest_dir = f"/var/lib/clickhouse/user_files/iceberg_data/default/{table_name}" + prepare_databricks_uniform_table(dest_dir) + upload_table(started_cluster_iceberg_with_spark, storage_type, table_name) + + expression = get_creation_expression( + storage_type, + table_name, + started_cluster_iceberg_with_spark, + run_on_cluster=run_on_cluster, + table_function=True, + format_version=3, + ) + settings = {"use_iceberg_metadata_files_cache": 0, "use_puffin_files_cache": 0} + + assert int(instance.query(f"SELECT count() FROM {expression}", settings=settings)) == 9000 + assert ( + int(instance.query(f"SELECT min(toInt64(Id)) FROM {expression}", settings=settings)) + == 1001 + ) + assert ( + int(instance.query(f"SELECT max(toInt64(Id)) FROM {expression}", settings=settings)) + == 10000 + ) + assert ( + int( + instance.query( + f"SELECT countIf(toInt64(Id) <= 1000) FROM {expression}", + settings=settings, + ) + ) + == 0 + ) + assert get_int_array( + instance.query(f"SELECT toInt64(Id) FROM {expression} ORDER BY toInt64(Id)", settings=settings) + ) == EXPECTED_LIVE_IDS + assert ( + int( + instance.query( + f"SELECT count() FROM {expression}", + settings={**settings, "optimize_trivial_count_query": 1}, + ) + ) + == 9000 + ) From 9c7da1c86cb06ea60e17166c8f8650fd5473fc61 Mon Sep 17 00:00:00 2001 From: Andrey Zvonov <32552679+zvonand@users.noreply.github.com> Date: Tue, 15 Sep 2026 21:51:06 +0200 Subject: [PATCH 4/5] Resolve conflicts in cherry-pick of #2271 The legacy Docusaurus doc docs/en/sql-reference/table-functions/icebergCluster.md was removed on antalya-26.8 (commit 9440ca729a3, new layout docs/reference/...mdx), so the cherry-pick re-added it as a modify/delete conflict. Applied the PR's one-line deletion-vector wording change to the current file docs/reference/functions/table-functions/icebergCluster.mdx instead and dropped the re-added legacy copy. Source-PR: #2271 (https://github.com/Altinity/ClickHouse/pull/2271) --- .../table-functions/icebergCluster.md | 136 ------------------ .../table-functions/icebergCluster.mdx | 2 +- .../tests/gtest_iceberg_count_shortcuts.cpp | 1 + .../gtest_puffin_dv_referenced_data_file.cpp | 1 + .../gtest_puffin_files_cache_metrics.cpp | 36 ++--- .../tests/gtest_puffin_footer_cache.cpp | 4 +- 6 files changed, 23 insertions(+), 157 deletions(-) delete mode 100644 docs/en/sql-reference/table-functions/icebergCluster.md diff --git a/docs/en/sql-reference/table-functions/icebergCluster.md b/docs/en/sql-reference/table-functions/icebergCluster.md deleted file mode 100644 index 72577f140e71..000000000000 --- a/docs/en/sql-reference/table-functions/icebergCluster.md +++ /dev/null @@ -1,136 +0,0 @@ ---- -description: 'An extension to the iceberg table function which allows processing files - from Apache Iceberg in parallel from many nodes in a specified cluster.' -sidebar_label: 'icebergCluster' -sidebar_position: 91 -slug: /sql-reference/table-functions/icebergCluster -title: 'icebergCluster' -doc_type: 'reference' ---- - -This is an extension to the [iceberg](/sql-reference/table-functions/iceberg.md) table function. - -Allows processing files from Apache [Iceberg](https://iceberg.apache.org/) in parallel from many nodes in a specified cluster. On initiator it creates a connection to all nodes in the cluster and dispatches each file dynamically. On the worker node it asks the initiator about the next task to process and processes it. This is repeated until all tasks are finished. - -## Deletion vectors on cluster reads {#deletion-vectors-cluster} - -Iceberg v3 [deletion vectors](https://iceberg.apache.org/spec/#deletion-vectors) are loaded on the **initiator** while it distributes tasks: for each data file the initiator reads the Puffin or Delta `.bin` object, validates the `deletion-vector-v1` envelope, materializes deleted row positions, and attaches the bitmap to the task sent to workers. Workers apply that bitmap when reading Parquet; they do not fetch or parse the deletion-vector file again for that path. See [Processing of tables with deleted rows](/sql-reference/table-functions/iceberg.md#deleted-rows) for format limits and caching. - -On wide tables with many deletion vectors, initiator-side decode and per-task bitmap serialization can become a bottleneck even when Parquet reads are well parallelized across the cluster. - -## Syntax {#syntax} - -```sql -icebergS3Cluster(cluster_name, url [, NOSIGN | access_key_id, secret_access_key, [session_token]] [,format] [,compression_method] [,extra_credentials]) -icebergS3Cluster(cluster_name, named_collection[, option=value [,..]]) - -icebergAzureCluster(cluster_name, connection_string|storage_account_url, container_name, blobpath, [,account_name], [,account_key] [,format] [,compression_method]) -icebergAzureCluster(cluster_name, named_collection[, option=value [,..]]) - -icebergHDFSCluster(cluster_name, path_to_table, [,format] [,compression_method]) -icebergHDFSCluster(cluster_name, named_collection[, option=value [,..]]) -``` - -## Arguments {#arguments} - -- `cluster_name` — Name of a cluster that is used to build a set of addresses and connection parameters to remote and local servers. -- Description of all other arguments coincides with description of arguments in equivalent [iceberg](/sql-reference/table-functions/iceberg.md) table function. -- An optional `extra_credentials` parameter can be used to pass a `role_arn` for role-based access in ClickHouse Cloud. See [Secure S3](/cloud/data-sources/secure-s3) for configuration steps. - -**Returned value** - -A table with the specified structure for reading data from cluster in the specified Iceberg table. - -**Examples** - -```sql -SELECT * FROM icebergS3Cluster('cluster_simple', 'http://test.s3.amazonaws.com/clickhouse-bucket/test_table', 'test', 'test') -``` - -## Virtual Columns {#virtual-columns} - -- `_path` — Path to the file. Type: `LowCardinality(String)`. -- `_file` — Name of the file. Type: `LowCardinality(String)`. -- `_size` — Size of the file in bytes. Type: `Nullable(UInt64)`. If the file size is unknown, the value is `NULL`. -- `_time` — Last modified time of the file. Type: `Nullable(DateTime)`. If the time is unknown, the value is `NULL`. -- `_etag` — The etag of the file. Type: `LowCardinality(String)`. If the etag is unknown, the value is `NULL`. - -## Altinity Antalya branch - -### `icebergLocalCluster` table function - -Only in the Altinity Antalya branch, `icebergLocalCluster` designed to make distributed cluster queries when Iceberg data is stored on shared network storage mounted with a local path. The path must be identical on all replicas. - -```sql -icebergLocalCluster(cluster_name, path_to_table, [,format] [,compression_method]) -``` - -### Specify storage type in function arguments - -Only in the Altinity Antalya branch, the `icebergCluster` table function supports all storage backends. The storage backend can be specified using the named argument `storage_type`. Valid values include `s3`, `azure`, `hdfs`, and `local`. - -```sql -icebergCluster(storage_type='s3', cluster_name, url [, NOSIGN | access_key_id, secret_access_key, [session_token]] [,format] [,compression_method]) - -icebergCluster(storage_type='azure', cluster_name, connection_string|storage_account_url, container_name, blobpath, [,account_name], [,account_key] [,format] [,compression_method]) - -icebergCluster(storage_type='hdfs', cluster_name, path_to_table, [,format] [,compression_method]) - -icebergCluster(storage_type='local', cluster_name, path_to_table, [,format] [,compression_method]) -``` - -### Specify storage type in a named collection - -Only in the Altinity Antalya branch, `storage_type` can be part of a named collection. - -```xml - - - - http://test.s3.amazonaws.com/clickhouse-bucket/ - test - test - auto - auto - s3 - - - -``` - -```sql -icebergCluster(iceberg_conf[, option=value [,..]]) -``` - -The default value for `storage_type` is `s3`. - -### `object_storage_cluster` setting. - -Only in the Altinity Antalya branch, an alternative syntax for `icebergCluster` table function is available. This allows the `iceberg` function to be used with the non-empty `object_storage_cluster` setting, specifying a cluster name. This enables distributed queries over Iceberg table across a ClickHouse cluster. - -```sql -icebergS3(url [, NOSIGN | access_key_id, secret_access_key, [session_token]] [,format] [,compression_method]) SETTINGS object_storage_cluster='cluster_name' - -icebergAzure(connection_string|storage_account_url, container_name, blobpath, [,account_name], [,account_key] [,format] [,compression_method]) SETTINGS object_storage_cluster='cluster_name' - -icebergHDFS(path_to_table, [,format] [,compression_method]) SETTINGS object_storage_cluster='cluster_name' - -icebergLocal(path_to_table, [,format] [,compression_method]) SETTINGS object_storage_cluster='cluster_name' - -icebergS3(option=value [,..]) SETTINGS object_storage_cluster='cluster_name' - -iceberg(storage_type='s3', url [, NOSIGN | access_key_id, secret_access_key, [session_token]] [,format] [,compression_method]) SETTINGS object_storage_cluster='cluster_name' - -iceberg(storage_type='azure', connection_string|storage_account_url, container_name, blobpath, [,account_name], [,account_key] [,format] [,compression_method]) SETTINGS object_storage_cluster='cluster_name' - -iceberg(storage_type='hdfs', path_to_table, [,format] [,compression_method]) SETTINGS object_storage_cluster='cluster_name' - -iceberg(storage_type='local', path_to_table, [,format] [,compression_method]) SETTINGS object_storage_cluster='cluster_name' - -iceberg(iceberg_conf[, option=value [,..]]) SETTINGS object_storage_cluster='cluster_name' -``` - -**See Also** - -- [Iceberg engine](/engines/table-engines/integrations/iceberg.md) -- [Iceberg table function](/sql-reference/table-functions/iceberg.md) diff --git a/docs/reference/functions/table-functions/icebergCluster.mdx b/docs/reference/functions/table-functions/icebergCluster.mdx index 72be387d8ce4..453bda2dc1e1 100644 --- a/docs/reference/functions/table-functions/icebergCluster.mdx +++ b/docs/reference/functions/table-functions/icebergCluster.mdx @@ -14,7 +14,7 @@ Allows processing files from Apache [Iceberg](https://iceberg.apache.org/) in pa ## Deletion vectors on cluster reads {#deletion-vectors-cluster} -Iceberg v3 [deletion vectors](https://iceberg.apache.org/spec/#deletion-vectors) are loaded on the **initiator** while it distributes tasks: for each data file the initiator reads the Puffin blob, validates it, materializes deleted row positions, and attaches the bitmap to the task sent to workers. Workers apply that bitmap when reading Parquet; they do not fetch or parse the Puffin file again for that path. See [Processing of tables with deleted rows](/sql-reference/table-functions/iceberg.md#deleted-rows) for format limits and caching. +Iceberg v3 [deletion vectors](https://iceberg.apache.org/spec/#deletion-vectors) are loaded on the **initiator** while it distributes tasks: for each data file the initiator reads the Puffin or Delta `.bin` object, validates the `deletion-vector-v1` envelope, materializes deleted row positions, and attaches the bitmap to the task sent to workers. Workers apply that bitmap when reading Parquet; they do not fetch or parse the deletion-vector file again for that path. See [Processing of tables with deleted rows](/sql-reference/table-functions/iceberg.md#deleted-rows) for format limits and caching. On wide tables with many deletion vectors, initiator-side decode and per-task bitmap serialization can become a bottleneck even when Parquet reads are well parallelized across the cluster. diff --git a/src/Storages/ObjectStorage/DataLakes/Iceberg/tests/gtest_iceberg_count_shortcuts.cpp b/src/Storages/ObjectStorage/DataLakes/Iceberg/tests/gtest_iceberg_count_shortcuts.cpp index 1c1a894e8018..1e00063187ba 100644 --- a/src/Storages/ObjectStorage/DataLakes/Iceberg/tests/gtest_iceberg_count_shortcuts.cpp +++ b/src/Storages/ObjectStorage/DataLakes/Iceberg/tests/gtest_iceberg_count_shortcuts.cpp @@ -124,6 +124,7 @@ ProcessedManifestFileEntryPtr makeDataEntryForRecordCount( /*row_number=*/0, ManifestEntryStatus::ADDED, /*written_sequence_number=*/std::nullopt, + /*written_file_sequence_number=*/std::nullopt, /*written_snapshot_id=*/std::nullopt, DB::Row{}, std::move(columns_infos), diff --git a/src/Storages/ObjectStorage/DataLakes/tests/gtest_puffin_dv_referenced_data_file.cpp b/src/Storages/ObjectStorage/DataLakes/tests/gtest_puffin_dv_referenced_data_file.cpp index 568ca89a0797..855a8ac25f7c 100644 --- a/src/Storages/ObjectStorage/DataLakes/tests/gtest_puffin_dv_referenced_data_file.cpp +++ b/src/Storages/ObjectStorage/DataLakes/tests/gtest_puffin_dv_referenced_data_file.cpp @@ -85,6 +85,7 @@ std::shared_ptr makePositionDeleteEntry( /*row_number=*/0, ManifestEntryStatus::ADDED, /*written_sequence_number=*/std::nullopt, + /*written_file_sequence_number=*/std::nullopt, /*written_snapshot_id=*/std::nullopt, DB::Row{}, std::unordered_map{}, diff --git a/src/Storages/ObjectStorage/DataLakes/tests/gtest_puffin_files_cache_metrics.cpp b/src/Storages/ObjectStorage/DataLakes/tests/gtest_puffin_files_cache_metrics.cpp index 91b730bf83d7..966869851857 100644 --- a/src/Storages/ObjectStorage/DataLakes/tests/gtest_puffin_files_cache_metrics.cpp +++ b/src/Storages/ObjectStorage/DataLakes/tests/gtest_puffin_files_cache_metrics.cpp @@ -40,8 +40,8 @@ TEST(PuffinFilesCacheMetrics, ClearDuringLoadCountsAsMissNotHit) ASSERT_TRUE(key.has_value()); auto & counters = CurrentThread::getProfileEvents(); - const auto hits_before = counters[ProfileEvents::PuffinFilesCacheHits].load(); - const auto misses_before = counters[ProfileEvents::PuffinFilesCacheMisses].load(); + const auto hits_before = counters[ProfileEvents::PuffinFilesCacheHits]; + const auto misses_before = counters[ProfileEvents::PuffinFilesCacheMisses]; size_t load_calls = 0; const auto result = cache.getOrSetDeletionVector( @@ -58,8 +58,8 @@ TEST(PuffinFilesCacheMetrics, ClearDuringLoadCountsAsMissNotHit) ASSERT_EQ(load_calls, 1u); ASSERT_NE(result, nullptr); EXPECT_TRUE(result->rb_contains(1)); - EXPECT_EQ(counters[ProfileEvents::PuffinFilesCacheHits].load() - hits_before, 0u); - EXPECT_EQ(counters[ProfileEvents::PuffinFilesCacheMisses].load() - misses_before, 1u); + EXPECT_EQ(counters[ProfileEvents::PuffinFilesCacheHits] - hits_before, 0u); + EXPECT_EQ(counters[ProfileEvents::PuffinFilesCacheMisses] - misses_before, 1u); /// Entry was not inserted after clear; the next lookup must load again. const auto second = cache.getOrSetDeletionVector(*key, [&]() @@ -69,8 +69,8 @@ TEST(PuffinFilesCacheMetrics, ClearDuringLoadCountsAsMissNotHit) }); ASSERT_EQ(load_calls, 2u); ASSERT_NE(second, nullptr); - EXPECT_EQ(counters[ProfileEvents::PuffinFilesCacheMisses].load() - misses_before, 2u); - EXPECT_EQ(counters[ProfileEvents::PuffinFilesCacheHits].load() - hits_before, 0u); + EXPECT_EQ(counters[ProfileEvents::PuffinFilesCacheMisses] - misses_before, 2u); + EXPECT_EQ(counters[ProfileEvents::PuffinFilesCacheHits] - hits_before, 0u); } TEST(PuffinFilesCacheMetrics, WaiterOfClearDiscardedLoadCountsAsMiss) @@ -81,8 +81,8 @@ TEST(PuffinFilesCacheMetrics, WaiterOfClearDiscardedLoadCountsAsMiss) "Local:////test-prefix", "puffin.bin", "etag-waiter", 100, 200, "data/file-w.parquet", 1, 100); ASSERT_TRUE(key.has_value()); - const auto hits_before = ProfileEvents::global_counters[ProfileEvents::PuffinFilesCacheHits].load(); - const auto misses_before = ProfileEvents::global_counters[ProfileEvents::PuffinFilesCacheMisses].load(); + const auto hits_before = ProfileEvents::global_counters[ProfileEvents::PuffinFilesCacheHits]; + const auto misses_before = ProfileEvents::global_counters[ProfileEvents::PuffinFilesCacheMisses]; std::promise load_started; auto load_started_future = load_started.get_future(); @@ -151,8 +151,8 @@ TEST(PuffinFilesCacheMetrics, OrdinaryHitAndMissCounters) ASSERT_TRUE(key.has_value()); auto & counters = CurrentThread::getProfileEvents(); - const auto hits_before = counters[ProfileEvents::PuffinFilesCacheHits].load(); - const auto misses_before = counters[ProfileEvents::PuffinFilesCacheMisses].load(); + const auto hits_before = counters[ProfileEvents::PuffinFilesCacheHits]; + const auto misses_before = counters[ProfileEvents::PuffinFilesCacheMisses]; size_t load_calls = 0; auto load_fn = [&]() @@ -165,8 +165,8 @@ TEST(PuffinFilesCacheMetrics, OrdinaryHitAndMissCounters) ASSERT_NE(cache.getOrSetDeletionVector(*key, load_fn), nullptr); EXPECT_EQ(load_calls, 1u); - EXPECT_EQ(counters[ProfileEvents::PuffinFilesCacheMisses].load() - misses_before, 1u); - EXPECT_EQ(counters[ProfileEvents::PuffinFilesCacheHits].load() - hits_before, 1u); + EXPECT_EQ(counters[ProfileEvents::PuffinFilesCacheMisses] - misses_before, 1u); + EXPECT_EQ(counters[ProfileEvents::PuffinFilesCacheHits] - hits_before, 1u); } TEST(PuffinFilesCacheMetrics, HitRemainsHitWhenCacheClearedAfterLookup) @@ -187,17 +187,17 @@ TEST(PuffinFilesCacheMetrics, HitRemainsHitWhenCacheClearedAfterLookup) ASSERT_NE(cache.getOrSetDeletionVector(*key, load_fn), nullptr); - const auto hits_before = counters[ProfileEvents::PuffinFilesCacheHits].load(); - const auto misses_before = counters[ProfileEvents::PuffinFilesCacheMisses].load(); + const auto hits_before = counters[ProfileEvents::PuffinFilesCacheHits]; + const auto misses_before = counters[ProfileEvents::PuffinFilesCacheMisses]; ASSERT_NE(cache.getOrSetDeletionVector(*key, load_fn), nullptr); EXPECT_EQ(load_calls, 1u); - EXPECT_EQ(counters[ProfileEvents::PuffinFilesCacheHits].load() - hits_before, 1u); - EXPECT_EQ(counters[ProfileEvents::PuffinFilesCacheMisses].load() - misses_before, 0u); + EXPECT_EQ(counters[ProfileEvents::PuffinFilesCacheHits] - hits_before, 1u); + EXPECT_EQ(counters[ProfileEvents::PuffinFilesCacheMisses] - misses_before, 0u); /// Clearing after the hit must not rewrite the already-recorded hit as a miss. The old /// contains()-after-getOrSet path could race here with SYSTEM DROP PUFFIN FILES CACHE. cache.clear(); - EXPECT_EQ(counters[ProfileEvents::PuffinFilesCacheHits].load() - hits_before, 1u); - EXPECT_EQ(counters[ProfileEvents::PuffinFilesCacheMisses].load() - misses_before, 0u); + EXPECT_EQ(counters[ProfileEvents::PuffinFilesCacheHits] - hits_before, 1u); + EXPECT_EQ(counters[ProfileEvents::PuffinFilesCacheMisses] - misses_before, 0u); } diff --git a/src/Storages/ObjectStorage/DataLakes/tests/gtest_puffin_footer_cache.cpp b/src/Storages/ObjectStorage/DataLakes/tests/gtest_puffin_footer_cache.cpp index 5aea99489283..cd7ea18fd553 100644 --- a/src/Storages/ObjectStorage/DataLakes/tests/gtest_puffin_footer_cache.cpp +++ b/src/Storages/ObjectStorage/DataLakes/tests/gtest_puffin_footer_cache.cpp @@ -84,7 +84,7 @@ TEST(PuffinFooterMemo, CoalescedSlicesShareOneFooterParse) ASSERT_TRUE(key_b.has_value()); auto & counters = ProfileEvents::global_counters; - const auto files_read_before = counters[ProfileEvents::PuffinFilesRead].load(); + const auto files_read_before = counters[ProfileEvents::PuffinFilesRead]; size_t footer_loads = 0; auto load_footer = [&]() @@ -122,7 +122,7 @@ TEST(PuffinFooterMemo, CoalescedSlicesShareOneFooterParse) EXPECT_EQ(footer_loads, 1u); /// One footer parse (`PuffinFilesRead` in readPuffinFooter) plus two blob reads. - EXPECT_EQ(counters[ProfileEvents::PuffinFilesRead].load() - files_read_before, 3u); + EXPECT_EQ(counters[ProfileEvents::PuffinFilesRead] - files_read_before, 3u); } TEST(PuffinFooterMemo, ClearDropsFooterEntries) From 950ee7ac487015f5950bf11f4ade7a2aba3b6aa7 Mon Sep 17 00:00:00 2001 From: Andrey Zvonov <32552679+zvonand@users.noreply.github.com> Date: Fri, 18 Sep 2026 19:33:24 +0200 Subject: [PATCH 5/5] Update SettingsChangesHistory.cpp --- src/Core/SettingsChangesHistory.cpp | 21 ++++++++++++++++----- 1 file changed, 16 insertions(+), 5 deletions(-) diff --git a/src/Core/SettingsChangesHistory.cpp b/src/Core/SettingsChangesHistory.cpp index c18a487d7b07..b8002b360cd4 100644 --- a/src/Core/SettingsChangesHistory.cpp +++ b/src/Core/SettingsChangesHistory.cpp @@ -41,10 +41,6 @@ const VersionToSettingsChangesMap & getSettingsChangesHistory() /// controls new feature and it's 'true' by default, use 'false' as previous_value). /// It's used to implement `compatibility` setting (see https://github.com/ClickHouse/ClickHouse/issues/35972) /// Note: please check if the key already exists to prevent duplicate entries. - addSettingsChanges(settings_changes_history, "26.8.1.20001.altinityantalya", - { - {"use_puffin_files_cache", false, true, "Enables cache of parsed Puffin file content such as deletion vectors."}, - }); addSettingsChanges(settings_changes_history, "26.8", { {"allow_experimental_ai_functions", false, false, "The setting is obsolete, AI functions are beta now and enabled by default."}, @@ -220,7 +216,10 @@ const VersionToSettingsChangesMap & getSettingsChangesHistory() {"join_runtime_filter_size_from_hash_table_stats", false, true, "Use hash table size statistics collected from previous executions to size the JOIN runtime filter. When disabled, fall back to the fixed `join_runtime_bloom_filter_bytes`."}, {"statistics_max_set_size_for_exact_selectivity_estimation", 10000, 10000, "The bound on the cost of estimating the selectivity of `IN` with a large set is kept under `compatibility` with an earlier version: the previous value is deliberately equal to the new one, so that the uncapped estimation, which could add hundreds of milliseconds to the planning of a single query, is not restored."}, }); - + addSettingsChanges(settings_changes_history, "26.6.2.20001.altinityantalya", + { + {"use_puffin_files_cache", false, true, "Enables cache of parsed Puffin file content such as deletion vectors."}, + }); addSettingsChanges(settings_changes_history, "26.6", { {"analyzer_compatibility_apply_final_to_all_joined_tables", true, false, "Fixed a bug in the analyzer where FINAL on the left-most table of a JOIN was incorrectly applied to the other joined tables as well. previous_value=true so `compatibility` with versions before 26.6 restores the old behavior."}, @@ -380,6 +379,18 @@ const VersionToSettingsChangesMap & getSettingsChangesHistory() {"variant_throw_on_type_mismatch", true, true, "New setting to control type mismatch behavior in default Variant implementation"}, {"dynamic_throw_on_type_mismatch", true, true, "New setting to control type mismatch behavior in default Dynamic implementation"}, }); + addSettingsChanges(settings_changes_history, "26.3.1.20001.altinityantalya", + { + // {"object_storage_cluster_join_mode", "allow", "allow", "New setting"}, + // {"export_merge_tree_partition_task_timeout_seconds", "3600", "86400", "Increase default value to make it more realistic"}, + // {"export_merge_tree_part_allow_lossy_cast", false, false, "New setting to gate lossy casts in EXPORT PART/PARTITION behind explicit acknowledgment"}, + // {"export_merge_tree_part_schema_match_mode", "POSITION", "POSITION", "New setting to control how EXPORT PART/EXPORT PARTITION matches source columns to destination columns"}, + // {"export_merge_tree_part_ignore_extra_source_columns", false, false, "New setting to allow EXPORT PART/EXPORT PARTITION when the source table has columns absent from the destination"}, + // {"export_merge_tree_partition_retry_initial_backoff_seconds", 5, 5, "New setting for exponential back-off between failed part export retries in an export partition task"}, + // {"export_merge_tree_partition_retry_max_backoff_seconds", 300, 300, "New setting capping the exponential back-off between failed part export retries in an export partition task"}, + // {"export_merge_tree_partition_max_retries", 3, 3, "Obsolete and ignored: export partition tasks now retry retryable failures until the task timeout and fail immediately on non-retryable errors, instead of using a fixed retry budget"}, + // {"allow_experimental_database_s3_tables", false, false, "New setting to enable experimental database S3 tables (AWS Iceberg REST catalog)."}, + }); addSettingsChanges(settings_changes_history, "26.3", { {"defer_partition_pruning_after_final", false, true, "Gates the FINAL planner's unconditional skipping of partition pruning when the partition-key column is not in the sorting key. The behavior change itself shipped silently in 26.3 via https://github.com/ClickHouse/ClickHouse/pull/98242; this entry retroactively documents it so `compatibility = '26.2'` restores the pre-regression behavior (0 = prune before FINAL, fast; 1 = defer pruning, correctness-safe)."},