diff --git a/dev/dv-fixtures/generate_go_fixtures.go b/dev/dv-fixtures/generate_go_fixtures.go new file mode 100644 index 000000000..67d26d6c0 --- /dev/null +++ b/dev/dv-fixtures/generate_go_fixtures.go @@ -0,0 +1,152 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +//go:build ignore + +package main + +import ( + "bytes" + "fmt" + "os" + "path/filepath" + "strconv" + + "github.com/apache/iceberg-go/puffin" + "github.com/apache/iceberg-go/table/dv" +) + +type fixtureBlob struct { + referencedDataFile string + positions []uint64 + ranges []positionRange +} + +type positionRange struct { + start uint64 + end uint64 +} + +func writeFixture(outputDir, fileName, createdBy string, blobs []fixtureBlob) error { + var output bytes.Buffer + writer, err := puffin.NewWriter(&output) + if err != nil { + return err + } + if err := writer.SetCreatedBy(createdBy); err != nil { + return err + } + + for _, blob := range blobs { + bitmap := dv.NewRoaringPositionBitmap() + for _, position := range blob.positions { + bitmap.Set(position) + } + for _, positionRange := range blob.ranges { + bitmap.SetRange(positionRange.start, positionRange.end) + } + payload, err := dv.SerializeDV(bitmap) + if err != nil { + return err + } + _, err = writer.AddBlob(puffin.BlobMetadataInput{ + Type: puffin.BlobTypeDeletionVector, + SnapshotID: -1, + SequenceNumber: -1, + Fields: []int32{}, + Properties: map[string]string{ + "referenced-data-file": blob.referencedDataFile, + "cardinality": strconv.FormatInt(bitmap.Cardinality(), 10), + }, + }, payload) + if err != nil { + return err + } + } + + if err := writer.Finish(); err != nil { + return err + } + return os.WriteFile(filepath.Join(outputDir, fileName), output.Bytes(), 0o644) +} + +func main() { + if len(os.Args) != 2 { + fmt.Fprintln(os.Stderr, "usage: go run generate_go_fixtures.go OUTPUT_DIR") + os.Exit(2) + } + outputDir := os.Args[1] + if err := os.MkdirAll(outputDir, 0o755); err != nil { + panic(err) + } + + err := writeFixture(outputDir, "single-blob-dv.puffin", + "iceberg-go test fixture", []fixtureBlob{{ + referencedDataFile: "data/test.parquet", + positions: []uint64{1, 3, 5, 7, 9}, + }}) + if err != nil { + panic(err) + } + + err = writeFixture(outputDir, "multi-blob-dv.puffin", + "iceberg-go cross-language fixture", []fixtureBlob{ + { + referencedDataFile: "s3://warehouse/db/table/data/go-file-001.parquet", + positions: []uint64{ + 0, 100, 200, (uint64(1) << 32) + 7, + }, + }, + { + referencedDataFile: "s3://warehouse/db/table/data/go-file-002.parquet", + positions: []uint64{ + 50, 150, (uint64(2) << 32) + 9, + }, + }, + }) + if err != nil { + panic(err) + } + + position := func(bucket, container, value uint64) uint64 { + return (bucket << 32) + (container << 16) + value + } + allContainerPositions := []uint64{ + position(0, 0, 5), + position(0, 0, 7), + position(1, 0, 10), + position(1, 0, 20), + } + for bucket := uint64(0); bucket < 2; bucket++ { + for value := uint64(0); value < 10000; value += 2 { + allContainerPositions = + append(allContainerPositions, position(bucket, 2, value)) + } + } + err = writeFixture(outputDir, "all-container-types-dv.puffin", + "iceberg-go cross-language fixture", []fixtureBlob{{ + referencedDataFile: "s3://warehouse/db/table/data/all-containers.parquet", + positions: allContainerPositions, + ranges: []positionRange{ + {start: position(0, 1, 1), end: position(0, 1, 1000)}, + {start: position(1, 1, 10), end: position(1, 1, 500)}, + }, + }}) + if err != nil { + panic(err) + } +} diff --git a/src/iceberg/CMakeLists.txt b/src/iceberg/CMakeLists.txt index 8a98274ff..7782ba164 100644 --- a/src/iceberg/CMakeLists.txt +++ b/src/iceberg/CMakeLists.txt @@ -26,6 +26,7 @@ set(ICEBERG_SOURCES catalog/memory/in_memory_catalog.cc catalog/session_catalog.cc catalog/session_context.cc + compaction_planner.cc delete_file_index.cc deletes/dv_util.cc deletes/dv_writer.cc @@ -237,11 +238,13 @@ if(MSVC_TOOLCHAIN) endif() set(ICEBERG_DATA_SOURCES + data/compaction_executor.cc data/data_writer.cc data/delete_filter.cc data/delete_loader.cc data/equality_delete_writer.cc data/file_scan_task_reader.cc + data/position_delete_update.cc data/position_delete_writer.cc data/writer.cc) diff --git a/src/iceberg/arrow/arrow_io.cc b/src/iceberg/arrow/arrow_io.cc index 4c795badf..11f905da6 100644 --- a/src/iceberg/arrow/arrow_io.cc +++ b/src/iceberg/arrow/arrow_io.cc @@ -591,7 +591,14 @@ Result> ArrowFileSystemFileIO::NewOutputFile( /// \brief Delete a file at the given location. Status ArrowFileSystemFileIO::DeleteFile(const std::string& file_location) { ICEBERG_ASSIGN_OR_RAISE(auto path, ResolvePath(file_location)); - ICEBERG_ARROW_RETURN_NOT_OK(arrow_fs_->DeleteFile(path)); + auto status = arrow_fs_->DeleteFile(path); + if (!status.ok()) { + auto info = arrow_fs_->GetFileInfo(path); + if (info.ok() && info->type() == ::arrow::fs::FileType::NotFound) { + return {}; + } + } + ICEBERG_ARROW_RETURN_NOT_OK(status); return {}; } diff --git a/src/iceberg/compaction_planner.cc b/src/iceberg/compaction_planner.cc new file mode 100644 index 000000000..c7e963ef6 --- /dev/null +++ b/src/iceberg/compaction_planner.cc @@ -0,0 +1,392 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#include "iceberg/compaction_planner.h" + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "iceberg/expression/literal.h" +#include "iceberg/manifest/manifest_entry.h" +#include "iceberg/table_scan.h" +#include "iceberg/util/content_file_util.h" +#include "iceberg/util/data_file_set.h" +#include "iceberg/util/int128.h" +#include "iceberg/util/macros.h" + +namespace iceberg { +namespace { + +struct CanonicalPartitionKey { + int32_t spec_id; + std::string values; + + auto operator<=>(const CanonicalPartitionKey&) const = default; +}; + +struct Candidate { + CompactionFile file; + bool has_delete_pressure; +}; + +struct PartitionCandidates { + int32_t spec_id; + std::vector files; +}; + +enum class Comparison { + kLess, + kEqual, + kGreater, +}; + +Status ValidateConfig(const CompactionPlannerConfig& config) { + ICEBERG_PRECHECK(config.target_file_size_bytes > 0, + "target_file_size_bytes must be greater than zero"); + ICEBERG_PRECHECK(std::isfinite(config.min_file_size_ratio) && + config.min_file_size_ratio >= 0 && config.min_file_size_ratio <= 1, + "min_file_size_ratio must be finite and in [0, 1]"); + ICEBERG_PRECHECK( + std::isfinite(config.max_file_size_ratio) && config.max_file_size_ratio >= 1, + "max_file_size_ratio must be finite and at least 1"); + ICEBERG_PRECHECK(config.min_input_files > 0, + "min_input_files must be greater than zero"); + ICEBERG_PRECHECK(config.delete_file_threshold >= 0, + "delete_file_threshold must not be negative"); + ICEBERG_PRECHECK(std::isfinite(config.delete_ratio_threshold) && + config.delete_ratio_threshold >= 0 && + config.delete_ratio_threshold <= 1, + "delete_ratio_threshold must be finite and in [0, 1]"); + return {}; +} + +Result CheckedAddNonNegative(int64_t lhs, int64_t rhs, + std::string_view description) { + ICEBERG_PRECHECK(lhs >= 0 && rhs >= 0, "{} must not be negative", description); + ICEBERG_PRECHECK(lhs <= std::numeric_limits::max() - rhs, "{} overflow", + description); + return lhs + rhs; +} + +Comparison Compare(uint128_t lhs, uint128_t rhs) { + if (lhs < rhs) { + return Comparison::kLess; + } + if (lhs > rhs) { + return Comparison::kGreater; + } + return Comparison::kEqual; +} + +Comparison CompareIntegerToScaled(int64_t value, int64_t scale, double factor) { + if (factor == 0) { + return Compare(static_cast(value), uint128_t{0}); + } + + constexpr int64_t kLargestExactDoubleInteger = int64_t{1} << 53; + if (value <= kLargestExactDoubleInteger && scale <= kLargestExactDoubleInteger) { + // Preserve normal double threshold semantics while both integers are exact. + const double scaled = static_cast(scale) * factor; + if (static_cast(value) < scaled) { + return Comparison::kLess; + } + if (static_cast(value) > scaled) { + return Comparison::kGreater; + } + return Comparison::kEqual; + } + + // Avoid lossy integer-to-double conversion by comparing against the exact binary + // rational represented by factor. + int exponent = 0; + const double fraction = std::frexp(factor, &exponent); + constexpr int kDoubleDigits = std::numeric_limits::digits; + const auto significand = static_cast(std::ldexp(fraction, kDoubleDigits)); + const int shift = exponent - kDoubleDigits; + const uint128_t product = + static_cast(scale) * static_cast(significand); + constexpr uint128_t kMax = ~uint128_t{0}; + if (product == 0) { + return Compare(static_cast(value), uint128_t{0}); + } + + if (shift >= 0) { + if (shift >= 128 || product > (kMax >> shift)) { + return Comparison::kLess; + } + return Compare(static_cast(value), product << shift); + } + + const int left_shift = -shift; + const uint128_t lhs = static_cast(value); + if (lhs == 0) { + return Comparison::kLess; + } + if (left_shift >= 128 || lhs > (kMax >> left_shift)) { + return Comparison::kGreater; + } + return Compare(lhs << left_shift, product); +} + +void AppendFramed(std::string& output, std::string_view value) { + output.append(std::to_string(value.size())); + output.push_back(':'); + output.append(value); +} + +Result MakePartitionKey(const DataFile& data_file) { + ICEBERG_PRECHECK(data_file.partition_spec_id.has_value(), + "Data file '{}' is missing partition_spec_id", data_file.file_path); + + std::string encoded; + encoded.append(std::to_string(data_file.partition.num_fields())); + encoded.push_back(':'); + for (const auto& literal : data_file.partition.values()) { + AppendFramed(encoded, literal.type()->ToString()); + if (literal.IsNull()) { + encoded.push_back('N'); + continue; + } + + encoded.push_back('V'); + if (literal.IsNaN()) { + encoded.push_back('N'); + continue; + } + + encoded.push_back('B'); + ICEBERG_ASSIGN_OR_RAISE(auto bytes, literal.Serialize()); + std::string_view serialized; + if (!bytes.empty()) { + serialized = + std::string_view(reinterpret_cast(bytes.data()), bytes.size()); + } + AppendFramed(encoded, serialized); + } + return CanonicalPartitionKey{.spec_id = *data_file.partition_spec_id, + .values = std::move(encoded)}; +} + +Result BuildCompactionFile( + const std::shared_ptr& scan_task) { + CompactionFile result{.scan_task = scan_task}; + const auto& data_file = scan_task->data_file(); + ICEBERG_PRECHECK(data_file->record_count >= 0, + "Data file '{}' has negative record count", data_file->file_path); + ICEBERG_PRECHECK(data_file->file_size_in_bytes >= 0, + "Data file '{}' has negative file size", data_file->file_path); + + DeleteFileSet applicable_deletes; + for (const auto& delete_file : scan_task->delete_files()) { + ICEBERG_PRECHECK(delete_file != nullptr, "Data file '{}' has a null delete file", + data_file->file_path); + if (delete_file->content != DataFile::Content::kPositionDeletes) { + continue; + } + + ICEBERG_ASSIGN_OR_RAISE(auto referenced_file, + ContentFileUtil::ReferencedDataFile(*delete_file)); + if (!referenced_file.has_value() || *referenced_file != data_file->file_path) { + continue; + } + + if (!applicable_deletes.insert(delete_file).second) { + continue; + } + + ICEBERG_PRECHECK(delete_file->record_count >= 0, + "Delete file '{}' has negative record count", + delete_file->file_path); + if (delete_file->IsDeletionVector()) { + ICEBERG_PRECHECK( + delete_file->record_count <= data_file->record_count, + "Deletion vector '{}' cardinality {} exceeds data file '{}' record count {}", + delete_file->file_path, delete_file->record_count, data_file->file_path, + data_file->record_count); + } + + ICEBERG_ASSIGN_OR_RAISE(result.file_scoped_delete_count, + CheckedAddNonNegative(result.file_scoped_delete_count, 1, + "File-scoped delete-file count")); + ICEBERG_ASSIGN_OR_RAISE(result.file_scoped_delete_record_count, + CheckedAddNonNegative(result.file_scoped_delete_record_count, + delete_file->record_count, + "File-scoped delete-record count")); + } + + result.file_scoped_delete_record_count = + std::min(result.file_scoped_delete_record_count, data_file->record_count); + return result; +} + +bool CompactionFileLess(const CompactionFile& lhs, const CompactionFile& rhs) { + return lhs.scan_task->data_file()->file_path < rhs.scan_task->data_file()->file_path; +} + +bool CandidateLess(const Candidate& lhs, const Candidate& rhs) { + return CompactionFileLess(lhs.file, rhs.file); +} + +bool PackingCandidateLess(const Candidate& lhs, const Candidate& rhs) { + const int64_t lhs_size = lhs.file.scan_task->data_file()->file_size_in_bytes; + const int64_t rhs_size = rhs.file.scan_task->data_file()->file_size_in_bytes; + return lhs_size != rhs_size ? lhs_size > rhs_size : CandidateLess(lhs, rhs); +} + +Result AddToGroup(CompactionGroup& group, Candidate candidate, + bool& has_delete_pressure) { + const auto& data_file = *candidate.file.scan_task->data_file(); + ICEBERG_ASSIGN_OR_RAISE( + group.data_file_size_bytes, + CheckedAddNonNegative(group.data_file_size_bytes, data_file.file_size_in_bytes, + "Compaction group data-file size")); + ICEBERG_ASSIGN_OR_RAISE( + group.file_scoped_delete_record_count, + CheckedAddNonNegative(group.file_scoped_delete_record_count, + candidate.file.file_scoped_delete_record_count, + "Compaction group delete-record count")); + has_delete_pressure |= candidate.has_delete_pressure; + group.files.push_back(std::move(candidate.file)); + return {}; +} + +struct PendingGroup { + CompactionGroup group; + bool has_delete_pressure = false; +}; + +Result> PackPartition( + PartitionCandidates partition, const CompactionPlannerConfig& config) { + const auto representative = std::ranges::min_element(partition.files, CandidateLess) + ->file.scan_task->data_file() + ->partition; + std::ranges::sort(partition.files, PackingCandidateLess); + + std::vector bins; + for (auto& candidate : partition.files) { + const int64_t file_size = candidate.file.scan_task->data_file()->file_size_in_bytes; + size_t best_bin = bins.size(); + int64_t best_remaining = std::numeric_limits::max(); + for (size_t i = 0; i < bins.size(); ++i) { + const int64_t bin_size = bins[i].group.data_file_size_bytes; + if (bin_size > config.target_file_size_bytes || + file_size > config.target_file_size_bytes - bin_size) { + continue; + } + + const int64_t remaining = config.target_file_size_bytes - bin_size - file_size; + if (remaining < best_remaining) { + best_bin = i; + best_remaining = remaining; + } + } + + if (best_bin == bins.size()) { + bins.push_back(PendingGroup{ + .group = CompactionGroup{.partition_spec_id = partition.spec_id, + .partition = representative}, + }); + best_bin = bins.size() - 1; + } + ICEBERG_RETURN_UNEXPECTED(AddToGroup(bins[best_bin].group, std::move(candidate), + bins[best_bin].has_delete_pressure)); + } + + std::vector groups; + for (auto& bin : bins) { + if (!bin.has_delete_pressure && bin.group.files.size() < config.min_input_files) { + continue; + } + std::ranges::sort(bin.group.files, CompactionFileLess); + groups.push_back(std::move(bin.group)); + } + std::ranges::sort(groups, [](const CompactionGroup& lhs, const CompactionGroup& rhs) { + return CompactionFileLess(lhs.files.front(), rhs.files.front()); + }); + return groups; +} + +} // namespace + +Result CompactionPlanner::Plan( + int64_t source_snapshot_id, std::span> scan_tasks, + const CompactionPlannerConfig& config) { + ICEBERG_PRECHECK(source_snapshot_id >= 0, "source_snapshot_id must not be negative"); + ICEBERG_RETURN_UNEXPECTED(ValidateConfig(config)); + std::map partitions; + DataFileSet seen_data_files; + + for (const auto& scan_task : scan_tasks) { + ICEBERG_PRECHECK(scan_task != nullptr, "File scan task must not be null"); + ICEBERG_PRECHECK(scan_task->data_file() != nullptr, + "File scan task data file must not be null"); + const auto& data_file = *scan_task->data_file(); + ICEBERG_PRECHECK(seen_data_files.insert(scan_task->data_file()).second, + "Duplicate scan task for data file '{}'", data_file.file_path); + ICEBERG_ASSIGN_OR_RAISE(auto partition_key, MakePartitionKey(data_file)); + ICEBERG_ASSIGN_OR_RAISE(auto file, BuildCompactionFile(scan_task)); + + const bool is_small = CompareIntegerToScaled( + data_file.file_size_in_bytes, config.target_file_size_bytes, + config.min_file_size_ratio) == Comparison::kLess; + const bool is_oversized = + CompareIntegerToScaled(data_file.file_size_in_bytes, + config.target_file_size_bytes, + config.max_file_size_ratio) == Comparison::kGreater; + const bool has_many_delete_files = + config.delete_file_threshold > 0 && + file.file_scoped_delete_count >= config.delete_file_threshold; + const bool has_high_delete_ratio = + config.delete_ratio_threshold > 0 && data_file.record_count > 0 && + CompareIntegerToScaled(file.file_scoped_delete_record_count, + data_file.record_count, + config.delete_ratio_threshold) != Comparison::kLess; + const bool has_delete_pressure = has_many_delete_files || has_high_delete_ratio; + const bool should_compact = + is_oversized ? has_delete_pressure : is_small || has_delete_pressure; + if (!should_compact) { + continue; + } + + auto partition = + partitions + .try_emplace(std::move(partition_key), + PartitionCandidates{.spec_id = *data_file.partition_spec_id}) + .first; + partition->second.files.push_back( + Candidate{.file = std::move(file), .has_delete_pressure = has_delete_pressure}); + } + + CompactionPlan plan{.source_snapshot_id = source_snapshot_id}; + for (auto& [_, partition] : partitions) { + ICEBERG_ASSIGN_OR_RAISE(auto groups, PackPartition(std::move(partition), config)); + plan.groups.insert(plan.groups.end(), std::make_move_iterator(groups.begin()), + std::make_move_iterator(groups.end())); + } + return plan; +} + +} // namespace iceberg diff --git a/src/iceberg/compaction_planner.h b/src/iceberg/compaction_planner.h new file mode 100644 index 000000000..599ffbc8a --- /dev/null +++ b/src/iceberg/compaction_planner.h @@ -0,0 +1,126 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#pragma once + +/// \file iceberg/compaction_planner.h +/// Plan data-file compaction without executing rewrites. + +#include +#include +#include +#include +#include + +#include "iceberg/iceberg_export.h" +#include "iceberg/result.h" +#include "iceberg/row/partition_values.h" +#include "iceberg/type_fwd.h" + +namespace iceberg { + +/// \brief Thresholds used to select data files for compaction. +struct ICEBERG_EXPORT CompactionPlannerConfig { + /// Desired aggregate data-file size for each compaction group. + /// + /// A single selected file may exceed this size. + int64_t target_file_size_bytes = int64_t{512} * 1024 * 1024; + + /// Files smaller than this ratio of the target size are small-file candidates. + double min_file_size_ratio = 0.75; + + /// Files larger than this ratio of the target size are oversized. + /// + /// Oversized files are selected only when they also meet an enabled delete-pressure + /// threshold. + double max_file_size_ratio = 1.8; + + /// Minimum number of files required for a group containing only small-file candidates. + size_t min_input_files = 5; + + /// Minimum number of applicable file-scoped position delete files. + /// + /// A value of zero disables this selection criterion. + int64_t delete_file_threshold = 2; + + /// Minimum ratio of deleted records to data-file records. + /// + /// A value of zero disables this selection criterion. + double delete_ratio_threshold = 0.3; +}; + +/// \brief A selected data file and its file-scoped delete pressure. +struct ICEBERG_EXPORT CompactionFile { + /// Scan task for the selected data file and its applicable delete files. + std::shared_ptr scan_task; + + /// Number of distinct applicable file-scoped position delete files. + int64_t file_scoped_delete_count = 0; + + /// Number of deleted records, capped at the data file's record count. + int64_t file_scoped_delete_record_count = 0; +}; + +/// \brief Selected files from one partition that can be rewritten together. +struct ICEBERG_EXPORT CompactionGroup { + /// Partition spec ID shared by every file in this group. + int32_t partition_spec_id; + + /// Partition tuple shared by every file in this group. + PartitionValues partition; + + /// Selected files in canonical file-path order. + std::vector files; + + /// Aggregate size of the selected data files. + int64_t data_file_size_bytes = 0; + + /// Aggregate file-scoped deleted-record count. + int64_t file_scoped_delete_record_count = 0; +}; + +/// \brief Result of compaction planning. +struct ICEBERG_EXPORT CompactionPlan { + /// Snapshot whose scan tasks were used to produce this plan. + /// + /// An executor must verify this snapshot is still valid before rewriting files. + int64_t source_snapshot_id; + + /// Compaction groups in canonical partition and file order. + std::vector groups; +}; + +/// \brief Select and group scan tasks for data-file compaction. +class ICEBERG_EXPORT CompactionPlanner { + public: + /// \brief Plan deterministic, partition-isolated compaction groups. + /// + /// \param source_snapshot_id Non-negative snapshot whose scan tasks are being planned. + /// \param scan_tasks Data-file scan tasks with applicable delete files. + /// \param config Candidate selection and group sizing thresholds. + /// \return A metadata-only compaction plan, or an error for invalid configuration, + /// snapshot ID, duplicate data-file tasks, metadata, partition keys, delete + /// cardinality, or aggregate overflow. + static Result Plan( + int64_t source_snapshot_id, + std::span> scan_tasks, + const CompactionPlannerConfig& config = {}); +}; + +} // namespace iceberg diff --git a/src/iceberg/data/compaction_executor.cc b/src/iceberg/data/compaction_executor.cc new file mode 100644 index 000000000..cf9c9e26a --- /dev/null +++ b/src/iceberg/data/compaction_executor.cc @@ -0,0 +1,340 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#include "iceberg/data/compaction_executor.h" + +#include +#include +#include +#include +#include +#include +#include + +#include "iceberg/arrow_c_data_guard_internal.h" +#include "iceberg/compaction_planner.h" +#include "iceberg/data/data_writer.h" +#include "iceberg/data/file_scan_task_reader.h" +#include "iceberg/file_format.h" +#include "iceberg/file_io.h" +#include "iceberg/location_provider.h" +#include "iceberg/manifest/manifest_entry.h" +#include "iceberg/metadata_columns.h" +#include "iceberg/partition_spec.h" +#include "iceberg/schema.h" +#include "iceberg/snapshot.h" +#include "iceberg/table.h" +#include "iceberg/table_metadata.h" +#include "iceberg/table_properties.h" +#include "iceberg/table_scan.h" +#include "iceberg/update/rewrite_files.h" +#include "iceberg/util/content_file_util.h" +#include "iceberg/util/data_file_set.h" +#include "iceberg/util/macros.h" +#include "iceberg/util/uuid.h" + +namespace iceberg { +namespace { + +class ArrowArrayStreamGuard { + public: + explicit ArrowArrayStreamGuard(ArrowArrayStream* stream) : stream_(stream) {} + ~ArrowArrayStreamGuard() { + if (stream_ != nullptr && stream_->release != nullptr) { + stream_->release(stream_); + } + } + + private: + ArrowArrayStream* stream_; +}; + +Result> Next(ArrowArrayStream& stream) { + ArrowArray batch{}; + if (stream.get_next(&stream, &batch) != 0) { + const char* detail = + stream.get_last_error == nullptr ? nullptr : stream.get_last_error(&stream); + return IOError("Failed to read compaction input: {}", + detail == nullptr ? "unknown stream error" : detail); + } + if (batch.release == nullptr) { + return std::nullopt; + } + return batch; +} + +Result> RewriteSchema(const Table& table) { + ICEBERG_ASSIGN_OR_RAISE(auto table_schema, table.schema()); + if (table.metadata()->format_version < 3) { + return table_schema; + } + + std::vector fields(table_schema->fields().begin(), + table_schema->fields().end()); + fields.push_back(MetadataColumns::kRowId); + fields.push_back(MetadataColumns::kLastUpdatedSequenceNumber); + ICEBERG_ASSIGN_OR_RAISE(auto schema, + Schema::Make(std::move(fields), table_schema->schema_id(), + table_schema->IdentifierFieldIds())); + return std::shared_ptr(std::move(schema)); +} + +} // namespace + +class CompactionExecutor::Impl { + public: + explicit Impl(std::shared_ptr table) : table_(std::move(table)) {} + + Status Execute(const CompactionPlan& plan) { + ICEBERG_PRECHECK(!terminal_, "Compaction executor is no longer usable"); + ICEBERG_RETURN_UNEXPECTED(CleanupOutput()); + ICEBERG_ASSIGN_OR_RAISE(auto inputs, ValidatePlan(plan)); + + ICEBERG_RETURN_UNEXPECTED(table_->Refresh()); + ICEBERG_ASSIGN_OR_RAISE(auto snapshot, table_->current_snapshot()); + if (snapshot->snapshot_id != plan.source_snapshot_id) { + return ValidationFailed( + "Compaction plan snapshot {} is stale; current snapshot is {}", + plan.source_snapshot_id, snapshot->snapshot_id); + } + ICEBERG_ASSIGN_OR_RAISE(auto table_schema, table_->schema()); + ICEBERG_ASSIGN_OR_RAISE(auto rewrite_schema, RewriteSchema(*table_)); + ICEBERG_ASSIGN_OR_RAISE( + auto format, FileFormatTypeFromString( + table_->properties().Get(TableProperties::kDefaultFileFormat))); + ICEBERG_PRECHECK(format != FileFormatType::kPuffin, + "Puffin is not a data file format"); + ICEBERG_ASSIGN_OR_RAISE(auto location_provider, table_->location_provider()); + + std::vector> schemas(table_->metadata()->schemas.begin(), + table_->metadata()->schemas.end()); + ICEBERG_ASSIGN_OR_RAISE(auto reader, FileScanTaskReader::Make({ + .io = table_->io(), + .table_schema = std::move(table_schema), + .schemas = std::move(schemas), + .projected_schema = rewrite_schema, + .properties = table_->properties().configs(), + })); + + std::vector> added_files; + for (size_t group_index = 0; group_index < plan.groups.size(); ++group_index) { + auto result = RewriteGroup(plan.groups[group_index], group_index, format, + rewrite_schema, *reader, *location_provider); + if (!result.has_value()) { + return ReturnAfterCleanup(std::move(result.error())); + } + if (result.value() != nullptr) { + added_files.push_back(std::move(result.value())); + } + } + + auto rewrite_result = table_->NewRewriteFiles(); + if (!rewrite_result.has_value()) { + return ReturnAfterCleanup(std::move(rewrite_result.error())); + } + auto rewrite = std::move(rewrite_result.value()); + rewrite->ValidateFromSnapshot(plan.source_snapshot_id) + .SetDataSequenceNumber(snapshot->sequence_number) + .Rewrite(inputs.data_files, inputs.delete_files, added_files, {}); + + auto status = rewrite->Commit(); + if (!status.has_value()) { + if (status.error().kind == ErrorKind::kCommitStateUnknown) { + terminal_ = true; + output_paths_.clear(); + } else { + return ReturnAfterCleanup(std::move(status.error())); + } + return status; + } + + output_paths_.clear(); + return {}; + } + + Status Cleanup() { return CleanupOutput(); } + + private: + struct PlanInputs { + std::vector> data_files; + std::vector> delete_files; + }; + + Result ValidatePlan(const CompactionPlan& plan) { + ICEBERG_PRECHECK(plan.source_snapshot_id >= 0, + "Compaction plan snapshot ID must be non-negative"); + ICEBERG_PRECHECK(!plan.groups.empty(), "Compaction plan must contain a group"); + + DataFileSet data_files; + DeleteFileSet delete_files; + for (const auto& group : plan.groups) { + ICEBERG_PRECHECK(!group.files.empty(), "Compaction group must contain a file"); + for (const auto& compaction_file : group.files) { + ICEBERG_PRECHECK(compaction_file.scan_task != nullptr, + "Compaction file is missing its scan task"); + const auto& data_file = compaction_file.scan_task->data_file(); + ICEBERG_PRECHECK(data_file != nullptr, + "Compaction scan task is missing data file"); + ICEBERG_PRECHECK(data_file->content == DataFile::Content::kData, + "Compaction input is not a data file: {}", data_file->file_path); + ICEBERG_PRECHECK(data_file->partition_spec_id == group.partition_spec_id && + data_file->partition == group.partition, + "Compaction input does not match its group: {}", + data_file->file_path); + ICEBERG_PRECHECK(data_files.insert(data_file).second, + "Duplicate compaction input data file: {}", + data_file->file_path); + + for (const auto& delete_file : compaction_file.scan_task->delete_files()) { + ICEBERG_PRECHECK(delete_file != nullptr, + "Compaction scan task contains a null delete file"); + if (delete_file->content != DataFile::Content::kPositionDeletes) { + continue; + } + ICEBERG_ASSIGN_OR_RAISE(auto referenced_file, + ContentFileUtil::ReferencedDataFile(*delete_file)); + if (referenced_file == data_file->file_path) { + delete_files.insert(delete_file); + } + } + } + } + + return PlanInputs{ + .data_files = + std::vector>(data_files.begin(), data_files.end()), + .delete_files = std::vector>(delete_files.begin(), + delete_files.end()), + }; + } + + Result> RewriteGroup( + const CompactionGroup& group, size_t group_index, FileFormatType format, + const std::shared_ptr& rewrite_schema, FileScanTaskReader& reader, + LocationProvider& location_provider) { + ICEBERG_PRECHECK(!group.files.empty(), "Compaction group must contain a file"); + ICEBERG_ASSIGN_OR_RAISE( + auto spec, table_->metadata()->PartitionSpecById(group.partition_spec_id)); + + std::unique_ptr writer; + for (const auto& compaction_file : group.files) { + const auto& data_file = compaction_file.scan_task->data_file(); + FileScanTask task(data_file, compaction_file.scan_task->delete_files()); + ICEBERG_ASSIGN_OR_RAISE(auto stream, reader.Open(task)); + ArrowArrayStreamGuard stream_guard(&stream); + while (true) { + ICEBERG_ASSIGN_OR_RAISE(auto batch, Next(stream)); + if (!batch.has_value()) { + break; + } + + internal::ArrowArrayGuard batch_guard(&batch.value()); + if (batch->length == 0) { + continue; + } + if (writer == nullptr) { + const auto filename = + std::format("compacted-{}-{}.{}", Uuid::GenerateV7().ToString(), + group_index, ToString(format)); + ICEBERG_ASSIGN_OR_RAISE( + auto output_path, + location_provider.NewDataLocation(*spec, group.partition, filename)); + output_paths_.insert(output_path); + ICEBERG_ASSIGN_OR_RAISE(writer, + DataWriter::Make({ + .path = output_path, + .schema = rewrite_schema, + .spec = spec, + .partition = group.partition, + .format = format, + .io = table_->io(), + .properties = table_->properties().configs(), + })); + } + batch_guard.Release(); + ICEBERG_RETURN_UNEXPECTED(writer->Write(&batch.value())); + } + } + + if (writer != nullptr) { + ICEBERG_RETURN_UNEXPECTED(writer->Close()); + ICEBERG_ASSIGN_OR_RAISE(auto metadata, writer->Metadata()); + ICEBERG_PRECHECK(metadata.data_files.size() == 1, + "Compaction writer produced {} data files", + metadata.data_files.size()); + return std::move(metadata.data_files.front()); + } + return nullptr; + } + + Status ReturnAfterCleanup(Error error) { + auto cleanup_status = CleanupOutput(); + if (!cleanup_status.has_value()) { + error.message += "; additionally failed to clean output files: "; + error.message += cleanup_status.error().message; + } + return std::unexpected(std::move(error)); + } + + Status CleanupOutput() { + std::optional first_error; + for (auto iter = output_paths_.begin(); iter != output_paths_.end();) { + auto status = table_->io()->DeleteFile(*iter); + if (status.has_value()) { + iter = output_paths_.erase(iter); + continue; + } + if (!first_error.has_value()) { + first_error = std::move(status.error()); + } else { + first_error->message += "; additionally failed to delete output file: "; + first_error->message += status.error().message; + } + ++iter; + } + if (first_error.has_value()) { + return std::unexpected(std::move(first_error.value())); + } + return {}; + } + + std::shared_ptr
table_; + std::unordered_set output_paths_; + bool terminal_ = false; +}; + +CompactionExecutor::CompactionExecutor(std::unique_ptr impl) + : impl_(std::move(impl)) {} + +CompactionExecutor::~CompactionExecutor() = default; + +Result> CompactionExecutor::Make( + std::shared_ptr
table) { + ICEBERG_PRECHECK(table != nullptr, "Cannot create compaction executor without table"); + return std::unique_ptr( + new CompactionExecutor(std::make_unique(std::move(table)))); +} + +Status CompactionExecutor::Execute(const CompactionPlan& plan) { + return impl_->Execute(plan); +} + +Status CompactionExecutor::Cleanup() { return impl_->Cleanup(); } + +} // namespace iceberg diff --git a/src/iceberg/data/compaction_executor.h b/src/iceberg/data/compaction_executor.h new file mode 100644 index 000000000..761c9cd0a --- /dev/null +++ b/src/iceberg/data/compaction_executor.h @@ -0,0 +1,81 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#pragma once + +/// \file iceberg/data/compaction_executor.h +/// Execute planned data-file compaction. + +#include + +#include "iceberg/iceberg_data_export.h" +#include "iceberg/result.h" +#include "iceberg/type_fwd.h" + +namespace iceberg { + +struct CompactionPlan; + +/// \brief Executes a snapshot-bound data-file compaction plan. +/// +/// The executor reads every planned source file through the delete-aware scan reader, +/// writes replacement data files, and commits all data and file-scoped position-delete +/// replacements in one table update. Generated files remain owned by the executor until +/// commit succeeds, commit state becomes unknown, or cleanup deletes them. +class ICEBERG_DATA_EXPORT CompactionExecutor { + public: + /// \brief Destroy the executor. + ~CompactionExecutor(); + + /// \brief Create an executor for a table. + /// + /// \param table Table whose files and metadata will be rewritten. + /// \return A new executor, or an error if table is null. + static Result> Make(std::shared_ptr
table); + + /// \brief Rewrite and atomically commit a snapshot-bound compaction plan. + /// + /// Execution refreshes the table and rejects plans whose snapshot is no longer + /// current before writing. Commit validation also rejects deletes added after the + /// plan snapshot. A definite failure attempts to clean generated files; cleanup + /// failures are appended to the original error and undeleted files remain owned by + /// this executor. + /// + /// \param plan Planner output containing the source snapshot and compaction groups. + /// \return Success after commit, or the original planning, IO, validation, or commit + /// error, including cleanup details when cleanup also fails. + Status Execute(const CompactionPlan& plan); + + /// \brief Retry deletion of uncommitted output files owned by this executor. + /// + /// Successfully deleted or absent paths are released. Paths whose deletion fails + /// remain owned and may be retried by calling Cleanup again. + /// + /// \return Success when every owned output is deleted, otherwise the first cleanup + /// error. + Status Cleanup(); + + private: + class Impl; + std::unique_ptr impl_; + + explicit CompactionExecutor(std::unique_ptr impl); +}; + +} // namespace iceberg diff --git a/src/iceberg/data/meson.build b/src/iceberg/data/meson.build index bbb26db27..b881f58ce 100644 --- a/src/iceberg/data/meson.build +++ b/src/iceberg/data/meson.build @@ -17,11 +17,13 @@ install_headers( [ + 'compaction_executor.h', 'data_writer.h', 'delete_filter.h', 'delete_loader.h', 'equality_delete_writer.h', 'file_scan_task_reader.h', + 'position_delete_update.h', 'position_delete_writer.h', 'writer.h', ], diff --git a/src/iceberg/data/position_delete_update.cc b/src/iceberg/data/position_delete_update.cc new file mode 100644 index 000000000..e206de7bc --- /dev/null +++ b/src/iceberg/data/position_delete_update.cc @@ -0,0 +1,324 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#include "iceberg/data/position_delete_update.h" + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "iceberg/data/delete_loader.h" +#include "iceberg/data/position_delete_writer.h" +#include "iceberg/deletes/dv_writer.h" +#include "iceberg/file_format.h" +#include "iceberg/file_io.h" +#include "iceberg/location_provider.h" +#include "iceberg/manifest/manifest_entry.h" +#include "iceberg/partition_spec.h" +#include "iceberg/schema.h" +#include "iceberg/table.h" +#include "iceberg/table_metadata.h" +#include "iceberg/table_properties.h" +#include "iceberg/table_scan.h" +#include "iceberg/update/row_delta.h" +#include "iceberg/util/macros.h" +#include "iceberg/util/string_util.h" +#include "iceberg/util/uuid.h" + +namespace iceberg { + +namespace { + +struct TargetFile { + std::shared_ptr data_file; + std::shared_ptr spec; + std::vector> position_delete_files; +}; + +struct PreparedDeletes { + std::vector> added_files; + std::vector> rewritten_files; + std::vector referenced_data_files; +}; + +} // namespace + +class PositionDeleteUpdate::Impl { + public: + explicit Impl(std::shared_ptr
table) : table_(std::move(table)) {} + + Status Delete(std::string_view data_file_path, int64_t pos) { + ICEBERG_PRECHECK(!terminal_, "Position delete update is no longer usable"); + ICEBERG_PRECHECK(!data_file_path.empty(), "Data file path cannot be empty"); + ICEBERG_PRECHECK(pos >= 0, "Position delete must be non-negative: {}", pos); + deletes_[std::string(data_file_path)].push_back(pos); + return {}; + } + + Status Commit() { + ICEBERG_PRECHECK(!terminal_, "Position delete update is no longer usable"); + ICEBERG_PRECHECK(!deletes_.empty(), "Position delete update is empty"); + ICEBERG_PRECHECK(table_->metadata()->format_version >= 2, + "Position deletes require table format version 2 or later"); + ICEBERG_RETURN_UNEXPECTED(CleanupOutput()); + + ICEBERG_ASSIGN_OR_RAISE(auto snapshot, table_->current_snapshot()); + ICEBERG_ASSIGN_OR_RAISE(auto targets, ResolveTargets()); + + auto prepared = table_->metadata()->format_version >= 3 + ? WriteDeletionVectors(targets) + : WriteParquetDeletes(targets); + if (!prepared.has_value()) { + return FailAfterCleanup(std::move(prepared.error())); + } + + auto row_delta_result = table_->NewRowDelta(); + if (!row_delta_result.has_value()) { + return FailAfterCleanup(std::move(row_delta_result.error())); + } + auto row_delta = std::move(row_delta_result.value()); + row_delta->ValidateFromSnapshot(snapshot->snapshot_id) + .ValidateDataFilesExist(prepared->referenced_data_files) + .ValidateDeletedFiles(); + for (const auto& file : prepared->added_files) { + row_delta->AddDeletes(file); + } + for (const auto& file : prepared->rewritten_files) { + row_delta->RemoveDeletes(file); + } + + auto status = row_delta->Commit(); + if (!status.has_value()) { + if (status.error().kind == ErrorKind::kCommitStateUnknown) { + terminal_ = true; + output_paths_.clear(); + } else { + return FailAfterCleanup(std::move(status.error())); + } + return status; + } + + terminal_ = true; + output_paths_.clear(); + return {}; + } + + private: + Result> ResolveTargets() const { + ICEBERG_ASSIGN_OR_RAISE(auto scan_builder, table_->NewScan()); + ICEBERG_ASSIGN_OR_RAISE(auto scan, scan_builder->Build()); + ICEBERG_ASSIGN_OR_RAISE(auto tasks, scan->PlanFiles()); + + std::unordered_map targets; + for (const auto& task : tasks) { + const auto& data_file = task->data_file(); + if (!deletes_.contains(data_file->file_path)) { + continue; + } + + ICEBERG_PRECHECK(data_file->partition_spec_id.has_value(), + "Data file is missing partition spec ID: {}", + data_file->file_path); + ICEBERG_ASSIGN_OR_RAISE(auto spec, table_->metadata()->PartitionSpecById( + *data_file->partition_spec_id)); + + TargetFile target{.data_file = data_file, .spec = std::move(spec)}; + for (const auto& delete_file : task->delete_files()) { + if (delete_file->content == DataFile::Content::kPositionDeletes) { + target.position_delete_files.push_back(delete_file); + } + } + + auto [_, inserted] = targets.emplace(data_file->file_path, std::move(target)); + ICEBERG_PRECHECK(inserted, "Duplicate live data file path: {}", + data_file->file_path); + } + + for (const auto& [path, _] : deletes_) { + ICEBERG_PRECHECK(targets.contains(path), "Cannot find live data file: {}", path); + } + return targets; + } + + Result WriteDeletionVectors( + const std::unordered_map& targets) { + ICEBERG_ASSIGN_OR_RAISE(auto location_provider, table_->location_provider()); + auto output_path = location_provider->NewDataLocation( + std::format("position-deletes-{}.puffin", Uuid::GenerateV7().ToString())); + output_paths_.insert(output_path); + + DeleteLoader loader(table_->io()); + ICEBERG_ASSIGN_OR_RAISE( + auto writer, + DVWriter::Make(DVWriterOptions{ + .path = output_path, + .io = table_->io(), + .load_previous_deletes = [&targets, &loader](std::string_view path) + -> Result> { + const auto target = targets.find(std::string(path)); + ICEBERG_CHECK(target != targets.end(), + "Missing target data file while loading deletes: {}", path); + if (target->second.position_delete_files.empty()) { + return std::nullopt; + } + ICEBERG_ASSIGN_OR_RAISE( + auto index, + loader.LoadPositionDeletes(target->second.position_delete_files, path)); + return std::optional(std::move(index)); + }, + })); + + for (const auto& [path, positions] : deletes_) { + const auto& target = targets.at(path); + for (int64_t pos : positions) { + ICEBERG_RETURN_UNEXPECTED( + writer->Delete(path, pos, target.spec, target.data_file->partition)); + } + } + ICEBERG_RETURN_UNEXPECTED(writer->Close()); + ICEBERG_ASSIGN_OR_RAISE(auto result, writer->Metadata()); + return PreparedDeletes{ + .added_files = std::move(result.data_files), + .rewritten_files = std::move(result.rewritten_delete_files), + .referenced_data_files = std::move(result.referenced_data_files), + }; + } + + Result WriteParquetDeletes( + const std::unordered_map& targets) { + ICEBERG_ASSIGN_OR_RAISE(auto location_provider, table_->location_provider()); + ICEBERG_ASSIGN_OR_RAISE(auto schema, table_->schema()); + + PreparedDeletes result; + result.added_files.reserve(deletes_.size()); + result.referenced_data_files.reserve(deletes_.size()); + auto properties = table_->properties().configs(); + properties[TableProperties::kParquetCompression.key()] = + table_->properties().Get(TableProperties::kDeleteParquetCompression); + properties[TableProperties::kParquetCompressionLevel.key()] = + table_->properties().Get(TableProperties::kDeleteParquetCompressionLevel); + const auto write_uuid = Uuid::GenerateV7().ToString(); + size_t file_number = 0; + + for (const auto& [path, positions] : deletes_) { + auto sorted_positions = positions; + std::ranges::sort(sorted_positions); + + const auto& target = targets.at(path); + const auto filename = + std::format("position-deletes-{}-{}.parquet", write_uuid, file_number++); + auto output_path = location_provider->NewDataLocation(filename); + output_paths_.insert(output_path); + + ICEBERG_ASSIGN_OR_RAISE(auto writer, + PositionDeleteWriter::Make(PositionDeleteWriterOptions{ + .path = output_path, + .schema = schema, + .spec = target.spec, + .partition = target.data_file->partition, + .format = FileFormatType::kParquet, + .io = table_->io(), + .properties = properties, + })); + for (int64_t pos : sorted_positions) { + ICEBERG_RETURN_UNEXPECTED(writer->WriteDelete(path, pos)); + } + ICEBERG_RETURN_UNEXPECTED(writer->Close()); + ICEBERG_ASSIGN_OR_RAISE(auto metadata, writer->Metadata()); + result.added_files.insert(result.added_files.end(), + std::make_move_iterator(metadata.data_files.begin()), + std::make_move_iterator(metadata.data_files.end())); + result.referenced_data_files.push_back(path); + } + return result; + } + + Status CleanupOutput() { + std::optional first_error; + for (auto it = output_paths_.begin(); it != output_paths_.end();) { + auto status = table_->io()->DeleteFile(*it); + if (status.has_value()) { + it = output_paths_.erase(it); + continue; + } + + if (!first_error.has_value()) { + first_error = std::move(status.error()); + } else { + first_error->message += "; additionally failed to delete output file: "; + first_error->message += status.error().message; + } + ++it; + } + if (first_error.has_value()) { + return std::unexpected(std::move(*first_error)); + } + return {}; + } + + Status FailAfterCleanup(Error error) { + auto cleanup_status = CleanupOutput(); + if (!cleanup_status.has_value()) { + error.message += "; additionally failed to clean output files: "; + error.message += cleanup_status.error().message; + } + return std::unexpected(std::move(error)); + } + + std::shared_ptr
table_; + std::map, StringLess> deletes_; + std::set output_paths_; + bool terminal_ = false; +}; + +PositionDeleteUpdate::PositionDeleteUpdate(std::unique_ptr impl) + : impl_(std::move(impl)) {} + +PositionDeleteUpdate::~PositionDeleteUpdate() = default; + +Result> PositionDeleteUpdate::Make( + std::shared_ptr
table) { + ICEBERG_PRECHECK(table != nullptr, + "Cannot create position delete update without table"); + return std::unique_ptr( + new PositionDeleteUpdate(std::make_unique(std::move(table)))); +} + +PositionDeleteUpdate& PositionDeleteUpdate::Delete(std::string_view data_file_path, + int64_t pos) { + ICEBERG_BUILDER_RETURN_IF_ERROR(impl_->Delete(data_file_path, pos)); + return *this; +} + +Status PositionDeleteUpdate::Commit() { + ICEBERG_RETURN_UNEXPECTED(CheckErrors()); + return impl_->Commit(); +} + +} // namespace iceberg diff --git a/src/iceberg/data/position_delete_update.h b/src/iceberg/data/position_delete_update.h new file mode 100644 index 000000000..3b531d8fa --- /dev/null +++ b/src/iceberg/data/position_delete_update.h @@ -0,0 +1,60 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#pragma once + +/// \file iceberg/data/position_delete_update.h +/// Table-aware position delete writing and commit. + +#include +#include +#include + +#include "iceberg/iceberg_data_export.h" +#include "iceberg/result.h" +#include "iceberg/type_fwd.h" +#include "iceberg/util/error_collector.h" + +namespace iceberg { + +/// \brief Writes and commits position deletes using the table format. +/// +/// Format v2 tables produce Parquet position delete files. Format v3 tables +/// produce deletion vectors and merge previous file-scoped position deletes. +class ICEBERG_DATA_EXPORT PositionDeleteUpdate : public ErrorCollector { + public: + ~PositionDeleteUpdate() override; + + /// \brief Create a position delete update for a table. + static Result> Make(std::shared_ptr
table); + + /// \brief Add a deleted row position for a live data file. + PositionDeleteUpdate& Delete(std::string_view data_file_path, int64_t pos); + + /// \brief Write and commit all added position deletes. + Status Commit(); + + private: + class Impl; + std::unique_ptr impl_; + + explicit PositionDeleteUpdate(std::unique_ptr impl); +}; + +} // namespace iceberg diff --git a/src/iceberg/delete_file_index.cc b/src/iceberg/delete_file_index.cc index a8c4ef126..7a59c2b26 100644 --- a/src/iceberg/delete_file_index.cc +++ b/src/iceberg/delete_file_index.cc @@ -41,6 +41,7 @@ #include "iceberg/util/content_file_util.h" #include "iceberg/util/executor_util_internal.h" #include "iceberg/util/macros.h" +#include "iceberg/util/struct_like_set.h" namespace iceberg { @@ -453,10 +454,20 @@ Result> DeleteFileIndex::FindDV( return nullptr; } - ICEBERG_CHECK(it->second.sequence_number.value() >= seq, - "DV data sequence number {} must be greater than or equal to data file " - "sequence number {}", - it->second.sequence_number.value(), seq); + const auto& dv = *it->second.data_file; + ICEBERG_PRECHECK(data_file.partition_spec_id.has_value(), + "Missing partition spec id from data file {}", data_file.file_path); + ICEBERG_PRECHECK(dv.partition_spec_id.has_value(), + "Missing partition spec id from DV {}", dv.file_path); + if (dv.partition_spec_id != data_file.partition_spec_id) { + return nullptr; + } + + ICEBERG_ASSIGN_OR_RAISE(auto partitions_match, + StructLikeEqual(dv.partition, data_file.partition)); + if (!partitions_match || it->second.sequence_number.value() < seq) { + return nullptr; + } return it->second.data_file; } diff --git a/src/iceberg/deletes/dv_util.cc b/src/iceberg/deletes/dv_util.cc index 670ccfedb..e9fb6449d 100644 --- a/src/iceberg/deletes/dv_util.cc +++ b/src/iceberg/deletes/dv_util.cc @@ -17,6 +17,7 @@ * under the License. */ +#include #include #include #include @@ -36,13 +37,63 @@ #include "iceberg/metadata_columns.h" #include "iceberg/partition_spec.h" #include "iceberg/puffin/file_metadata.h" +#include "iceberg/puffin/puffin_reader.h" #include "iceberg/result.h" #include "iceberg/util/content_file_util.h" #include "iceberg/util/macros.h" +#include "iceberg/util/string_util.h" #include "iceberg/version.h" namespace iceberg { +namespace { + +constexpr std::string_view kReferencedDataFileProperty = "referenced-data-file"; +constexpr std::string_view kCardinalityProperty = "cardinality"; + +Status ValidateDVBlobMetadata(const puffin::BlobMetadata& blob, + const DataFile& delete_file) { + ICEBERG_PRECHECK(blob.type == puffin::StandardBlobTypes::kDeletionVectorV1, + "Invalid deletion vector blob type '{}', expected '{}'", blob.type, + puffin::StandardBlobTypes::kDeletionVectorV1); + ICEBERG_PRECHECK(blob.snapshot_id == -1, + "Deletion vector requires snapshot-id -1, got {}", blob.snapshot_id); + ICEBERG_PRECHECK(blob.sequence_number == -1, + "Deletion vector requires sequence-number -1, got {}", + blob.sequence_number); + ICEBERG_PRECHECK(blob.compression_codec.empty(), + "Deletion vector must not be compressed, got '{}'", + blob.compression_codec); + + auto referenced_data_file = + blob.properties.find(std::string(kReferencedDataFileProperty)); + ICEBERG_PRECHECK(referenced_data_file != blob.properties.end() && + !referenced_data_file->second.empty(), + "Deletion vector blob requires non-empty '{}' property", + kReferencedDataFileProperty); + ICEBERG_PRECHECK( + referenced_data_file->second == *delete_file.referenced_data_file, + "Manifest referenced_data_file '{}' does not match Puffin '{}' property '{}'", + *delete_file.referenced_data_file, kReferencedDataFileProperty, + referenced_data_file->second); + + auto cardinality = blob.properties.find(std::string(kCardinalityProperty)); + ICEBERG_PRECHECK(cardinality != blob.properties.end(), + "Deletion vector blob requires '{}' property", kCardinalityProperty); + ICEBERG_ASSIGN_OR_RAISE(auto parsed_cardinality, + StringUtils::ParseNumber(cardinality->second)); + ICEBERG_PRECHECK(parsed_cardinality >= 0, + "Deletion vector cardinality must be non-negative, got {}", + parsed_cardinality); + ICEBERG_PRECHECK( + parsed_cardinality == delete_file.record_count, + "Manifest record_count {} does not match Puffin cardinality property {}", + delete_file.record_count, parsed_cardinality); + return {}; +} + +} // namespace + Result>> DVUtil::MergeAndWriteDVs( std::span groups, std::string_view output_path, const std::shared_ptr& io) { @@ -84,6 +135,10 @@ Result DVUtil::ReadDV(const std::shared_ptr& dele delete_file->content_size_in_bytes.has_value(), "Deletion vector requires content_offset and content_size_in_bytes: {}", delete_file->file_path); + ICEBERG_PRECHECK(delete_file->referenced_data_file.has_value() && + !delete_file->referenced_data_file->empty(), + "Deletion vector requires referenced_data_file: {}", + delete_file->file_path); const int64_t offset = delete_file->content_offset.value(); const int64_t length = delete_file->content_size_in_bytes.value(); @@ -94,14 +149,21 @@ Result DVUtil::ReadDV(const std::shared_ptr& dele "Cannot read deletion vector larger than 2GB: {}", length); ICEBERG_ASSIGN_OR_RAISE(auto input_file, io->NewInputFile(delete_file->file_path)); - ICEBERG_ASSIGN_OR_RAISE(auto stream, input_file->Open()); - - std::vector bytes(static_cast(length)); - ICEBERG_RETURN_UNEXPECTED(stream->ReadFully(offset, bytes)); - ICEBERG_RETURN_UNEXPECTED(stream->Close()); - - std::span blob(reinterpret_cast(bytes.data()), - bytes.size()); + ICEBERG_ASSIGN_OR_RAISE(auto reader, puffin::PuffinReader::Make(std::move(input_file))); + ICEBERG_ASSIGN_OR_RAISE(auto metadata, reader->ReadFileMetadata()); + auto blob_metadata = std::ranges::find_if( + metadata.blobs, [offset](const auto& blob) { return blob.offset == offset; }); + ICEBERG_PRECHECK(blob_metadata != metadata.blobs.end(), + "No Puffin blob starts at manifest content_offset {}", offset); + ICEBERG_PRECHECK( + blob_metadata->length == length, + "Puffin blob at offset {} has length {}, manifest content_size_in_bytes is {}", + offset, blob_metadata->length, length); + ICEBERG_RETURN_UNEXPECTED(ValidateDVBlobMetadata(*blob_metadata, *delete_file)); + + ICEBERG_ASSIGN_OR_RAISE(auto blob_data, reader->ReadBlob(*blob_metadata)); + std::span blob(reinterpret_cast(blob_data.second.data()), + blob_data.second.size()); return PositionDeleteIndex::Deserialize(blob, delete_file); } @@ -118,8 +180,9 @@ Result DVUtil::WriteDVBlob(puffin::PuffinWriter& writer, .data = std::move(data), .requested_compression = puffin::PuffinCompressionCodec::kNone, }; - blob.properties.emplace("referenced-data-file", std::string(referenced_data_file)); - blob.properties.emplace("cardinality", std::format("{}", positions.Cardinality())); + blob.properties.emplace(kReferencedDataFileProperty, std::string(referenced_data_file)); + blob.properties.emplace(kCardinalityProperty, + std::format("{}", positions.Cardinality())); return writer.Write(blob); } diff --git a/src/iceberg/deletes/roaring_position_bitmap.cc b/src/iceberg/deletes/roaring_position_bitmap.cc index a2827d4bb..2020160f1 100644 --- a/src/iceberg/deletes/roaring_position_bitmap.cc +++ b/src/iceberg/deletes/roaring_position_bitmap.cc @@ -271,6 +271,7 @@ Result RoaringPositionBitmap::Deserialize(std::string_vie --remaining_count; } + ICEBERG_PRECHECK(remaining == 0, "Trailing data after bitmaps: {} bytes", remaining); return RoaringPositionBitmap(std::move(impl)); } diff --git a/src/iceberg/file_io.h b/src/iceberg/file_io.h index 3ea4afa49..84bfff62f 100644 --- a/src/iceberg/file_io.h +++ b/src/iceberg/file_io.h @@ -162,6 +162,9 @@ class ICEBERG_EXPORT FileIO { /// \brief Delete a file at the given location. /// + /// Deletion is idempotent: implementations must return success when the file does + /// not exist. + /// /// \param file_location The location of the file to delete. /// \return void if the delete succeeded, an error code if the delete failed. virtual Status DeleteFile(const std::string& file_location) { diff --git a/src/iceberg/meson.build b/src/iceberg/meson.build index 989f4ae03..bb6cc213d 100644 --- a/src/iceberg/meson.build +++ b/src/iceberg/meson.build @@ -79,6 +79,7 @@ iceberg_sources = files( 'catalog/memory/in_memory_catalog.cc', 'catalog/session_catalog.cc', 'catalog/session_context.cc', + 'compaction_planner.cc', 'delete_file_index.cc', 'deletes/dv_util.cc', 'deletes/dv_writer.cc', @@ -218,11 +219,13 @@ iceberg_sources = files( ) iceberg_data_sources = files( + 'data/compaction_executor.cc', 'data/data_writer.cc', 'data/delete_filter.cc', 'data/delete_loader.cc', 'data/equality_delete_writer.cc', 'data/file_scan_task_reader.cc', + 'data/position_delete_update.cc', 'data/position_delete_writer.cc', 'data/writer.cc', ) @@ -319,6 +322,7 @@ install_headers( [ 'arrow_c_data.h', 'catalog.h', + 'compaction_planner.h', 'constants.h', 'delete_file_index.h', 'exception.h', diff --git a/src/iceberg/test/CMakeLists.txt b/src/iceberg/test/CMakeLists.txt index 5ca9fd915..8547c6a32 100644 --- a/src/iceberg/test/CMakeLists.txt +++ b/src/iceberg/test/CMakeLists.txt @@ -158,10 +158,14 @@ add_iceberg_test(util_test add_iceberg_test(puffin_test USE_DATA SOURCES + dv_util_test.cc + puffin_dv_interop_test.cc puffin_format_test.cc puffin_json_test.cc puffin_reader_writer_test.cc) +add_iceberg_test(compaction_planner_test SOURCES compaction_planner_test.cc) + if(ICEBERG_BUILD_BUNDLE) add_iceberg_test(avro_test USE_BUNDLE @@ -253,12 +257,14 @@ if(ICEBERG_BUILD_BUNDLE) SOURCES arrow_c_data_util_test.cc arrow_row_builder_test.cc + compaction_executor_test.cc data_writer_test.cc default_value_test.cc delete_filter_test.cc delete_loader_test.cc dv_writer_test.cc file_scan_task_reader_test.cc + position_delete_update_test.cc literal_util_test.cc) endif() diff --git a/src/iceberg/test/arrow_io_test.cc b/src/iceberg/test/arrow_io_test.cc index 7bc9ebba5..6b3f6e4dd 100644 --- a/src/iceberg/test/arrow_io_test.cc +++ b/src/iceberg/test/arrow_io_test.cc @@ -364,8 +364,7 @@ TEST_F(LocalFileIOTest, DeleteFile) { EXPECT_THAT(del_res, IsOk()); del_res = file_io_->DeleteFile(temp_filepath_); - EXPECT_THAT(del_res, IsError(ErrorKind::kIOError)); - EXPECT_THAT(del_res, HasErrorMessage("Cannot delete file")); + EXPECT_THAT(del_res, IsOk()); } TEST_F(LocalFileIOTest, DeleteFiles) { @@ -414,6 +413,13 @@ TEST_F(LocalFileIOTest, StdReadFullyReadsFromAbsolutePosition) { VerifyReadFullyReadsFromAbsolutePosition(file_io, temp_filepath_)); } +TEST_F(LocalFileIOTest, StdDeleteFileIsIdempotent) { + auto file_io = std::make_shared(); + ASSERT_THAT(file_io->WriteFile(temp_filepath_, "abc"), IsOk()); + EXPECT_THAT(file_io->DeleteFile(temp_filepath_), IsOk()); + EXPECT_THAT(file_io->DeleteFile(temp_filepath_), IsOk()); +} + TEST_F(LocalFileIOTest, StdReadKeepsPositionAvailableAtEof) { auto file_io = std::make_shared(); ASSERT_THAT(file_io->WriteFile(temp_filepath_, "abc"), IsOk()); diff --git a/src/iceberg/test/compaction_executor_test.cc b/src/iceberg/test/compaction_executor_test.cc new file mode 100644 index 000000000..f66662333 --- /dev/null +++ b/src/iceberg/test/compaction_executor_test.cc @@ -0,0 +1,860 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#include "iceberg/data/compaction_executor.h" + +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include + +#include "iceberg/arrow/arrow_io_internal.h" +#include "iceberg/avro/avro_register.h" +#include "iceberg/compaction_planner.h" +#include "iceberg/data/data_writer.h" +#include "iceberg/data/file_scan_task_reader.h" +#include "iceberg/data/position_delete_update.h" +#include "iceberg/file_writer.h" +#include "iceberg/manifest/manifest_entry.h" +#include "iceberg/manifest/manifest_reader.h" +#include "iceberg/metadata_columns.h" +#include "iceberg/parquet/parquet_register.h" +#include "iceberg/partition_spec.h" +#include "iceberg/schema.h" +#include "iceberg/schema_internal.h" +#include "iceberg/snapshot.h" +#include "iceberg/table.h" +#include "iceberg/table_metadata.h" +#include "iceberg/table_properties.h" +#include "iceberg/table_scan.h" +#include "iceberg/test/matchers.h" +#include "iceberg/test/mock_catalog.h" +#include "iceberg/test/update_test_base.h" +#include "iceberg/update/fast_append.h" +#include "iceberg/update/update_properties.h" +#include "iceberg/util/macros.h" + +namespace iceberg { +namespace { + +using Row = std::array; + +class FailingDeleteFileIO : public FileIO { + public: + explicit FailingDeleteFileIO(std::shared_ptr delegate) + : delegate_(std::move(delegate)) {} + + Result> NewInputFile(std::string file_location) override { + return delegate_->NewInputFile(std::move(file_location)); + } + + Result> NewInputFile(std::string file_location, + size_t length) override { + return delegate_->NewInputFile(std::move(file_location), length); + } + + Result> NewOutputFile(std::string file_location) override { + return delegate_->NewOutputFile(std::move(file_location)); + } + + Status DeleteFile(const std::string& file_location) override { + if (fail_compacted_deletes_ && + file_location.find("compacted-") != std::string::npos) { + return IOError("injected cleanup failure: {}", file_location); + } + return delegate_->DeleteFile(file_location); + } + + void AllowCompactedDeletes() { fail_compacted_deletes_ = false; } + + private: + std::shared_ptr delegate_; + bool fail_compacted_deletes_ = true; +}; + +class FailBeforeCreateFileIO : public FileIO { + public: + explicit FailBeforeCreateFileIO(std::shared_ptr delegate) + : delegate_(std::move(delegate)), + fail_before_create_(std::make_shared(true)) {} + + Result> NewInputFile(std::string file_location) override { + return delegate_->NewInputFile(std::move(file_location)); + } + + Result> NewInputFile(std::string file_location, + size_t length) override { + return delegate_->NewInputFile(std::move(file_location), length); + } + + Result> NewOutputFile(std::string file_location) override { + ICEBERG_ASSIGN_OR_RAISE(auto output, delegate_->NewOutputFile(file_location)); + if (!IsCompacted(file_location)) { + return output; + } + return std::unique_ptr(new FailingOutputFile( + std::move(output), std::move(file_location), fail_before_create_)); + } + + Status DeleteFile(const std::string& file_location) override { + return delegate_->DeleteFile(file_location); + } + + private: + class FailingOutputFile : public OutputFile { + public: + FailingOutputFile(std::unique_ptr delegate, std::string location, + std::shared_ptr fail_before_create) + : delegate_(std::move(delegate)), + location_(std::move(location)), + fail_before_create_(std::move(fail_before_create)) {} + + std::string_view location() const override { return location_; } + + Result> Create() override { + return CreateOrFail([this] { return delegate_->Create(); }); + } + + Result> CreateOrOverwrite() override { + return CreateOrFail([this] { return delegate_->CreateOrOverwrite(); }); + } + + private: + template + Result> CreateOrFail(CreateFn&& create) { + if (*fail_before_create_) { + *fail_before_create_ = false; + return IOError("injected output creation failure before materialization"); + } + return std::forward(create)(); + } + + std::unique_ptr delegate_; + std::string location_; + std::shared_ptr fail_before_create_; + }; + + static bool IsCompacted(std::string_view location) { + return location.find("compacted-") != std::string_view::npos; + } + + std::shared_ptr delegate_; + std::shared_ptr fail_before_create_; +}; + +class FailAfterCreateFileIO : public FileIO { + public: + explicit FailAfterCreateFileIO(std::shared_ptr delegate) + : delegate_(std::move(delegate)), + fail_after_create_(std::make_shared(true)) {} + + Result> NewInputFile(std::string file_location) override { + return delegate_->NewInputFile(std::move(file_location)); + } + + Result> NewInputFile(std::string file_location, + size_t length) override { + return delegate_->NewInputFile(std::move(file_location), length); + } + + Result> NewOutputFile(std::string file_location) override { + ICEBERG_ASSIGN_OR_RAISE(auto output, delegate_->NewOutputFile(file_location)); + if (!IsCompacted(file_location)) { + return output; + } + return std::unique_ptr(new FailingOutputFile( + std::move(output), std::move(file_location), fail_after_create_)); + } + + Status DeleteFile(const std::string& file_location) override { + return delegate_->DeleteFile(file_location); + } + + private: + class FailingOutputFile : public OutputFile { + public: + FailingOutputFile(std::unique_ptr delegate, std::string location, + std::shared_ptr fail_after_create) + : delegate_(std::move(delegate)), + location_(std::move(location)), + fail_after_create_(std::move(fail_after_create)) {} + + std::string_view location() const override { return location_; } + + Result> Create() override { + ICEBERG_ASSIGN_OR_RAISE(auto stream, delegate_->Create()); + return Track(std::move(stream)); + } + + Result> CreateOrOverwrite() override { + ICEBERG_ASSIGN_OR_RAISE(auto stream, delegate_->CreateOrOverwrite()); + return Track(std::move(stream)); + } + + private: + Result> Track( + std::unique_ptr stream) { + if (*fail_after_create_) { + *fail_after_create_ = false; + return IOError("injected writer failure after output creation"); + } + return stream; + } + + std::unique_ptr delegate_; + std::string location_; + std::shared_ptr fail_after_create_; + }; + + static bool IsCompacted(std::string_view location) { + return location.find("compacted-") != std::string_view::npos; + } + + std::shared_ptr delegate_; + std::shared_ptr fail_after_create_; +}; + +class CompactionExecutorTest : public MinimalUpdateTestBase { + protected: + static void SetUpTestSuite() { + avro::RegisterAll(); + parquet::RegisterAll(); + } + + int8_t format_version() const override { return 3; } + + void SetUp() override { + MinimalUpdateTestBase::SetUp(); + ICEBERG_UNWRAP_OR_FAIL(auto properties, table_->NewUpdateProperties()); + properties->Set(TableProperties::kParquetCompression.key(), "uncompressed") + .Set(TableProperties::kDeleteParquetCompression.key(), "uncompressed") + .Set(TableProperties::kCommitNumRetries.key(), "0"); + ASSERT_THAT(properties->Commit(), IsOk()); + ASSERT_THAT(table_->Refresh(), IsOk()); + ICEBERG_UNWRAP_OR_FAIL(schema_, table_->schema()); + ICEBERG_UNWRAP_OR_FAIL(spec_, table_->spec()); + auto arrow_io = std::dynamic_pointer_cast(file_io_); + ASSERT_NE(arrow_io, nullptr); + ASSERT_TRUE(arrow_io->fs()->CreateDir(table_location_ + "/data/x=10").ok()); + ASSERT_TRUE(arrow_io->fs()->CreateDir(table_location_ + "/data/x=20").ok()); + } + + Result> WriteDataFile(std::string_view name, + std::string_view json, + int64_t partition = 10) { + const auto path = std::format("{}/data/{}", table_location_, name); + ICEBERG_ASSIGN_OR_RAISE( + auto writer, + DataWriter::Make({ + .path = path, + .schema = schema_, + .spec = spec_, + .partition = PartitionValues({Literal::Long(partition)}), + .format = FileFormatType::kParquet, + .io = file_io_, + .properties = {{"write.parquet.compression-codec", "uncompressed"}}, + })); + + ArrowSchema c_schema{}; + ICEBERG_RETURN_UNEXPECTED(ToArrowSchema(*schema_, &c_schema)); + auto arrow_type = ::arrow::ImportType(&c_schema); + if (!arrow_type.ok()) { + return UnknownError(arrow_type.status().ToString()); + } + auto array = ::arrow::json::ArrayFromJSONString( + ::arrow::struct_(arrow_type.ValueOrDie()->fields()), std::string(json)); + if (!array.ok()) { + return UnknownError(array.status().ToString()); + } + + ArrowArray c_array{}; + auto export_status = ::arrow::ExportArray(*array.ValueOrDie(), &c_array); + if (!export_status.ok()) { + return UnknownError(export_status.ToString()); + } + ICEBERG_RETURN_UNEXPECTED(writer->Write(&c_array)); + ICEBERG_RETURN_UNEXPECTED(writer->Close()); + ICEBERG_ASSIGN_OR_RAISE(auto metadata, writer->Metadata()); + ICEBERG_CHECK(metadata.data_files.size() == 1, "Expected one data file, found {}", + metadata.data_files.size()); + return metadata.data_files.front(); + } + + Status Append(const std::vector>& files) { + ICEBERG_ASSIGN_OR_RAISE(auto append, table_->NewFastAppend()); + for (const auto& file : files) { + append->AppendFile(file); + } + ICEBERG_RETURN_UNEXPECTED(append->Commit()); + return table_->Refresh(); + } + + Result>> Tasks( + const std::shared_ptr
& table) { + ICEBERG_ASSIGN_OR_RAISE(auto builder, table->NewScan()); + ICEBERG_ASSIGN_OR_RAISE(auto scan, builder->Build()); + return scan->PlanFiles(); + } + + Result Plan(const std::vector>& tasks) { + ICEBERG_ASSIGN_OR_RAISE(auto snapshot, table_->current_snapshot()); + ICEBERG_ASSIGN_OR_RAISE( + auto plan, CompactionPlanner::Plan(snapshot->snapshot_id, tasks, + CompactionPlannerConfig{ + .target_file_size_bytes = 1024 * 1024, + .min_file_size_ratio = 1, + .max_file_size_ratio = 2, + .min_input_files = 1, + })); + return plan; + } + + std::shared_ptr LineageProjection() { + std::vector fields(schema_->fields().begin(), schema_->fields().end()); + fields.push_back(MetadataColumns::kRowId); + fields.push_back(MetadataColumns::kLastUpdatedSequenceNumber); + return std::make_shared(std::move(fields), schema_->schema_id()); + } + + Result> ReadRows(const FileScanTask& task) { + ICEBERG_ASSIGN_OR_RAISE(auto reader, FileScanTaskReader::Make({ + .io = file_io_, + .table_schema = schema_, + .schemas = table_->metadata()->schemas, + .projected_schema = LineageProjection(), + })); + ICEBERG_ASSIGN_OR_RAISE(auto stream, reader->Open(task)); + auto batch_reader = ::arrow::ImportRecordBatchReader(&stream); + if (!batch_reader.ok()) { + return UnknownError(batch_reader.status().ToString()); + } + + std::vector rows; + while (true) { + auto batch_result = batch_reader.ValueOrDie()->Next(); + if (!batch_result.ok()) { + return UnknownError(batch_result.status().ToString()); + } + auto batch = batch_result.ValueOrDie(); + if (batch == nullptr) { + break; + } + ICEBERG_CHECK(batch->num_columns() == 5, "Expected five projected columns"); + std::array, 5> columns; + for (size_t i = 0; i < columns.size(); ++i) { + columns[i] = std::static_pointer_cast<::arrow::Int64Array>(batch->column(i)); + } + for (int64_t row = 0; row < batch->num_rows(); ++row) { + Row values; + for (size_t column = 0; column < columns.size(); ++column) { + ICEBERG_CHECK(!columns[column]->IsNull(row), + "Compaction row lineage column is null"); + values[column] = columns[column]->Value(row); + } + rows.push_back(values); + } + } + return rows; + } + + Result> LiveDeleteEntries() { + ICEBERG_ASSIGN_OR_RAISE(auto snapshot, table_->current_snapshot()); + SnapshotCache cache(snapshot.get()); + ICEBERG_ASSIGN_OR_RAISE(auto manifests, cache.DeleteManifests(file_io_)); + std::vector result; + for (const auto& manifest : manifests) { + ICEBERG_ASSIGN_OR_RAISE( + auto spec, table_->metadata()->PartitionSpecById(manifest.partition_spec_id)); + ICEBERG_ASSIGN_OR_RAISE( + auto reader, + ManifestReader::Make(manifest, file_io_, schema_, std::move(spec))); + ICEBERG_ASSIGN_OR_RAISE(auto entries, reader->LiveEntries()); + result.insert(result.end(), std::make_move_iterator(entries.begin()), + std::make_move_iterator(entries.end())); + } + return result; + } + + std::vector CompactedFiles() { + auto arrow_io = std::dynamic_pointer_cast(file_io_); + EXPECT_NE(arrow_io, nullptr); + ::arrow::fs::FileSelector selector; + selector.base_dir = table_location_ + "/data"; + selector.recursive = true; + auto infos = arrow_io->fs()->GetFileInfo(selector); + EXPECT_TRUE(infos.ok()) << infos.status().ToString(); + std::vector result; + if (!infos.ok()) { + return result; + } + for (const auto& info : *infos) { + if (info.path().find("compacted-") != std::string::npos) { + result.push_back(info.path()); + } + } + return result; + } + + std::shared_ptr schema_; + std::shared_ptr spec_; +}; + +class CompactionExecutorV2UpgradeTest : public CompactionExecutorTest { + protected: + int8_t format_version() const override { return 2; } +}; + +TEST_F(CompactionExecutorTest, AppliesDeletesAndPreservesRowLineage) { + ICEBERG_UNWRAP_OR_FAIL( + auto data_file, + WriteDataFile("input.parquet", + R"([[10, 100, 1000], [10, 200, 2000], [10, 300, 3000]])")); + ASSERT_THAT(Append({data_file}), IsOk()); + + ICEBERG_UNWRAP_OR_FAIL(auto deletes, PositionDeleteUpdate::Make(table_)); + deletes->Delete(data_file->file_path, 1); + ASSERT_THAT(deletes->Commit(), IsOk()); + ASSERT_THAT(table_->Refresh(), IsOk()); + + ICEBERG_UNWRAP_OR_FAIL(auto tasks, Tasks(table_)); + ASSERT_EQ(tasks.size(), 1); + ICEBERG_UNWRAP_OR_FAIL(auto expected_rows, ReadRows(*tasks.front())); + ICEBERG_UNWRAP_OR_FAIL(auto base_snapshot, table_->current_snapshot()); + ICEBERG_UNWRAP_OR_FAIL(auto plan, Plan(tasks)); + + ICEBERG_UNWRAP_OR_FAIL(auto executor, CompactionExecutor::Make(table_)); + ASSERT_THAT(executor->Execute(plan), IsOk()); + ASSERT_THAT(table_->Refresh(), IsOk()); + + ICEBERG_UNWRAP_OR_FAIL(auto rewritten_tasks, Tasks(table_)); + ASSERT_EQ(rewritten_tasks.size(), 1); + const auto& rewritten = rewritten_tasks.front(); + EXPECT_NE(rewritten->data_file()->file_path, data_file->file_path); + EXPECT_EQ(rewritten->data_file()->record_count, 2); + EXPECT_EQ(rewritten->data_file()->data_sequence_number, base_snapshot->sequence_number); + EXPECT_TRUE(rewritten->delete_files().empty()); + ICEBERG_UNWRAP_OR_FAIL(auto actual_rows, ReadRows(*rewritten)); + EXPECT_EQ(actual_rows, expected_rows); + ICEBERG_UNWRAP_OR_FAIL(auto live_deletes, LiveDeleteEntries()); + EXPECT_TRUE(live_deletes.empty()); +} + +TEST_F(CompactionExecutorTest, KeepsSharedPuffinForUncompactedDataFile) { + ICEBERG_UNWRAP_OR_FAIL( + auto first, + WriteDataFile("first.parquet", R"([[10, 100, 1000], [10, 200, 2000]])")); + ICEBERG_UNWRAP_OR_FAIL( + auto second, + WriteDataFile("second.parquet", R"([[10, 300, 3000], [10, 400, 4000]])")); + ASSERT_THAT(Append({first, second}), IsOk()); + + ICEBERG_UNWRAP_OR_FAIL(auto deletes, PositionDeleteUpdate::Make(table_)); + deletes->Delete(first->file_path, 0).Delete(second->file_path, 1); + ASSERT_THAT(deletes->Commit(), IsOk()); + ASSERT_THAT(table_->Refresh(), IsOk()); + + ICEBERG_UNWRAP_OR_FAIL(auto tasks, Tasks(table_)); + auto first_task = std::ranges::find_if(tasks, [&](const auto& task) { + return task->data_file()->file_path == first->file_path; + }); + auto second_task = std::ranges::find_if(tasks, [&](const auto& task) { + return task->data_file()->file_path == second->file_path; + }); + ASSERT_NE(first_task, tasks.end()); + ASSERT_NE(second_task, tasks.end()); + ASSERT_EQ((*first_task)->delete_files().size(), 1); + ASSERT_EQ((*second_task)->delete_files().size(), 1); + const auto shared_puffin = (*first_task)->delete_files().front()->file_path; + ASSERT_EQ(shared_puffin, (*second_task)->delete_files().front()->file_path); + + ICEBERG_UNWRAP_OR_FAIL(auto plan, + Plan(std::vector>{*first_task})); + ICEBERG_UNWRAP_OR_FAIL(auto executor, CompactionExecutor::Make(table_)); + ASSERT_THAT(executor->Execute(plan), IsOk()); + ASSERT_THAT(table_->Refresh(), IsOk()); + + ICEBERG_UNWRAP_OR_FAIL(auto rewritten_tasks, Tasks(table_)); + auto remaining = std::ranges::find_if(rewritten_tasks, [&](const auto& task) { + return task->data_file()->file_path == second->file_path; + }); + ASSERT_NE(remaining, rewritten_tasks.end()); + ASSERT_EQ((*remaining)->delete_files().size(), 1); + EXPECT_EQ((*remaining)->delete_files().front()->file_path, shared_puffin); + auto arrow_io = std::dynamic_pointer_cast(file_io_); + ASSERT_NE(arrow_io, nullptr); + auto info = arrow_io->fs()->GetFileInfo(shared_puffin); + ASSERT_TRUE(info.ok()) << info.status().ToString(); + EXPECT_EQ(info->type(), ::arrow::fs::FileType::File); + + ICEBERG_UNWRAP_OR_FAIL(auto live_deletes, LiveDeleteEntries()); + ASSERT_EQ(live_deletes.size(), 1); + EXPECT_EQ(live_deletes.front().data_file->referenced_data_file, second->file_path); +} + +TEST_F(CompactionExecutorTest, FailedCommitCleansRewrittenDataFile) { + ICEBERG_UNWRAP_OR_FAIL( + auto data_file, + WriteDataFile("input.parquet", R"([[10, 100, 1000], [10, 200, 2000]])")); + ASSERT_THAT(Append({data_file}), IsOk()); + ICEBERG_UNWRAP_OR_FAIL(auto tasks, Tasks(table_)); + ICEBERG_UNWRAP_OR_FAIL(auto plan, Plan(tasks)); + + auto mock_catalog = std::make_shared<::testing::NiceMock>(); + ON_CALL(*mock_catalog, UpdateTable(::testing::_, ::testing::_, ::testing::_)) + .WillByDefault([](const TableIdentifier&, + const std::vector>&, + const std::vector>&) + -> Result> { + return CommitFailed("injected failure"); + }); + ICEBERG_UNWRAP_OR_FAIL(auto mock_table, + Table::Make(table_->name(), table_->metadata(), + std::string(table_->metadata_file_location()), + table_->io(), mock_catalog)); + ON_CALL(*mock_catalog, LoadTable(::testing::_)) + .WillByDefault( + [&mock_table](const TableIdentifier&) -> Result> { + return mock_table; + }); + + ICEBERG_UNWRAP_OR_FAIL(auto executor, CompactionExecutor::Make(mock_table)); + EXPECT_THAT(executor->Execute(plan), IsError(ErrorKind::kCommitFailed)); + EXPECT_TRUE(CompactedFiles().empty()); +} + +TEST_F(CompactionExecutorTest, CommitStateUnknownRelinquishesOutputOwnership) { + ICEBERG_UNWRAP_OR_FAIL( + auto data_file, + WriteDataFile("input.parquet", R"([[10, 100, 1000], [10, 200, 2000]])")); + ASSERT_THAT(Append({data_file}), IsOk()); + ICEBERG_UNWRAP_OR_FAIL(auto tasks, Tasks(table_)); + ICEBERG_UNWRAP_OR_FAIL(auto plan, Plan(tasks)); + + auto mock_catalog = std::make_shared<::testing::NiceMock>(); + int update_calls = 0; + ON_CALL(*mock_catalog, UpdateTable(::testing::_, ::testing::_, ::testing::_)) + .WillByDefault( + [&update_calls](const TableIdentifier&, + const std::vector>&, + const std::vector>&) + -> Result> { + ++update_calls; + return CommitStateUnknown("injected unknown state"); + }); + ICEBERG_UNWRAP_OR_FAIL(auto mock_table, + Table::Make(table_->name(), table_->metadata(), + std::string(table_->metadata_file_location()), + table_->io(), mock_catalog)); + ON_CALL(*mock_catalog, LoadTable(::testing::_)) + .WillByDefault( + [&mock_table](const TableIdentifier&) -> Result> { + return mock_table; + }); + + ICEBERG_UNWRAP_OR_FAIL(auto executor, CompactionExecutor::Make(mock_table)); + EXPECT_THAT(executor->Execute(plan), IsError(ErrorKind::kCommitStateUnknown)); + ASSERT_EQ(CompactedFiles().size(), 1); + + EXPECT_THAT(executor->Execute(plan), IsError(ErrorKind::kInvalidArgument)); + EXPECT_EQ(update_calls, 1); + EXPECT_EQ(CompactedFiles().size(), 1); +} + +TEST_F(CompactionExecutorTest, RejectsDuplicateSourceFilesBeforeWriting) { + ICEBERG_UNWRAP_OR_FAIL( + auto data_file, + WriteDataFile("input.parquet", R"([[10, 100, 1000], [10, 200, 2000]])")); + ASSERT_THAT(Append({data_file}), IsOk()); + ICEBERG_UNWRAP_OR_FAIL(auto tasks, Tasks(table_)); + ICEBERG_UNWRAP_OR_FAIL(auto plan, Plan(tasks)); + ASSERT_EQ(plan.groups.size(), 1); + ASSERT_EQ(plan.groups.front().files.size(), 1); + + auto within_group = plan; + within_group.groups.front().files.push_back(within_group.groups.front().files.front()); + ICEBERG_UNWRAP_OR_FAIL(auto executor, CompactionExecutor::Make(table_)); + EXPECT_THAT(executor->Execute(within_group), IsError(ErrorKind::kInvalidArgument)); + EXPECT_TRUE(CompactedFiles().empty()); + + auto across_groups = plan; + across_groups.groups.push_back(across_groups.groups.front()); + EXPECT_THAT(executor->Execute(across_groups), IsError(ErrorKind::kInvalidArgument)); + EXPECT_TRUE(CompactedFiles().empty()); +} + +TEST_F(CompactionExecutorTest, RejectsPlanAfterNewDeleteSnapshot) { + ICEBERG_UNWRAP_OR_FAIL( + auto data_file, + WriteDataFile("input.parquet", + R"([[10, 100, 1000], [10, 200, 2000], [10, 300, 3000]])")); + ASSERT_THAT(Append({data_file}), IsOk()); + ICEBERG_UNWRAP_OR_FAIL(auto tasks, Tasks(table_)); + ICEBERG_UNWRAP_OR_FAIL(auto plan, Plan(tasks)); + + ICEBERG_UNWRAP_OR_FAIL(auto deletes, PositionDeleteUpdate::Make(table_)); + deletes->Delete(data_file->file_path, 1); + ASSERT_THAT(deletes->Commit(), IsOk()); + + ICEBERG_UNWRAP_OR_FAIL(auto executor, CompactionExecutor::Make(table_)); + EXPECT_THAT(executor->Execute(plan), IsError(ErrorKind::kValidationFailed)); + EXPECT_TRUE(CompactedFiles().empty()); + + ASSERT_THAT(table_->Refresh(), IsOk()); + ICEBERG_UNWRAP_OR_FAIL(auto current_tasks, Tasks(table_)); + ASSERT_EQ(current_tasks.size(), 1); + ASSERT_EQ(current_tasks.front()->delete_files().size(), 1); + ICEBERG_UNWRAP_OR_FAIL(auto rows, ReadRows(*current_tasks.front())); + EXPECT_EQ(rows.size(), 2); +} + +TEST_F(CompactionExecutorTest, MultiFileMultiGroupCommitIsAtomicOnFailure) { + ICEBERG_UNWRAP_OR_FAIL( + auto first, + WriteDataFile("first.parquet", R"([[10, 100, 1000], [10, 200, 2000]])")); + ICEBERG_UNWRAP_OR_FAIL( + auto second, + WriteDataFile("second.parquet", R"([[10, 300, 3000], [10, 400, 4000]])")); + ICEBERG_UNWRAP_OR_FAIL( + auto third, + WriteDataFile("third.parquet", R"([[20, 500, 5000], [20, 600, 6000]])", 20)); + ASSERT_THAT(Append({first, second, third}), IsOk()); + ICEBERG_UNWRAP_OR_FAIL(auto tasks, Tasks(table_)); + ICEBERG_UNWRAP_OR_FAIL(auto plan, Plan(tasks)); + ASSERT_EQ(plan.groups.size(), 2); + ASSERT_EQ(plan.groups.front().files.size(), 2); + + auto mock_catalog = std::make_shared<::testing::NiceMock>(); + int update_calls = 0; + ON_CALL(*mock_catalog, UpdateTable(::testing::_, ::testing::_, ::testing::_)) + .WillByDefault( + [&update_calls](const TableIdentifier&, + const std::vector>&, + const std::vector>&) + -> Result> { + ++update_calls; + return CommitFailed("injected failure"); + }); + ICEBERG_UNWRAP_OR_FAIL(auto mock_table, + Table::Make(table_->name(), table_->metadata(), + std::string(table_->metadata_file_location()), + table_->io(), mock_catalog)); + ON_CALL(*mock_catalog, LoadTable(::testing::_)) + .WillByDefault( + [&mock_table](const TableIdentifier&) -> Result> { + return mock_table; + }); + + ICEBERG_UNWRAP_OR_FAIL(auto executor, CompactionExecutor::Make(mock_table)); + EXPECT_THAT(executor->Execute(plan), IsError(ErrorKind::kCommitFailed)); + EXPECT_EQ(update_calls, 1); + EXPECT_TRUE(CompactedFiles().empty()); + + ICEBERG_UNWRAP_OR_FAIL(auto current_tasks, Tasks(table_)); + EXPECT_EQ(current_tasks.size(), 3); +} + +TEST_F(CompactionExecutorTest, CleanupFailureRetainsOutputForExplicitRetry) { + ICEBERG_UNWRAP_OR_FAIL( + auto data_file, + WriteDataFile("input.parquet", R"([[10, 100, 1000], [10, 200, 2000]])")); + ASSERT_THAT(Append({data_file}), IsOk()); + ICEBERG_UNWRAP_OR_FAIL(auto tasks, Tasks(table_)); + ICEBERG_UNWRAP_OR_FAIL(auto plan, Plan(tasks)); + + auto failing_io = std::make_shared(table_->io()); + auto mock_catalog = std::make_shared<::testing::NiceMock>(); + ON_CALL(*mock_catalog, UpdateTable(::testing::_, ::testing::_, ::testing::_)) + .WillByDefault([](const TableIdentifier&, + const std::vector>&, + const std::vector>&) + -> Result> { + return CommitFailed("injected commit failure"); + }); + ICEBERG_UNWRAP_OR_FAIL(auto mock_table, + Table::Make(table_->name(), table_->metadata(), + std::string(table_->metadata_file_location()), + failing_io, mock_catalog)); + ON_CALL(*mock_catalog, LoadTable(::testing::_)) + .WillByDefault( + [&mock_table](const TableIdentifier&) -> Result> { + return mock_table; + }); + + ICEBERG_UNWRAP_OR_FAIL(auto executor, CompactionExecutor::Make(mock_table)); + auto status = executor->Execute(plan); + EXPECT_THAT(status, IsError(ErrorKind::kCommitFailed)); + EXPECT_THAT(status, HasErrorMessage("injected commit failure")); + EXPECT_THAT(status, HasErrorMessage("injected cleanup failure")); + ASSERT_EQ(CompactedFiles().size(), 1); + + failing_io->AllowCompactedDeletes(); + EXPECT_THAT(executor->Cleanup(), IsOk()); + EXPECT_TRUE(CompactedFiles().empty()); +} + +TEST_F(CompactionExecutorTest, WriterFailureBeforeFileCreationLeavesExecutorReusable) { + ICEBERG_UNWRAP_OR_FAIL( + auto data_file, + WriteDataFile("input.parquet", R"([[10, 100, 1000], [10, 200, 2000]])")); + ASSERT_THAT(Append({data_file}), IsOk()); + ICEBERG_UNWRAP_OR_FAIL(auto tasks, Tasks(table_)); + ICEBERG_UNWRAP_OR_FAIL(auto plan, Plan(tasks)); + + ICEBERG_UNWRAP_OR_FAIL(auto invalid_properties, table_->NewUpdateProperties()); + invalid_properties->Set(WriterProperties::kParquetMaxRowGroupRows.key(), "0"); + ASSERT_THAT(invalid_properties->Commit(), IsOk()); + ASSERT_THAT(table_->Refresh(), IsOk()); + + ICEBERG_UNWRAP_OR_FAIL(auto executor, CompactionExecutor::Make(table_)); + auto status = executor->Execute(plan); + EXPECT_THAT(status, IsError(ErrorKind::kInvalidArgument)); + EXPECT_THAT(status, HasErrorMessage("Parquet max row group rows")); + EXPECT_TRUE(CompactedFiles().empty()); + EXPECT_THAT(executor->Cleanup(), IsOk()); + + ICEBERG_UNWRAP_OR_FAIL(auto valid_properties, table_->NewUpdateProperties()); + valid_properties->Set(WriterProperties::kParquetMaxRowGroupRows.key(), "100"); + ASSERT_THAT(valid_properties->Commit(), IsOk()); + ASSERT_THAT(table_->Refresh(), IsOk()); + + ASSERT_THAT(executor->Execute(plan), IsOk()); + ASSERT_THAT(table_->Refresh(), IsOk()); + ICEBERG_UNWRAP_OR_FAIL(auto rewritten_tasks, Tasks(table_)); + ASSERT_EQ(rewritten_tasks.size(), 1); + EXPECT_NE(rewritten_tasks.front()->data_file()->file_path, data_file->file_path); +} + +TEST_F(CompactionExecutorTest, OutputCreateFailureCleansAndRemainsReusable) { + ICEBERG_UNWRAP_OR_FAIL( + auto data_file, + WriteDataFile("input.parquet", R"([[10, 100, 1000], [10, 200, 2000]])")); + ASSERT_THAT(Append({data_file}), IsOk()); + ICEBERG_UNWRAP_OR_FAIL(auto tasks, Tasks(table_)); + ICEBERG_UNWRAP_OR_FAIL(auto plan, Plan(tasks)); + + auto writer_io = std::make_shared(table_->io()); + ICEBERG_UNWRAP_OR_FAIL( + auto writer_table, + Table::Make(table_->name(), table_->metadata(), + std::string(table_->metadata_file_location()), writer_io, catalog_)); + ICEBERG_UNWRAP_OR_FAIL(auto executor, CompactionExecutor::Make(writer_table)); + + auto status = executor->Execute(plan); + EXPECT_THAT(status, IsError(ErrorKind::kIOError)); + EXPECT_THAT(status, HasErrorMessage("output creation failure before materialization")); + EXPECT_TRUE(CompactedFiles().empty()); + EXPECT_THAT(executor->Cleanup(), IsOk()); + + ASSERT_THAT(executor->Execute(plan), IsOk()); + ASSERT_THAT(table_->Refresh(), IsOk()); + ICEBERG_UNWRAP_OR_FAIL(auto rewritten_tasks, Tasks(table_)); + ASSERT_EQ(rewritten_tasks.size(), 1); + EXPECT_NE(rewritten_tasks.front()->data_file()->file_path, data_file->file_path); +} + +TEST_F(CompactionExecutorTest, WriterFailureAfterFileCreationCleansAndRemainsReusable) { + ICEBERG_UNWRAP_OR_FAIL( + auto data_file, + WriteDataFile("input.parquet", R"([[10, 100, 1000], [10, 200, 2000]])")); + ASSERT_THAT(Append({data_file}), IsOk()); + ICEBERG_UNWRAP_OR_FAIL(auto tasks, Tasks(table_)); + ICEBERG_UNWRAP_OR_FAIL(auto plan, Plan(tasks)); + + auto writer_io = std::make_shared(table_->io()); + ICEBERG_UNWRAP_OR_FAIL( + auto writer_table, + Table::Make(table_->name(), table_->metadata(), + std::string(table_->metadata_file_location()), writer_io, catalog_)); + ICEBERG_UNWRAP_OR_FAIL(auto executor, CompactionExecutor::Make(writer_table)); + + auto status = executor->Execute(plan); + EXPECT_THAT(status, IsError(ErrorKind::kIOError)); + EXPECT_THAT(status, HasErrorMessage("injected writer failure after output creation")); + EXPECT_TRUE(CompactedFiles().empty()); + EXPECT_THAT(executor->Cleanup(), IsOk()); + + ASSERT_THAT(executor->Execute(plan), IsOk()); + ASSERT_THAT(table_->Refresh(), IsOk()); + ICEBERG_UNWRAP_OR_FAIL(auto rewritten_tasks, Tasks(table_)); + ASSERT_EQ(rewritten_tasks.size(), 1); + EXPECT_NE(rewritten_tasks.front()->data_file()->file_path, data_file->file_path); +} + +TEST_F(CompactionExecutorV2UpgradeTest, + RemovesParquetDeleteAndAssignsUpgradedRowLineage) { + ICEBERG_UNWRAP_OR_FAIL( + auto data_file, + WriteDataFile("input.parquet", + R"([[10, 100, 1000], [10, 200, 2000], [10, 300, 3000]])")); + ASSERT_THAT(Append({data_file}), IsOk()); + + ICEBERG_UNWRAP_OR_FAIL(auto deletes, PositionDeleteUpdate::Make(table_)); + deletes->Delete(data_file->file_path, 1); + ASSERT_THAT(deletes->Commit(), IsOk()); + ASSERT_THAT(table_->Refresh(), IsOk()); + ICEBERG_UNWRAP_OR_FAIL(auto v2_tasks, Tasks(table_)); + ASSERT_EQ(v2_tasks.size(), 1); + ASSERT_EQ(v2_tasks.front()->delete_files().size(), 1); + EXPECT_EQ(v2_tasks.front()->delete_files().front()->file_format, + FileFormatType::kParquet); + + ICEBERG_UNWRAP_OR_FAIL(auto properties, table_->NewUpdateProperties()); + properties->Set(TableProperties::kFormatVersion.key(), "3"); + ASSERT_THAT(properties->Commit(), IsOk()); + ASSERT_THAT(table_->Refresh(), IsOk()); + + ICEBERG_UNWRAP_OR_FAIL(auto tasks, Tasks(table_)); + ASSERT_EQ(tasks.size(), 1); + ASSERT_FALSE(tasks.front()->data_file()->first_row_id.has_value()); + ASSERT_TRUE(tasks.front()->data_file()->data_sequence_number.has_value()); + const auto original_sequence = *tasks.front()->data_file()->data_sequence_number; + ICEBERG_UNWRAP_OR_FAIL(auto plan, Plan(tasks)); + + ICEBERG_UNWRAP_OR_FAIL(auto executor, CompactionExecutor::Make(table_)); + ASSERT_THAT(executor->Execute(plan), IsOk()); + ASSERT_THAT(table_->Refresh(), IsOk()); + + ICEBERG_UNWRAP_OR_FAIL(auto rewritten_tasks, Tasks(table_)); + ASSERT_EQ(rewritten_tasks.size(), 1); + ASSERT_TRUE(rewritten_tasks.front()->data_file()->first_row_id.has_value()); + const auto first_row_id = *rewritten_tasks.front()->data_file()->first_row_id; + ICEBERG_UNWRAP_OR_FAIL(auto rows, ReadRows(*rewritten_tasks.front())); + ASSERT_EQ(rows.size(), 2); + EXPECT_EQ(rows[0][3], first_row_id); + EXPECT_EQ(rows[1][3], first_row_id + 1); + EXPECT_EQ(rows[0][4], original_sequence); + EXPECT_EQ(rows[1][4], original_sequence); + EXPECT_TRUE(rewritten_tasks.front()->delete_files().empty()); + ICEBERG_UNWRAP_OR_FAIL(auto live_deletes, LiveDeleteEntries()); + EXPECT_TRUE(live_deletes.empty()); +} + +} // namespace +} // namespace iceberg diff --git a/src/iceberg/test/compaction_planner_test.cc b/src/iceberg/test/compaction_planner_test.cc new file mode 100644 index 000000000..65eff74ca --- /dev/null +++ b/src/iceberg/test/compaction_planner_test.cc @@ -0,0 +1,420 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#include "iceberg/compaction_planner.h" + +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include + +#include "iceberg/expression/literal.h" +#include "iceberg/file_format.h" +#include "iceberg/manifest/manifest_entry.h" +#include "iceberg/table_scan.h" +#include "iceberg/test/matchers.h" +#include "iceberg/type.h" + +namespace iceberg { +namespace { + +using ::testing::ElementsAre; +using ::testing::IsEmpty; +using ::testing::SizeIs; + +class CompactionPlannerTest : public testing::Test { + protected: + static constexpr int64_t kSnapshotId = 1234; + + static std::shared_ptr Data(std::string path, PartitionValues partition, + int64_t size, std::optional spec_id = 1, + int64_t records = 100) { + return std::make_shared(DataFile{ + .file_path = std::move(path), + .partition = std::move(partition), + .record_count = records, + .file_size_in_bytes = size, + .partition_spec_id = spec_id, + }); + } + + static std::shared_ptr Data(std::string path, int32_t partition, int64_t size, + std::optional spec_id = 1, + int64_t records = 100) { + return Data(std::move(path), PartitionValues(Literal::Int(partition)), size, spec_id, + records); + } + + static std::shared_ptr PositionDelete(std::string path, + const std::string& referenced_file, + int64_t records, + bool file_scoped = true) { + return std::make_shared(DataFile{ + .content = DataFile::Content::kPositionDeletes, + .file_path = std::move(path), + .record_count = records, + .referenced_data_file = + file_scoped ? std::make_optional(referenced_file) : std::nullopt, + }); + } + + static std::shared_ptr DeletionVector(std::string path, + const std::string& referenced_file, + int64_t records, int64_t offset = 0, + int64_t size = 10) { + return std::make_shared(DataFile{ + .content = DataFile::Content::kPositionDeletes, + .file_path = std::move(path), + .file_format = FileFormatType::kPuffin, + .record_count = records, + .referenced_data_file = referenced_file, + .content_offset = offset, + .content_size_in_bytes = size, + }); + } + + static std::shared_ptr EqualityDelete(std::string path, int64_t records) { + return std::make_shared(DataFile{ + .content = DataFile::Content::kEqualityDeletes, + .file_path = std::move(path), + .record_count = records, + }); + } + + static std::shared_ptr Task( + std::shared_ptr data_file, + std::vector> deletes = {}) { + return std::make_shared(std::move(data_file), std::move(deletes)); + } + + static std::vector Paths(const CompactionGroup& group) { + std::vector paths; + for (const auto& file : group.files) { + paths.push_back(file.scan_task->data_file()->file_path); + } + return paths; + } + + static CompactionPlannerConfig Config() { + return CompactionPlannerConfig{ + .target_file_size_bytes = 100, + .min_file_size_ratio = 0.5, + .max_file_size_ratio = 2.0, + .min_input_files = 2, + .delete_file_threshold = 2, + .delete_ratio_threshold = 0.2, + }; + } +}; + +TEST_F(CompactionPlannerTest, ValidatesConfig) { + std::vector invalid; + + auto config = Config(); + config.target_file_size_bytes = 0; + invalid.push_back(config); + config = Config(); + config.min_file_size_ratio = -0.1; + invalid.push_back(config); + config = Config(); + config.min_file_size_ratio = 1.1; + invalid.push_back(config); + config = Config(); + config.min_file_size_ratio = std::numeric_limits::quiet_NaN(); + invalid.push_back(config); + config = Config(); + config.max_file_size_ratio = 0.9; + invalid.push_back(config); + config = Config(); + config.max_file_size_ratio = std::numeric_limits::infinity(); + invalid.push_back(config); + config = Config(); + config.min_input_files = 0; + invalid.push_back(config); + config = Config(); + config.delete_file_threshold = -1; + invalid.push_back(config); + config = Config(); + config.delete_ratio_threshold = -0.1; + invalid.push_back(config); + config = Config(); + config.delete_ratio_threshold = 1.1; + invalid.push_back(config); + config = Config(); + config.delete_ratio_threshold = std::numeric_limits::quiet_NaN(); + invalid.push_back(config); + + for (const auto& invalid_config : invalid) { + EXPECT_THAT(CompactionPlanner::Plan(kSnapshotId, {}, invalid_config), + IsError(ErrorKind::kInvalidArgument)); + } +} + +TEST_F(CompactionPlannerTest, ZeroDeleteThresholdsDisableDeleteSelection) { + auto config = Config(); + config.delete_file_threshold = 0; + config.delete_ratio_threshold = 0; + std::vector> tasks{ + Task(Data("data", 1, 100), {PositionDelete("delete-a", "data", 100), + PositionDelete("delete-b", "data", 100)})}; + + ICEBERG_UNWRAP_OR_FAIL(auto plan, CompactionPlanner::Plan(kSnapshotId, tasks, config)); + EXPECT_THAT(plan.groups, IsEmpty()); +} + +TEST_F(CompactionPlannerTest, BindsPlanToSourceSnapshot) { + ICEBERG_UNWRAP_OR_FAIL(auto plan, CompactionPlanner::Plan(kSnapshotId, {}, Config())); + + EXPECT_EQ(plan.source_snapshot_id, kSnapshotId); + EXPECT_THAT(CompactionPlanner::Plan(-1, {}, Config()), + IsError(ErrorKind::kInvalidArgument)); +} + +TEST_F(CompactionPlannerTest, SelectsThresholdBoundariesAndIgnoresOtherDeletes) { + std::vector> tasks{ + Task(Data("small-a", 1, 49)), + Task(Data("small-b", 1, 49)), + Task(Data("at-min-size", 1, 50)), + Task(Data("below-ratio", 2, 100), + {DeletionVector("below-ratio.dv", "below-ratio", 19)}), + Task(Data("at-ratio", 2, 100), {DeletionVector("at-ratio.dv", "at-ratio", 20)}), + Task(Data("at-count", 3, 100), {PositionDelete("count-a", "at-count", 0), + PositionDelete("count-b", "at-count", 0)}), + Task(Data("ignored", 4, 100), + {EqualityDelete("equality", 100), + PositionDelete("partition-scoped", "ignored", 100, false)})}; + + ICEBERG_UNWRAP_OR_FAIL(auto plan, + CompactionPlanner::Plan(kSnapshotId, tasks, Config())); + ASSERT_THAT(plan.groups, SizeIs(3)); + EXPECT_THAT(Paths(plan.groups[0]), ElementsAre("small-a", "small-b")); + EXPECT_THAT(Paths(plan.groups[1]), ElementsAre("at-ratio")); + EXPECT_EQ(plan.groups[1].file_scoped_delete_record_count, 20); + EXPECT_THAT(Paths(plan.groups[2]), ElementsAre("at-count")); +} + +TEST_F(CompactionPlannerTest, GroupsNullPartitionsTogether) { + auto null_partition = [] { return PartitionValues(Literal::Null(int32())); }; + std::vector> tasks{ + Task(Data("null-b", null_partition(), 10)), Task(Data("value-a", 1, 10)), + Task(Data("null-a", null_partition(), 10)), Task(Data("value-b", 1, 10))}; + + ICEBERG_UNWRAP_OR_FAIL(auto plan, + CompactionPlanner::Plan(kSnapshotId, tasks, Config())); + ASSERT_THAT(plan.groups, SizeIs(2)); + EXPECT_THAT(Paths(plan.groups[0]), ElementsAre("null-a", "null-b")); + EXPECT_THAT(Paths(plan.groups[1]), ElementsAre("value-a", "value-b")); +} + +TEST_F(CompactionPlannerTest, GroupsCanonicalNaNPartitionsTogether) { + auto quiet_nan = + PartitionValues(Literal::Double(std::numeric_limits::quiet_NaN())); + auto signaling_nan = + PartitionValues(Literal::Double(std::numeric_limits::signaling_NaN())); + std::vector> tasks{ + Task(Data("nan-b", std::move(signaling_nan), 10)), + Task(Data("nan-a", std::move(quiet_nan), 10))}; + + ICEBERG_UNWRAP_OR_FAIL(auto plan, + CompactionPlanner::Plan(kSnapshotId, tasks, Config())); + ASSERT_THAT(plan.groups, SizeIs(1)); + EXPECT_THAT(Paths(plan.groups[0]), ElementsAre("nan-a", "nan-b")); +} + +TEST_F(CompactionPlannerTest, RejectsMissingPartitionSpecId) { + std::vector> tasks{ + Task(Data("missing-spec", 1, 10, std::nullopt))}; + + EXPECT_THAT(CompactionPlanner::Plan(kSnapshotId, tasks, Config()), + IsError(ErrorKind::kInvalidArgument)); +} + +TEST_F(CompactionPlannerTest, RejectsDuplicateDataFileTasks) { + std::vector> tasks{Task(Data("duplicate", 1, 10)), + Task(Data("duplicate", 1, 10))}; + + EXPECT_THAT(CompactionPlanner::Plan(kSnapshotId, tasks, Config()), + IsError(ErrorKind::kInvalidArgument)); +} + +TEST_F(CompactionPlannerTest, ProducesCanonicalPartitionAndFileOrder) { + std::vector> tasks{ + Task(Data("s2-z", 1, 10, 2)), Task(Data("p2-z", 2, 10)), Task(Data("p1-z", 1, 10)), + Task(Data("s2-a", 1, 10, 2)), Task(Data("p2-a", 2, 10)), Task(Data("p1-a", 1, 10))}; + + ICEBERG_UNWRAP_OR_FAIL(auto first, + CompactionPlanner::Plan(kSnapshotId, tasks, Config())); + std::ranges::reverse(tasks); + ICEBERG_UNWRAP_OR_FAIL(auto second, + CompactionPlanner::Plan(kSnapshotId, tasks, Config())); + + ASSERT_THAT(first.groups, SizeIs(3)); + ASSERT_THAT(second.groups, SizeIs(3)); + for (size_t i = 0; i < first.groups.size(); ++i) { + EXPECT_EQ(first.groups[i].partition_spec_id, second.groups[i].partition_spec_id); + EXPECT_EQ(Paths(first.groups[i]), Paths(second.groups[i])); + } + EXPECT_EQ(first.groups[0].partition_spec_id, 1); + EXPECT_THAT(Paths(first.groups[0]), ElementsAre("p1-a", "p1-z")); + EXPECT_THAT(Paths(first.groups[1]), ElementsAre("p2-a", "p2-z")); + EXPECT_EQ(first.groups[2].partition_spec_id, 2); + EXPECT_THAT(Paths(first.groups[2]), ElementsAre("s2-a", "s2-z")); +} + +TEST_F(CompactionPlannerTest, ComparesLargeFileSizeRatiosExactly) { + constexpr int64_t kMax = std::numeric_limits::max(); + auto config = Config(); + config.target_file_size_bytes = kMax; + config.min_file_size_ratio = std::nextafter(1.0, 0.0); + config.max_file_size_ratio = 1.0; + config.min_input_files = 1; + config.delete_file_threshold = 0; + config.delete_ratio_threshold = 0; + std::vector> tasks{ + Task(Data("below-boundary", 1, kMax - 1024)), + Task(Data("above-boundary", 2, kMax - 1023))}; + + ICEBERG_UNWRAP_OR_FAIL(auto plan, CompactionPlanner::Plan(kSnapshotId, tasks, config)); + ASSERT_THAT(plan.groups, SizeIs(1)); + EXPECT_THAT(Paths(plan.groups[0]), ElementsAre("below-boundary")); +} + +TEST_F(CompactionPlannerTest, ComparesLargeDeleteRatiosExactly) { + constexpr int64_t kMax = std::numeric_limits::max(); + auto config = Config(); + config.min_file_size_ratio = 0; + config.delete_file_threshold = 0; + config.delete_ratio_threshold = std::nextafter(1.0, 0.0); + std::vector> tasks{ + Task(Data("below-boundary", 1, 100, 1, kMax), + {PositionDelete("below.delete", "below-boundary", kMax - 1024)}), + Task(Data("at-boundary", 2, 100, 1, kMax), + {PositionDelete("at.delete", "at-boundary", kMax - 1023)})}; + + ICEBERG_UNWRAP_OR_FAIL(auto plan, CompactionPlanner::Plan(kSnapshotId, tasks, config)); + ASSERT_THAT(plan.groups, SizeIs(1)); + EXPECT_THAT(Paths(plan.groups[0]), ElementsAre("at-boundary")); +} + +TEST_F(CompactionPlannerTest, DeduplicatesDeleteReferencesAndDistinguishesDvRanges) { + std::vector> tasks{ + Task(Data("data", 1, 100), + {PositionDelete("deletes", "data", 30), PositionDelete("deletes", "data", 30), + DeletionVector("deletes", "data", 20, 0, 10), + DeletionVector("deletes", "data", 20, 0, 10), + DeletionVector("deletes", "data", 5, 10, 10)})}; + auto config = Config(); + config.delete_file_threshold = 3; + config.delete_ratio_threshold = 0; + + ICEBERG_UNWRAP_OR_FAIL(auto plan, CompactionPlanner::Plan(kSnapshotId, tasks, config)); + ASSERT_THAT(plan.groups, SizeIs(1)); + ASSERT_THAT(plan.groups[0].files, SizeIs(1)); + EXPECT_EQ(plan.groups[0].files[0].file_scoped_delete_count, 3); + EXPECT_EQ(plan.groups[0].files[0].file_scoped_delete_record_count, 55); +} + +TEST_F(CompactionPlannerTest, CapsPositionDeleteCardinalityAtDataRows) { + std::vector> tasks{Task( + Data("data", 1, 100), + {PositionDelete("delete-a", "data", 80), PositionDelete("delete-b", "data", 40)})}; + + ICEBERG_UNWRAP_OR_FAIL(auto plan, + CompactionPlanner::Plan(kSnapshotId, tasks, Config())); + ASSERT_THAT(plan.groups, SizeIs(1)); + EXPECT_EQ(plan.groups[0].files[0].file_scoped_delete_record_count, 100); + EXPECT_EQ(plan.groups[0].file_scoped_delete_record_count, 100); +} + +TEST_F(CompactionPlannerTest, RejectsDvCardinalityAboveDataRows) { + std::vector> tasks{ + Task(Data("data", 1, 100), {DeletionVector("dv", "data", 101)})}; + + EXPECT_THAT(CompactionPlanner::Plan(kSnapshotId, tasks, Config()), + IsError(ErrorKind::kInvalidArgument)); +} + +TEST_F(CompactionPlannerTest, ReturnsErrorsForAggregateOverflow) { + constexpr int64_t kMax = std::numeric_limits::max(); + std::vector> file_overflow{ + Task(Data("data", 1, 100, 1, kMax), {PositionDelete("delete-a", "data", kMax), + PositionDelete("delete-b", "data", kMax)})}; + EXPECT_THAT(CompactionPlanner::Plan(kSnapshotId, file_overflow, Config()), + IsError(ErrorKind::kInvalidArgument)); + + std::vector> group_overflow{ + Task(Data("a", 1, 0, 1, kMax), {PositionDelete("delete-a", "a", kMax)}), + Task(Data("b", 1, 0, 1, kMax), {PositionDelete("delete-b", "b", kMax)})}; + EXPECT_THAT(CompactionPlanner::Plan(kSnapshotId, group_overflow, Config()), + IsError(ErrorKind::kInvalidArgument)); +} + +TEST_F(CompactionPlannerTest, RequiresDeletePressureForOversizedFiles) { + std::vector> tasks{ + Task(Data("no-deletes", 1, 201)), + Task(Data("below-ratio", 1, 201), {DeletionVector("below.dv", "below-ratio", 19)}), + Task(Data("at-ratio", 1, 201), {DeletionVector("at.dv", "at-ratio", 20)})}; + + ICEBERG_UNWRAP_OR_FAIL(auto plan, + CompactionPlanner::Plan(kSnapshotId, tasks, Config())); + ASSERT_THAT(plan.groups, SizeIs(1)); + EXPECT_THAT(Paths(plan.groups[0]), ElementsAre("at-ratio")); +} + +TEST_F(CompactionPlannerTest, SplitsPartitionIntoTargetSizedGroups) { + std::vector> tasks; + for (const auto* path : {"f", "e", "d", "c", "b", "a"}) { + tasks.push_back(Task(Data(path, 1, 40))); + } + + ICEBERG_UNWRAP_OR_FAIL(auto plan, + CompactionPlanner::Plan(kSnapshotId, tasks, Config())); + ASSERT_THAT(plan.groups, SizeIs(3)); + EXPECT_THAT(Paths(plan.groups[0]), ElementsAre("a", "b")); + EXPECT_THAT(Paths(plan.groups[1]), ElementsAre("c", "d")); + EXPECT_THAT(Paths(plan.groups[2]), ElementsAre("e", "f")); + for (const auto& group : plan.groups) { + EXPECT_EQ(group.data_file_size_bytes, 80); + } +} + +TEST_F(CompactionPlannerTest, BestFitAvoidsDroppingCompatibleSmallFiles) { + auto config = Config(); + config.min_file_size_ratio = 1.0; + std::vector> tasks{ + Task(Data("d40", 1, 40)), Task(Data("b60", 1, 60)), Task(Data("c40", 1, 40)), + Task(Data("a60", 1, 60))}; + + ICEBERG_UNWRAP_OR_FAIL(auto plan, CompactionPlanner::Plan(kSnapshotId, tasks, config)); + ASSERT_THAT(plan.groups, SizeIs(2)); + EXPECT_THAT(Paths(plan.groups[0]), ElementsAre("a60", "c40")); + EXPECT_THAT(Paths(plan.groups[1]), ElementsAre("b60", "d40")); + EXPECT_EQ(plan.groups[0].data_file_size_bytes, 100); + EXPECT_EQ(plan.groups[1].data_file_size_bytes, 100); +} + +} // namespace +} // namespace iceberg diff --git a/src/iceberg/test/delete_file_index_test.cc b/src/iceberg/test/delete_file_index_test.cc index 75c82bc39..340688632 100644 --- a/src/iceberg/test/delete_file_index_test.cc +++ b/src/iceberg/test/delete_file_index_test.cc @@ -65,6 +65,11 @@ class DeleteFileIndexTest : public testing::TestWithParam { PartitionSpec::Make( /*spec_id=*/1, {PartitionField(/*source_id=*/2, /*field_id=*/1000, "data_bucket", Transform::Bucket(16))})); + ICEBERG_UNWRAP_OR_FAIL( + equivalent_partitioned_spec_, + PartitionSpec::Make( + /*spec_id=*/2, {PartitionField(/*source_id=*/2, /*field_id=*/1000, + "data_bucket", Transform::Bucket(16))})); // Unpartitioned spec unpartitioned_spec_ = PartitionSpec::Unpartitioned(); @@ -187,6 +192,7 @@ class DeleteFileIndexTest : public testing::TestWithParam { std::unordered_map> GetSpecsById() { return {{partitioned_spec_->spec_id(), partitioned_spec_}, + {equivalent_partitioned_spec_->spec_id(), equivalent_partitioned_spec_}, {unpartitioned_spec_->spec_id(), unpartitioned_spec_}}; } @@ -209,6 +215,7 @@ class DeleteFileIndexTest : public testing::TestWithParam { std::shared_ptr file_io_; std::shared_ptr schema_; std::shared_ptr partitioned_spec_; + std::shared_ptr equivalent_partitioned_spec_; std::shared_ptr unpartitioned_spec_; std::shared_ptr file_a_; @@ -1041,8 +1048,9 @@ TEST_P(DeleteFileIndexTest, TestMixDeleteFilesAndDVs) { auto partition_b = PartitionValues({Literal::Int(1)}); // Position delete for file_a_ - auto pos_delete_a = MakePositionDeleteFile("/path/to/pos-delete-a.parquet", partition_a, - partitioned_spec_->spec_id()); + auto pos_delete_a = + MakePositionDeleteFile("/path/to/pos-delete-a.parquet", partition_a, + partitioned_spec_->spec_id(), file_a_->file_path); // DV for file_a_ (should take precedence) auto dv_a = MakeDV("/path/to/dv-a.puffin", partition_a, partitioned_spec_->spec_id(), file_a_->file_path); @@ -1113,7 +1121,82 @@ TEST_P(DeleteFileIndexTest, TestMultipleDVs) { EXPECT_THAT(index_result, HasErrorMessage(file_a_->file_path)); } -TEST_P(DeleteFileIndexTest, TestInvalidDVSequenceNumber) { +TEST_P(DeleteFileIndexTest, TestDVApplicability) { + auto version = GetParam(); + if (version < 3) { + GTEST_SKIP() << "DVs only supported in V3+"; + } + + const auto null_partition = PartitionValues({Literal::Null(int32())}); + auto null_partition_file = MakeDataFile("/path/to/data-null.parquet", null_partition, + partitioned_spec_->spec_id()); + + struct TestCase { + std::string name; + PartitionValues dv_partition; + std::shared_ptr dv_spec; + std::shared_ptr data_file; + bool applies; + }; + const std::vector cases = { + { + .name = "equal-partition", + .dv_partition = file_a_->partition, + .dv_spec = partitioned_spec_, + .data_file = file_a_, + .applies = true, + }, + { + .name = "different-spec", + .dv_partition = file_a_->partition, + .dv_spec = equivalent_partitioned_spec_, + .data_file = file_a_, + .applies = false, + }, + { + .name = "different-partition-value", + .dv_partition = file_b_->partition, + .dv_spec = partitioned_spec_, + .data_file = file_a_, + .applies = false, + }, + { + .name = "equal-null-partition", + .dv_partition = null_partition, + .dv_spec = partitioned_spec_, + .data_file = null_partition_file, + .applies = true, + }, + { + .name = "null-partition-mismatch", + .dv_partition = file_a_->partition, + .dv_spec = partitioned_spec_, + .data_file = null_partition_file, + .applies = false, + }, + }; + + for (const auto& test_case : cases) { + SCOPED_TRACE(test_case.name); + auto dv = MakeDV("/path/to/" + test_case.name + ".puffin", test_case.dv_partition, + test_case.dv_spec->spec_id(), test_case.data_file->file_path); + std::vector entries; + entries.push_back(MakeDeleteEntry(/*snapshot_id=*/1000L, /*sequence_number=*/2, dv)); + auto manifest = WriteDeleteManifest(version, /*snapshot_id=*/1000L, + std::move(entries), test_case.dv_spec); + ICEBERG_UNWRAP_OR_FAIL(auto index, BuildIndex({manifest})); + ICEBERG_UNWRAP_OR_FAIL(auto deletes, index->ForDataFile(1, *test_case.data_file)); + + if (test_case.applies) { + ASSERT_EQ(deletes.size(), 1); + EXPECT_EQ(deletes[0]->file_path, dv->file_path); + } else { + EXPECT_TRUE(deletes.empty()); + } + } +} + +TEST_P(DeleteFileIndexTest, TestInapplicableDVSequenceNumber) { auto version = GetParam(); if (version < 3) { GTEST_SKIP() << "DVs only supported in V3+"; @@ -1123,20 +1206,23 @@ TEST_P(DeleteFileIndexTest, TestInvalidDVSequenceNumber) { auto dv = MakeDV("/path/to/dv.puffin", partition_a, partitioned_spec_->spec_id(), file_a_->file_path); + auto pos_delete = + MakePositionDeleteFile("/path/to/pos-delete.parquet", partition_a, + partitioned_spec_->spec_id(), file_a_->file_path); std::vector entries; entries.push_back(MakeDeleteEntry(/*snapshot_id=*/1000L, /*sequence_number=*/1, dv)); + entries.push_back( + MakeDeleteEntry(/*snapshot_id=*/1000L, /*sequence_number=*/2, pos_delete)); auto manifest = WriteDeleteManifest(version, /*snapshot_id=*/1000L, std::move(entries), partitioned_spec_); ICEBERG_UNWRAP_OR_FAIL(auto index, BuildIndex({manifest})); - // Querying with sequence number > DV sequence number should fail - auto result = index->ForDataFile(2, *file_a_); - EXPECT_THAT(result, IsError(ErrorKind::kValidationFailed)); - EXPECT_THAT(result, HasErrorMessage( - "must be greater than or equal to data file sequence number")); + ICEBERG_UNWRAP_OR_FAIL(auto deletes, index->ForDataFile(2, *file_a_)); + ASSERT_EQ(deletes.size(), 1); + EXPECT_EQ(deletes[0]->file_path, pos_delete->file_path); } TEST_P(DeleteFileIndexTest, TestReferencedDeleteFiles) { diff --git a/src/iceberg/test/dv_util_test.cc b/src/iceberg/test/dv_util_test.cc new file mode 100644 index 000000000..d84eb0784 --- /dev/null +++ b/src/iceberg/test/dv_util_test.cc @@ -0,0 +1,303 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include + +#include "iceberg/deletes/dv_util_internal.h" +#include "iceberg/deletes/position_delete_index.h" +#include "iceberg/file_format.h" +#include "iceberg/manifest/manifest_entry.h" +#include "iceberg/metadata_columns.h" +#include "iceberg/puffin/file_metadata.h" +#include "iceberg/puffin/json_serde_internal.h" +#include "iceberg/puffin/puffin_format.h" +#include "iceberg/test/matchers.h" +#include "iceberg/test/mock_io.h" +#include "iceberg/util/endian.h" +#include "iceberg/util/macros.h" + +namespace iceberg { + +namespace { + +constexpr std::string_view kReferencedDataFile = "data.parquet"; +constexpr std::string_view kReferencedDataFileProperty = "referenced-data-file"; +constexpr std::string_view kCardinalityProperty = "cardinality"; + +struct MetadataTestCase { + std::string name; + std::function&)> mutate_data; + std::function mutate_blob; + std::function mutate_manifest; + std::string expected_error; +}; + +void AddTrailingBitmapData(std::vector& data) { + constexpr size_t kLengthPrefixBytes = 4; + constexpr size_t kCrcBytes = 4; + + data.insert(data.end() - kCrcBytes, 0); + const auto length = static_cast(data.size() - kLengthPrefixBytes - kCrcBytes); + WriteBigEndian(length, data.data()); + + uLong crc = crc32(0L, Z_NULL, 0); + crc = crc32(crc, reinterpret_cast(data.data() + kLengthPrefixBytes), + static_cast(length)); + WriteBigEndian(static_cast(crc), data.data() + data.size() - kCrcBytes); +} + +Result> WriteDVFixture(const std::shared_ptr& io, + const MetadataTestCase& test_case) { + PositionDeleteIndex positions; + positions.Delete(1); + positions.Delete(3); + positions.Delete(5); + ICEBERG_ASSIGN_OR_RAISE(auto data, positions.Serialize()); + if (test_case.mutate_data) { + test_case.mutate_data(data); + } + + puffin::Blob blob{ + .type = std::string(puffin::StandardBlobTypes::kDeletionVectorV1), + .input_fields = {MetadataColumns::kFilePositionColumnId}, + .snapshot_id = -1, + .sequence_number = -1, + .data = std::move(data), + .requested_compression = puffin::PuffinCompressionCodec::kNone, + .properties = + { + {std::string(kReferencedDataFileProperty), + std::string(kReferencedDataFile)}, + {std::string(kCardinalityProperty), "3"}, + }, + }; + if (test_case.mutate_blob) { + test_case.mutate_blob(blob); + } + + const std::string path = "memory://" + test_case.name + ".puffin"; + const auto codec = + blob.requested_compression.value_or(puffin::PuffinCompressionCodec::kNone); + puffin::BlobMetadata blob_metadata{ + .type = blob.type, + .input_fields = blob.input_fields, + .snapshot_id = blob.snapshot_id, + .sequence_number = blob.sequence_number, + .offset = puffin::PuffinFormat::kMagicLength, + .length = static_cast(blob.data.size()), + .compression_codec = std::string(puffin::CodecName(codec)), + .properties = blob.properties, + }; + const std::string footer = + puffin::ToJsonString(puffin::FileMetadata{.blobs = {blob_metadata}}); + + std::vector file_data; + auto append = [&file_data](const void* data, size_t size) { + auto bytes = std::span(reinterpret_cast(data), size); + file_data.insert(file_data.end(), bytes.begin(), bytes.end()); + }; + append(puffin::PuffinFormat::kMagicV1.data(), puffin::PuffinFormat::kMagicV1.size()); + append(blob.data.data(), blob.data.size()); + append(puffin::PuffinFormat::kMagicV1.data(), puffin::PuffinFormat::kMagicV1.size()); + append(footer.data(), footer.size()); + + std::array footer_struct{}; + WriteLittleEndian(static_cast(footer.size()), footer_struct.data()); + std::memcpy(footer_struct.data() + puffin::PuffinFormat::kFooterStructMagicOffset, + puffin::PuffinFormat::kMagicV1.data(), + puffin::PuffinFormat::kMagicV1.size()); + append(footer_struct.data(), footer_struct.size()); + io->AddFile(path, file_data); + + auto delete_file = std::make_shared(DataFile{ + .content = DataFile::Content::kPositionDeletes, + .file_path = path, + .file_format = FileFormatType::kPuffin, + .record_count = 3, + .file_size_in_bytes = static_cast(file_data.size()), + .referenced_data_file = std::string(kReferencedDataFile), + .content_offset = blob_metadata.offset, + .content_size_in_bytes = blob_metadata.length, + }); + if (test_case.mutate_manifest) { + test_case.mutate_manifest(*delete_file); + } + return delete_file; +} + +} // namespace + +TEST(DVUtilTest, RejectsMalformedPuffinMetadata) { + const std::vector cases = { + { + .name = "wrong-offset", + .mutate_manifest = [](DataFile& file) { ++*file.content_offset; }, + .expected_error = "No Puffin blob starts", + }, + { + .name = "wrong-length", + .mutate_manifest = [](DataFile& file) { --*file.content_size_in_bytes; }, + .expected_error = "manifest content_size_in_bytes", + }, + { + .name = "missing-manifest-reference", + .mutate_manifest = [](DataFile& file) { file.referenced_data_file.reset(); }, + .expected_error = "requires referenced_data_file", + }, + { + .name = "empty-manifest-reference", + .mutate_manifest = [](DataFile& file) { file.referenced_data_file = ""; }, + .expected_error = "requires referenced_data_file", + }, + { + .name = "wrong-type", + .mutate_blob = [](puffin::Blob& blob) { blob.type = "test-blob"; }, + .expected_error = "Invalid deletion vector blob type", + }, + { + .name = "wrong-snapshot", + .mutate_blob = [](puffin::Blob& blob) { blob.snapshot_id = 0; }, + .expected_error = "snapshot-id -1", + }, + { + .name = "wrong-sequence", + .mutate_blob = [](puffin::Blob& blob) { blob.sequence_number = 0; }, + .expected_error = "sequence-number -1", + }, + { + .name = "compressed", + .mutate_blob = + [](puffin::Blob& blob) { + blob.requested_compression = puffin::PuffinCompressionCodec::kZstd; + }, + .expected_error = "must not be compressed", + }, + { + .name = "missing-reference", + .mutate_blob = + [](puffin::Blob& blob) { + blob.properties.erase(std::string(kReferencedDataFileProperty)); + }, + .expected_error = "requires non-empty 'referenced-data-file'", + }, + { + .name = "empty-reference", + .mutate_blob = + [](puffin::Blob& blob) { + blob.properties[std::string(kReferencedDataFileProperty)] = ""; + }, + .expected_error = "requires non-empty 'referenced-data-file'", + }, + { + .name = "mismatched-reference", + .mutate_blob = + [](puffin::Blob& blob) { + blob.properties[std::string(kReferencedDataFileProperty)] = + "other.parquet"; + }, + .expected_error = "does not match Puffin", + }, + { + .name = "missing-cardinality", + .mutate_blob = + [](puffin::Blob& blob) { + blob.properties.erase(std::string(kCardinalityProperty)); + }, + .expected_error = "requires 'cardinality'", + }, + { + .name = "invalid-cardinality", + .mutate_blob = + [](puffin::Blob& blob) { + blob.properties[std::string(kCardinalityProperty)] = "three"; + }, + .expected_error = "Failed to parse", + }, + { + .name = "negative-cardinality", + .mutate_blob = + [](puffin::Blob& blob) { + blob.properties[std::string(kCardinalityProperty)] = "-1"; + }, + .expected_error = "must be non-negative", + }, + { + .name = "mismatched-cardinality", + .mutate_blob = + [](puffin::Blob& blob) { + blob.properties[std::string(kCardinalityProperty)] = "4"; + }, + .expected_error = "Manifest record_count 3", + }, + { + .name = "trailing-bitmap-data", + .mutate_data = AddTrailingBitmapData, + .expected_error = "Trailing data after bitmaps", + }, + }; + + for (const auto& test_case : cases) { + SCOPED_TRACE(test_case.name); + auto io = std::make_shared(); + ICEBERG_UNWRAP_OR_FAIL(auto delete_file, WriteDVFixture(io, test_case)); + EXPECT_THAT(DVUtil::ReadDV(delete_file, io), + HasErrorMessage(test_case.expected_error)); + } +} + +TEST(DVUtilTest, ReadsValidPuffinMetadata) { + auto io = std::make_shared(); + MetadataTestCase test_case{.name = "valid"}; + ICEBERG_UNWRAP_OR_FAIL(auto delete_file, WriteDVFixture(io, test_case)); + ICEBERG_UNWRAP_OR_FAIL(auto positions, DVUtil::ReadDV(delete_file, io)); + + EXPECT_EQ(positions.Cardinality(), 3); + EXPECT_TRUE(positions.IsDeleted(1)); + EXPECT_TRUE(positions.IsDeleted(3)); + EXPECT_TRUE(positions.IsDeleted(5)); +} + +TEST(DVUtilTest, ReadsUsingActualFileSize) { + auto io = std::make_shared(); + MetadataTestCase test_case{ + .name = "stale-manifest-file-size", + .mutate_manifest = [](DataFile& file) { file.file_size_in_bytes = 0; }, + }; + ICEBERG_UNWRAP_OR_FAIL(auto delete_file, WriteDVFixture(io, test_case)); + ICEBERG_UNWRAP_OR_FAIL(auto positions, DVUtil::ReadDV(delete_file, io)); + + EXPECT_EQ(positions.Cardinality(), 3); +} + +} // namespace iceberg diff --git a/src/iceberg/test/meson.build b/src/iceberg/test/meson.build index 6844df5e9..0dc79db82 100644 --- a/src/iceberg/test/meson.build +++ b/src/iceberg/test/meson.build @@ -63,6 +63,11 @@ iceberg_tests = { 'update_schema_test.cc', ), }, + 'compaction_planner_test': {'sources': files('compaction_planner_test.cc')}, + 'compaction_executor_test': { + 'sources': files('compaction_executor_test.cc'), + 'use_data': true, + }, 'logging_test': { 'sources': files( 'cerr_logger_test.cc', diff --git a/src/iceberg/test/position_delete_update_test.cc b/src/iceberg/test/position_delete_update_test.cc new file mode 100644 index 000000000..8bf2e3d4f --- /dev/null +++ b/src/iceberg/test/position_delete_update_test.cc @@ -0,0 +1,605 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#include "iceberg/data/position_delete_update.h" + +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include + +#include "iceberg/arrow/arrow_io_internal.h" +#include "iceberg/avro/avro_register.h" +#include "iceberg/data/delete_loader.h" +#include "iceberg/data/position_delete_writer.h" +#include "iceberg/deletes/position_delete_index.h" +#include "iceberg/file_reader.h" +#include "iceberg/manifest/manifest_reader.h" +#include "iceberg/metadata_columns.h" +#include "iceberg/parquet/parquet_register.h" +#include "iceberg/partition_spec.h" +#include "iceberg/schema.h" +#include "iceberg/schema_field.h" +#include "iceberg/schema_internal.h" +#include "iceberg/snapshot.h" +#include "iceberg/table.h" +#include "iceberg/table_metadata.h" +#include "iceberg/table_properties.h" +#include "iceberg/table_scan.h" +#include "iceberg/test/matchers.h" +#include "iceberg/test/mock_catalog.h" +#include "iceberg/test/update_test_base.h" +#include "iceberg/update/fast_append.h" +#include "iceberg/update/row_delta.h" +#include "iceberg/update/update_partition_spec.h" +#include "iceberg/update/update_properties.h" +#include "iceberg/util/uuid.h" + +namespace iceberg { + +namespace { + +struct RoutingCase { + int8_t format_version; + bool unpartitioned; + FileFormatType expected_format; +}; + +class FailOncePuffinDeleteFileIO : public arrow::ArrowFileSystemFileIO { + public: + explicit FailOncePuffinDeleteFileIO( + std::shared_ptr<::arrow::fs::FileSystem> file_system) + : ArrowFileSystemFileIO(std::move(file_system)) {} + + Status DeleteFile(const std::string& file_location) override { + if (!file_location.ends_with(".puffin")) { + return ArrowFileSystemFileIO::DeleteFile(file_location); + } + + puffin_delete_attempts.push_back(file_location); + if (fail_next_puffin_delete_) { + fail_next_puffin_delete_ = false; + return IOError("injected cleanup failure for {}", file_location); + } + return ArrowFileSystemFileIO::DeleteFile(file_location); + } + + std::vector puffin_delete_attempts; + + private: + bool fail_next_puffin_delete_ = true; +}; + +class PositionDeleteUpdateTest : public MinimalUpdateTestBase, + public ::testing::WithParamInterface { + protected: + static void SetUpTestSuite() { + avro::RegisterAll(); + parquet::RegisterAll(); + } + + int8_t format_version() const override { return GetParam().format_version; } + + void SetUp() override { + MinimalUpdateTestBase::SetUp(); + if (GetParam().unpartitioned) { + RegisterUnpartitionedTable(); + } + if (GetParam().format_version == 2) { + ICEBERG_UNWRAP_OR_FAIL(auto properties, table_->NewUpdateProperties()); + properties->Set(TableProperties::kDeleteParquetCompression.key(), "uncompressed"); + ASSERT_THAT(properties->Commit(), IsOk()); + ASSERT_THAT(table_->Refresh(), IsOk()); + } + ICEBERG_UNWRAP_OR_FAIL(spec_, table_->spec()); + data_file_ = MakeDataFile(); + AppendDataFile(); + } + + void RegisterUnpartitionedTable() { + ICEBERG_UNWRAP_OR_FAIL( + auto metadata, ReadTableMetadataFromResource("TableMetadataV3ValidMinimal.json")); + metadata->location = table_location_; + metadata->partition_specs = {PartitionSpec::Unpartitioned()}; + metadata->default_spec_id = PartitionSpec::kInitialSpecId; + + const auto metadata_location = + std::format("{}/metadata/00001-{}.metadata.json", table_location_, + Uuid::GenerateV7().ToString()); + ASSERT_THAT(TableMetadataUtil::Write(*file_io_, metadata_location, *metadata), + IsOk()); + ASSERT_THAT(catalog_->DropTable(table_ident_, /*purge=*/false), IsOk()); + ICEBERG_UNWRAP_OR_FAIL(table_, + catalog_->RegisterTable(table_ident_, metadata_location)); + } + + std::shared_ptr MakeDataFile() const { + auto file = std::make_shared(); + file->content = DataFile::Content::kData; + file->file_path = table_location_ + "/data/file.parquet"; + file->file_format = FileFormatType::kParquet; + file->partition = GetParam().unpartitioned ? PartitionValues{} + : PartitionValues({Literal::Long(10)}); + file->file_size_in_bytes = 1024; + file->record_count = 10; + file->partition_spec_id = spec_->spec_id(); + return file; + } + + void AppendDataFile() { + ICEBERG_UNWRAP_OR_FAIL(auto append, table_->NewFastAppend()); + append->AppendFile(data_file_); + ASSERT_THAT(append->Commit(), IsOk()); + ASSERT_THAT(table_->Refresh(), IsOk()); + } + + Result> CurrentTask() { + ICEBERG_ASSIGN_OR_RAISE(auto builder, table_->NewScan()); + ICEBERG_ASSIGN_OR_RAISE(auto scan, builder->Build()); + ICEBERG_ASSIGN_OR_RAISE(auto tasks, scan->PlanFiles()); + ICEBERG_CHECK(tasks.size() == 1, "Expected one file scan task, found {}", + tasks.size()); + return tasks.front(); + } + + Result LoadPositions(const FileScanTask& task) { + std::vector> deletes; + std::ranges::copy_if(task.delete_files(), std::back_inserter(deletes), + [](const auto& file) { + return file->content == DataFile::Content::kPositionDeletes; + }); + DeleteLoader loader(file_io_); + return loader.LoadPositionDeletes(deletes, task.data_file()->file_path); + } + + std::shared_ptr spec_; + std::shared_ptr data_file_; +}; + +TEST_P(PositionDeleteUpdateTest, HandlesDescendingInputAndSortsV2Positions) { + ICEBERG_UNWRAP_OR_FAIL(auto update, PositionDeleteUpdate::Make(table_)); + update->Delete(data_file_->file_path, 7).Delete(data_file_->file_path, 2); + ASSERT_THAT(update->Commit(), IsOk()); + ASSERT_THAT(table_->Refresh(), IsOk()); + + ICEBERG_UNWRAP_OR_FAIL(auto task, CurrentTask()); + ASSERT_EQ(task->delete_files().size(), 1); + const auto& delete_file = task->delete_files().front(); + ASSERT_EQ(delete_file->file_format, GetParam().expected_format); + if (GetParam().format_version != 2) { + return; + } + + auto delete_schema = std::make_shared(std::vector{ + MetadataColumns::kDeleteFilePath, MetadataColumns::kDeleteFilePos}); + ICEBERG_UNWRAP_OR_FAIL( + auto reader, + ReaderFactoryRegistry::Open( + FileFormatType::kParquet, + {.path = delete_file->file_path, .io = file_io_, .projection = delete_schema})); + ICEBERG_UNWRAP_OR_FAIL(auto batch, reader->Next()); + ASSERT_TRUE(batch.has_value()); + + ArrowSchema arrow_schema; + ASSERT_THAT(ToArrowSchema(*delete_schema, &arrow_schema), IsOk()); + auto arrow_type = ::arrow::ImportType(&arrow_schema).ValueOrDie(); + auto rows = ::arrow::ImportArray(&batch.value(), arrow_type).ValueOrDie(); + auto struct_rows = std::static_pointer_cast<::arrow::StructArray>(rows); + auto positions = std::static_pointer_cast<::arrow::Int64Array>(struct_rows->field(1)); + ASSERT_EQ(positions->length(), 2); + EXPECT_EQ(positions->Value(0), 2); + EXPECT_EQ(positions->Value(1), 7); +} + +TEST_P(PositionDeleteUpdateTest, RoutesAndCommitsPositionDeletes) { + ICEBERG_UNWRAP_OR_FAIL(auto update, PositionDeleteUpdate::Make(table_)); + update->Delete(data_file_->file_path, 2).Delete(data_file_->file_path, 7); + ASSERT_THAT(update->Commit(), IsOk()); + ASSERT_THAT(table_->Refresh(), IsOk()); + + ICEBERG_UNWRAP_OR_FAIL(auto task, CurrentTask()); + ASSERT_EQ(task->delete_files().size(), 1); + const auto& delete_file = task->delete_files().front(); + EXPECT_EQ(delete_file->file_format, GetParam().expected_format); + EXPECT_EQ(delete_file->partition_spec_id, data_file_->partition_spec_id); + EXPECT_EQ(delete_file->partition, data_file_->partition); + + ICEBERG_UNWRAP_OR_FAIL(auto positions, LoadPositions(*task)); + EXPECT_EQ(positions.Cardinality(), 2); + EXPECT_TRUE(positions.IsDeleted(2)); + EXPECT_TRUE(positions.IsDeleted(7)); +} + +INSTANTIATE_TEST_SUITE_P( + FormatAndPartitioning, PositionDeleteUpdateTest, + ::testing::Values(RoutingCase{.format_version = 2, + .unpartitioned = false, + .expected_format = FileFormatType::kParquet}, + RoutingCase{.format_version = 3, + .unpartitioned = false, + .expected_format = FileFormatType::kPuffin}, + RoutingCase{.format_version = 3, + .unpartitioned = true, + .expected_format = FileFormatType::kPuffin})); + +class PositionDeleteV3Test : public MinimalUpdateTestBase { + protected: + static void SetUpTestSuite() { + avro::RegisterAll(); + parquet::RegisterAll(); + } + + int8_t format_version() const override { return 3; } + + void SetUp() override { + MinimalUpdateTestBase::SetUp(); + ICEBERG_UNWRAP_OR_FAIL(spec_, table_->spec()); + ICEBERG_UNWRAP_OR_FAIL(schema_, table_->schema()); + data_file_ = MakeDataFile(); + AppendDataFile(); + } + + std::shared_ptr MakeDataFile() const { + auto file = std::make_shared(); + file->content = DataFile::Content::kData; + file->file_path = table_location_ + "/data/file.parquet"; + file->file_format = FileFormatType::kParquet; + file->partition = PartitionValues({Literal::Long(10)}); + file->file_size_in_bytes = 1024; + file->record_count = 10; + file->partition_spec_id = spec_->spec_id(); + return file; + } + + void AppendDataFile() { + ICEBERG_UNWRAP_OR_FAIL(auto append, table_->NewFastAppend()); + append->AppendFile(data_file_); + ASSERT_THAT(append->Commit(), IsOk()); + ASSERT_THAT(table_->Refresh(), IsOk()); + } + + Result> CurrentTask(const std::shared_ptr
& table) { + ICEBERG_ASSIGN_OR_RAISE(auto builder, table->NewScan()); + ICEBERG_ASSIGN_OR_RAISE(auto scan, builder->Build()); + ICEBERG_ASSIGN_OR_RAISE(auto tasks, scan->PlanFiles()); + ICEBERG_CHECK(tasks.size() == 1, "Expected one file scan task, found {}", + tasks.size()); + return tasks.front(); + } + + Result LoadPositions(const FileScanTask& task) { + DeleteLoader loader(file_io_); + return loader.LoadPositionDeletes(task.delete_files(), task.data_file()->file_path); + } + + Result> WritePositionDeletes( + std::span positions) { + const auto path = table_location_ + "/data/existing-position-deletes.parquet"; + ICEBERG_ASSIGN_OR_RAISE( + auto writer, + PositionDeleteWriter::Make(PositionDeleteWriterOptions{ + .path = path, + .schema = schema_, + .spec = spec_, + .partition = data_file_->partition, + .format = FileFormatType::kParquet, + .io = file_io_, + .properties = {{"write.parquet.compression-codec", "uncompressed"}}, + })); + for (int64_t pos : positions) { + ICEBERG_RETURN_UNEXPECTED(writer->WriteDelete(data_file_->file_path, pos)); + } + ICEBERG_RETURN_UNEXPECTED(writer->Close()); + ICEBERG_ASSIGN_OR_RAISE(auto result, writer->Metadata()); + ICEBERG_CHECK(result.data_files.size() == 1, + "Expected one position delete file, found {}", + result.data_files.size()); + return result.data_files.front(); + } + + Result> CurrentDeleteEntries() { + ICEBERG_ASSIGN_OR_RAISE(auto snapshot, table_->current_snapshot()); + SnapshotCache cache(snapshot.get()); + ICEBERG_ASSIGN_OR_RAISE(auto manifests, cache.DeleteManifests(file_io_)); + std::vector entries; + for (const auto& manifest : manifests) { + ICEBERG_ASSIGN_OR_RAISE( + auto spec, table_->metadata()->PartitionSpecById(manifest.partition_spec_id)); + ICEBERG_ASSIGN_OR_RAISE( + auto reader, + ManifestReader::Make(manifest, file_io_, schema_, std::move(spec))); + ICEBERG_ASSIGN_OR_RAISE(auto manifest_entries, reader->Entries()); + entries.insert(entries.end(), std::make_move_iterator(manifest_entries.begin()), + std::make_move_iterator(manifest_entries.end())); + } + return entries; + } + + void ConfigureRetries(int32_t retries) { + ICEBERG_UNWRAP_OR_FAIL(auto properties, table_->NewUpdateProperties()); + properties->Set(TableProperties::kCommitNumRetries.key(), std::to_string(retries)) + .Set(TableProperties::kCommitMinRetryWaitMs.key(), "1") + .Set(TableProperties::kCommitMaxRetryWaitMs.key(), "1") + .Set(TableProperties::kCommitTotalRetryTimeMs.key(), "1000"); + ASSERT_THAT(properties->Commit(), IsOk()); + ASSERT_THAT(table_->Refresh(), IsOk()); + } + + std::vector PuffinFiles() { + auto arrow_io = std::dynamic_pointer_cast(file_io_); + EXPECT_NE(arrow_io, nullptr); + ::arrow::fs::FileSelector selector; + selector.base_dir = table_location_ + "/data"; + selector.recursive = true; + auto infos = arrow_io->fs()->GetFileInfo(selector); + EXPECT_TRUE(infos.ok()) << infos.status().ToString(); + std::vector paths; + if (!infos.ok()) { + return paths; + } + for (const auto& info : *infos) { + if (info.path().ends_with(".puffin")) { + paths.push_back(info.path()); + } + } + return paths; + } + + std::shared_ptr spec_; + std::shared_ptr schema_; + std::shared_ptr data_file_; +}; + +TEST_F(PositionDeleteV3Test, SecondDeleteMergesAndSupersedesPreviousDV) { + ICEBERG_UNWRAP_OR_FAIL(auto first, PositionDeleteUpdate::Make(table_)); + first->Delete(data_file_->file_path, 1); + ASSERT_THAT(first->Commit(), IsOk()); + ASSERT_THAT(table_->Refresh(), IsOk()); + ICEBERG_UNWRAP_OR_FAIL(auto old_task, CurrentTask(table_)); + const auto old_dv = old_task->delete_files().front(); + + ICEBERG_UNWRAP_OR_FAIL(auto second, PositionDeleteUpdate::Make(table_)); + second->Delete(data_file_->file_path, 3); + ASSERT_THAT(second->Commit(), IsOk()); + ASSERT_THAT(table_->Refresh(), IsOk()); + + ICEBERG_UNWRAP_OR_FAIL(auto task, CurrentTask(table_)); + ASSERT_EQ(task->delete_files().size(), 1); + EXPECT_NE(task->delete_files().front()->file_path, old_dv->file_path); + ICEBERG_UNWRAP_OR_FAIL(auto positions, LoadPositions(*task)); + EXPECT_EQ(positions.Cardinality(), 2); + EXPECT_TRUE(positions.IsDeleted(1)); + EXPECT_TRUE(positions.IsDeleted(3)); + + ICEBERG_UNWRAP_OR_FAIL(auto entries, CurrentDeleteEntries()); + EXPECT_TRUE(std::ranges::any_of(entries, [&old_dv](const ManifestEntry& entry) { + return entry.status == ManifestStatus::kDeleted && entry.data_file != nullptr && + entry.data_file->file_path == old_dv->file_path && + entry.data_file->content_offset == old_dv->content_offset; + })); +} + +TEST_F(PositionDeleteV3Test, MergesAndSupersedesFileScopedParquetDelete) { + RegisterTableFromResource("TableMetadataV2ValidMinimal.json"); + ICEBERG_UNWRAP_OR_FAIL(spec_, table_->spec()); + ICEBERG_UNWRAP_OR_FAIL(schema_, table_->schema()); + data_file_ = MakeDataFile(); + AppendDataFile(); + + const std::vector existing_positions{1, 3}; + ICEBERG_UNWRAP_OR_FAIL(auto old_delete, WritePositionDeletes(existing_positions)); + ASSERT_EQ(old_delete->referenced_data_file, data_file_->file_path); + ICEBERG_UNWRAP_OR_FAIL(auto row_delta, table_->NewRowDelta()); + row_delta->AddDeletes(old_delete); + ASSERT_THAT(row_delta->Commit(), IsOk()); + ASSERT_THAT(table_->Refresh(), IsOk()); + + ICEBERG_UNWRAP_OR_FAIL(auto properties, table_->NewUpdateProperties()); + properties->Set(TableProperties::kFormatVersion.key(), "3"); + ASSERT_THAT(properties->Commit(), IsOk()); + ASSERT_THAT(table_->Refresh(), IsOk()); + + ICEBERG_UNWRAP_OR_FAIL(auto update, PositionDeleteUpdate::Make(table_)); + update->Delete(data_file_->file_path, 5); + ASSERT_THAT(update->Commit(), IsOk()); + ASSERT_THAT(table_->Refresh(), IsOk()); + + ICEBERG_UNWRAP_OR_FAIL(auto task, CurrentTask(table_)); + ASSERT_EQ(task->delete_files().size(), 1); + EXPECT_EQ(task->delete_files().front()->file_format, FileFormatType::kPuffin); + ICEBERG_UNWRAP_OR_FAIL(auto positions, LoadPositions(*task)); + EXPECT_EQ(positions.Cardinality(), 3); + EXPECT_TRUE(positions.IsDeleted(1)); + EXPECT_TRUE(positions.IsDeleted(3)); + EXPECT_TRUE(positions.IsDeleted(5)); + + ICEBERG_UNWRAP_OR_FAIL(auto entries, CurrentDeleteEntries()); + EXPECT_TRUE(std::ranges::any_of(entries, [&old_delete](const ManifestEntry& entry) { + return entry.status == ManifestStatus::kDeleted && entry.data_file != nullptr && + entry.data_file->file_path == old_delete->file_path; + })); +} + +TEST_F(PositionDeleteV3Test, UsesTargetDataFileSpecAfterPartitionEvolution) { + ICEBERG_UNWRAP_OR_FAIL(auto spec_update, table_->NewUpdatePartitionSpec()); + spec_update->RemoveField("x"); + ASSERT_THAT(spec_update->Commit(), IsOk()); + ASSERT_THAT(table_->Refresh(), IsOk()); + ASSERT_NE(table_->metadata()->default_spec_id, data_file_->partition_spec_id); + + ICEBERG_UNWRAP_OR_FAIL(auto update, PositionDeleteUpdate::Make(table_)); + update->Delete(data_file_->file_path, 4); + ASSERT_THAT(update->Commit(), IsOk()); + ASSERT_THAT(table_->Refresh(), IsOk()); + + ICEBERG_UNWRAP_OR_FAIL(auto task, CurrentTask(table_)); + const auto& dv = task->delete_files().front(); + EXPECT_EQ(dv->partition_spec_id, data_file_->partition_spec_id); + EXPECT_EQ(dv->partition, data_file_->partition); +} + +TEST_F(PositionDeleteV3Test, CommitRetryReusesWrittenDV) { + ConfigureRetries(1); + auto mock_catalog = std::make_shared<::testing::NiceMock>(); + std::weak_ptr weak_catalog = mock_catalog; + int update_calls = 0; + ON_CALL(*mock_catalog, LoadTable(::testing::_)) + .WillByDefault( + [this, weak_catalog](const TableIdentifier&) -> Result> { + auto retry_catalog = weak_catalog.lock(); + ICEBERG_CHECK(retry_catalog != nullptr, "Mock catalog expired"); + ICEBERG_ASSIGN_OR_RAISE(auto loaded, catalog_->LoadTable(table_ident_)); + return Table::Make(loaded->name(), loaded->metadata(), + std::string(loaded->metadata_file_location()), + loaded->io(), std::move(retry_catalog)); + }); + ON_CALL(*mock_catalog, UpdateTable(::testing::_, ::testing::_, ::testing::_)) + .WillByDefault( + [this, &update_calls, weak_catalog]( + const TableIdentifier& identifier, + const std::vector>& requirements, + const std::vector>& updates) + -> Result> { + if (++update_calls == 1) { + return CommitFailed("injected conflict"); + } + ICEBERG_ASSIGN_OR_RAISE( + auto committed, catalog_->UpdateTable(identifier, requirements, updates)); + auto retry_catalog = weak_catalog.lock(); + ICEBERG_CHECK(retry_catalog != nullptr, "Mock catalog expired"); + return Table::Make(committed->name(), committed->metadata(), + std::string(committed->metadata_file_location()), + committed->io(), std::move(retry_catalog)); + }); + ICEBERG_UNWRAP_OR_FAIL(auto mock_table, + Table::Make(table_->name(), table_->metadata(), + std::string(table_->metadata_file_location()), + table_->io(), mock_catalog)); + + ICEBERG_UNWRAP_OR_FAIL(auto update, PositionDeleteUpdate::Make(mock_table)); + update->Delete(data_file_->file_path, 5); + ASSERT_THAT(update->Commit(), IsOk()); + + EXPECT_EQ(update_calls, 2); + EXPECT_EQ(PuffinFiles().size(), 1); +} + +TEST_F(PositionDeleteV3Test, FailedCommitCleansWrittenDV) { + ConfigureRetries(0); + auto mock_catalog = std::make_shared<::testing::NiceMock>(); + ON_CALL(*mock_catalog, UpdateTable(::testing::_, ::testing::_, ::testing::_)) + .WillByDefault([](const TableIdentifier&, + const std::vector>&, + const std::vector>&) + -> Result> { + return CommitFailed("injected failure"); + }); + ICEBERG_UNWRAP_OR_FAIL(auto mock_table, + Table::Make(table_->name(), table_->metadata(), + std::string(table_->metadata_file_location()), + table_->io(), mock_catalog)); + + ICEBERG_UNWRAP_OR_FAIL(auto update, PositionDeleteUpdate::Make(mock_table)); + update->Delete(data_file_->file_path, 6); + EXPECT_THAT(update->Commit(), IsError(ErrorKind::kCommitFailed)); + EXPECT_TRUE(PuffinFiles().empty()); +} + +TEST_F(PositionDeleteV3Test, CleanupFailureIsReportedAndRetried) { + ConfigureRetries(0); + auto mock_catalog = std::make_shared<::testing::NiceMock>(); + int update_calls = 0; + ON_CALL(*mock_catalog, UpdateTable(::testing::_, ::testing::_, ::testing::_)) + .WillByDefault( + [this, &update_calls]( + const TableIdentifier& identifier, + const std::vector>& requirements, + const std::vector>& updates) + -> Result> { + if (++update_calls == 1) { + return CommitFailed("injected commit failure"); + } + return catalog_->UpdateTable(identifier, requirements, updates); + }); + auto arrow_io = std::dynamic_pointer_cast(file_io_); + ASSERT_NE(arrow_io, nullptr); + auto failing_io = std::make_shared(arrow_io->fs()); + ICEBERG_UNWRAP_OR_FAIL(auto mock_table, + Table::Make(table_->name(), table_->metadata(), + std::string(table_->metadata_file_location()), + failing_io, mock_catalog)); + + ICEBERG_UNWRAP_OR_FAIL(auto update, PositionDeleteUpdate::Make(mock_table)); + update->Delete(data_file_->file_path, 6); + auto first_status = update->Commit(); + EXPECT_THAT(first_status, IsError(ErrorKind::kCommitFailed)); + EXPECT_THAT(first_status, HasErrorMessage("injected commit failure")); + EXPECT_THAT(first_status, HasErrorMessage("injected cleanup failure")); + ASSERT_EQ(PuffinFiles().size(), 1); + ASSERT_EQ(failing_io->puffin_delete_attempts.size(), 1); + const auto retained_path = failing_io->puffin_delete_attempts.front(); + + EXPECT_THAT(update->Commit(), IsOk()); + EXPECT_EQ(update_calls, 2); + EXPECT_EQ(PuffinFiles().size(), 1); + ASSERT_EQ(failing_io->puffin_delete_attempts.size(), 2); + EXPECT_EQ(std::ranges::count(failing_io->puffin_delete_attempts, retained_path), 2); +} + +TEST_F(PositionDeleteV3Test, CommitStateUnknownRelinquishesOutputOwnership) { + ConfigureRetries(1); + auto mock_catalog = std::make_shared<::testing::NiceMock>(); + int update_calls = 0; + ON_CALL(*mock_catalog, UpdateTable(::testing::_, ::testing::_, ::testing::_)) + .WillByDefault( + [&update_calls](const TableIdentifier&, + const std::vector>&, + const std::vector>&) + -> Result> { + ++update_calls; + return CommitStateUnknown("injected unknown state"); + }); + ICEBERG_UNWRAP_OR_FAIL(auto mock_table, + Table::Make(table_->name(), table_->metadata(), + std::string(table_->metadata_file_location()), + table_->io(), mock_catalog)); + + ICEBERG_UNWRAP_OR_FAIL(auto update, PositionDeleteUpdate::Make(mock_table)); + update->Delete(data_file_->file_path, 6); + EXPECT_THAT(update->Commit(), IsError(ErrorKind::kCommitStateUnknown)); + ASSERT_EQ(PuffinFiles().size(), 1); + + EXPECT_THAT(update->Commit(), IsError(ErrorKind::kInvalidArgument)); + EXPECT_EQ(update_calls, 1); + EXPECT_EQ(PuffinFiles().size(), 1); +} + +} // namespace + +} // namespace iceberg diff --git a/src/iceberg/test/puffin_dv_interop_test.cc b/src/iceberg/test/puffin_dv_interop_test.cc new file mode 100644 index 000000000..b322fe0ae --- /dev/null +++ b/src/iceberg/test/puffin_dv_interop_test.cc @@ -0,0 +1,247 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#include +#include +#include +#include +#include +#include + +#include + +#include "iceberg/deletes/dv_util_internal.h" +#include "iceberg/deletes/position_delete_index.h" +#include "iceberg/file_format.h" +#include "iceberg/manifest/manifest_entry.h" +#include "iceberg/metadata_columns.h" +#include "iceberg/puffin/file_metadata.h" +#include "iceberg/puffin/puffin_reader.h" +#include "iceberg/test/matchers.h" +#include "iceberg/test/std_io.h" +#include "iceberg/test/test_resource.h" + +namespace iceberg::puffin { + +namespace { + +constexpr std::string_view kReferencedDataFileProperty = "referenced-data-file"; +constexpr std::string_view kCardinalityProperty = "cardinality"; + +int64_t Position(int64_t bucket, int64_t container, int64_t value) { + return (bucket << 32) + (container << 16) + value; +} + +std::vector AllContainerPositions() { + std::vector positions = { + Position(0, 0, 5), + Position(0, 0, 7), + Position(1, 0, 10), + Position(1, 0, 20), + }; + positions.reserve(10004); + for (int64_t bucket = 0; bucket < 2; ++bucket) { + for (int64_t value = 0; value < 10000; value += 2) { + positions.push_back(Position(bucket, 2, value)); + } + } + return positions; +} + +struct PositionRange { + int64_t begin; + int64_t end; +}; + +struct ExpectedBlob { + std::string referenced_data_file; + std::vector input_fields; + int64_t offset; + int64_t length; + int64_t cardinality; + std::vector positions; + std::vector ranges; +}; + +std::shared_ptr MakeDeleteFile(const ExpectedBlob& expected, + const std::string& fixture_path, + int64_t file_size) { + return std::make_shared(DataFile{ + .content = DataFile::Content::kPositionDeletes, + .file_path = fixture_path, + .file_format = FileFormatType::kPuffin, + .record_count = expected.cardinality, + .file_size_in_bytes = file_size, + .referenced_data_file = expected.referenced_data_file, + .content_offset = expected.offset, + .content_size_in_bytes = expected.length, + }); +} + +void AssertPositions(const ExpectedBlob& expected, + const std::shared_ptr& delete_file, + const std::shared_ptr& io) { + ASSERT_TRUE(delete_file->content_offset.has_value()); + ASSERT_TRUE(delete_file->content_size_in_bytes.has_value()); + ICEBERG_UNWRAP_OR_FAIL(auto positions, DVUtil::ReadDV(delete_file, io)); + EXPECT_EQ(positions.Cardinality(), expected.cardinality); + int64_t expected_cardinality = static_cast(expected.positions.size()); + for (const auto& range : expected.ranges) { + expected_cardinality += range.end - range.begin; + } + ASSERT_EQ(expected_cardinality, expected.cardinality); + for (int64_t position : expected.positions) { + EXPECT_TRUE(positions.IsDeleted(position)) << "Missing position " << position; + } + for (const auto& range : expected.ranges) { + for (int64_t position = range.begin; position < range.end; ++position) { + ASSERT_TRUE(positions.IsDeleted(position)) << "Missing position " << position; + } + } +} + +void AssertFixture(const std::string& resource_name, + const std::vector& expected_blobs) { + const std::string fixture_path = GetResourcePath(resource_name); + const auto file_size = static_cast(std::filesystem::file_size(fixture_path)); + auto io = std::make_shared(); + ICEBERG_UNWRAP_OR_FAIL(auto input_file, io->NewInputFile(fixture_path)); + ICEBERG_UNWRAP_OR_FAIL(auto reader, PuffinReader::Make(std::move(input_file))); + ICEBERG_UNWRAP_OR_FAIL(auto metadata, reader->ReadFileMetadata()); + + ASSERT_EQ(metadata.blobs.size(), expected_blobs.size()); + std::unordered_set seen; + for (const auto& blob : metadata.blobs) { + const auto& referenced_data_file = + blob.properties.at(std::string(kReferencedDataFileProperty)); + auto expected = std::ranges::find_if(expected_blobs, [&](const auto& candidate) { + return candidate.referenced_data_file == referenced_data_file; + }); + ASSERT_NE(expected, expected_blobs.end()) + << "Unexpected referenced data file " << referenced_data_file; + ASSERT_TRUE(seen.insert(referenced_data_file).second) + << "Duplicate referenced data file " << referenced_data_file; + + EXPECT_EQ(blob.type, StandardBlobTypes::kDeletionVectorV1); + EXPECT_EQ(blob.input_fields, expected->input_fields); + EXPECT_EQ(blob.snapshot_id, -1); + EXPECT_EQ(blob.sequence_number, -1); + EXPECT_EQ(blob.offset, expected->offset); + EXPECT_EQ(blob.length, expected->length); + EXPECT_TRUE(blob.compression_codec.empty()); + EXPECT_EQ(blob.properties.at(std::string(kCardinalityProperty)), + std::to_string(expected->cardinality)); + + AssertPositions(*expected, MakeDeleteFile(*expected, fixture_path, file_size), io); + } + EXPECT_EQ(seen.size(), expected_blobs.size()); +} + +} // namespace + +TEST(PuffinDVInteropTest, ReadsJavaSingleBlobFixture) { + AssertFixture( + "deletion_vectors/java/single-blob-dv.puffin", + {{ + .referenced_data_file = "s3://warehouse/db/table/data/00000-0-abc.parquet", + .input_fields = {MetadataColumns::kFilePositionColumnId}, + .offset = 4, + .length = 50, + .cardinality = 5, + .positions = {1, 3, 5, 7, 9}, + }}); +} + +TEST(PuffinDVInteropTest, ReadsJavaMultiBlobFixture) { + AssertFixture( + "deletion_vectors/java/multi-blob-dv.puffin", + { + { + .referenced_data_file = "s3://warehouse/db/table/data/file-001.parquet", + .input_fields = {MetadataColumns::kFilePositionColumnId}, + .offset = 4, + .length = 46, + .cardinality = 3, + .positions = {0, 100, 200}, + }, + { + .referenced_data_file = "s3://warehouse/db/table/data/file-002.parquet", + .input_fields = {MetadataColumns::kFilePositionColumnId}, + .offset = 50, + .length = 44, + .cardinality = 2, + .positions = {50, 150}, + }, + }); +} + +TEST(PuffinDVInteropTest, ReadsGoSingleBlobFixture) { + AssertFixture("deletion_vectors/go/single-blob-dv.puffin", + {{ + .referenced_data_file = "data/test.parquet", + .input_fields = {}, + .offset = 4, + .length = 50, + .cardinality = 5, + .positions = {1, 3, 5, 7, 9}, + }}); +} + +TEST(PuffinDVInteropTest, ReadsGoMultiBlobFixture) { + AssertFixture( + "deletion_vectors/go/multi-blob-dv.puffin", + { + { + .referenced_data_file = "s3://warehouse/db/table/data/go-file-001.parquet", + .input_fields = {}, + .offset = 4, + .length = 68, + .cardinality = 4, + .positions = {0, 100, 200, (int64_t{1} << 32) + 7}, + }, + { + .referenced_data_file = "s3://warehouse/db/table/data/go-file-002.parquet", + .input_fields = {}, + .offset = 72, + .length = 66, + .cardinality = 3, + .positions = {50, 150, (int64_t{2} << 32) + 9}, + }, + }); +} + +TEST(PuffinDVInteropTest, ReadsGoAllContainerTypesFixture) { + AssertFixture( + "deletion_vectors/go/all-container-types-dv.puffin", + {{ + .referenced_data_file = "s3://warehouse/db/table/data/all-containers.parquet", + .input_fields = {}, + .offset = 4, + .length = 16466, + .cardinality = 11493, + .positions = AllContainerPositions(), + .ranges = + { + {Position(0, 1, 1), Position(0, 1, 1000)}, + {Position(1, 1, 10), Position(1, 1, 500)}, + }, + }}); +} + +} // namespace iceberg::puffin diff --git a/src/iceberg/test/resources/deletion_vectors/README.md b/src/iceberg/test/resources/deletion_vectors/README.md new file mode 100644 index 000000000..983cbfe3e --- /dev/null +++ b/src/iceberg/test/resources/deletion_vectors/README.md @@ -0,0 +1,44 @@ +# Deletion vector interoperability fixtures + +These fixtures verify deletion-vector compatibility against bytes produced by +implementations other than Iceberg C++. + +## Java fixtures + +The files under `java/` were copied byte-for-byte from Apache Iceberg Go commit +`3020adbbc3faff047da6f483f739f1b5e1de611b`, where they are consumed by +`table/dv/dv_cross_client_test.go`. + +- `single-blob-dv.puffin` and `multi-blob-dv.puffin` were produced by Apache + Iceberg Java using + `iceberg-go/dev/dv-fixtures/GenerateDVFixtures.java`. + +The committed Puffin bytes were reproduced against Apache Iceberg Java commit +`76d35b1e40f77edcad19646bb6afdd9f05249964`. To regenerate them, follow +`iceberg-go/dev/dv-fixtures/README.md` at the Iceberg Go commit above using +that Java revision, then copy the generated files into `java/`. + +## Go fixtures + +The files under `go/` are produced with the Iceberg Go Puffin and deletion +vector writers. From an Iceberg Go checkout at commit +`3020adbbc3faff047da6f483f739f1b5e1de611b`, run: + +```bash +go run /path/to/iceberg-cpp/dev/dv-fixtures/generate_go_fixtures.go \ + /path/to/iceberg-cpp/src/iceberg/test/resources/deletion_vectors/go +``` + +The multi-blob fixture includes positions in three distinct high-32-bit +buckets. `all-container-types-dv.puffin` covers array, run, and bitmap +containers inside a complete Puffin file. + +## Checksums + +```text +ab8309671c0c5ef1956f4f1d7b907f4a69ffeb154ab77d5a4855d7dd4779108c java/single-blob-dv.puffin +fed7edbb5a343a6c6fc4706ff4c213ab3f0a50916baeb228619f1f2c956f3f27 java/multi-blob-dv.puffin +dd293e827439cd22053a9356ae6d56f0d5d69e648b1850d39663cdc5c7ec5a77 go/single-blob-dv.puffin +000a77697d5ddce01e0242786b91fc15482a8b8f4fa3c8e214a936d7c2fc978b go/multi-blob-dv.puffin +d9f374706891b4780f8a59f155a7cc4cae4161b6415578abb4790f6eed793a52 go/all-container-types-dv.puffin +``` diff --git a/src/iceberg/test/resources/deletion_vectors/go/all-container-types-dv.puffin b/src/iceberg/test/resources/deletion_vectors/go/all-container-types-dv.puffin new file mode 100644 index 000000000..5582a008e Binary files /dev/null and b/src/iceberg/test/resources/deletion_vectors/go/all-container-types-dv.puffin differ diff --git a/src/iceberg/test/resources/deletion_vectors/go/multi-blob-dv.puffin b/src/iceberg/test/resources/deletion_vectors/go/multi-blob-dv.puffin new file mode 100644 index 000000000..4404137af Binary files /dev/null and b/src/iceberg/test/resources/deletion_vectors/go/multi-blob-dv.puffin differ diff --git a/src/iceberg/test/resources/deletion_vectors/go/single-blob-dv.puffin b/src/iceberg/test/resources/deletion_vectors/go/single-blob-dv.puffin new file mode 100644 index 000000000..84c6baf79 Binary files /dev/null and b/src/iceberg/test/resources/deletion_vectors/go/single-blob-dv.puffin differ diff --git a/src/iceberg/test/resources/deletion_vectors/java/multi-blob-dv.puffin b/src/iceberg/test/resources/deletion_vectors/java/multi-blob-dv.puffin new file mode 100644 index 000000000..b9037900c Binary files /dev/null and b/src/iceberg/test/resources/deletion_vectors/java/multi-blob-dv.puffin differ diff --git a/src/iceberg/test/resources/deletion_vectors/java/single-blob-dv.puffin b/src/iceberg/test/resources/deletion_vectors/java/single-blob-dv.puffin new file mode 100644 index 000000000..5c0007e21 Binary files /dev/null and b/src/iceberg/test/resources/deletion_vectors/java/single-blob-dv.puffin differ diff --git a/src/iceberg/test/std_io.h b/src/iceberg/test/std_io.h index 725fc7ba5..c7d5b312b 100644 --- a/src/iceberg/test/std_io.h +++ b/src/iceberg/test/std_io.h @@ -319,11 +319,9 @@ class StdFileIO : public FileIO { Status DeleteFile(const std::string& file_location) override { std::error_code ec; - if (!std::filesystem::remove(file_location, ec)) { - if (ec) { - return IOError("Failed to delete file {}: {}", file_location, ec.message()); - } - return IOError("File does not exist: {}", file_location); + std::filesystem::remove(file_location, ec); + if (ec) { + return IOError("Failed to delete file {}: {}", file_location, ec.message()); } return {}; }