diff --git a/core/src/main/java/org/apache/iceberg/puffin/StandardBlobTypes.java b/core/src/main/java/org/apache/iceberg/puffin/StandardBlobTypes.java index ce78375c4b1a..da4b60f5349e 100644 --- a/core/src/main/java/org/apache/iceberg/puffin/StandardBlobTypes.java +++ b/core/src/main/java/org/apache/iceberg/puffin/StandardBlobTypes.java @@ -27,6 +27,12 @@ private StandardBlobTypes() {} */ public static final String APACHE_DATASKETCHES_THETA_V1 = "apache-datasketches-theta-v1"; + /** + * A serialized form of a "compact" HyperLogLog sketch produced by the Apache DataSketches library + */ + public static final String APACHE_DATASKETCHES_HLL_V1 = "apache-datasketches-hll-v1"; + /** A serialized deletion vector according to the Iceberg spec */ public static final String DV_V1 = "deletion-vector-v1"; } diff --git a/spark/v4.2/spark/src/main/java/org/apache/iceberg/spark/actions/NDVSketchUtil.java b/spark/v4.2/spark/src/main/java/org/apache/iceberg/spark/actions/NDVSketchUtil.java index 11b14fafd916..5c0eda7a8613 100644 --- a/spark/v4.2/spark/src/main/java/org/apache/iceberg/spark/actions/NDVSketchUtil.java +++ b/spark/v4.2/spark/src/main/java/org/apache/iceberg/spark/actions/NDVSketchUtil.java @@ -20,6 +20,7 @@ import java.nio.ByteBuffer; import java.util.List; +import org.apache.datasketches.hll.HllSketch; import org.apache.datasketches.memory.Memory; import org.apache.datasketches.theta.CompactSketch; import org.apache.datasketches.theta.Sketch; @@ -38,7 +39,9 @@ import org.apache.spark.sql.Dataset; import org.apache.spark.sql.Row; import org.apache.spark.sql.SparkSession; +import org.apache.spark.sql.catalyst.expressions.aggregate.AggregateFunction; import org.apache.spark.sql.classic.ExpressionColumnNode; +import org.apache.spark.sql.stats.HLLSketchAgg; import org.apache.spark.sql.stats.ThetaSketchAgg; public class NDVSketchUtil { @@ -49,43 +52,65 @@ private NDVSketchUtil() {} static List generateBlobs( SparkSession spark, Table table, Snapshot snapshot, List columns) { - Row sketches = computeNDVSketches(spark, table, snapshot, columns); + return generateBlobs( + spark, table, snapshot, columns, StandardBlobTypes.APACHE_DATASKETCHES_THETA_V1); + } + + static List generateBlobs( + SparkSession spark, Table table, Snapshot snapshot, List columns, String blobType) { + Row sketches = computeNDVSketches(spark, table, snapshot, columns, blobType); Schema schema = table.schemas().get(snapshot.schemaId()); List blobs = Lists.newArrayListWithExpectedSize(columns.size()); for (int i = 0; i < columns.size(); i++) { Types.NestedField field = schema.findField(columns.get(i)); - Sketch sketch = CompactSketch.wrap(Memory.wrap((byte[]) sketches.get(i))); - blobs.add(toBlob(field, sketch, snapshot)); + byte[] bytes = (byte[]) sketches.get(i); + blobs.add(toBlob(field, blobType, bytes, estimate(blobType, bytes), snapshot)); } return blobs; } - private static Blob toBlob(Types.NestedField field, Sketch sketch, Snapshot snapshot) { + private static long estimate(String blobType, byte[] bytes) { + return switch (blobType) { + case StandardBlobTypes.APACHE_DATASKETCHES_THETA_V1 -> { + Sketch sketch = CompactSketch.wrap(Memory.wrap(bytes)); + yield (long) sketch.getEstimate(); + } + case StandardBlobTypes.APACHE_DATASKETCHES_HLL_V1 -> + (long) HllSketch.heapify(Memory.wrap(bytes)).getEstimate(); + default -> throw new IllegalArgumentException("Unsupported NDV blob type: " + blobType); + }; + } + + private static Blob toBlob( + Types.NestedField field, String blobType, byte[] bytes, long ndv, Snapshot snapshot) { return new Blob( - StandardBlobTypes.APACHE_DATASKETCHES_THETA_V1, + blobType, ImmutableList.of(field.fieldId()), snapshot.snapshotId(), snapshot.sequenceNumber(), - ByteBuffer.wrap(sketch.toByteArray()), + ByteBuffer.wrap(bytes), PuffinCompressionCodec.ZSTD, - ImmutableMap.of( - APACHE_DATASKETCHES_THETA_V1_NDV_PROPERTY, - String.valueOf((long) sketch.getEstimate()))); + ImmutableMap.of(APACHE_DATASKETCHES_THETA_V1_NDV_PROPERTY, String.valueOf(ndv))); } private static Row computeNDVSketches( - SparkSession spark, Table table, Snapshot snapshot, List colNames) { + SparkSession spark, Table table, Snapshot snapshot, List colNames, String blobType) { Dataset inputDF = SparkTableUtil.loadTable(spark, table, snapshot.snapshotId()); - return inputDF.select(toAggColumns(colNames)).first(); + return inputDF.select(toAggColumns(colNames, blobType)).first(); } - private static Column[] toAggColumns(List colNames) { - return colNames.stream().map(NDVSketchUtil::toAggColumn).toArray(Column[]::new); + private static Column[] toAggColumns(List colNames, String blobType) { + return colNames.stream().map(name -> toAggColumn(name, blobType)).toArray(Column[]::new); } - private static Column toAggColumn(String colName) { - ThetaSketchAgg agg = new ThetaSketchAgg(colName); + private static Column toAggColumn(String colName, String blobType) { + AggregateFunction agg = + switch (blobType) { + case StandardBlobTypes.APACHE_DATASKETCHES_THETA_V1 -> new ThetaSketchAgg(colName); + case StandardBlobTypes.APACHE_DATASKETCHES_HLL_V1 -> new HLLSketchAgg(colName); + default -> throw new IllegalArgumentException("Unsupported NDV blob type: " + blobType); + }; return new Column(ExpressionColumnNode.apply(agg.toAggregateExpression())); } } diff --git a/spark/v4.2/spark/src/main/scala/org/apache/spark/sql/stats/HLLSketchAgg.scala b/spark/v4.2/spark/src/main/scala/org/apache/spark/sql/stats/HLLSketchAgg.scala new file mode 100644 index 000000000000..16b3e4cc72c7 --- /dev/null +++ b/spark/v4.2/spark/src/main/scala/org/apache/spark/sql/stats/HLLSketchAgg.scala @@ -0,0 +1,122 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.spark.sql.stats + +import java.nio.ByteBuffer +import org.apache.datasketches.hll.HllSketch +import org.apache.datasketches.hll.Union +import org.apache.datasketches.memory.Memory +import org.apache.iceberg.spark.SparkSchemaUtil +import org.apache.iceberg.types.Conversions +import org.apache.spark.sql.catalyst.InternalRow +import org.apache.spark.sql.catalyst.analysis +import org.apache.spark.sql.catalyst.expressions.Expression +import org.apache.spark.sql.catalyst.expressions.aggregate.ImperativeAggregate +import org.apache.spark.sql.catalyst.expressions.aggregate.TypedImperativeAggregate +import org.apache.spark.sql.catalyst.trees.UnaryLike +import org.apache.spark.sql.types.BinaryType +import org.apache.spark.sql.types.DataType +import org.apache.spark.sql.types.Decimal +import org.apache.spark.unsafe.types.UTF8String + +/** + * HLLSketchAgg generates HLL sketch with default seed and default lgK. + * The values fed to the sketch are converted to bytes using Iceberg's single value serialization. + * The result returned is an array of bytes of Compact HLL sketch of Datasketches library. + * + * See [[https://iceberg.apache.org/puffin-spec/]] for more information. + */ +case class HLLSketchAgg( + child: Expression, + mutableAggBufferOffset: Int = 0, + inputAggBufferOffset: Int = 0) + extends TypedImperativeAggregate[HllSketch] + with UnaryLike[Expression] { + + private lazy val icebergType = SparkSchemaUtil.convert(child.dataType) + + def this(colName: String) = { + this(analysis.UnresolvedAttribute.quotedString(colName), 0, 0) + } + + override def dataType: DataType = BinaryType + + override def nullable: Boolean = false + + override def createAggregationBuffer(): HllSketch = { + new HllSketch() + } + + override def update(buffer: HllSketch, input: InternalRow): HllSketch = { + val value = child.eval(input) + if (value != null) { + val icebergValue = toIcebergValue(value) + val byteBuffer = Conversions.toByteBuffer(icebergType, icebergValue) + buffer.update(toBytes(byteBuffer)) + } + buffer + } + + private def toIcebergValue(value: Any): Any = { + value match { + case s: UTF8String => s.toString + case d: Decimal => d.toJavaBigDecimal + case b: Array[Byte] => ByteBuffer.wrap(b) + case _ => value + } + } + + private def toBytes(byteBuffer: ByteBuffer): Array[Byte] = { + val bytes = new Array[Byte](byteBuffer.remaining()) + byteBuffer.duplicate().get(bytes) + bytes + } + + override def merge(buffer: HllSketch, input: HllSketch): HllSketch = { + val union = new Union(math.max(buffer.getLgConfigK, input.getLgConfigK)) + union.update(buffer) + union.update(input) + union.getResult + } + + override def eval(buffer: HllSketch): Any = { + buffer.toCompactByteArray + } + + override def serialize(buffer: HllSketch): Array[Byte] = { + buffer.toCompactByteArray + } + + override def deserialize(storageFormat: Array[Byte]): HllSketch = { + HllSketch.heapify(Memory.wrap(storageFormat)) + } + + override def withNewMutableAggBufferOffset( + newMutableAggBufferOffset: Int): ImperativeAggregate = { + copy(mutableAggBufferOffset = newMutableAggBufferOffset) + } + + override def withNewInputAggBufferOffset(newInputAggBufferOffset: Int): ImperativeAggregate = { + copy(inputAggBufferOffset = newInputAggBufferOffset) + } + + override protected def withNewChildInternal(newChild: Expression): Expression = { + copy(child = newChild) + } +} diff --git a/spark/v4.2/spark/src/test/java/org/apache/iceberg/spark/actions/TestComputeTableStatsAction.java b/spark/v4.2/spark/src/test/java/org/apache/iceberg/spark/actions/TestComputeTableStatsAction.java index 2252ccda62fe..91f94c081fef 100644 --- a/spark/v4.2/spark/src/test/java/org/apache/iceberg/spark/actions/TestComputeTableStatsAction.java +++ b/spark/v4.2/spark/src/test/java/org/apache/iceberg/spark/actions/TestComputeTableStatsAction.java @@ -38,7 +38,10 @@ import org.apache.iceberg.data.FileHelpers; import org.apache.iceberg.data.GenericRecord; import org.apache.iceberg.data.Record; +import org.apache.iceberg.puffin.Blob; +import org.apache.iceberg.puffin.StandardBlobTypes; import org.apache.iceberg.relocated.com.google.common.base.Preconditions; +import org.apache.iceberg.relocated.com.google.common.collect.ImmutableList; import org.apache.iceberg.relocated.com.google.common.collect.ImmutableMap; import org.apache.iceberg.relocated.com.google.common.collect.Lists; import org.apache.iceberg.spark.CatalogTestBase; @@ -139,6 +142,38 @@ public void testComputeTableStatsAction() throws NoSuchTableException, ParseExce }); } + @TestTemplate + void generateHLLBlobs() throws Exception { + sql("CREATE TABLE %s (id int, data string) USING iceberg", tableName); + Table table = Spark3Util.loadIcebergTable(spark, tableName); + List records = + Lists.newArrayList( + new SimpleRecord(1, "a"), + new SimpleRecord(1, "a"), + new SimpleRecord(2, "b"), + new SimpleRecord(3, "c"), + new SimpleRecord(4, "d")); + spark.createDataset(records, Encoders.bean(SimpleRecord.class)).writeTo(tableName).append(); + table.refresh(); + + List blobs = + NDVSketchUtil.generateBlobs( + spark, + table, + table.currentSnapshot(), + ImmutableList.of("id", "data"), + StandardBlobTypes.APACHE_DATASKETCHES_HLL_V1); + + assertThat(blobs) + .hasSize(2) + .allSatisfy( + blob -> { + assertThat(blob.type()).isEqualTo(StandardBlobTypes.APACHE_DATASKETCHES_HLL_V1); + assertThat(blob.properties()) + .containsEntry(APACHE_DATASKETCHES_THETA_V1_NDV_PROPERTY, "4"); + }); + } + @TestTemplate public void testComputeTableStatsActionWithoutExplicitColumns() throws NoSuchTableException, ParseException {