diff --git a/core/src/main/java/org/apache/iceberg/puffin/StandardBlobTypes.java b/core/src/main/java/org/apache/iceberg/puffin/StandardBlobTypes.java
index ce78375c4b1a..da4b60f5349e 100644
--- a/core/src/main/java/org/apache/iceberg/puffin/StandardBlobTypes.java
+++ b/core/src/main/java/org/apache/iceberg/puffin/StandardBlobTypes.java
@@ -27,6 +27,12 @@ private StandardBlobTypes() {}
*/
public static final String APACHE_DATASKETCHES_THETA_V1 = "apache-datasketches-theta-v1";
+ /**
+ * A serialized form of a "compact" HyperLogLog sketch produced by the Apache DataSketches library
+ */
+ public static final String APACHE_DATASKETCHES_HLL_V1 = "apache-datasketches-hll-v1";
+
/** A serialized deletion vector according to the Iceberg spec */
public static final String DV_V1 = "deletion-vector-v1";
}
diff --git a/spark/v4.2/spark/src/main/java/org/apache/iceberg/spark/actions/NDVSketchUtil.java b/spark/v4.2/spark/src/main/java/org/apache/iceberg/spark/actions/NDVSketchUtil.java
index 11b14fafd916..5c0eda7a8613 100644
--- a/spark/v4.2/spark/src/main/java/org/apache/iceberg/spark/actions/NDVSketchUtil.java
+++ b/spark/v4.2/spark/src/main/java/org/apache/iceberg/spark/actions/NDVSketchUtil.java
@@ -20,6 +20,7 @@
import java.nio.ByteBuffer;
import java.util.List;
+import org.apache.datasketches.hll.HllSketch;
import org.apache.datasketches.memory.Memory;
import org.apache.datasketches.theta.CompactSketch;
import org.apache.datasketches.theta.Sketch;
@@ -38,7 +39,9 @@
import org.apache.spark.sql.Dataset;
import org.apache.spark.sql.Row;
import org.apache.spark.sql.SparkSession;
+import org.apache.spark.sql.catalyst.expressions.aggregate.AggregateFunction;
import org.apache.spark.sql.classic.ExpressionColumnNode;
+import org.apache.spark.sql.stats.HLLSketchAgg;
import org.apache.spark.sql.stats.ThetaSketchAgg;
public class NDVSketchUtil {
@@ -49,43 +52,65 @@ private NDVSketchUtil() {}
static List generateBlobs(
SparkSession spark, Table table, Snapshot snapshot, List columns) {
- Row sketches = computeNDVSketches(spark, table, snapshot, columns);
+ return generateBlobs(
+ spark, table, snapshot, columns, StandardBlobTypes.APACHE_DATASKETCHES_THETA_V1);
+ }
+
+ static List generateBlobs(
+ SparkSession spark, Table table, Snapshot snapshot, List columns, String blobType) {
+ Row sketches = computeNDVSketches(spark, table, snapshot, columns, blobType);
Schema schema = table.schemas().get(snapshot.schemaId());
List blobs = Lists.newArrayListWithExpectedSize(columns.size());
for (int i = 0; i < columns.size(); i++) {
Types.NestedField field = schema.findField(columns.get(i));
- Sketch sketch = CompactSketch.wrap(Memory.wrap((byte[]) sketches.get(i)));
- blobs.add(toBlob(field, sketch, snapshot));
+ byte[] bytes = (byte[]) sketches.get(i);
+ blobs.add(toBlob(field, blobType, bytes, estimate(blobType, bytes), snapshot));
}
return blobs;
}
- private static Blob toBlob(Types.NestedField field, Sketch sketch, Snapshot snapshot) {
+ private static long estimate(String blobType, byte[] bytes) {
+ return switch (blobType) {
+ case StandardBlobTypes.APACHE_DATASKETCHES_THETA_V1 -> {
+ Sketch sketch = CompactSketch.wrap(Memory.wrap(bytes));
+ yield (long) sketch.getEstimate();
+ }
+ case StandardBlobTypes.APACHE_DATASKETCHES_HLL_V1 ->
+ (long) HllSketch.heapify(Memory.wrap(bytes)).getEstimate();
+ default -> throw new IllegalArgumentException("Unsupported NDV blob type: " + blobType);
+ };
+ }
+
+ private static Blob toBlob(
+ Types.NestedField field, String blobType, byte[] bytes, long ndv, Snapshot snapshot) {
return new Blob(
- StandardBlobTypes.APACHE_DATASKETCHES_THETA_V1,
+ blobType,
ImmutableList.of(field.fieldId()),
snapshot.snapshotId(),
snapshot.sequenceNumber(),
- ByteBuffer.wrap(sketch.toByteArray()),
+ ByteBuffer.wrap(bytes),
PuffinCompressionCodec.ZSTD,
- ImmutableMap.of(
- APACHE_DATASKETCHES_THETA_V1_NDV_PROPERTY,
- String.valueOf((long) sketch.getEstimate())));
+ ImmutableMap.of(APACHE_DATASKETCHES_THETA_V1_NDV_PROPERTY, String.valueOf(ndv)));
}
private static Row computeNDVSketches(
- SparkSession spark, Table table, Snapshot snapshot, List colNames) {
+ SparkSession spark, Table table, Snapshot snapshot, List colNames, String blobType) {
Dataset inputDF = SparkTableUtil.loadTable(spark, table, snapshot.snapshotId());
- return inputDF.select(toAggColumns(colNames)).first();
+ return inputDF.select(toAggColumns(colNames, blobType)).first();
}
- private static Column[] toAggColumns(List colNames) {
- return colNames.stream().map(NDVSketchUtil::toAggColumn).toArray(Column[]::new);
+ private static Column[] toAggColumns(List colNames, String blobType) {
+ return colNames.stream().map(name -> toAggColumn(name, blobType)).toArray(Column[]::new);
}
- private static Column toAggColumn(String colName) {
- ThetaSketchAgg agg = new ThetaSketchAgg(colName);
+ private static Column toAggColumn(String colName, String blobType) {
+ AggregateFunction agg =
+ switch (blobType) {
+ case StandardBlobTypes.APACHE_DATASKETCHES_THETA_V1 -> new ThetaSketchAgg(colName);
+ case StandardBlobTypes.APACHE_DATASKETCHES_HLL_V1 -> new HLLSketchAgg(colName);
+ default -> throw new IllegalArgumentException("Unsupported NDV blob type: " + blobType);
+ };
return new Column(ExpressionColumnNode.apply(agg.toAggregateExpression()));
}
}
diff --git a/spark/v4.2/spark/src/main/scala/org/apache/spark/sql/stats/HLLSketchAgg.scala b/spark/v4.2/spark/src/main/scala/org/apache/spark/sql/stats/HLLSketchAgg.scala
new file mode 100644
index 000000000000..16b3e4cc72c7
--- /dev/null
+++ b/spark/v4.2/spark/src/main/scala/org/apache/spark/sql/stats/HLLSketchAgg.scala
@@ -0,0 +1,122 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.spark.sql.stats
+
+import java.nio.ByteBuffer
+import org.apache.datasketches.hll.HllSketch
+import org.apache.datasketches.hll.Union
+import org.apache.datasketches.memory.Memory
+import org.apache.iceberg.spark.SparkSchemaUtil
+import org.apache.iceberg.types.Conversions
+import org.apache.spark.sql.catalyst.InternalRow
+import org.apache.spark.sql.catalyst.analysis
+import org.apache.spark.sql.catalyst.expressions.Expression
+import org.apache.spark.sql.catalyst.expressions.aggregate.ImperativeAggregate
+import org.apache.spark.sql.catalyst.expressions.aggregate.TypedImperativeAggregate
+import org.apache.spark.sql.catalyst.trees.UnaryLike
+import org.apache.spark.sql.types.BinaryType
+import org.apache.spark.sql.types.DataType
+import org.apache.spark.sql.types.Decimal
+import org.apache.spark.unsafe.types.UTF8String
+
+/**
+ * HLLSketchAgg generates HLL sketch with default seed and default lgK.
+ * The values fed to the sketch are converted to bytes using Iceberg's single value serialization.
+ * The result returned is an array of bytes of Compact HLL sketch of Datasketches library.
+ *
+ * See [[https://iceberg.apache.org/puffin-spec/]] for more information.
+ */
+case class HLLSketchAgg(
+ child: Expression,
+ mutableAggBufferOffset: Int = 0,
+ inputAggBufferOffset: Int = 0)
+ extends TypedImperativeAggregate[HllSketch]
+ with UnaryLike[Expression] {
+
+ private lazy val icebergType = SparkSchemaUtil.convert(child.dataType)
+
+ def this(colName: String) = {
+ this(analysis.UnresolvedAttribute.quotedString(colName), 0, 0)
+ }
+
+ override def dataType: DataType = BinaryType
+
+ override def nullable: Boolean = false
+
+ override def createAggregationBuffer(): HllSketch = {
+ new HllSketch()
+ }
+
+ override def update(buffer: HllSketch, input: InternalRow): HllSketch = {
+ val value = child.eval(input)
+ if (value != null) {
+ val icebergValue = toIcebergValue(value)
+ val byteBuffer = Conversions.toByteBuffer(icebergType, icebergValue)
+ buffer.update(toBytes(byteBuffer))
+ }
+ buffer
+ }
+
+ private def toIcebergValue(value: Any): Any = {
+ value match {
+ case s: UTF8String => s.toString
+ case d: Decimal => d.toJavaBigDecimal
+ case b: Array[Byte] => ByteBuffer.wrap(b)
+ case _ => value
+ }
+ }
+
+ private def toBytes(byteBuffer: ByteBuffer): Array[Byte] = {
+ val bytes = new Array[Byte](byteBuffer.remaining())
+ byteBuffer.duplicate().get(bytes)
+ bytes
+ }
+
+ override def merge(buffer: HllSketch, input: HllSketch): HllSketch = {
+ val union = new Union(math.max(buffer.getLgConfigK, input.getLgConfigK))
+ union.update(buffer)
+ union.update(input)
+ union.getResult
+ }
+
+ override def eval(buffer: HllSketch): Any = {
+ buffer.toCompactByteArray
+ }
+
+ override def serialize(buffer: HllSketch): Array[Byte] = {
+ buffer.toCompactByteArray
+ }
+
+ override def deserialize(storageFormat: Array[Byte]): HllSketch = {
+ HllSketch.heapify(Memory.wrap(storageFormat))
+ }
+
+ override def withNewMutableAggBufferOffset(
+ newMutableAggBufferOffset: Int): ImperativeAggregate = {
+ copy(mutableAggBufferOffset = newMutableAggBufferOffset)
+ }
+
+ override def withNewInputAggBufferOffset(newInputAggBufferOffset: Int): ImperativeAggregate = {
+ copy(inputAggBufferOffset = newInputAggBufferOffset)
+ }
+
+ override protected def withNewChildInternal(newChild: Expression): Expression = {
+ copy(child = newChild)
+ }
+}
diff --git a/spark/v4.2/spark/src/test/java/org/apache/iceberg/spark/actions/TestComputeTableStatsAction.java b/spark/v4.2/spark/src/test/java/org/apache/iceberg/spark/actions/TestComputeTableStatsAction.java
index 2252ccda62fe..91f94c081fef 100644
--- a/spark/v4.2/spark/src/test/java/org/apache/iceberg/spark/actions/TestComputeTableStatsAction.java
+++ b/spark/v4.2/spark/src/test/java/org/apache/iceberg/spark/actions/TestComputeTableStatsAction.java
@@ -38,7 +38,10 @@
import org.apache.iceberg.data.FileHelpers;
import org.apache.iceberg.data.GenericRecord;
import org.apache.iceberg.data.Record;
+import org.apache.iceberg.puffin.Blob;
+import org.apache.iceberg.puffin.StandardBlobTypes;
import org.apache.iceberg.relocated.com.google.common.base.Preconditions;
+import org.apache.iceberg.relocated.com.google.common.collect.ImmutableList;
import org.apache.iceberg.relocated.com.google.common.collect.ImmutableMap;
import org.apache.iceberg.relocated.com.google.common.collect.Lists;
import org.apache.iceberg.spark.CatalogTestBase;
@@ -139,6 +142,38 @@ public void testComputeTableStatsAction() throws NoSuchTableException, ParseExce
});
}
+ @TestTemplate
+ void generateHLLBlobs() throws Exception {
+ sql("CREATE TABLE %s (id int, data string) USING iceberg", tableName);
+ Table table = Spark3Util.loadIcebergTable(spark, tableName);
+ List records =
+ Lists.newArrayList(
+ new SimpleRecord(1, "a"),
+ new SimpleRecord(1, "a"),
+ new SimpleRecord(2, "b"),
+ new SimpleRecord(3, "c"),
+ new SimpleRecord(4, "d"));
+ spark.createDataset(records, Encoders.bean(SimpleRecord.class)).writeTo(tableName).append();
+ table.refresh();
+
+ List blobs =
+ NDVSketchUtil.generateBlobs(
+ spark,
+ table,
+ table.currentSnapshot(),
+ ImmutableList.of("id", "data"),
+ StandardBlobTypes.APACHE_DATASKETCHES_HLL_V1);
+
+ assertThat(blobs)
+ .hasSize(2)
+ .allSatisfy(
+ blob -> {
+ assertThat(blob.type()).isEqualTo(StandardBlobTypes.APACHE_DATASKETCHES_HLL_V1);
+ assertThat(blob.properties())
+ .containsEntry(APACHE_DATASKETCHES_THETA_V1_NDV_PROPERTY, "4");
+ });
+ }
+
@TestTemplate
public void testComputeTableStatsActionWithoutExplicitColumns()
throws NoSuchTableException, ParseException {