From a9954b988bd610c674458083dfcaba2cad172db3 Mon Sep 17 00:00:00 2001 From: Lee Rhodes Date: Thu, 24 Sep 2026 16:20:27 -0700 Subject: [PATCH 1/2] Changes to allow cross-language binaries to be the same as C++. --- .../theta/EmptyCompactSketch.java | 21 ++++++++------- .../datasketches/theta/PreambleUtil.java | 12 ++++----- .../apache/datasketches/common/UtilityIO.java | 2 +- .../hll/HllSketchCrossLanguageTest.java | 26 +++++++++++++++++++ .../theta/ThetaSketchCrossLanguageTest.java | 21 +++++++++++++++ 5 files changed, 65 insertions(+), 17 deletions(-) diff --git a/src/main/java/org/apache/datasketches/theta/EmptyCompactSketch.java b/src/main/java/org/apache/datasketches/theta/EmptyCompactSketch.java index a5ffed2f7..1ebc47afe 100644 --- a/src/main/java/org/apache/datasketches/theta/EmptyCompactSketch.java +++ b/src/main/java/org/apache/datasketches/theta/EmptyCompactSketch.java @@ -27,29 +27,30 @@ import org.apache.datasketches.common.SketchesArgumentException; /** - * Singleton empty CompactThetaSketch. + * Singleton Empty CompactThetaSketch. * * @author Lee Rhodes */ final class EmptyCompactSketch extends CompactThetaSketch { - //For backward compatibility, a candidate long must have Flags= compact, read-only, - // COMPACT-Family=3, SerVer=3, PreLongs=1, and be exactly 8 bytes long. The seedHash is ignored. - // NOTE: The empty and ordered flags may or may not be set - private static final long EMPTY_SKETCH_MASK = 0X00_00_EB_00_00_FF_FF_FFL; - private static final long EMPTY_SKETCH_TEST = 0X00_00_0A_00_00_03_03_01L; - //When returning a byte array the empty and ordered bits are also set + // For backward compatibility, a candidate EmptyCompactSketch + // must have Flags: compact, empty, read-only; + // and bytes: COMPACT-Family=3, SerVer=3, PreLongs=1; and be exactly 8 bytes long. + // NOTE: The seedHash bytes are ignored + private static final long EMPTY_SKETCH_MASK = 0X00_00_FF_FF_FF_FF_FF_FFL; + private static final long EMPTY_SKETCH_TEST = 0X00_00_1E_00_00_03_03_01L; + //When returning a byte array the following flags are set: ordered, compact, empty, read-only static final byte[] EMPTY_COMPACT_SKETCH_ARR = { 1, 3, 3, 0, 0, 0x1E, 0, 0 }; private static final EmptyCompactSketch EMPTY_COMPACT_SKETCH = new EmptyCompactSketch(); private EmptyCompactSketch() {} - static synchronized EmptyCompactSketch getInstance() { + static EmptyCompactSketch getInstance() { return EMPTY_COMPACT_SKETCH; } - //This should be a heapify - static synchronized EmptyCompactSketch getHeapInstance(final MemorySegment srcSeg) { + //This is effectively a heapify + static EmptyCompactSketch getHeapInstance(final MemorySegment srcSeg) { final long pre0 = srcSeg.get(JAVA_LONG_UNALIGNED, 0); if (testCandidatePre0(pre0)) { return EMPTY_COMPACT_SKETCH; diff --git a/src/main/java/org/apache/datasketches/theta/PreambleUtil.java b/src/main/java/org/apache/datasketches/theta/PreambleUtil.java index fcb7f24a4..f8239511f 100644 --- a/src/main/java/org/apache/datasketches/theta/PreambleUtil.java +++ b/src/main/java/org/apache/datasketches/theta/PreambleUtil.java @@ -204,12 +204,12 @@ private PreambleUtil() {} static final int UNION_THETA_LONG = 24; //8-byte aligned, only used by ThetaUnion // flag byte bit masks - static final int RESERVED_FLAG_MASK = 1; //Bit 0: Reserved, no longer used. Was BigEndian - static final int READ_ONLY_FLAG_MASK = 2; //Bit 1: Reserved, Set but not read. - static final int EMPTY_FLAG_MASK = 4; //Bit 2: - static final int COMPACT_FLAG_MASK = 8; //Bit 3: - static final int ORDERED_FLAG_MASK = 16;//Bit 4: - static final int SINGLEITEM_FLAG_MASK = 32;//Bit 5: + static final int RESERVED_FLAG_MASK = 1; //Bit 0: 0x1: Reserved, no longer used. Was BigEndian + static final int READ_ONLY_FLAG_MASK = 2; //Bit 1: 0x2: + static final int EMPTY_FLAG_MASK = 4; //Bit 2: 0x4: + static final int COMPACT_FLAG_MASK = 8; //Bit 3: 0x8: + static final int ORDERED_FLAG_MASK = 16;//Bit 4: 0x10: + static final int SINGLEITEM_FLAG_MASK = 32;//Bit 5: 0x20: //The last 2 bits (Bit 6,7) of the flags byte are reserved and assumed to be zero. //Other constants diff --git a/src/test/java/org/apache/datasketches/common/UtilityIO.java b/src/test/java/org/apache/datasketches/common/UtilityIO.java index 923dc20c2..eb8ff6a94 100644 --- a/src/test/java/org/apache/datasketches/common/UtilityIO.java +++ b/src/test/java/org/apache/datasketches/common/UtilityIO.java @@ -31,7 +31,7 @@ * Utilities common to testing */ public final class UtilityIO { - private static final String LS = System.getProperty("line.separator"); + public static final String LS = System.getProperty("line.separator"); private static final Class clazz = UtilityIO.class; private static final ClassLoader CL = clazz.getClassLoader(); private static final String TEST_DATA_ROOT_PROP = "test.data.root"; diff --git a/src/test/java/org/apache/datasketches/hll/HllSketchCrossLanguageTest.java b/src/test/java/org/apache/datasketches/hll/HllSketchCrossLanguageTest.java index 8595391a4..84b3af145 100644 --- a/src/test/java/org/apache/datasketches/hll/HllSketchCrossLanguageTest.java +++ b/src/test/java/org/apache/datasketches/hll/HllSketchCrossLanguageTest.java @@ -23,6 +23,7 @@ import static org.apache.datasketches.common.UtilityIO.CHECK_GO_FILES; import static org.apache.datasketches.common.UtilityIO.CHECK_JAVA_FILES; import static org.apache.datasketches.common.UtilityIO.GENERATE_JAVA_FILES; +import static org.apache.datasketches.common.UtilityIO.LS; import static org.apache.datasketches.common.UtilityIO.getFileBytes; import static org.apache.datasketches.common.UtilityIO.putBytesToJavaPath; import static org.apache.datasketches.hll.TgtHllType.HLL_4; @@ -34,6 +35,7 @@ import java.io.IOException; import java.lang.foreign.MemorySegment; +import org.apache.datasketches.common.SketchesArgumentException; import org.apache.datasketches.common.UtilityIO.GroupLanguage; import org.testng.annotations.Test; @@ -68,16 +70,20 @@ public void generateBinariesForCompatibilityTesting() throws IOException { @Test(groups = {CHECK_JAVA_FILES}, priority = 1) public void checkJava() { deserializeHll(GroupLanguage.JAVA); + deserializeHll_crossBinary(GroupLanguage.JAVA, GroupLanguage.CPP); + } @Test(groups = {CHECK_CPP_FILES}) public void checkCpp() { deserializeHll(GroupLanguage.CPP); + deserializeHll_crossBinary(GroupLanguage.JAVA, GroupLanguage.CPP); } @Test(groups = {CHECK_GO_FILES}) public void checkGo() { deserializeHll(GroupLanguage.GO); + deserializeHll_crossBinary(GroupLanguage.JAVA, GroupLanguage.GO); } private static void deserializeHll(final GroupLanguage lang) { @@ -97,4 +103,24 @@ private static void deserializeHll(final GroupLanguage lang) { } } + private static void deserializeHll_crossBinary(final GroupLanguage lang1, final GroupLanguage lang2) { + final String[] sArr = {"hll4", "hll6", "hll8"}; + final int[] nArr = {0, 1, 10, 100, 1000, 10000, 100000, 1000000}; + for (final String s: sArr) { + for (final int n: nArr) { + final String fileName1 = s + "_n" + n + lang1.sfx + ".sk"; + final byte[] bytes1 = getFileBytes(lang1.pth, fileName1); + final String fileName2 = s + "_n" + n + lang2.sfx + ".sk"; + final byte[] bytes2 = getFileBytes(lang2.pth, fileName2); + if (bytes1.length == 0 || bytes2.length == 0) { continue;} + try { assertEquals(bytes1, bytes2); } + catch (final AssertionError e) { + String err = "FileName1: " + fileName1 + ", FileName2: " + fileName2 + + ", HllType: " + s + ", n: " + n; + throw new SketchesArgumentException(err + LS + e); + } + } + } + } + } diff --git a/src/test/java/org/apache/datasketches/theta/ThetaSketchCrossLanguageTest.java b/src/test/java/org/apache/datasketches/theta/ThetaSketchCrossLanguageTest.java index 5aba332ff..9d64f201f 100644 --- a/src/test/java/org/apache/datasketches/theta/ThetaSketchCrossLanguageTest.java +++ b/src/test/java/org/apache/datasketches/theta/ThetaSketchCrossLanguageTest.java @@ -23,6 +23,7 @@ import static org.apache.datasketches.common.UtilityIO.CHECK_GO_FILES; import static org.apache.datasketches.common.UtilityIO.CHECK_JAVA_FILES; import static org.apache.datasketches.common.UtilityIO.GENERATE_JAVA_FILES; +import static org.apache.datasketches.common.UtilityIO.LS; import static org.apache.datasketches.common.UtilityIO.getFileBytes; import static org.apache.datasketches.common.UtilityIO.putBytesToJavaPath; import static org.testng.Assert.assertEquals; @@ -32,6 +33,7 @@ import java.io.IOException; import java.lang.foreign.MemorySegment; +import org.apache.datasketches.common.SketchesArgumentException; import org.apache.datasketches.common.UtilityIO.GroupLanguage; import org.testng.annotations.Test; @@ -92,6 +94,7 @@ public void checkCpp() { deserializeCompressedFromFile(GroupLanguage.CPP); deserializeNonEmptyNoEntriesUsingSegment(GroupLanguage.CPP); deserializeNonEmptyNoEntriesFromFile(GroupLanguage.CPP); + deserializeSketchesFromFile_crossBinary(GroupLanguage.CPP, GroupLanguage.JAVA); } @Test(groups = {CHECK_GO_FILES}) @@ -102,6 +105,7 @@ public void checkGo() { deserializeCompressedFromFile(GroupLanguage.GO); deserializeNonEmptyNoEntriesUsingSegment(GroupLanguage.GO); deserializeNonEmptyNoEntriesFromFile(GroupLanguage.GO); + //deserializeSketchesFromFile_crossBinary(GroupLanguage.GO, GroupLanguage.JAVA); } private static void deserializeSketchesUsingSegment(final GroupLanguage lang) { @@ -146,6 +150,23 @@ private static void deserializeSketchesFromFile(final GroupLanguage lang) { } } + private static void deserializeSketchesFromFile_crossBinary(final GroupLanguage lang1, final GroupLanguage lang2) { + final int[] nArr = {0, 1, 10, 100, 1000, 10000, 100000, 1000000}; + for (final int n: nArr) { + final String fileName1 = "theta_n" + n + lang1.sfx + ".sk"; + final byte[] bytes1 = getFileBytes(lang1.pth, fileName1); + final String fileName2 = "theta_n" + n + lang2.sfx + ".sk"; + final byte[] bytes2 = getFileBytes(lang2.pth, fileName2); + if (bytes1.length == 0 || bytes2.length == 0) { continue; } + try { assertEquals(bytes1, bytes2); } + catch (final AssertionError e) { + String err = "FileName1: " + fileName1 + ", FileName2: " + fileName2 + + ", n: " + n; + throw new SketchesArgumentException(err + LS + e); + } + } + } + private static void deserializeCompressedUsingSegment(final GroupLanguage lang) { final int[] nArr = {10, 100, 1000, 10000, 100000, 1000000}; for (final int n: nArr) { From 9951cf834cee6fe74b0817be7328841f3f429489 Mon Sep 17 00:00:00 2001 From: Lee Rhodes Date: Fri, 25 Sep 2026 10:50:28 -0700 Subject: [PATCH 2/2] Improved Cross-lang tests --- pom.xml | 27 +++++++- .../apache/datasketches/common/UtilityIO.java | 1 + .../hll/HllSketchCrossLanguageTest.java | 61 ++++++++++++------- .../theta/ThetaSketchCrossLanguageTest.java | 57 +++++++++++------ 4 files changed, 103 insertions(+), 43 deletions(-) diff --git a/pom.xml b/pom.xml index 2792edacf..bdd818a48 100644 --- a/pom.xml +++ b/pom.xml @@ -140,8 +140,15 @@ under the License. check_cpp_files check_go_files check_rust_files - check_cpp_historical_files - ${testng.generate_java_files},${testng.check_java_files},${testng.check_cpp_files},${testng.check_go_files},${testng.check_rust_files},${testng.check_cpp_historical_files} + + check_CL_binary_files + check_cpp_historical_files + + ${testng.generate_java_files},${testng.check_java_files},${testng.check_cpp_files} + ${testng.check_go_files},${testng.check_rust_files} + ${testng.check_CL_binaries}${testng.check_cpp_historicals} + + ${testng.check_java_cpp}${testng.check_go_rust}${testng.check_specials} ${project.basedir} false @@ -855,6 +862,22 @@ under the License. + + check_CL_binary_files + + + + org.apache.maven.plugins + maven-surefire-plugin + + ${testng.check_CL_binary_files} + + + + + + + check_cpp_historical_files diff --git a/src/test/java/org/apache/datasketches/common/UtilityIO.java b/src/test/java/org/apache/datasketches/common/UtilityIO.java index eb8ff6a94..f74f58824 100644 --- a/src/test/java/org/apache/datasketches/common/UtilityIO.java +++ b/src/test/java/org/apache/datasketches/common/UtilityIO.java @@ -44,6 +44,7 @@ public final class UtilityIO { public static final String CHECK_CPP_FILES = "check_cpp_files"; public static final String CHECK_GO_FILES = "check_go_files"; public static final String CHECK_RUST_FILES = "check_rust_files"; + public static final String CHECK_CL_BINARY_FILES = "check_CL_binary_files"; public static final String CHECK_CPP_HISTORICAL_FILES = "check_cpp_historical_files"; /** diff --git a/src/test/java/org/apache/datasketches/hll/HllSketchCrossLanguageTest.java b/src/test/java/org/apache/datasketches/hll/HllSketchCrossLanguageTest.java index 84b3af145..f63a50156 100644 --- a/src/test/java/org/apache/datasketches/hll/HllSketchCrossLanguageTest.java +++ b/src/test/java/org/apache/datasketches/hll/HllSketchCrossLanguageTest.java @@ -19,6 +19,7 @@ package org.apache.datasketches.hll; +import static org.apache.datasketches.common.UtilityIO.CHECK_CL_BINARY_FILES; import static org.apache.datasketches.common.UtilityIO.CHECK_CPP_FILES; import static org.apache.datasketches.common.UtilityIO.CHECK_GO_FILES; import static org.apache.datasketches.common.UtilityIO.CHECK_JAVA_FILES; @@ -70,42 +71,37 @@ public void generateBinariesForCompatibilityTesting() throws IOException { @Test(groups = {CHECK_JAVA_FILES}, priority = 1) public void checkJava() { deserializeHll(GroupLanguage.JAVA); - deserializeHll_crossBinary(GroupLanguage.JAVA, GroupLanguage.CPP); - } @Test(groups = {CHECK_CPP_FILES}) public void checkCpp() { deserializeHll(GroupLanguage.CPP); - deserializeHll_crossBinary(GroupLanguage.JAVA, GroupLanguage.CPP); } @Test(groups = {CHECK_GO_FILES}) public void checkGo() { deserializeHll(GroupLanguage.GO); - deserializeHll_crossBinary(GroupLanguage.JAVA, GroupLanguage.GO); } - private static void deserializeHll(final GroupLanguage lang) { - final String[] sArr = {"hll4", "hll6", "hll8"}; - final int[] nArr = {0, 1, 10, 100, 1000, 10000, 100000, 1000000}; - for (final String s: sArr) { - for (final int n: nArr) { - final String fileName = s + "_n" + n + lang.sfx + ".sk"; - final byte[] bytes = getFileBytes(lang.pth, fileName); - if (bytes.length == 0) { continue;} - //System.out.println(fileName); - final HllSketch sketch = HllSketch.heapify(MemorySegment.ofArray(bytes)); - assertEquals(sketch.getLgConfigK(), 12); - assertTrue(n == 0 ? sketch.isEmpty() : !sketch.isEmpty()); - assertEquals(sketch.getEstimate(), n, n * 0.02); - } - } + @Test(groups = {CHECK_CL_BINARY_FILES}) + public void checkBinaries1() { + deserializeHll_crossBinary(GroupLanguage.CPP, GroupLanguage.JAVA); + } + + @Test(groups = {CHECK_CL_BINARY_FILES}) + public void checkBinaries2() { + deserializeHll_crossBinary(GroupLanguage.GO, GroupLanguage.JAVA); + } + + @Test(groups = {CHECK_CL_BINARY_FILES}) + public void checkBinaries3() { + deserializeHll_crossBinary(GroupLanguage.GO, GroupLanguage.CPP); } private static void deserializeHll_crossBinary(final GroupLanguage lang1, final GroupLanguage lang2) { final String[] sArr = {"hll4", "hll6", "hll8"}; final int[] nArr = {0, 1, 10, 100, 1000, 10000, 100000, 1000000}; + String id = null; for (final String s: sArr) { for (final int n: nArr) { final String fileName1 = s + "_n" + n + lang1.sfx + ".sk"; @@ -113,14 +109,35 @@ private static void deserializeHll_crossBinary(final GroupLanguage lang1, final final String fileName2 = s + "_n" + n + lang2.sfx + ".sk"; final byte[] bytes2 = getFileBytes(lang2.pth, fileName2); if (bytes1.length == 0 || bytes2.length == 0) { continue;} + id = "CLB: " + fileName1 + " vs " + fileName2 + ", HllType: " + s + ", n: " + n; try { assertEquals(bytes1, bytes2); } catch (final AssertionError e) { - String err = "FileName1: " + fileName1 + ", FileName2: " + fileName2 - + ", HllType: " + s + ", n: " + n; - throw new SketchesArgumentException(err + LS + e); + throw new SketchesArgumentException(id + LS + e); } + println("PASSED: " + id); } } } + private static void deserializeHll(final GroupLanguage lang) { + final String[] sArr = {"hll4", "hll6", "hll8"}; + final int[] nArr = {0, 1, 10, 100, 1000, 10000, 100000, 1000000}; + for (final String s: sArr) { + for (final int n: nArr) { + final String fileName = s + "_n" + n + lang.sfx + ".sk"; + final byte[] bytes = getFileBytes(lang.pth, fileName); + if (bytes.length == 0) { continue;} + //System.out.println(fileName); + final HllSketch sketch = HllSketch.heapify(MemorySegment.ofArray(bytes)); + assertEquals(sketch.getLgConfigK(), 12); + assertTrue(n == 0 ? sketch.isEmpty() : !sketch.isEmpty()); + assertEquals(sketch.getEstimate(), n, n * 0.02); + } + } + } + + static void println(Object o) { + //System.out.println(o.toString()); + } + } diff --git a/src/test/java/org/apache/datasketches/theta/ThetaSketchCrossLanguageTest.java b/src/test/java/org/apache/datasketches/theta/ThetaSketchCrossLanguageTest.java index 9d64f201f..8617cd6b9 100644 --- a/src/test/java/org/apache/datasketches/theta/ThetaSketchCrossLanguageTest.java +++ b/src/test/java/org/apache/datasketches/theta/ThetaSketchCrossLanguageTest.java @@ -19,6 +19,7 @@ package org.apache.datasketches.theta; +import static org.apache.datasketches.common.UtilityIO.CHECK_CL_BINARY_FILES; import static org.apache.datasketches.common.UtilityIO.CHECK_CPP_FILES; import static org.apache.datasketches.common.UtilityIO.CHECK_GO_FILES; import static org.apache.datasketches.common.UtilityIO.CHECK_JAVA_FILES; @@ -94,7 +95,6 @@ public void checkCpp() { deserializeCompressedFromFile(GroupLanguage.CPP); deserializeNonEmptyNoEntriesUsingSegment(GroupLanguage.CPP); deserializeNonEmptyNoEntriesFromFile(GroupLanguage.CPP); - deserializeSketchesFromFile_crossBinary(GroupLanguage.CPP, GroupLanguage.JAVA); } @Test(groups = {CHECK_GO_FILES}) @@ -105,7 +105,39 @@ public void checkGo() { deserializeCompressedFromFile(GroupLanguage.GO); deserializeNonEmptyNoEntriesUsingSegment(GroupLanguage.GO); deserializeNonEmptyNoEntriesFromFile(GroupLanguage.GO); - //deserializeSketchesFromFile_crossBinary(GroupLanguage.GO, GroupLanguage.JAVA); + } + + @Test(groups = {CHECK_CL_BINARY_FILES}) + public void checkBinaries1() { + deserializeSketchesFromFile_crossBinary(GroupLanguage.CPP, GroupLanguage.JAVA); + } + + @Test(groups = {CHECK_CL_BINARY_FILES}) + public void checkBinaries2() { + deserializeSketchesFromFile_crossBinary(GroupLanguage.GO, GroupLanguage.JAVA); + } + + @Test(groups = {CHECK_CL_BINARY_FILES}) + public void checkBinaries3() { + deserializeSketchesFromFile_crossBinary(GroupLanguage.GO, GroupLanguage.CPP); + } + + private static void deserializeSketchesFromFile_crossBinary(final GroupLanguage lang1, final GroupLanguage lang2) { + final int[] nArr = {0, 1, 10, 100, 1000, 10000, 100000, 1000000}; + String id = null; + for (final int n: nArr) { + final String fileName1 = "theta_n" + n + lang1.sfx + ".sk"; + final byte[] bytes1 = getFileBytes(lang1.pth, fileName1); + final String fileName2 = "theta_n" + n + lang2.sfx + ".sk"; + final byte[] bytes2 = getFileBytes(lang2.pth, fileName2); + if (bytes1.length == 0 || bytes2.length == 0) { continue; } + id = "CLB: " + fileName1 + " vs " + fileName2 + ", n: " + n; + try { assertEquals(bytes1, bytes2); } + catch (final AssertionError e) { + throw new SketchesArgumentException(id + LS + e); + } + println("PASSED: " + id); + } } private static void deserializeSketchesUsingSegment(final GroupLanguage lang) { @@ -150,23 +182,6 @@ private static void deserializeSketchesFromFile(final GroupLanguage lang) { } } - private static void deserializeSketchesFromFile_crossBinary(final GroupLanguage lang1, final GroupLanguage lang2) { - final int[] nArr = {0, 1, 10, 100, 1000, 10000, 100000, 1000000}; - for (final int n: nArr) { - final String fileName1 = "theta_n" + n + lang1.sfx + ".sk"; - final byte[] bytes1 = getFileBytes(lang1.pth, fileName1); - final String fileName2 = "theta_n" + n + lang2.sfx + ".sk"; - final byte[] bytes2 = getFileBytes(lang2.pth, fileName2); - if (bytes1.length == 0 || bytes2.length == 0) { continue; } - try { assertEquals(bytes1, bytes2); } - catch (final AssertionError e) { - String err = "FileName1: " + fileName1 + ", FileName2: " + fileName2 - + ", n: " + n; - throw new SketchesArgumentException(err + LS + e); - } - } - } - private static void deserializeCompressedUsingSegment(final GroupLanguage lang) { final int[] nArr = {10, 100, 1000, 10000, 100000, 1000000}; for (final int n: nArr) { @@ -229,4 +244,8 @@ private static void deserializeNonEmptyNoEntriesFromFile(final GroupLanguage lan assertEquals(sketch.getRetainedEntries(), 0); } + static void println(Object o) { + //System.out.println(o.toString()); + } + }