Skip to content

Bind HLL get_composite_estimate #70

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
jaideeppyne wants to merge 1 commit into
base: main
Choose a base branch
from
Open

Bind HLL get_composite_estimate #70

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
8 changes: 8 additions & 0 deletions src/hll_wrapper.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -53,6 +53,10 @@ void init_hll(nb::module_ &m) {
.def_prop_ro("tgt_type", &hll_sketch::get_target_type, "The HLL type (4, 6, or 8) when in estimation mode")
.def("get_estimate", &hll_sketch::get_estimate,
"Estimate of the distinct count of the input stream")
.def("get_composite_estimate", &hll_sketch::get_composite_estimate,
"Composite estimate of the distinct count. Less accurate than get_estimate() when the HIP estimator "
"is available, but order-independent, and the estimator used once a sketch has been unioned. "
"Intended for error characterization rather than normal use.")
.def("get_lower_bound", &hll_sketch::get_lower_bound, nb::arg("num_std_devs"),
"Returns the approximate lower error bound given the specified number of standard deviations in {1, 2, 3}")
.def("get_upper_bound", &hll_sketch::get_upper_bound, nb::arg("num_std_devs"),
Expand Down Expand Up @@ -111,6 +115,10 @@ void init_hll(nb::module_ &m) {
.def_prop_ro("lg_config_k", &hll_union::get_lg_config_k, "Configured lg_k value for the union")
.def("get_estimate", &hll_union::get_estimate,
"Estimate of the distinct count of the input stream")
.def("get_composite_estimate", &hll_union::get_composite_estimate,
"Composite estimate of the distinct count. This is less accurate than get_estimate() and is used "
"automatically when a union cannot keep the HIP estimator. Intended for error characterization "
"rather than normal use.")
.def("get_lower_bound", &hll_union::get_lower_bound, nb::arg("num_std_devs"),
"Returns the approximate lower error bound given the specified number of standard deviations in {1, 2, 3}")
.def("get_upper_bound", &hll_union::get_upper_bound, nb::arg("num_std_devs"),
Expand Down
20 changes: 19 additions & 1 deletion tests/hll_test.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -122,7 +122,25 @@ def test_hll_union(self):
sk = union.get_result()
self.assertTrue(isinstance(sk, hll_sketch))
self.assertEqual(sk.tgt_type, tgt_hll_type.HLL_4)


def test_hll_composite_estimate_is_order_independent(self):
lgk = 12
n = 5000
forward = hll_sketch(lgk)
reverse = hll_sketch(lgk)
for i in range(n):
forward.update(i)
reverse.update(n - 1 - i)

# HIP estimates can depend on insertion order. The composite estimator must not.
self.assertEqual(forward.get_composite_estimate(), reverse.get_composite_estimate())
self.assertGreater(forward.get_composite_estimate(), 0)

union = hll_union(lgk)
union.update(forward)
union.update(reverse)
self.assertEqual(union.get_composite_estimate(), union.get_result().get_composite_estimate())

def generate_sketch(self, n, lgk, sk_type=tgt_hll_type.HLL_4, st_idx=0):
sk = hll_sketch(lgk, sk_type)
for i in range(st_idx, st_idx + n):
Expand Down