From edc73920a5eb73f245a7d712c61a1068bd182552 Mon Sep 17 00:00:00 2001 From: jgjesdal Date: Mon, 12 Oct 2026 14:14:26 +0200 Subject: [PATCH 1/8] Read datapoints through the binary endpoint, decoded in parallel retrieve_datapoints_binary posts the JSON retrieve body to POST /timeseries/data/list/binary and parses the frames that come back into one columnar BinaryDatapoints per series. A body without the DHDE end marker was cut short and is a 502, never fewer points. Aggregate frames (codec 2) carry avg, min, max and sum; decimals stay exact. The envelopes are walked first and the frames then decompressed and decoded on scoped threads: zstd was 90% of a single-threaded parse. For 100M float rows the parse went from 1.46s to 0.60s and the SDK's share of an end-to-end read from 1.57s to 0.90s. In Python, the sync client's retrieve_datapoints_binary returns objects implementing the Arrow PyCapsule stream protocol, so polars, pyarrow and pandas 3 take them without copying. The package adds no runtime dependency. The offline tests read fixtures written by the platform's own frame writer. Co-Authored-By: Claude Opus 5.5 Signed-off-by: jgjesdal --- AGENTS.md | 6 +- README.md | 35 + datahub_python_bindings/Cargo.toml | 3 + .../intellistream_datahub_sdk/__init__.pyi | 70 ++ .../src/timeseries/binary_read.rs | 165 ++++ datahub_python_bindings/src/timeseries/mod.rs | 2 + .../src/timeseries/sync_service.rs | 27 + python_tests/requirements.txt | 2 + python_tests/test_timeseries_binary_read.py | 212 +++++ src/blocking.rs | 3 +- src/generic.rs | 51 ++ src/timeseries/binary.rs | 14 + src/timeseries/binary_read.rs | 836 ++++++++++++++++++ .../fixtures/GenerateReadFixtures.java | 115 +++ src/timeseries/fixtures/read_aggregates.bin | Bin 0 -> 401 bytes src/timeseries/fixtures/read_values.bin | Bin 0 -> 2594 bytes src/timeseries/mod.rs | 2 + src/timeseries/test.rs | 160 +++- 18 files changed, 1699 insertions(+), 4 deletions(-) create mode 100644 datahub_python_bindings/src/timeseries/binary_read.rs create mode 100644 python_tests/test_timeseries_binary_read.py create mode 100644 src/timeseries/binary_read.rs create mode 100644 src/timeseries/fixtures/GenerateReadFixtures.java create mode 100644 src/timeseries/fixtures/read_aggregates.bin create mode 100644 src/timeseries/fixtures/read_values.bin diff --git a/AGENTS.md b/AGENTS.md index 7b432db..6a153fa 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -199,9 +199,11 @@ at-least-once path is still `SubscriptionsService::listen`. Two things differ fr `recommend_value_type` sits beside it: a bare `ValueTypeRecommendation`, and an unknown unit is the generic default with `recognized: false`, not an error. -### Binary datapoint ingest (`src/timeseries/binary.rs`) +### Binary datapoints (`src/timeseries/binary.rs`, `src/timeseries/binary_read.rs`) -`TimeSeriesService::insert_datapoints_binary` is the second ingest path, to `POST /timeseries/data/binary`: the same `DatapointsCollection` input, resolved through `/timeseries/byids` (cached per service instance; needs read access on the dataset), checked against each series' value type locally, cut into Arrow IPC frames at the contract's caps, zstd-compressed per frame (mandatory: level 1, 3 or 9, default 9) and posted through `execute_post_bytes_request`. Every refusal on this path is a problem document typed for its status — `invalid-frame` (400), `unknown-timeseries` (404), `request-too-large` (413), `unsupported-media-type` (415), `value-type-mismatch` and `external-id-mismatch` (422), `too-many-in-flight` (429) — with the kebab-case sub-case in a `reason` extension beside it. It was one type, `datapoint-block-rejected`, answering with all six statuses until the api split it (platform #120); the SDK matches the new slugs only, and nothing here recognises the old one — the binary path has never been in a release, so no caller can be on the other side of that change. `is_stale_series_rejection` reads the slug to decide the one re-resolve-and-retry, and reads it from the problem rather than the body text because the SDK's own pre-flight 404 names the series it could not resolve — an external id spelling `unknown-timeseries` used to trigger the retry. `FrameWriter` builds one frame and is public; `cut_into_writers` and `pack_requests` hold the caps. The byte layout is the platform's `binary_datapoints_format.md`. The arrow-rs crates (`arrow-array`, `arrow-schema`, `arrow-ipc`) exist for this path and are the seed of the Arrow read path. In Python it is `insert_datapoints_binary` and `insert_from_lists_binary` on the **sync** client only, each taking an optional `zstd_level`; the async client has no binary methods. The ignored `timeseries::tests::test_datapoints_binary` is the live twin of `test_datapoints` and needs a backend that serves the endpoint; the writer's own tests in `binary.rs` run offline. `python_tests/test_timeseries_binary.py` round-trips the Python binding through the JSON read path. +`TimeSeriesService::insert_datapoints_binary` is the second ingest path, to `POST /timeseries/data/binary`: the same `DatapointsCollection` input, resolved through `/timeseries/byids` (cached per service instance; needs read access on the dataset), checked against each series' value type locally, cut into Arrow IPC frames at the contract's caps, zstd-compressed per frame (mandatory: level 1, 3 or 9, default 9) and posted through `execute_post_bytes_request`. Every refusal on this path is a problem document typed for its status — `invalid-frame` (400), `unknown-timeseries` (404), `request-too-large` (413), `unsupported-media-type` (415), `value-type-mismatch` and `external-id-mismatch` (422), `too-many-in-flight` (429) — with the kebab-case sub-case in a `reason` extension beside it. It was one type, `datapoint-block-rejected`, answering with all six statuses until the api split it (platform #120); the SDK matches the new slugs only, and nothing here recognises the old one — the binary path has never been in a release, so no caller can be on the other side of that change. `is_stale_series_rejection` reads the slug to decide the one re-resolve-and-retry, and reads it from the problem rather than the body text because the SDK's own pre-flight 404 names the series it could not resolve — an external id spelling `unknown-timeseries` used to trigger the retry. `FrameWriter` builds one frame and is public; `cut_into_writers` and `pack_requests` hold the caps. The byte layout is the platform's `binary_datapoints_format.md`. The arrow-rs crates (`arrow-array`, `arrow-schema`, `arrow-ipc`) serve both directions. In Python it is `insert_datapoints_binary` and `insert_from_lists_binary` on the **sync** client only, each taking an optional `zstd_level`; the async client has no binary methods. The ignored `timeseries::tests::test_datapoints_binary` is the live twin of `test_datapoints` and needs a backend that serves the endpoint; the writer's own tests in `binary.rs` run offline. `python_tests/test_timeseries_binary.py` round-trips the Python binding through the JSON read path. + +The read is `TimeSeriesService::retrieve_datapoints_binary`, to `POST /timeseries/data/list/binary`: the JSON retrieve body in, the same frames out, one series per frame, then the 4-byte marker `DHDE`. A body without the marker was cut short (the server streams, so a failure after the first bytes can only end the body early) and is a **502** saying so, never fewer points; bytes after it are refused too. Codec 2 frames carry bucketed aggregates in their own schema (`avg`/`min`/`max`/`sum`, Float64, 50 000 buckets per frame) while the envelope's type byte stays the series' own. `parse_read_response` returns one columnar `BinaryDatapoints` per series, merging consecutive frames of one series; decimals stay exact as unscaled `i128` plus scale. It walks the envelopes first (cheap, and where a cut-short body is caught), then decompresses and decodes the frames on scoped threads — zstd was 90% of a single-threaded parse, 1.3 s per 100M float rows — and `retrieve_datapoints_binary` runs it on `spawn_blocking`. What remains (about 0.6 s per 100M rows) is allocation and copying, not zstd; streaming the read would hide it behind the download. Differences from the JSON read worth knowing: no `cursor` (a 400; advance `start` past the last timestamp), a series with no points is absent rather than empty, and an unknown or unreadable series is a 400 for the whole request — `DataWrapper` serialises only `items`, so the request-level `ignoreUnknownIds` and `limit` are not reachable. `src/timeseries/fixtures/read_*.bin` are response bodies written by the platform's own `DatapointFrameWriter.wrap`, so the offline tests in `binary_read.rs` prove arrow-rs reads what the Java writer emits; regenerate them when the platform's writer changes. The ignored `timeseries::tests::test_datapoints_binary_read` is the live check against the JSON read. In Python it is `retrieve_datapoints_binary` on the sync client, returning `BinaryDatapoints` objects that implement the Arrow PyCapsule stream protocol (`__arrow_c_stream__`, via `arrow-array`'s `ffi` feature enabled in the bindings crate only), so polars, pyarrow and pandas 3 read them zero-copy; the package still declares no runtime dependency, and `to_polars`/`to_pandas` import theirs lazily. `python_tests/test_timeseries_binary_read.py` covers it offline against a stand-in server and live when the backend serves the endpoint. ### The `ApiServiceProvider` trait (`src/generic.rs`) diff --git a/README.md b/README.md index 0653c0b..a228fe0 100644 --- a/README.md +++ b/README.md @@ -139,6 +139,41 @@ and sends again, once. Resolving by external id goes through The durable spool does not cover this path. `binary::FrameWriter` is public for producers that build frames themselves. +## Binary datapoint read + +`time_series.retrieve_datapoints_binary(&filters)` takes the request `retrieve_datapoints` +takes and reads the same frames back from `POST /timeseries/data/list/binary`, so a large read +costs no JSON and no object per point. It returns one `BinaryDatapoints` per series that has +points, in the order of the request's items: `timestamps` in epoch milliseconds and `values` as +one typed column (`BinaryValues::Float(Vec)`, exact `Decimal { unscaled, scale }` for +`numeric` and `decimal32`, two optional columns for `mixed`, or `Aggregates { avg, min, max, +sum }` for an item with `aggregates` and a `granularity`). + +```rust +let mut filter = RetrieveFilter::default(); +filter.external_id = Some("engine.temp".to_string()); +filter.start = Some(Utc.with_ymd_and_hms(2026, 10, 1, 0, 0, 0).unwrap()); +filter.end = Some(Utc.with_ymd_and_hms(2026, 10, 2, 0, 0, 0).unwrap()); +for series in api.time_series.retrieve_datapoints_binary(&DataWrapper::from_vec(vec![filter])).await? { + if let BinaryValues::Float(values) = &series.values { + println!("{}: {} points", series.external_id, values.len()); + } +} +``` + +An item with a `limit` (at most 100 000) returns that many of the oldest points from `start`; +an item without one returns its whole window, so bound a long series with `end`, as the whole +answer is held in memory. There is no cursor: to read on in pieces, send the request again with +`start` one millisecond past the last timestamp you got. A series that does not exist is a 400 for the whole request, where +the JSON read answers empty. A response the server could not finish is a 502 rather than fewer +points, and is safe to send again. + +In Python, `client.timeseries.retrieve_datapoints_binary(filter_or_list)` on the sync client +returns objects that implement the Arrow PyCapsule stream protocol, so +`polars.DataFrame(series)`, `pyarrow.table(series)` and `pandas.DataFrame.from_arrow(series)` +take them without copying; `series.to_polars()` and `series.to_pandas()` do the same. The package +adds no dependency for this — bring polars, or pyarrow for pandas. + ## Python bindings `datahub_python_bindings/` wraps this SDK as the Python package `intellistream-datahub-sdk` (import name diff --git a/datahub_python_bindings/Cargo.toml b/datahub_python_bindings/Cargo.toml index 96212f4..cae7243 100644 --- a/datahub_python_bindings/Cargo.toml +++ b/datahub_python_bindings/Cargo.toml @@ -25,3 +25,6 @@ pyo3-async-runtimes = { version = "0.29", features = ["tokio-runtime"] } numpy = "0.29" uuid = { version = "1.22.0", features = ["v4"] } strum = { version = "0.28", features = ["derive"] } +# `ffi` for the Arrow PyCapsule stream; the SDK already depends on both crates. +arrow-array = { version = "59.3", features = ["ffi"] } +arrow-schema = "59.3" diff --git a/datahub_python_bindings/python/intellistream_datahub_sdk/__init__.pyi b/datahub_python_bindings/python/intellistream_datahub_sdk/__init__.pyi index 7a233fa..0359bba 100644 --- a/datahub_python_bindings/python/intellistream_datahub_sdk/__init__.pyi +++ b/datahub_python_bindings/python/intellistream_datahub_sdk/__init__.pyi @@ -590,6 +590,43 @@ class DatapointsCollectionDatapoints: def id(self) -> int | None: ... +class BinaryDatapoints: + """One series' rows from ``TimeSeriesServiceSync.retrieve_datapoints_binary``, as an Arrow table. + + Implements the Arrow PyCapsule stream protocol, so ``polars.DataFrame(series)``, + ``pyarrow.table(series)`` and ``pandas.DataFrame.from_arrow(series)`` (pandas 3) take it + without copying. Every call makes a new stream, so one object converts more than once. + + Columns are ``timestamp`` (``timestamp[ms, UTC]``; the bucket start for aggregates) and then, + by value type: ``value`` as int64 (``bigint``), float64 (``float``), float32 (``float32``), + ``decimal128(18, 6)`` (``numeric``), ``decimal128(9, 4)`` (``decimal32``) or utf8 (``text``); + nullable ``value_numeric`` and ``value_text`` for ``mixed``, each row setting one; or + ``avg``, ``min``, ``max`` and ``sum`` as float64 when :attr:`is_aggregates`. + """ + + @property + def id(self) -> int: ... + @property + def external_id(self) -> str: ... + @property + def value_type(self) -> str: + """The series' value type: ``bigint``, ``float``, ``float32``, ``numeric``, ``decimal32``, + ``text`` or ``mixed``. An aggregate read reports the series' own type.""" + @property + def is_aggregates(self) -> bool: + """Whether the rows are buckets with ``avg``, ``min``, ``max`` and ``sum``.""" + @property + def columns(self) -> list[str]: ... + def __len__(self) -> int: ... + def __arrow_c_stream__(self, requested_schema: object | None = None) -> object: + """The Arrow PyCapsule stream protocol. ``requested_schema`` is ignored.""" + def to_polars(self) -> Any: + """A ``polars.DataFrame``. Needs polars; raises ``ImportError`` without it.""" + def to_pandas(self) -> Any: + """A ``pandas.DataFrame``, converted through pyarrow. Needs pandas and pyarrow; raises + ``ImportError`` without pyarrow.""" + + class RetrieveFilter: def __init__( self, @@ -748,6 +785,39 @@ class TimeSeriesServiceSync: ts: Identifiable, ) -> list[str]: ... def retrieve_datapoints(self, input: RetrieveFilter) -> list[DatapointsCollectionDatapoints]: ... + def retrieve_datapoints_binary( + self, input: RetrieveFilter | list[RetrieveFilter] + ) -> list[BinaryDatapoints]: + """Read datapoints through ``POST /timeseries/data/list/binary``: the request + ``retrieve_datapoints`` takes, answered as compressed Arrow frames instead of JSON. + + Parameters + ---------- + input + One filter or a list. Each needs a ``start`` or an ``end``. With a ``limit`` (at most + 100 000) it returns that many of the oldest points from there; without one it returns + every point in its window, so bound a long series with ``end``, as the whole answer is + held in memory. With ``aggregates`` and a ``granularity`` + the series comes back as buckets carrying all four aggregates, whichever were named. + + Returns + ------- + list[BinaryDatapoints] + One per series that has points in its window, in the order of ``input``; a series + with none is left out rather than returned empty. + + Raises + ------ + DataHubException + 400 for a ``cursor`` (there is no paging: send the request again with ``start`` one + millisecond past the last timestamp read), for aggregates on a text series, and for + a series that does not exist or that you cannot read, where the JSON read answers + empty. 502 when the response was cut short; sending it again is safe. + + Notes + ----- + Sync client only, like the binary insert. + """ def delete_datapoints(self, input: list[DeleteFilter]) -> None: ... def retrieve_latest_datapoints( self, input: list[Identifiable] diff --git a/datahub_python_bindings/src/timeseries/binary_read.rs b/datahub_python_bindings/src/timeseries/binary_read.rs new file mode 100644 index 0000000..8a044f6 --- /dev/null +++ b/datahub_python_bindings/src/timeseries/binary_read.rs @@ -0,0 +1,165 @@ +use std::sync::Arc; + +use arrow_array::ffi_stream::FFI_ArrowArrayStream; +use arrow_array::{ + ArrayRef, Decimal128Array, Float32Array, Float64Array, Int64Array, RecordBatch, + RecordBatchIterator, StringArray, TimestampMillisecondArray, +}; +use arrow_schema::{ArrowError, DataType, Field, Schema, TimeUnit}; +use intellistream_datahub_sdk::timeseries::{BinaryDatapoints, BinaryValues, DatapointValueType}; +use pyo3::exceptions::{PyImportError, PyValueError}; +use pyo3::prelude::*; +use pyo3::types::PyCapsule; + +/// One series' rows from `retrieve_datapoints_binary`, as an Arrow table: `timestamp` and the +/// value columns of its type. Read it with anything that takes the Arrow PyCapsule protocol. +#[pyclass(module = "intellistream_datahub_sdk", name = "BinaryDatapoints", frozen)] +pub struct PyBinaryDatapoints { + id: u64, + external_id: String, + value_type: DatapointValueType, + aggregates: bool, + batch: RecordBatch, +} + +impl TryFrom for PyBinaryDatapoints { + type Error = ArrowError; + + fn try_from(series: BinaryDatapoints) -> Result { + let aggregates = series.is_aggregates(); + let timestamp = Field::new( + "timestamp", + DataType::Timestamp(TimeUnit::Millisecond, Some("UTC".into())), + false, + ); + let mut fields = vec![timestamp]; + let mut columns: Vec = + vec![Arc::new(TimestampMillisecondArray::from(series.timestamps).with_timezone("UTC"))]; + let mut push = |name: &str, nullable: bool, array: ArrayRef| { + fields.push(Field::new(name, array.data_type().clone(), nullable)); + columns.push(array); + }; + match series.values { + BinaryValues::Bigint(v) => push("value", false, Arc::new(Int64Array::from(v))), + BinaryValues::Float(v) => push("value", false, Arc::new(Float64Array::from(v))), + BinaryValues::Float32(v) => push("value", false, Arc::new(Float32Array::from(v))), + BinaryValues::Decimal { unscaled, scale } => { + let precision = if series.value_type == DatapointValueType::Decimal32 { 9 } else { 18 }; + let array = Decimal128Array::from(unscaled).with_precision_and_scale(precision, scale)?; + push("value", false, Arc::new(array)) + } + BinaryValues::Text(v) => push("value", false, Arc::new(StringArray::from(v))), + BinaryValues::Mixed { numeric, text } => { + push("value_numeric", true, Arc::new(Float64Array::from(numeric))); + push("value_text", true, Arc::new(StringArray::from(text))); + } + BinaryValues::Aggregates { avg, min, max, sum } => { + push("avg", false, Arc::new(Float64Array::from(avg))); + push("min", false, Arc::new(Float64Array::from(min))); + push("max", false, Arc::new(Float64Array::from(max))); + push("sum", false, Arc::new(Float64Array::from(sum))); + } + other => { + return Err(ArrowError::NotYetImplemented(format!( + "binary values {other:?} have no Arrow mapping" + ))) + } + } + let batch = RecordBatch::try_new(Arc::new(Schema::new(fields)), columns)?; + Ok(Self { + id: series.id, + external_id: series.external_id, + value_type: series.value_type, + aggregates, + batch, + }) + } +} + +#[pymethods] +impl PyBinaryDatapoints { + #[getter] + fn id(&self) -> u64 { + self.id + } + + #[getter] + fn external_id(&self) -> &str { + &self.external_id + } + + /// The series' value type, as a timeseries read reports it: `float`, `bigint`, `text`, ... + #[getter] + fn value_type(&self) -> String { + format!("{:?}", self.value_type).to_lowercase() + } + + /// Whether the rows are buckets with `avg`, `min`, `max` and `sum` rather than values. + #[getter] + fn is_aggregates(&self) -> bool { + self.aggregates + } + + /// The column names, in order. + #[getter] + fn columns(&self) -> Vec { + self.batch.schema().fields().iter().map(|f| f.name().clone()).collect() + } + + fn __len__(&self) -> usize { + self.batch.num_rows() + } + + fn __repr__(&self) -> String { + format!( + "BinaryDatapoints(external_id={:?}, value_type={:?}, rows={}{})", + self.external_id, + self.value_type(), + self.batch.num_rows(), + if self.aggregates { ", aggregates" } else { "" } + ) + } + + /// The Arrow PyCapsule stream protocol. Each call is a new stream over the same buffers, so + /// the object can be converted more than once. `requested_schema` is ignored, as the protocol + /// allows: the columns come in their own types. + #[pyo3(signature = (requested_schema=None))] + fn __arrow_c_stream__<'py>( + &self, + py: Python<'py>, + requested_schema: Option>, + ) -> PyResult> { + let _ = requested_schema; + let reader = RecordBatchIterator::new(vec![Ok(self.batch.clone())], self.batch.schema()); + let stream = FFI_ArrowArrayStream::new(Box::new(reader)); + PyCapsule::new_with_value(py, stream, c"arrow_array_stream") + } + + /// A `polars.DataFrame`, without copying. Needs polars. + fn to_polars<'py>(slf: &Bound<'py, Self>) -> PyResult> { + let polars = import(slf.py(), "polars", "to_polars")?; + polars.call_method1("DataFrame", (slf,)) + } + + /// A `pandas.DataFrame`. Needs pandas and pyarrow, which pandas converts Arrow through. + fn to_pandas<'py>(slf: &Bound<'py, Self>) -> PyResult> { + let py = slf.py(); + let pyarrow = import(py, "pyarrow", "to_pandas")?; + pyarrow.call_method1("table", (slf,))?.call_method0("to_pandas") + } +} + +fn import<'py>(py: Python<'py>, module: &str, method: &str) -> PyResult> { + py.import(module).map_err(|_| { + PyImportError::new_err(format!( + "BinaryDatapoints.{method}() needs {module}; install it with `pip install {module}`" + )) + }) +} + +pub(crate) fn to_py(series: Vec) -> PyResult> { + series + .into_iter() + .map(|s| PyBinaryDatapoints::try_from(s).map_err(|e| PyValueError::new_err(e.to_string()))) + .collect() +} diff --git a/datahub_python_bindings/src/timeseries/mod.rs b/datahub_python_bindings/src/timeseries/mod.rs index 5cd6d94..53422a7 100644 --- a/datahub_python_bindings/src/timeseries/mod.rs +++ b/datahub_python_bindings/src/timeseries/mod.rs @@ -33,6 +33,7 @@ use std::sync::Arc; use strum::{Display, EnumString}; pub mod async_service; +pub mod binary_read; mod construction; pub mod datapoints; pub mod general; @@ -469,6 +470,7 @@ pub fn register(m: &Bound<'_, PyModule>) -> PyResult<()> { m.add_class::()?; m.add_class::()?; m.add_class::()?; + m.add_class::()?; m.add_class::()?; m.add_class::()?; m.add_class::()?; diff --git a/datahub_python_bindings/src/timeseries/sync_service.rs b/datahub_python_bindings/src/timeseries/sync_service.rs index 0d84537..4303ea2 100644 --- a/datahub_python_bindings/src/timeseries/sync_service.rs +++ b/datahub_python_bindings/src/timeseries/sync_service.rs @@ -4,11 +4,18 @@ use crate::timeseries::datapoints::{ }; use crate::{DatahubIdentity, Identifiable}; use crate::{PyIdCollection, PyRetrieveFilter}; +use crate::timeseries::binary_read::{self, PyBinaryDatapoints}; use intellistream_datahub_sdk::generic::{DataWrapper, IdAndExtId}; use intellistream_datahub_sdk::{ApiService, TimeSeriesUpdateCollection}; use pyo3_async_runtimes::tokio::future_into_py; use std::sync::Arc; +#[derive(FromPyObject)] +enum RetrieveFilters { + One(PyRetrieveFilter), + Many(Vec), +} + #[pyclass(module = "intellistream_datahub_sdk", name = "TimeSeriesServiceSync")] pub struct PyTimeSeriesServiceSync { pub api_service: Arc, @@ -355,6 +362,26 @@ impl PyTimeSeriesServiceSync { Ok(result) }) } + /// `POST /timeseries/data/list/binary`: the request `retrieve_datapoints` takes, one filter or + /// a list, answered as Arrow frames. One `BinaryDatapoints` per series that has points. + fn retrieve_datapoints_binary<'py>( + &self, + py: Python<'py>, + input: RetrieveFilters, + ) -> PyResult> { + let service = self.api_service.clone(); + let filters: Vec = match input { + RetrieveFilters::One(f) => vec![f.into()], + RetrieveFilters::Many(fs) => fs.into_iter().map(Into::into).collect(), + }; + let wrapper = DataWrapper::::from_vec(filters); + let series = py.detach(|| { + self.runtime + .block_on(service.time_series.retrieve_datapoints_binary(&wrapper)) + .map_err(crate::datahub_err) + })?; + binary_read::to_py(series) + } fn delete_datapoints<'py>(&self, py: Python<'py>, input: Vec) -> PyResult<()> { let service = self.api_service.clone(); let wrapper = diff --git a/python_tests/requirements.txt b/python_tests/requirements.txt index c73f18a..5d91ece 100644 --- a/python_tests/requirements.txt +++ b/python_tests/requirements.txt @@ -14,3 +14,5 @@ pytest-asyncio numpy pandas polars +# pandas converts Arrow through pyarrow (BinaryDatapoints.to_pandas). +pyarrow diff --git a/python_tests/test_timeseries_binary_read.py b/python_tests/test_timeseries_binary_read.py new file mode 100644 index 0000000..8e9c03d --- /dev/null +++ b/python_tests/test_timeseries_binary_read.py @@ -0,0 +1,212 @@ +"""Binary datapoint read (`POST /timeseries/data/list/binary`) through the Python bindings. + +Two halves. The offline one answers the client from a stand-in server with response bodies the +platform's own Java frame writer produced (`src/timeseries/fixtures/`), so it checks the binding +and the Arrow PyCapsule stream without a backend. The live one round-trips through a backend and +skips when that backend does not serve the endpoint yet. +""" +import datetime +import decimal +import http.server +import pathlib +import threading + +import pytest + +import intellistream_datahub_sdk +from intellistream_datahub_sdk import DataHubException +from polling import poll_until +from python_tests.fixtures import * # noqa: F401,F403 + +FIXTURES = pathlib.Path(__file__).resolve().parent.parent / "src" / "timeseries" / "fixtures" +T0 = datetime.datetime.fromtimestamp(1_700_000_000, tz=datetime.timezone.utc) +MEDIA_TYPE = "application/vnd.intellistream.datapoint-block" + + +class _StandIn(http.server.BaseHTTPRequestHandler): + status = 200 + body = b"" + content_type = MEDIA_TYPE + seen = [] + + def do_POST(self): + length = int(self.headers.get("Content-Length", 0)) + type(self).seen.append((self.path, self.headers.get("Accept"), self.rfile.read(length))) + self.send_response(type(self).status) + self.send_header("Content-Type", type(self).content_type) + self.send_header("Content-Length", str(len(type(self).body))) + self.end_headers() + self.wfile.write(type(self).body) + + def log_message(self, *args): + pass + + +@pytest.fixture +def stand_in(): + _StandIn.status, _StandIn.content_type, _StandIn.seen = 200, MEDIA_TYPE, [] + server = http.server.ThreadingHTTPServer(("127.0.0.1", 0), _StandIn) + threading.Thread(target=server.serve_forever, daemon=True).start() + client = intellistream_datahub_sdk.DataHubClient( + base_url=f"http://127.0.0.1:{server.server_address[1]}", token="test-token" + ) + yield client, _StandIn + server.shutdown() + + +def _filter(ts="rd_float"): + return intellistream_datahub_sdk.RetrieveFilter(ts=ts, start=T0, limit=1000) + + +def test_every_value_type_converts_through_the_arrow_stream(stand_in): + pl = pytest.importorskip("polars") + client, server = stand_in + server.body = (FIXTURES / "read_values.bin").read_bytes() + + series = client.timeseries.retrieve_datapoints_binary([_filter("rd_bigint"), _filter("rd_float")]) + + path, accept, _ = server.seen[0] + assert path == "/timeseries/data/list/binary" + assert accept.startswith(MEDIA_TYPE) and "application/problem+json" in accept + assert [s.external_id for s in series] == [ + "rd_bigint", "rd_float", "rd_float32", "rd_numeric", "rd_decimal32", "rd_text", "rd_mixed", "rd_mixed_numbers", + ] + by_name = {s.external_id: s for s in series} + + bigint = by_name["rd_bigint"] + assert (bigint.id, bigint.value_type, len(bigint), bigint.is_aggregates) == (11, "bigint", 3, False) + assert bigint.columns == ["timestamp", "value"] + frame = pl.DataFrame(bigint) + assert frame["value"].to_list() == [1, 42, -9_000_000_000] + assert frame["timestamp"].dtype == pl.Datetime("ms", "UTC") + assert frame["timestamp"][0] == T0 + # A stream is made per call, so one object converts more than once. + assert bigint.to_polars().equals(frame) + + assert pl.DataFrame(by_name["rd_float"])["value"].to_list() == [1.5, -2.25] + assert pl.DataFrame(by_name["rd_float32"])["value"].dtype == pl.Float32 + numeric = pl.DataFrame(by_name["rd_numeric"])["value"] + assert numeric.dtype == pl.Decimal(18, 6) + assert numeric.to_list() == [decimal.Decimal("123456789012.345678"), decimal.Decimal("-0.000001")] + assert pl.DataFrame(by_name["rd_decimal32"])["value"].dtype == pl.Decimal(9, 4) + assert pl.DataFrame(by_name["rd_text"])["value"].to_list() == ["open", "blåbær"] + + mixed = by_name["rd_mixed"] + assert mixed.columns == ["timestamp", "value_numeric", "value_text"] + mixed_frame = pl.DataFrame(mixed) + assert mixed_frame["value_numeric"].to_list() == [3.5, None, -7.0] + assert mixed_frame["value_text"].to_list() == [None, "fault", None] + + +def test_aggregates_convert_to_pandas(stand_in): + pytest.importorskip("pyarrow") + pytest.importorskip("pandas") + client, server = stand_in + server.body = (FIXTURES / "read_aggregates.bin").read_bytes() + + (series,) = client.timeseries.retrieve_datapoints_binary(_filter("rd_aggregates")) + + assert series.is_aggregates and series.value_type == "float" + assert series.columns == ["timestamp", "avg", "min", "max", "sum"] + frame = series.to_pandas() + assert frame["avg"].tolist() == [1.5, 2.5, 3.5] + assert frame["sum"].tolist() == [6.0, 10.0, 14.0] + assert frame["timestamp"].iloc[1] == T0 + datetime.timedelta(hours=1) + + +def test_no_points_is_no_series(stand_in): + client, server = stand_in + server.body = b"DHDE" + assert client.timeseries.retrieve_datapoints_binary(_filter()) == [] + + +def test_a_response_cut_short_raises_rather_than_returning_fewer_points(stand_in): + client, server = stand_in + full = (FIXTURES / "read_values.bin").read_bytes() + server.body = full[:-4] # every frame intact, only the end marker missing + with pytest.raises(DataHubException, match="cut short") as excinfo: + client.timeseries.retrieve_datapoints_binary(_filter()) + assert excinfo.value.status_code == 502 + + +def test_a_refusal_is_the_problem_the_server_sent(stand_in): + client, server = stand_in + server.status, server.content_type = 400, "application/problem+json" + server.body = ( + b'{"type":"https://intellistream.ai/errors/bad-request","title":"Bad Request","status":400,' + b'"detail":"The binary read has no cursors; page by moving `start` past the last timestamp read."}' + ) + with pytest.raises(DataHubException, match="no cursors") as excinfo: + client.timeseries.retrieve_datapoints_binary(_filter()) + assert excinfo.value.status_code == 400 + + +# --- live --------------------------------------------------------------------------------------- + +START = datetime.datetime(2024, 3, 1, tzinfo=datetime.timezone.utc) +TIMESTAMPS = [START + datetime.timedelta(minutes=i) for i in range(120)] + + +@pytest.fixture +def binary_read(sync_client, make_ts): + """Skips unless the backend serves the binary read: it is newer than the binary insert.""" + probe = intellistream_datahub_sdk.RetrieveFilter(ts=make_ts(value_type="float"), start=START, limit=1) + try: + sync_client.timeseries.retrieve_datapoints_binary(probe) + except DataHubException as e: + if e.status_code in (404, 405): + pytest.skip("the backend does not serve POST /timeseries/data/list/binary") + raise + return sync_client.timeseries.retrieve_datapoints_binary + + +def _wait_for(client, ts, expected): + rf = intellistream_datahub_sdk.RetrieveFilter(ts=ts, start=START, limit=10_000) + poll_until( + lambda: client.timeseries.retrieve_datapoints(rf), + lambda collections: bool(collections) and len(collections[0].get_datapoints()) >= expected, + ) + + +def test_binary_read_matches_the_json_read(sync_client, make_ts, binary_read): + pl = pytest.importorskip("polars") + ts = make_ts(value_type="float") + values = [i * 0.5 - 30.0 for i in range(len(TIMESTAMPS))] + sync_client.timeseries.insert_from_lists_binary(TIMESTAMPS, values, ts) + _wait_for(sync_client, ts, len(TIMESTAMPS)) + + (series,) = binary_read(intellistream_datahub_sdk.RetrieveFilter(ts=ts, start=START, limit=10_000)) + + assert series.external_id == ts.external_id and series.value_type == "float" + frame = series.to_polars() + assert frame["timestamp"].to_list() == TIMESTAMPS + assert frame["value"].to_list() == values + + +def test_binary_read_of_hourly_aggregates(sync_client, make_ts, binary_read): + pytest.importorskip("pyarrow") + ts = make_ts(value_type="float") + sync_client.timeseries.insert_from_lists_binary(TIMESTAMPS, [float(i) for i in range(len(TIMESTAMPS))], ts) + _wait_for(sync_client, ts, len(TIMESTAMPS)) + + rf = intellistream_datahub_sdk.RetrieveFilter( + ts=ts, start=START, end=TIMESTAMPS[-1] + datetime.timedelta(minutes=1), + aggregates=["avg", "min", "max", "sum"], granularity="1h", + ) + (series,) = binary_read(rf) + json_buckets = sync_client.timeseries.retrieve_datapoints(rf)[0].get_datapoints() + + frame = series.to_pandas() + assert series.is_aggregates and len(frame) == len(json_buckets) == 2 + # Zero-order hold with carry-in, so the second hour's min is the 59 carried into it. + assert frame["min"].tolist() == [dp.min for dp in json_buckets] + assert frame["max"].tolist() == [dp.max for dp in json_buckets] + assert frame["sum"].tolist() == pytest.approx([dp.sum for dp in json_buckets]) + assert frame["avg"].tolist() == pytest.approx([dp.average for dp in json_buckets]) + + +def test_binary_read_of_an_unknown_series_is_a_bad_request(make_ts, binary_read): + missing = unique_id("ts_missing") + with pytest.raises(DataHubException, match=missing) as excinfo: + binary_read(intellistream_datahub_sdk.RetrieveFilter(ts=missing, start=START)) + assert excinfo.value.status_code == 400 diff --git a/src/blocking.rs b/src/blocking.rs index 1dd827a..057baf7 100644 --- a/src/blocking.rs +++ b/src/blocking.rs @@ -49,7 +49,7 @@ use crate::resources::{ ResourceNetwork, ResourceUpdate, }; use crate::timeseries::{ - BinaryIngestOptions, TimeSeries, TimeSeriesFilter, TimeSeriesUpdateCollection, + BinaryDatapoints, BinaryIngestOptions, TimeSeries, TimeSeriesFilter, TimeSeriesUpdateCollection, ValueTypeRecommendation, }; use crate::unit::Unit; @@ -182,6 +182,7 @@ impl TimeSeriesService { fn insert_datapoints(json: &mut DataWrapper>) -> Result, ResponseError>; fn insert_datapoints_binary(json: &DataWrapper>, options: &BinaryIngestOptions) -> Result, ResponseError>; fn retrieve_datapoints(json: &DataWrapper) -> Result>, ResponseError>; + fn retrieve_datapoints_binary(json: &DataWrapper) -> Result, ResponseError>; fn delete_datapoints(json: &DataWrapper) -> Result, ResponseError>; fn retrieve_latest_datapoint(json: &DataWrapper) -> Result>, ResponseError>; } diff --git a/src/generic.rs b/src/generic.rs index 7faad32..0220427 100644 --- a/src/generic.rs +++ b/src/generic.rs @@ -882,6 +882,57 @@ pub trait ApiServiceProvider { } } + /// `POST` a JSON body to an endpoint that answers with bytes of type `accept` rather than JSON + /// (currently only `/timeseries/data/list/binary`), and return the whole body. + /// + /// `application/problem+json` is offered beside `accept` so a refusal reaches the caller as the + /// problem document it is, rather than as Spring's 406 for a type the handler does not produce. + /// A body that breaks off while it is being read is a 502: the server answered 200 and then + /// failed to finish, which is not the caller's fault and is safe to send again. + async fn execute_post_request_for_bytes( + &self, + path: &str, + json: &J, + accept: &str, + ) -> Result, ResponseError> { + let token = self.get_token().await?; + let response = self + .get_api_service() + .http_client + .post(path) + .json(json) + .header(http::header::ACCEPT, format!("{accept}, application/problem+json")) + .bearer_auth(token.clone()) + .send() + .await + .map_err(ResponseError::from_err)?; + let status = response.status(); + if !status.is_success() { + let content_type = response + .headers() + .get(reqwest::header::CONTENT_TYPE) + .and_then(|value| value.to_str().ok()) + .map(str::to_string); + let error = ResponseError { + status, + message: response + .text() + .await + .unwrap_or_else(|_| "Failed to read response body".to_string()), + content_type, + }; + return Err(self.on_request_error(error, &token).await); + } + match response.bytes().await { + Ok(body) => Ok(body.into()), + Err(error) => Err(ResponseError { + status: http::StatusCode::BAD_GATEWAY, + message: format!("the response was cut short while it was being read: {error}"), + content_type: None, + }), + } + } + /// `GET` an endpoint that answers with bytes rather than JSON (currently only /// `/files/download/{id}`). /// diff --git a/src/timeseries/binary.rs b/src/timeseries/binary.rs index 45cbfac..f156c22 100644 --- a/src/timeseries/binary.rs +++ b/src/timeseries/binary.rs @@ -86,6 +86,20 @@ impl DatapointValueType { self as u8 } + /// The value type a frame envelope numbers `id`. + pub fn from_id(id: u8) -> Option { + Some(match id { + 1 => Self::Bigint, + 2 => Self::Float, + 3 => Self::Numeric, + 4 => Self::Text, + 5 => Self::Decimal32, + 6 => Self::Mixed, + 7 => Self::Float32, + _ => return None, + }) + } + pub fn carries_text(self) -> bool { matches!(self, Self::Text | Self::Mixed) } diff --git a/src/timeseries/binary_read.rs b/src/timeseries/binary_read.rs new file mode 100644 index 0000000..c57a76a --- /dev/null +++ b/src/timeseries/binary_read.rs @@ -0,0 +1,836 @@ +//! The binary datapoint read behind `POST /timeseries/data/list/binary`. +//! +//! The request is the JSON retrieve body; the answer is the frames `POST /timeseries/data/binary` +//! takes, one series per frame, concatenated, and ended by the 4-byte marker `DHDE`. A body without +//! the marker was cut short, whether inside a frame or between two, and is an error rather than a +//! shorter answer. Codec 2 frames carry bucketed aggregates instead of values. +//! +//! [`parse_read_response`] turns a body into one [`BinaryDatapoints`] per series, columnar, with +//! the frames of one series merged. [`TimeSeriesService::retrieve_datapoints_binary`] posts the +//! request and parses the answer. + +use std::io::Cursor; +use std::sync::atomic::{AtomicUsize, Ordering}; +use std::sync::Arc; + +use arrow_array::{ + Array, Decimal128Array, Float32Array, Float64Array, Int64Array, RecordBatch, StringArray, + TimestampMillisecondArray, +}; +use arrow_ipc::reader::StreamReader; +use arrow_schema::{DataType, Field, Schema, SchemaRef, TimeUnit}; +use oauth2::http::StatusCode; + +use crate::generic::{ApiServiceProvider, DataWrapper, RetrieveFilter}; +use crate::http::ResponseError; +use crate::timeseries::binary::{ + DatapointValueType, CODEC_ARROW_IPC, COMPRESSION_ZSTD, HEADER_BYTES, MAGIC, MAX_FRAME_RAW_BYTES, + MEDIA_TYPE, VERSION, +}; +use crate::timeseries::TimeSeriesService; + +/// Ends a binary read's response, after the last frame. +pub const END_OF_RESPONSE: &[u8; 4] = b"DHDE"; +/// A response frame whose payload holds bucketed aggregates in [`aggregates_schema`]. +pub const CODEC_ARROW_IPC_AGGREGATES: u8 = 2; +/// Buckets per aggregate frame, whatever the series' type. +pub const MAX_AGGREGATE_ROWS: usize = 50_000; + +/// The schema of an aggregate frame: the bucket start and four Float64s, for every value type. +pub fn aggregates_schema() -> SchemaRef { + let float = |name: &str| Field::new(name, DataType::Float64, false); + Arc::new(Schema::new(vec![ + Field::new("timeseries_id", DataType::Int64, false), + Field::new( + "timestamp", + DataType::Timestamp(TimeUnit::Millisecond, Some("UTC".into())), + false, + ), + float("avg"), + float("min"), + float("max"), + float("sum"), + ])) +} + +/// One series' rows from a binary read, a column per field rather than an object per point. +#[derive(Debug, Clone, PartialEq)] +pub struct BinaryDatapoints { + pub id: u64, + pub external_id: String, + pub value_type: DatapointValueType, + /// Epoch milliseconds, oldest first; for aggregates, the start of each bucket. + pub timestamps: Vec, + pub values: BinaryValues, +} + +impl BinaryDatapoints { + pub fn len(&self) -> usize { + self.timestamps.len() + } + + pub fn is_empty(&self) -> bool { + self.timestamps.is_empty() + } + + pub fn is_aggregates(&self) -> bool { + matches!(self.values, BinaryValues::Aggregates { .. }) + } +} + +/// The value column of a [`BinaryDatapoints`], typed by the series' value type. +#[derive(Debug, Clone, PartialEq)] +#[non_exhaustive] +pub enum BinaryValues { + Bigint(Vec), + Float(Vec), + Float32(Vec), + /// `numeric` and `decimal32`, exact: each value is `unscaled / 10^scale`. + Decimal { unscaled: Vec, scale: i8 }, + Text(Vec), + /// Each row sets exactly one side. + Mixed { + numeric: Vec>, + text: Vec>, + }, + /// Time-weighted `avg`, `min`, `max` and `sum` per bucket. All four come back whichever the + /// request named. + Aggregates { + avg: Vec, + min: Vec, + max: Vec, + sum: Vec, + }, +} + +impl BinaryValues { + fn reserve_exact(&mut self, additional: usize) { + match self { + Self::Bigint(v) => v.reserve_exact(additional), + Self::Float(v) => v.reserve_exact(additional), + Self::Float32(v) => v.reserve_exact(additional), + Self::Decimal { unscaled, .. } => unscaled.reserve_exact(additional), + Self::Text(v) => v.reserve_exact(additional), + Self::Mixed { numeric, text } => { + numeric.reserve_exact(additional); + text.reserve_exact(additional); + } + Self::Aggregates { avg, min, max, sum } => { + for v in [avg, min, max, sum] { + v.reserve_exact(additional); + } + } + } + } + + /// Appends `other`, which must be the same variant; false if it is not. + fn append(&mut self, other: BinaryValues) -> bool { + match (self, other) { + (Self::Bigint(a), Self::Bigint(b)) => a.extend(b), + (Self::Float(a), Self::Float(b)) => a.extend(b), + (Self::Float32(a), Self::Float32(b)) => a.extend(b), + ( + Self::Decimal { unscaled: a, scale: sa }, + Self::Decimal { unscaled: b, scale: sb }, + ) if *sa == sb => a.extend(b), + (Self::Text(a), Self::Text(b)) => a.extend(b), + ( + Self::Mixed { numeric: an, text: at }, + Self::Mixed { numeric: bn, text: bt }, + ) => { + an.extend(bn); + at.extend(bt); + } + ( + Self::Aggregates { avg: aa, min: ai, max: ax, sum: asum }, + Self::Aggregates { avg: ba, min: bi, max: bx, sum: bsum }, + ) => { + aa.extend(ba); + ai.extend(bi); + ax.extend(bx); + asum.extend(bsum); + } + _ => return false, + } + true + } +} + +fn u32_at(body: &[u8], offset: usize) -> usize { + u32::from_le_bytes(body[offset..offset + 4].try_into().unwrap()) as usize +} + +fn read_varint(body: &[u8], pos: &mut usize) -> Option { + let mut value = 0usize; + for shift in (0..35).step_by(7) { + let byte = *body.get(*pos)?; + *pos += 1; + value |= ((byte & 0x7F) as usize) << shift; + if byte & 0x80 == 0 { + return Some(value); + } + } + None +} + +/// Parses a binary read's whole response body: one entry per series in the order the frames came, +/// which is the order of the request's items, with consecutive frames of one series merged. +/// +/// The envelopes are walked first, which is cheap and finds a body that was cut short; the frames +/// are then decompressed and decoded on several threads, since zstd is most of the cost. +/// +/// Errors on anything short of a complete, well-formed answer: a missing end marker, bytes after +/// it, a frame that breaks the envelope, or a payload outside its canon. +pub fn parse_read_response(body: &[u8]) -> Result, String> { + let frames = scan_frames(body)?; + let decoded = decode_frames(&frames)?; + // Grouped before merging so each series is allocated once at its final size: extending part + // by part regrows and recopies a long series dozens of times. + let mut groups: Vec> = Vec::new(); + for (index, parts) in decoded.into_iter().enumerate() { + for part in parts { + let continues = groups.last().and_then(|g| g.last()).is_some_and(|(_, last)| { + last.id == part.id + && last.is_aggregates() == part.is_aggregates() + && last.timestamps.last() < part.timestamps.first() + }); + if continues { + groups.last_mut().unwrap().push((index, part)); + } else { + groups.push(vec![(index, part)]); + } + } + } + groups.into_iter().map(merge).collect() +} + +/// One series from its consecutive parts, in order. +fn merge(parts: Vec<(usize, BinaryDatapoints)>) -> Result { + let rows: usize = parts.iter().map(|(_, p)| p.len()).sum(); + let mut parts = parts.into_iter(); + let (_, mut series) = parts.next().expect("a group has at least one part"); + series.timestamps.reserve_exact(rows - series.len()); + series.values.reserve_exact(rows - series.len()); + for (index, part) in parts { + let BinaryDatapoints { timestamps, values, .. } = part; + series.timestamps.extend(timestamps); + if !series.values.append(values) { + return Err(format!( + "frame {index}: series {} changes shape between frames", + series.id + )); + } + } + Ok(series) +} + +fn cut_short(index: usize) -> String { + format!("the response ends without its end marker, at frame {index}; it was cut short") +} + +/// One frame's envelope, checked, with its payload still compressed. +struct FrameView<'a> { + index: usize, + value_type: DatapointValueType, + aggregates: bool, + rows: usize, + names: Vec<(i64, String)>, + payload: &'a [u8], + raw_len: usize, +} + +/// Every frame up to the end marker, in order. +fn scan_frames(body: &[u8]) -> Result>, String> { + let mut frames = Vec::new(); + let mut pos = 0; + loop { + let rest = &body[pos..]; + if rest.starts_with(END_OF_RESPONSE) { + if rest.len() != END_OF_RESPONSE.len() { + return Err(format!( + "{} bytes after the end marker", + rest.len() - END_OF_RESPONSE.len() + )); + } + return Ok(frames); + } + if rest.len() < HEADER_BYTES { + return Err(cut_short(frames.len())); + } + let (consumed, frame) = scan_frame(rest, frames.len())?; + frames.push(frame); + pos += consumed; + } +} + +/// One frame's envelope and directory: how many bytes it takes, and the view of it. +fn scan_frame(body: &[u8], index: usize) -> Result<(usize, FrameView<'_>), String> { + let fail = |message: String| format!("frame {index}: {message}"); + if &body[0..4] != MAGIC { + return Err(fail("does not start with DHDP".to_string())); + } + if body[4] != VERSION { + return Err(fail(format!("version {}", body[4]))); + } + let value_type = DatapointValueType::from_id(body[5]) + .ok_or_else(|| fail(format!("value type id {}", body[5])))?; + let aggregates = match body[6] { + CODEC_ARROW_IPC => false, + CODEC_ARROW_IPC_AGGREGATES => true, + codec => return Err(fail(format!("codec {codec}"))), + }; + if body[7] != COMPRESSION_ZSTD { + return Err(fail(format!("compression {}", body[7]))); + } + let rows = u32_at(body, 8); + let series_count = u32_at(body, 12); + let directory_len = u32_at(body, 16); + let payload_len = u32_at(body, 20); + let raw_len = u32_at(body, 24); + let max_rows = if aggregates { + MAX_AGGREGATE_ROWS + } else { + value_type.max_rows() + }; + if rows < 1 || rows > max_rows { + return Err(fail(format!("{rows} rows, allowed 1 to {max_rows}"))); + } + if raw_len > MAX_FRAME_RAW_BYTES { + return Err(fail(format!( + "payload is {raw_len} bytes raw, the cap is {MAX_FRAME_RAW_BYTES}" + ))); + } + let total = HEADER_BYTES + directory_len + payload_len; + if body.len() < total { + return Err(cut_short(index)); + } + + let directory = &body[HEADER_BYTES..HEADER_BYTES + directory_len]; + let mut names: Vec<(i64, String)> = Vec::with_capacity(series_count); + let mut at = 0; + for _ in 0..series_count { + let id = directory + .get(at..at + 8) + .map(|b| i64::from_le_bytes(b.try_into().unwrap())) + .ok_or_else(|| fail("directory is shorter than its series count".to_string()))?; + at += 8; + let len = read_varint(directory, &mut at) + .ok_or_else(|| fail("directory is shorter than its series count".to_string()))?; + let name = directory + .get(at..at + len) + .ok_or_else(|| fail("directory is shorter than its series count".to_string()))?; + at += len; + let name = String::from_utf8(name.to_vec()) + .map_err(|_| fail(format!("external id of series {id} is not UTF-8")))?; + names.push((id, name)); + } + if at != directory_len { + return Err(fail(format!("{} bytes after the directory", directory_len - at))); + } + let frame = FrameView { + index, + value_type, + aggregates, + rows, + names, + payload: &body[HEADER_BYTES + directory_len..total], + raw_len, + }; + Ok((total, frame)) +} + +/// Decodes every frame, on as many threads as there are cores and frames, keeping their order. +/// On failure, the error of the earliest frame that failed. +fn decode_frames(frames: &[FrameView<'_>]) -> Result>, String> { + let threads = std::thread::available_parallelism() + .map_or(1, |n| n.get()) + .min(frames.len()); + if threads <= 1 { + return frames.iter().map(decode_frame).collect(); + } + let next = AtomicUsize::new(0); + let mut decoded: Vec<(usize, Result, String>)> = std::thread::scope(|scope| { + let workers: Vec<_> = (0..threads) + .map(|_| { + scope.spawn(|| { + let mut done = Vec::new(); + loop { + let i = next.fetch_add(1, Ordering::Relaxed); + let Some(frame) = frames.get(i) else { return done }; + done.push((i, decode_frame(frame))); + } + }) + }) + .collect(); + workers + .into_iter() + .flat_map(|w| w.join().expect("a frame decoder panicked")) + .collect() + }); + decoded.sort_unstable_by_key(|(i, _)| *i); + decoded.into_iter().map(|(_, result)| result).collect() +} + +/// One frame's rows, split into one part per series it names. +fn decode_frame(frame: &FrameView<'_>) -> Result, String> { + let fail = |message: String| format!("frame {}: {message}", frame.index); + let raw = zstd::bulk::decompress(frame.payload, frame.raw_len) + .map_err(|e| fail(format!("zstd: {e}")))?; + if raw.len() != frame.raw_len { + return Err(fail(format!( + "payload decompressed to {} bytes, the envelope says {}", + raw.len(), + frame.raw_len + ))); + } + let reader = + StreamReader::try_new(Cursor::new(raw), None).map_err(|e| fail(format!("arrow: {e}")))?; + let expected = if frame.aggregates { + aggregates_schema() + } else { + frame.value_type.schema() + }; + let schema = reader.schema(); + let same_columns = schema.fields().len() == expected.fields().len() + && schema + .fields() + .iter() + .zip(expected.fields()) + .all(|(got, want)| got.name() == want.name() && got.data_type() == want.data_type()); + if !same_columns { + return Err(fail(format!("schema {schema:?} is not the canon {expected:?}"))); + } + + let mut parts: Vec = Vec::new(); + let mut read = 0; + for batch in reader { + let batch = batch.map_err(|e| fail(format!("arrow: {e}")))?; + read += batch.num_rows(); + split_batch(&batch, frame.value_type, frame.aggregates, &frame.names, &mut parts) + .map_err(fail)?; + } + if read != frame.rows { + return Err(fail(format!( + "{read} rows in the payload, the envelope says {}", + frame.rows + ))); + } + Ok(parts) +} + +/// Appends a batch's rows to `parts`, one part per run of a series id. +fn split_batch( + batch: &RecordBatch, + value_type: DatapointValueType, + aggregates: bool, + names: &[(i64, String)], + parts: &mut Vec, +) -> Result<(), String> { + let ids = column::(batch, 0)?; + let timestamps = column::(batch, 1)?; + let values = batch_values(batch, value_type, aggregates)?; + let mut start = 0; + while start < batch.num_rows() { + let id = ids.value(start); + let mut end = start + 1; + while end < batch.num_rows() && ids.value(end) == id { + end += 1; + } + let external_id = names + .iter() + .find(|(named, _)| *named == id) + .map(|(_, name)| name.clone()) + .ok_or_else(|| format!("series {id} is not in the directory"))?; + let part = BinaryDatapoints { + id: id as u64, + external_id, + value_type, + timestamps: timestamps.values()[start..end].to_vec(), + values: slice_values(&values, start, end), + }; + match parts.last_mut() { + Some(last) if last.id == part.id => { + last.timestamps.extend(part.timestamps); + last.values.append(part.values); + } + _ => parts.push(part), + } + start = end; + } + Ok(()) +} + +fn column(batch: &RecordBatch, index: usize) -> Result<&T, String> { + batch + .column(index) + .as_any() + .downcast_ref::() + .ok_or_else(|| format!("column {index} is not of its canonical type")) +} + +fn batch_values( + batch: &RecordBatch, + value_type: DatapointValueType, + aggregates: bool, +) -> Result { + let floats = |index| column::(batch, index).map(|a| a.values().to_vec()); + if aggregates { + return Ok(BinaryValues::Aggregates { + avg: floats(2)?, + min: floats(3)?, + max: floats(4)?, + sum: floats(5)?, + }); + } + Ok(match value_type { + DatapointValueType::Bigint => { + BinaryValues::Bigint(column::(batch, 2)?.values().to_vec()) + } + DatapointValueType::Float => BinaryValues::Float(floats(2)?), + DatapointValueType::Float32 => { + BinaryValues::Float32(column::(batch, 2)?.values().to_vec()) + } + DatapointValueType::Numeric | DatapointValueType::Decimal32 => { + let array = column::(batch, 2)?; + BinaryValues::Decimal { + unscaled: array.values().to_vec(), + scale: array.scale(), + } + } + DatapointValueType::Text => { + let array = column::(batch, 2)?; + BinaryValues::Text(array.iter().map(|v| v.unwrap_or_default().to_string()).collect()) + } + DatapointValueType::Mixed => { + let numeric = column::(batch, 2)?; + let text = column::(batch, 3)?; + BinaryValues::Mixed { + numeric: numeric.iter().collect(), + text: text.iter().map(|v| v.map(str::to_string)).collect(), + } + } + }) +} + +fn slice_values(values: &BinaryValues, start: usize, end: usize) -> BinaryValues { + match values { + BinaryValues::Bigint(v) => BinaryValues::Bigint(v[start..end].to_vec()), + BinaryValues::Float(v) => BinaryValues::Float(v[start..end].to_vec()), + BinaryValues::Float32(v) => BinaryValues::Float32(v[start..end].to_vec()), + BinaryValues::Decimal { unscaled, scale } => BinaryValues::Decimal { + unscaled: unscaled[start..end].to_vec(), + scale: *scale, + }, + BinaryValues::Text(v) => BinaryValues::Text(v[start..end].to_vec()), + BinaryValues::Mixed { numeric, text } => BinaryValues::Mixed { + numeric: numeric[start..end].to_vec(), + text: text[start..end].to_vec(), + }, + BinaryValues::Aggregates { avg, min, max, sum } => BinaryValues::Aggregates { + avg: avg[start..end].to_vec(), + min: min[start..end].to_vec(), + max: max[start..end].to_vec(), + sum: sum[start..end].to_vec(), + }, + } +} + +impl TimeSeriesService { + /// Reads datapoints through `POST /timeseries/data/list/binary`, the high-throughput twin of + /// [`retrieve_datapoints`](Self::retrieve_datapoints): the same request, answered as + /// zstd-compressed Arrow frames instead of JSON, and returned as one columnar + /// [`BinaryDatapoints`] per series, in the order of the request's items. + /// + /// Each item needs a `start` or an `end`. With a `limit` (at most 100 000) it returns that many + /// of the oldest points from there; without one it returns **every** point in its window, + /// however many, so bound a long series with `end`: the whole answer is held in memory. A + /// series with no points in its window is left out of the answer rather than returned empty. + /// There is no cursor (sending one is a 400): to read on, send the request again with `start` + /// one millisecond past the last timestamp received. An item with `aggregates` and a + /// `granularity` comes back as [`BinaryValues::Aggregates`], carrying all four whichever were + /// named; text series have none. + /// + /// Unlike the JSON read, a series that does not exist or sits in a dataset the caller cannot + /// read is a 400 for the whole request, not an empty result. A response the server could not + /// finish is a 502 whose message says it was cut short, never fewer points; sending the + /// request again is safe. + pub async fn retrieve_datapoints_binary( + &self, + json: &DataWrapper, + ) -> Result, ResponseError> { + let path = format!("{}/data/list/binary", self.base_url); + let body = self + .execute_post_request_for_bytes(&path, json, MEDIA_TYPE) + .await?; + let parsed = tokio::task::spawn_blocking(move || parse_read_response(&body)) + .await + .map_err(|e| ResponseError { + status: StatusCode::INTERNAL_SERVER_ERROR, + message: format!("decoding the binary read failed: {e}"), + content_type: None, + })?; + parsed.map_err(|message| ResponseError { + status: StatusCode::BAD_GATEWAY, + message: format!("POST {path} returned an unreadable answer: {message}"), + content_type: None, + }) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::timeseries::binary::FrameWriter; + use arrow_ipc::writer::StreamWriter; + + /// Written by the platform's own frame writer (`DatapointFrameWriter.wrap`, the code that + /// answers a binary read), from `feat/binary-datapoint-read`: one series per value type, then + /// the end marker. What the Rust reader must accept from the Java writer. + const JAVA_VALUES: &[u8] = include_bytes!("fixtures/read_values.bin"); + /// One codec-2 frame of three hourly buckets for series 21, `rd_aggregates`. + const JAVA_AGGREGATES: &[u8] = include_bytes!("fixtures/read_aggregates.bin"); + + const T0: i64 = 1_700_000_000_000; + + fn response(frames: &[Vec]) -> Vec { + let mut body: Vec = frames.concat(); + body.extend_from_slice(END_OF_RESPONSE); + body + } + + fn float_frame(id: u64, external_id: &str, timestamps: std::ops::Range) -> Vec { + let mut writer = FrameWriter::new(DatapointValueType::Float); + writer.series(id, external_id).unwrap(); + for t in timestamps { + writer.add_float(id, t, t as f64 / 2.0).unwrap(); + } + writer.build(1).unwrap().bytes + } + + /// A frame around any batch, with any codec byte: how a test makes a codec-2 frame. + fn frame_around(value_type: DatapointValueType, codec: u8, id: i64, name: &str, batch: &RecordBatch) -> Vec { + let mut raw = Vec::new(); + let mut writer = StreamWriter::try_new(&mut raw, batch.schema().as_ref()).unwrap(); + writer.write(batch).unwrap(); + writer.finish().unwrap(); + drop(writer); + let compressed = zstd::bulk::compress(&raw, 1).unwrap(); + let mut directory = id.to_le_bytes().to_vec(); + directory.push(name.len() as u8); + directory.extend_from_slice(name.as_bytes()); + let mut frame = MAGIC.to_vec(); + frame.extend_from_slice(&[VERSION, value_type.id(), codec, COMPRESSION_ZSTD]); + for n in [batch.num_rows(), 1, directory.len(), compressed.len(), raw.len()] { + frame.extend_from_slice(&(n as u32).to_le_bytes()); + } + frame.extend_from_slice(&directory); + frame.extend_from_slice(&compressed); + frame + } + + #[test] + fn reads_every_value_type_the_java_writer_answers_with() { + let series = parse_read_response(JAVA_VALUES).unwrap(); + let names: Vec<&str> = series.iter().map(|s| s.external_id.as_str()).collect(); + assert_eq!( + names, + ["rd_bigint", "rd_float", "rd_float32", "rd_numeric", "rd_decimal32", "rd_text", "rd_mixed", "rd_mixed_numbers"] + ); + + let bigint = &series[0]; + assert_eq!((bigint.id, bigint.value_type), (11, DatapointValueType::Bigint)); + assert_eq!(bigint.timestamps, [T0, T0 + 1000, T0 + 2000]); + assert_eq!(bigint.values, BinaryValues::Bigint(vec![1, 42, -9_000_000_000])); + + assert_eq!(series[1].values, BinaryValues::Float(vec![1.5, -2.25])); + assert_eq!(series[2].values, BinaryValues::Float32(vec![0.5])); + assert_eq!( + series[3].values, + BinaryValues::Decimal { unscaled: vec![123_456_789_012_345_678, -1], scale: 6 } + ); + assert_eq!( + series[4].values, + BinaryValues::Decimal { unscaled: vec![123_456_789], scale: 4 } + ); + assert_eq!( + series[5].values, + BinaryValues::Text(vec!["open".to_string(), "blåbær".to_string()]) + ); + assert_eq!( + series[6].values, + BinaryValues::Mixed { + numeric: vec![Some(3.5), None, Some(-7.0)], + text: vec![None, Some("fault".to_string()), None], + } + ); + // No text at all: the writer leaves the validity bitmap out of the numeric column. + assert_eq!( + series[7].values, + BinaryValues::Mixed { numeric: vec![Some(1.0), Some(2.0)], text: vec![None, None] } + ); + } + + #[test] + fn reads_an_aggregate_frame_the_java_writer_answers_with() { + let series = parse_read_response(JAVA_AGGREGATES).unwrap(); + assert_eq!(series.len(), 1); + let s = &series[0]; + assert_eq!((s.id, s.external_id.as_str()), (21, "rd_aggregates")); + assert_eq!(s.value_type, DatapointValueType::Float, "the series' own type, not the canon's"); + assert!(s.is_aggregates()); + assert_eq!(s.timestamps, [T0, T0 + 3_600_000, T0 + 7_200_000]); + assert_eq!( + s.values, + BinaryValues::Aggregates { + avg: vec![1.5, 2.5, 3.5], + min: vec![1.0, 2.0, 3.0], + max: vec![2.0, 3.0, 4.0], + sum: vec![6.0, 10.0, 14.0], + } + ); + } + + #[test] + fn consecutive_frames_of_one_series_merge_in_order() { + let body = response(&[ + float_frame(1, "a", 0..3), + float_frame(1, "a", 3..5), + float_frame(2, "b", 0..2), + ]); + let series = parse_read_response(&body).unwrap(); + assert_eq!(series.len(), 2); + assert_eq!(series[0].timestamps, [0, 1, 2, 3, 4]); + assert_eq!(series[0].values, BinaryValues::Float(vec![0.0, 0.5, 1.0, 1.5, 2.0])); + assert_eq!(series[1].external_id, "b"); + } + + /// Decoded on several threads, merged in the order the frames came, and a failure reported for + /// the earliest frame that has one, whichever thread reached it first. + #[test] + fn many_frames_decode_in_parallel_and_keep_their_order() { + let mut frames: Vec> = Vec::new(); + for id in 1..=64u64 { + let start = id as i64 * 1_000; + frames.push(float_frame(id, &format!("s{id}"), start..start + 3)); + frames.push(float_frame(id, &format!("s{id}"), start + 3..start + 5)); + } + let series = parse_read_response(&response(&frames)).unwrap(); + assert_eq!(series.len(), 64); + for (i, s) in series.iter().enumerate() { + let id = i as u64 + 1; + assert_eq!((s.id, s.external_id.as_str()), (id, format!("s{id}").as_str())); + let start = id as i64 * 1_000; + assert_eq!(s.timestamps, (start..start + 5).collect::>()); + } + + let batch = RecordBatch::try_new( + DatapointValueType::Float.schema(), + vec![ + Arc::new(Int64Array::from(vec![1])), + Arc::new(TimestampMillisecondArray::from(vec![0]).with_timezone("UTC")), + Arc::new(Float64Array::from(vec![1.0])), + ], + ) + .unwrap(); + let bad = frame_around(DatapointValueType::Float, CODEC_ARROW_IPC_AGGREGATES, 1, "a", &batch); + frames[100] = bad.clone(); + frames[17] = bad; + let error = parse_read_response(&response(&frames)).unwrap_err(); + assert!(error.starts_with("frame 17:"), "{error}"); + } + + /// The same series asked for twice, the second window starting before the first ended: two + /// entries, since merging them would put the timestamps out of order. + #[test] + fn a_series_whose_timestamps_restart_is_a_second_entry() { + let body = response(&[float_frame(1, "a", 0..3), float_frame(1, "a", 1..2)]); + let series = parse_read_response(&body).unwrap(); + assert_eq!(series.len(), 2); + assert_eq!(series[1].timestamps, [1]); + } + + #[test] + fn the_end_marker_alone_is_no_points() { + assert!(parse_read_response(END_OF_RESPONSE).unwrap().is_empty()); + } + + #[test] + fn a_response_cut_short_is_an_error_not_fewer_points() { + let first = float_frame(1, "a", 0..3); + let full = response(&[first.clone(), float_frame(2, "b", 0..3)]); + + // Between two frames: every byte that arrived parses, only the marker is missing. + let error = parse_read_response(&full[..first.len()]).unwrap_err(); + assert!(error.contains("cut short"), "{error}"); + // Inside a frame, in the header and in the payload. + for cut in [first.len() + 10, first.len() + HEADER_BYTES + 12, full.len() - 5] { + let error = parse_read_response(&full[..cut]).unwrap_err(); + assert!(error.contains("cut short"), "cut at {cut}: {error}"); + } + // Nothing at all. + assert!(parse_read_response(&[]).unwrap_err().contains("cut short")); + } + + #[test] + fn bytes_after_the_end_marker_are_refused() { + let mut body = response(&[float_frame(1, "a", 0..2)]); + body.push(0); + assert!(parse_read_response(&body).unwrap_err().contains("after the end marker")); + } + + #[test] + fn a_payload_outside_its_canon_is_refused() { + // Values in a codec-2 frame: right envelope, wrong schema. + let batch = RecordBatch::try_new( + DatapointValueType::Float.schema(), + vec![ + Arc::new(Int64Array::from(vec![1])), + Arc::new(TimestampMillisecondArray::from(vec![0]).with_timezone("UTC")), + Arc::new(Float64Array::from(vec![1.0])), + ], + ) + .unwrap(); + let body = response(&[frame_around(DatapointValueType::Float, CODEC_ARROW_IPC_AGGREGATES, 1, "a", &batch)]); + assert!(parse_read_response(&body).unwrap_err().contains("canon")); + + let body = response(&[frame_around(DatapointValueType::Float, 9, 1, "a", &batch)]); + assert!(parse_read_response(&body).unwrap_err().contains("codec 9")); + } + + #[test] + fn a_row_not_named_in_the_directory_is_refused() { + let batch = RecordBatch::try_new( + DatapointValueType::Float.schema(), + vec![ + Arc::new(Int64Array::from(vec![2])), + Arc::new(TimestampMillisecondArray::from(vec![0]).with_timezone("UTC")), + Arc::new(Float64Array::from(vec![1.0])), + ], + ) + .unwrap(); + let body = response(&[frame_around(DatapointValueType::Float, CODEC_ARROW_IPC, 1, "a", &batch)]); + assert!(parse_read_response(&body).unwrap_err().contains("not in the directory")); + } + + #[test] + fn the_aggregate_row_cap_is_its_own() { + let n = MAX_AGGREGATE_ROWS; + let floats = || Arc::new(Float64Array::from(vec![1.0; n])) as Arc; + let batch = RecordBatch::try_new( + aggregates_schema(), + vec![ + Arc::new(Int64Array::from(vec![1; n])), + Arc::new(TimestampMillisecondArray::from((0..n as i64).collect::>()).with_timezone("UTC")), + floats(), + floats(), + floats(), + floats(), + ], + ) + .unwrap(); + // Text's value cap is 10 000; an aggregate frame of a mixed series may still hold 50 000. + let body = response(&[frame_around(DatapointValueType::Mixed, CODEC_ARROW_IPC_AGGREGATES, 1, "a", &batch)]); + let series = parse_read_response(&body).unwrap(); + assert_eq!(series[0].len(), n); + } +} diff --git a/src/timeseries/fixtures/GenerateReadFixtures.java b/src/timeseries/fixtures/GenerateReadFixtures.java new file mode 100644 index 0000000..aab854a --- /dev/null +++ b/src/timeseries/fixtures/GenerateReadFixtures.java @@ -0,0 +1,115 @@ +// Regenerates read_values.bin and read_aggregates.bin with the platform's own frame writer, the +// code that answers POST /timeseries/data/list/binary. Run from a platform checkout that has built +// datahub-api-model, with arrow-format, flatbuffers-java, zero-allocation-hashing, zstd-jni and +// slf4j-api on the classpath: +// +// java -cp ":" \ +// GenerateReadFixtures.java src/timeseries/fixtures +// +// The Rust tests in binary_read.rs assert on the exact values written here. +import ai.intellistream.datahub.api.binary.*; +import ai.intellistream.datahub.api.binary.ArrowIpc.*; + +import java.io.ByteArrayOutputStream; +import java.math.BigDecimal; +import java.nio.ByteBuffer; +import java.nio.ByteOrder; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.ArrayList; +import java.util.List; + +/** Writes binary-read response bodies with the platform's own writer, for the Rust reader to parse. */ +public class GenerateReadFixtures { + static final long T0 = 1_700_000_000_000L; + static final PayloadCodec CODEC = new ZstdPayloadCodec(3); + + /** An ingest frame's batches, rewrapped the way the API answers a read. */ + static byte[] response(DatapointValueType type, long id, String ext, DatapointFrameWriter w) { + byte[] ingest = w.build(CODEC); + int dirLen = ByteBuffer.wrap(ingest).order(ByteOrder.LITTLE_ENDIAN).getInt(16); + int payLen = ByteBuffer.wrap(ingest).order(ByteOrder.LITTLE_ENDIAN).getInt(20); + int rawLen = ByteBuffer.wrap(ingest).order(ByteOrder.LITTLE_ENDIAN).getInt(24); + byte[] payload = java.util.Arrays.copyOfRange(ingest, 28 + dirLen, 28 + dirLen + payLen); + byte[] raw = CODEC.decompress(payload, rawLen); + List data = new ArrayList<>(); + for (Batch b : ArrowIpc.readStream(ArrowSchemaCanon.of(type), raw, 0, 100_000)) { + data.add(b.toData()); + } + return DatapointFrameWriter.wrap(type, false, id, ext, data, CODEC); + } + + static ByteBuffer longs(long... v) { + ByteBuffer b = ByteBuffer.allocate(v.length * 8).order(ByteOrder.LITTLE_ENDIAN); + for (long x : v) b.putLong(x); + return b.flip(); + } + + static ByteBuffer doubles(double... v) { + ByteBuffer b = ByteBuffer.allocate(v.length * 8).order(ByteOrder.LITTLE_ENDIAN); + for (double x : v) b.putDouble(x); + return b.flip(); + } + + public static void main(String[] args) throws Exception { + Path out = Path.of(args[0]); + ByteArrayOutputStream values = new ByteArrayOutputStream(); + + var bigint = DatapointFrameWriter.forType(DatapointValueType.BIGINT).series(11, "rd_bigint"); + bigint.addBigint(11, T0 + 2000, -9_000_000_000L); + bigint.addBigint(11, T0, 1); + bigint.addBigint(11, T0 + 1000, 42); + values.writeBytes(response(DatapointValueType.BIGINT, 11, "rd_bigint", bigint)); + + var flt = DatapointFrameWriter.forType(DatapointValueType.FLOAT).series(12, "rd_float"); + flt.addFloat(12, T0, 1.5); + flt.addFloat(12, T0 + 1000, -2.25); + values.writeBytes(response(DatapointValueType.FLOAT, 12, "rd_float", flt)); + + var f32 = DatapointFrameWriter.forType(DatapointValueType.FLOAT32).series(13, "rd_float32"); + f32.addFloat32(13, T0, 0.5f); + values.writeBytes(response(DatapointValueType.FLOAT32, 13, "rd_float32", f32)); + + var numeric = DatapointFrameWriter.forType(DatapointValueType.NUMERIC).series(14, "rd_numeric"); + numeric.addNumeric(14, T0, new BigDecimal("123456789012.345678")); + numeric.addNumeric(14, T0 + 1000, new BigDecimal("-0.000001")); + values.writeBytes(response(DatapointValueType.NUMERIC, 14, "rd_numeric", numeric)); + + var dec32 = DatapointFrameWriter.forType(DatapointValueType.DECIMAL32).series(15, "rd_decimal32"); + dec32.addDecimal32(15, T0, new BigDecimal("12345.6789")); + values.writeBytes(response(DatapointValueType.DECIMAL32, 15, "rd_decimal32", dec32)); + + var text = DatapointFrameWriter.forType(DatapointValueType.TEXT).series(16, "rd_text"); + text.addText(16, T0, "open"); + text.addText(16, T0 + 1000, "blåbær"); + values.writeBytes(response(DatapointValueType.TEXT, 16, "rd_text", text)); + + var mixed = DatapointFrameWriter.forType(DatapointValueType.MIXED).series(17, "rd_mixed"); + mixed.addMixed(17, T0, "3.5"); + mixed.addMixed(17, T0 + 1000, "fault"); + mixed.addMixed(17, T0 + 2000, "-7"); + values.writeBytes(response(DatapointValueType.MIXED, 17, "rd_mixed", mixed)); + + // Only numbers: the API leaves the validity bitmap out of a column with no nulls. + var mixedNumbers = DatapointFrameWriter.forType(DatapointValueType.MIXED).series(18, "rd_mixed_numbers"); + mixedNumbers.addMixed(18, T0, "1"); + mixedNumbers.addMixed(18, T0 + 1000, "2"); + values.writeBytes(response(DatapointValueType.MIXED, 18, "rd_mixed_numbers", mixedNumbers)); + + values.writeBytes(FrameLimits.END_OF_RESPONSE); + Files.write(out.resolve("read_values.bin"), values.toByteArray()); + + ByteArrayOutputStream aggregates = new ByteArrayOutputStream(); + BatchData buckets = new BatchData(3, new ColumnData[]{ + new ColumnData(null, 0, null, longs(21, 21, 21)), + new ColumnData(null, 0, null, longs(T0, T0 + 3_600_000, T0 + 7_200_000)), + new ColumnData(null, 0, null, doubles(1.5, 2.5, 3.5)), + new ColumnData(null, 0, null, doubles(1, 2, 3)), + new ColumnData(null, 0, null, doubles(2, 3, 4)), + new ColumnData(null, 0, null, doubles(6, 10, 14)), + }); + aggregates.writeBytes(DatapointFrameWriter.wrap(DatapointValueType.FLOAT, true, 21, "rd_aggregates", List.of(buckets), CODEC)); + aggregates.writeBytes(FrameLimits.END_OF_RESPONSE); + Files.write(out.resolve("read_aggregates.bin"), aggregates.toByteArray()); + } +} diff --git a/src/timeseries/fixtures/read_aggregates.bin b/src/timeseries/fixtures/read_aggregates.bin new file mode 100644 index 0000000000000000000000000000000000000000..adab85f38dd0ec815623a6129f14ca2bd4af309f GIT binary patch literal 401 zcmV;C0dD?8NJLNp0s;X8000010000M000160RRA~0{{RO000000000Da%5j&XJ>L{ zXJK?@b11bh{a~m9jS2ulPAxHTYtk6xSicJtN&Wq+D$#iCK3(;1O=W+dTYj{$~@XBizK5mzefr4Ndr7=0jb z-Mw*pegZ#s?e4CD4ps#zs(^oDQ%;NIh$1}^sf|@dxcLD#P*`A}7uOlWf`U#0jV0iR z=8K9P1;+~Of0khU?e+P0SHk=aKj-f}lYZ#ipXdK>%$-k$e!HV*oZa15s9{-TGhqsv z!ZHT%0-2d94-!BlF(eidC5nvg{~L_1HS3W}gWNCXeVGVOQy+$HMi{rY-}ZdOXdk|U zYYQ%8e(BG<8NGf~!V*^U8}cT1tFTZRN4qcTbYG>UBg3@xIrOA$)ioTkr*K}d#0}#O v>M*nJQ6L%4B_x9@Z*S~->O;vYS)2n}i%)xZ8LiK!|E>c5QGXhyL`XzMbOx+Yocf{mjgzRepu7A{z=F(q&_FUukkixtj_(xg=Q%$t6b_ zAxy{2y(FR1$(%WQNaa?XBZu_+PMt^Z$K(C@d>)@apO5GJ_5P4tNS**#FbD(!fUN?y z5_oZeLLf-+E-3_yej*?=IP~Nh(?+W=R2RTb79w;rvPcT&!imjh|TP_ZMNa>uSl|w1gd7KgVa=GGt zL+ZTf`T2QIyO64k6zDZN%;WJJmO<*PkScQH4u%lg$vt))!z`-4DDZV)ak^t01WWu* zLUT`P+5K5&!|jOMbc@Ee2@%{LBS=9Td?2LuT*~O6HlY6<%ZRx$8eW|RC zbIb{cL0l)qn`Pdqu@)GBB%ucd01(zDJ!SBOgY*dCgS-SCSh66! zpomknZ}13!NCq;DG&IY$cmlM%eb(rnTlj2tRc#rObZIn_xP6J@moWDpRyLIKW+OM^ z8@kF4hnbk>8mv7*2j8@}tch`tx~&vmaE439@W(~WYntZ6M*46ar?jK60qOKhC1gvr zbiCgdoDEG!Nt-Fw^fo-o=TL*W_weH^S7vg`H&&OkUDf%k+ zVHjyep3I}|Xq&uxi_Rb|0d#dR;_Cg^=pZyS*rcl>0O5nFps}~tDb*hyh%ay-my4>W z4zGV zBqM1Q*lS>8Kw=8uDJi-Ch}qybg~I?@GLZU57_%g|VJ#cX#hn~$n1%mT^UU^|vqigp z8y!N+*>HwQSt#-0{WaU){uiH1l6C;(Uutgy}j! znk_U*fuwTFxy85WQB6)SvNj}fb`(@_u3B?-hqLXvs&g8YKdjmmq zHXE?Ul%yzY--k&7wtZb_*NIG~O{RL^gEWn4;&cb@^!Z%Z!#K?9d9$+$A&c|-EIFgL zoH=H7md~4sdSc6`tW}itqn>DP+0vnq*%a9a7jTs^PwJo=sS^{G&ayfmay@8EHM8Y+ zq?fKbLpR-m{cuY{3JD^UOy-tNPTxqKjAk-ep@H8=Q$ZGD;AXfvnj=Xt*TSrW*Yn>U zEz5MV)E5Tp#bzi*Lxl7hG-6KM?|Pr;VLv#U;f64$GO^(#ZK-u;F_KW&;A`C1a_dla z9Ok;9sfB-z*qvQ!j*!DVD)J(slUqXYsujaWKRME93d$}zmBtvX>Sdj?j*m_CE+(+m zrF#i)-ddhpvozC2Fv~|WMIyK@h6mLG^}0r?;4CD6qVx>ho_)kC1dwmtpwMZzX&@lB zw?t_y9J1({pDJJP%3IiHQ8RfyVDVH~OOJWi)`G^i*CvU3BB&b8INg%wuDTa{d&V16 z6;}QluArX#{Kg0r2_k;IOc&1WWH2CMJtq9naHlxw3T+( z(AH6e7GG|BqLw$9;(^c36u)og^NFL>WCMh$V`J}r*L~im8c6SSEJ6^`MdvJXi9hj36Tz(~40%9ImW*maR1G}mBcHEz zWWL4U(VC&5(zGWbM~C-nUHA7Sx0U@x1ufhcwJ5|UEU6!wDS?1cpk|&REXiF`G9auo zjF@i;<_S<_AuObkxg`E?)Zl7PJLeg=c%F)#yFSp{^SLLm))gnMbtffkdBbJ4mihCM zJTUzWI%YhvJJk<#adMKxzCIxl4p($-p=Qh1Mh@)LplgrqoS)z6t!3-A%?9D%lvbaz zACW=D=4A!7dfO{j@dPD7Ioc$2v9To>Wa0S^?Yoeis4cNK7^ey{>OGQ}-bWttkpxNl zbEg2#xGV1=a0ap~JI+986pLhC3LAp1^;=pS4O|$2)nTlq6n%Fy`~Qj!h842@yXH)j+AokKwCpIG1`!-qB3-x6nX z(0*Tip65d@6gTNFb0U1EM;7g#LAhxyLHAaF$2qZ2zWybT(nJQnqHOjmg%QPO3nuo; zdbOH5x&_-*%Adl+hU7>?FwOp$PSN94_<94Cy^@%%PPtO}S#>MR&~s0kDur_8&AOq9 zC2sxK1e?h4^(}Z*xhs^~J?60i?APy`{fB;}D>NNtTpgpJ_eCT!UDrtLt!)==tQ=ym z6@K}JSeq{rxmTS&$Rg5I{Lb4ALAj;xl`(gh+-c-Nplo$?wV`Q|m$+DS%lgdn(Z7Qu z=1DyfH@sl)zqrRw&~%dLa%O<2?s2byKC7xvlDgDR%1elKj3|7=8`!V1c0wT;Zvy`~Ibj=gsa_Z!BEIMy}d{A#Z zDzkR?pr+d8{{eTYF+^09{rp5j|Ac=9xJ+UfKZ;NDhi1TZbOj!st!eDs-@b@G{5%DmHZWI+@H> zGrMu9KEvYiQIwKa#xuFw@oepD{zX$WBk!J99jweAHhf7af;h}}E7yM|Od+)?|I6Cn5uJ#R%NUot3mZ9V6k99%ja-Vq zbb4vS@v9&7T6590e~y$17zTT~LzMGcHR5=xbMM$2NX9cw`K#U9T^+UMHx-%A5e3n` V7xw2RkCR+JDds7XCI6ef{|B(lG0y-1 literal 0 HcmV?d00001 diff --git a/src/timeseries/mod.rs b/src/timeseries/mod.rs index e34231f..f8a64dc 100644 --- a/src/timeseries/mod.rs +++ b/src/timeseries/mod.rs @@ -1,8 +1,10 @@ pub mod binary; +pub mod binary_read; pub mod datapoint_listen; mod test; pub use binary::{BinaryIngestOptions, DatapointValueType, Frame, FrameWriter, ResolvedSeries}; +pub use binary_read::{BinaryDatapoints, BinaryValues}; pub use datapoint_listen::{DatapointListener, LiveDatapoint}; use crate::buffer::DurableSpool; diff --git a/src/timeseries/test.rs b/src/timeseries/test.rs index 2f7bb38..18fba9e 100644 --- a/src/timeseries/test.rs +++ b/src/timeseries/test.rs @@ -12,7 +12,8 @@ mod tests { use crate::generic::{DataWrapper, Datapoint, DatapointString, DatapointsCollection, DeleteFilter, IdAndExtId, RetrieveFilter}; use crate::http::ResponseError; use crate::timeseries::{TimeSeries, TimeSeriesFilter, TimeSeriesFilterForm, TimeSeriesUpdate, TimeSeriesUpdateCollection, TimeSeriesUpdateFields}; - use crate::timeseries::binary::BinaryIngestOptions; + use crate::timeseries::binary::{BinaryIngestOptions, DatapointValueType}; + use crate::timeseries::binary_read::{BinaryDatapoints, BinaryValues}; use crate::tests::cleanup::cleanup_timeseries; use crate::tests::ids::{unique_id, unique_token}; use crate::tests::polling::poll_until_for; @@ -712,6 +713,163 @@ mod tests { Ok(()) } + /// `POST /timeseries/data/list/binary` against the JSON read of the same series: every point + /// of a float series past one frame's row cap, hourly aggregates, a text series, an empty + /// window, and the two refusals the JSON read does not have. Needs a backend that serves the + /// binary read. + #[tokio::test] + #[ignore] + async fn test_datapoints_binary_read() -> Result<(), Box> { + let api_service = create_api_service(); + let float_ext_id = unique_id("bin_read_float"); + let text_ext_id = unique_id("bin_read_text"); + let mut ts_collection = DataWrapper::new(); + for (external_id, value_type) in [(&float_ext_id, "float"), (&text_ext_id, "text")] { + ts_collection.add_item( + TimeSeries::builder() + .set_external_id(external_id) + .set_name(external_id) + .set_unit("celsius") + .set_value_type(value_type) + .clone(), + ); + } + let mut cleanup = cleanup_timeseries(vec![float_ext_id.clone(), text_ext_id.clone()]); + api_service.time_series.create(&ts_collection).await.expect("could not create the series"); + + // 150 000 one-second floats: past the 100 000-row numeric frame cap and past the 100 000 + // an item's `limit` may ask for, so they come back whole only from an item without one, + // as two frames. 25 000 texts: past the 10 000-row text frame cap, so three frames. + let start = Utc.with_ymd_and_hms(2025, 1, 1, 0, 0, 0).unwrap(); + let count: i64 = 150_000; + let value = |i: i64| (i % 3600) as f64 * 0.25 - 100.0; + let mut floats = DatapointsCollection::from_external_id(&float_ext_id); + floats.datapoints = (0..count) + .map(|i| DatapointString::new(&(start.timestamp_millis() + i * 1000).to_string(), &value(i).to_string())) + .collect(); + let mut texts = DatapointsCollection::from_external_id(&text_ext_id); + let text_count: i64 = 25_000; + let text = |i: i64| format!("blåbær_{}", i % 97); + texts.datapoints = (0..text_count) + .map(|i| DatapointString::new(&(start.timestamp_millis() + i * 1000).to_string(), &text(i))) + .collect(); + api_service.time_series + .insert_datapoints_binary(&DataWrapper::from_vec(vec![floats, texts]), &BinaryIngestOptions::default()) + .await + .unwrap_or_else(|e| panic!("binary insert failed with {}: {}", e.get_status(), e.get_message())); + let item = |external_id: &str, limit: u64| { + let mut rf = RetrieveFilter::new(); + rf.set_external_id(external_id); + rf.set_start(start); + rf.set_limit(limit); + rf + }; + poll_all_points(&api_service, &float_ext_id, count as usize).await; + // Not poll_all_points: the JSON `Datapoint` holds only numbers and cannot read text. + let text_request = DataWrapper::from_vec(vec![item(&text_ext_id, text_count as u64)]); + poll_until_for( + std::time::Duration::from_secs(60), + || api_service.time_series.retrieve_datapoints_binary(&text_request), + |read| matches!(read, Ok(series) if series.first().is_some_and(|s| s.len() == text_count as usize)), + ) + .await; + + println!("Every point, in order, across frames..."); + let mut whole_window = RetrieveFilter::new(); + whole_window.set_external_id(&float_ext_id); + whole_window.set_start(start); + whole_window.set_end(start + Duration::seconds(count)); + let first = api_service.time_series + .retrieve_datapoints_binary(&DataWrapper::from_vec(vec![whole_window, item(&text_ext_id, text_count as u64)])) + .await + .unwrap_or_else(|e| panic!("binary read failed with {}: {}", e.get_status(), e.get_message())); + assert_eq!(first.len(), 2, "one entry per series, the frames of one merged"); + let float = first[0].clone(); + assert_eq!(float.external_id, float_ext_id); + assert_eq!(float.value_type, DatapointValueType::Float); + let want_timestamps: Vec = (0..count).map(|i| start.timestamp_millis() + i * 1000).collect(); + assert_eq!(float.timestamps, want_timestamps); + let want_values: Vec = (0..count).map(value).collect(); + assert_eq!(float.values, BinaryValues::Float(want_values)); + + let texts = &first[1]; + assert_eq!(texts.external_id, text_ext_id); + assert_eq!(texts.timestamps, want_timestamps[..text_count as usize]); + assert_eq!(texts.values, BinaryValues::Text((0..text_count).map(text).collect())); + + println!("The same window as the JSON read..."); + let json = api_service.time_series + .retrieve_datapoints(&DataWrapper::from_vec(vec![item(&float_ext_id, 1000)])) + .await?; + let json_points = &json.get_items()[0].datapoints; + assert_eq!(json_points.len(), 1000); + for (i, dp) in json_points.iter().enumerate() { + assert_eq!(dp.timestamp.timestamp_millis(), float.timestamps[i]); + assert_eq!(dp.value, Some(want_value_at(&float, i))); + } + + println!("Hourly aggregates against the JSON aggregates..."); + let hourly = |mut rf: RetrieveFilter| { + rf.set_aggregates(vec!["avg".to_string(), "min".to_string(), "max".to_string(), "sum".to_string()]); + rf.set_granularity("1h"); + rf.set_end(start + Duration::seconds(count)); + DataWrapper::from_vec(vec![rf]) + }; + let mut rf = RetrieveFilter::new(); + rf.set_external_id(&float_ext_id); + rf.set_start(start); + let binary_aggs = api_service.time_series.retrieve_datapoints_binary(&hourly(rf.clone())).await?; + let json_aggs = api_service.time_series.retrieve_datapoints(&hourly(rf)).await?; + let json_buckets = &json_aggs.get_items()[0].datapoints; + let BinaryValues::Aggregates { avg, min, max, sum } = &binary_aggs[0].values else { + panic!("an aggregate read came back as {:?}", binary_aggs[0].values); + }; + assert_eq!(binary_aggs[0].len(), json_buckets.len(), "the two reads answered a different number of buckets"); + assert_eq!(json_buckets.len(), (count as usize).div_ceil(3600)); + for (i, bucket) in json_buckets.iter().enumerate() { + assert_eq!(binary_aggs[0].timestamps[i], bucket.timestamp.timestamp_millis()); + assert_eq!((Some(min[i]), Some(max[i])), (bucket.min, bucket.max), "bucket {i}"); + assert!((avg[i] - bucket.average.unwrap()).abs() < AVG_TOLERANCE, "bucket {i}: avg"); + assert!((sum[i] - bucket.sum.unwrap()).abs() < 1e-6 * sum[i].abs().max(1.0), "bucket {i}: sum"); + } + + println!("An empty window is no entry..."); + let mut empty = item(&float_ext_id, 10); + empty.set_start(start - Duration::days(10)); + empty.set_end(start - Duration::days(9)); + let none = api_service.time_series.retrieve_datapoints_binary(&DataWrapper::from_vec(vec![empty])).await?; + assert!(none.is_empty(), "{none:?}"); + + println!("Refusals..."); + let mut with_cursor = item(&float_ext_id, 10); + with_cursor.cursor = Some("anything".to_string()); + let error = api_service.time_series + .retrieve_datapoints_binary(&DataWrapper::from_vec(vec![with_cursor])) + .await + .unwrap_err(); + assert_eq!(error.get_status(), StatusCode::BAD_REQUEST, "{}", error.get_message()); + assert!(error.problem().is_some(), "a refusal is a problem document: {}", error.get_message()); + + let unknown = unique_id("bin_read_missing"); + let error = api_service.time_series + .retrieve_datapoints_binary(&DataWrapper::from_vec(vec![item(&unknown, 10)])) + .await + .unwrap_err(); + assert_eq!(error.get_status(), StatusCode::BAD_REQUEST, "{}", error.get_message()); + assert!(error.get_message().contains(&unknown), "{}", error.get_message()); + + delete_timeseries(&api_service, &[&float_ext_id, &text_ext_id]).await; + cleanup.disarm(); + Ok(()) + } + + fn want_value_at(series: &BinaryDatapoints, i: usize) -> f64 { + match &series.values { + BinaryValues::Float(values) => values[i], + other => panic!("not a float series: {other:?}"), + } + } + /// JSON against binary ingest from Rust, reporting the same columns as the Java benchmark in /// the platform's `datahub-e2e` module and the Python one in `python_tests`, so the three /// clients can be compared. From 0b974183e229f7d08c74bb5316f6dfcfc6e6ce68 Mon Sep 17 00:00:00 2001 From: jgjesdal Date: Mon, 12 Oct 2026 14:35:30 +0200 Subject: [PATCH 2/8] Stream the binary read frame by frame as it arrives stream_datapoints_binary returns a Stream of BinaryDatapoints, one per frame. A FrameAssembler cuts the body into frames however the chunks fall, and frames are decoded on spawn_blocking, at most min(cores, 8) at once and kept in order, while the rest downloads. A refusal is the Err before there is a stream; a response that breaks off is an error item after the items that arrived. The blocking client has it as an iterator. retrieve_datapoints_binary now collects that stream and merges each series once at its final size, so it too decodes during the download. In Python, retrieve_datapoints_binary_stream returns a single-use BinaryDatapointReader: iterate it per frame, or read it as an Arrow stream of one long table (timeseries_id, timestamp, values), with external_ids mapping the ids. An external_id dictionary column cost polars 0.8s per 100M rows, so the table leaves it out. 100M float rows over 10 series: the server takes ~6.5s; the buffered call adds 0.24s (was 0.90s) and the stream into polars 0.02s. Co-Authored-By: Claude Opus 5.5 Signed-off-by: jgjesdal --- AGENTS.md | 2 +- Cargo.toml | 1 + README.md | 14 + datahub_python_bindings/Cargo.toml | 1 + .../intellistream_datahub_sdk/__init__.pyi | 66 +- .../src/timeseries/binary_read.rs | 290 +++++++-- datahub_python_bindings/src/timeseries/mod.rs | 1 + .../src/timeseries/sync_service.rs | 22 +- python_tests/test_timeseries_binary_read.py | 162 +++++ src/blocking.rs | 33 +- src/generic.rs | 50 +- src/timeseries/binary_read.rs | 585 ++++++++++++++---- src/timeseries/mod.rs | 2 +- src/timeseries/test.rs | 16 +- 14 files changed, 1030 insertions(+), 215 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index 6a153fa..6636093 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -203,7 +203,7 @@ the generic default with `recognized: false`, not an error. `TimeSeriesService::insert_datapoints_binary` is the second ingest path, to `POST /timeseries/data/binary`: the same `DatapointsCollection` input, resolved through `/timeseries/byids` (cached per service instance; needs read access on the dataset), checked against each series' value type locally, cut into Arrow IPC frames at the contract's caps, zstd-compressed per frame (mandatory: level 1, 3 or 9, default 9) and posted through `execute_post_bytes_request`. Every refusal on this path is a problem document typed for its status — `invalid-frame` (400), `unknown-timeseries` (404), `request-too-large` (413), `unsupported-media-type` (415), `value-type-mismatch` and `external-id-mismatch` (422), `too-many-in-flight` (429) — with the kebab-case sub-case in a `reason` extension beside it. It was one type, `datapoint-block-rejected`, answering with all six statuses until the api split it (platform #120); the SDK matches the new slugs only, and nothing here recognises the old one — the binary path has never been in a release, so no caller can be on the other side of that change. `is_stale_series_rejection` reads the slug to decide the one re-resolve-and-retry, and reads it from the problem rather than the body text because the SDK's own pre-flight 404 names the series it could not resolve — an external id spelling `unknown-timeseries` used to trigger the retry. `FrameWriter` builds one frame and is public; `cut_into_writers` and `pack_requests` hold the caps. The byte layout is the platform's `binary_datapoints_format.md`. The arrow-rs crates (`arrow-array`, `arrow-schema`, `arrow-ipc`) serve both directions. In Python it is `insert_datapoints_binary` and `insert_from_lists_binary` on the **sync** client only, each taking an optional `zstd_level`; the async client has no binary methods. The ignored `timeseries::tests::test_datapoints_binary` is the live twin of `test_datapoints` and needs a backend that serves the endpoint; the writer's own tests in `binary.rs` run offline. `python_tests/test_timeseries_binary.py` round-trips the Python binding through the JSON read path. -The read is `TimeSeriesService::retrieve_datapoints_binary`, to `POST /timeseries/data/list/binary`: the JSON retrieve body in, the same frames out, one series per frame, then the 4-byte marker `DHDE`. A body without the marker was cut short (the server streams, so a failure after the first bytes can only end the body early) and is a **502** saying so, never fewer points; bytes after it are refused too. Codec 2 frames carry bucketed aggregates in their own schema (`avg`/`min`/`max`/`sum`, Float64, 50 000 buckets per frame) while the envelope's type byte stays the series' own. `parse_read_response` returns one columnar `BinaryDatapoints` per series, merging consecutive frames of one series; decimals stay exact as unscaled `i128` plus scale. It walks the envelopes first (cheap, and where a cut-short body is caught), then decompresses and decodes the frames on scoped threads — zstd was 90% of a single-threaded parse, 1.3 s per 100M float rows — and `retrieve_datapoints_binary` runs it on `spawn_blocking`. What remains (about 0.6 s per 100M rows) is allocation and copying, not zstd; streaming the read would hide it behind the download. Differences from the JSON read worth knowing: no `cursor` (a 400; advance `start` past the last timestamp), a series with no points is absent rather than empty, and an unknown or unreadable series is a 400 for the whole request — `DataWrapper` serialises only `items`, so the request-level `ignoreUnknownIds` and `limit` are not reachable. `src/timeseries/fixtures/read_*.bin` are response bodies written by the platform's own `DatapointFrameWriter.wrap`, so the offline tests in `binary_read.rs` prove arrow-rs reads what the Java writer emits; regenerate them when the platform's writer changes. The ignored `timeseries::tests::test_datapoints_binary_read` is the live check against the JSON read. In Python it is `retrieve_datapoints_binary` on the sync client, returning `BinaryDatapoints` objects that implement the Arrow PyCapsule stream protocol (`__arrow_c_stream__`, via `arrow-array`'s `ffi` feature enabled in the bindings crate only), so polars, pyarrow and pandas 3 read them zero-copy; the package still declares no runtime dependency, and `to_polars`/`to_pandas` import theirs lazily. `python_tests/test_timeseries_binary_read.py` covers it offline against a stand-in server and live when the backend serves the endpoint. +The read is `TimeSeriesService::retrieve_datapoints_binary`, to `POST /timeseries/data/list/binary`: the JSON retrieve body in, the same frames out, one series per frame, then the 4-byte marker `DHDE`. A body without the marker was cut short (the server streams, so a failure after the first bytes can only end the body early) and is a **502** saying so, never fewer points; bytes after it are refused too. Codec 2 frames carry bucketed aggregates in their own schema (`avg`/`min`/`max`/`sum`, Float64, 50 000 buckets per frame) while the envelope's type byte stays the series' own. `parse_read_response` returns one columnar `BinaryDatapoints` per series, merging consecutive frames of one series; decimals stay exact as unscaled `i128` plus scale. `TimeSeriesService::stream_datapoints_binary` is the primitive: a `FrameAssembler` cuts the body into frames as chunks arrive (a frame may be split anywhere), each frame is an owned `Bytes` decoded on `spawn_blocking`, at most `min(cores, 8)` in flight through a `FuturesOrdered` so items stay in frame order, and a producer task feeds a bounded channel behind the `BinaryDatapointStream` (dropping it aborts the task and closes the connection). One item per frame, so a long series is several consecutive items. A refusal is the `Err` before there is a stream; a response that breaks off is an error item after the items that arrived, and that is the semantic difference stream consumers must handle. `retrieve_datapoints_binary` collects the stream and merges consecutive parts (each series allocated once at its final size); `parse_read_response` does the same for a body in memory, decoding on scoped threads. Measured on 100M float rows, 10 series: the server takes ~6.5 s; the buffered call adds ~0.24 s (the merge copy) and the stream into polars ~0.02 s. Single-threaded zstd used to add 1.3 s on its own. Differences from the JSON read worth knowing: no `cursor` (a 400; advance `start` past the last timestamp), a series with no points is absent rather than empty, and an unknown or unreadable series is a 400 for the whole request — `DataWrapper` serialises only `items`, so the request-level `ignoreUnknownIds` and `limit` are not reachable. `src/timeseries/fixtures/read_*.bin` are response bodies written by the platform's own `DatapointFrameWriter.wrap`, so the offline tests in `binary_read.rs` prove arrow-rs reads what the Java writer emits; regenerate them when the platform's writer changes. The ignored `timeseries::tests::test_datapoints_binary_read` is the live check against the JSON read. In Python it is `retrieve_datapoints_binary` on the sync client, plus `retrieve_datapoints_binary_stream`, whose single-use `BinaryDatapointReader` iterates per frame or is an Arrow stream of one long table (`timeseries_id`, `timestamp`, values; one schema, so a read mixing value types must be iterated). The long table carries no external id column on purpose: as a dictionary column, polars spent ~0.8 s per 100M rows making it categorical, more than everything else; `reader.external_ids` maps the ids instead. `retrieve_datapoints_binary` returns `BinaryDatapoints` objects that implement the Arrow PyCapsule stream protocol (`__arrow_c_stream__`, via `arrow-array`'s `ffi` feature enabled in the bindings crate only), so polars, pyarrow and pandas 3 read them zero-copy; the package still declares no runtime dependency, and `to_polars`/`to_pandas` import theirs lazily. `python_tests/test_timeseries_binary_read.py` covers it offline against a stand-in server and live when the backend serves the endpoint. ### The `ApiServiceProvider` trait (`src/generic.rs`) diff --git a/Cargo.toml b/Cargo.toml index 885d051..fc2916f 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -30,6 +30,7 @@ reqwest = { version = "0.12", default-features = false, features = ["blocking", chrono = { version ="0.4", features = ["serde"] } rand = "0.9" futures = "0.3" +bytes = "1" dotenv = "0.15" uuid = { version = "1.16", features = ["v4", "v7", "serde"] } regex = "1.11.1" diff --git a/README.md b/README.md index a228fe0..4e9f04d 100644 --- a/README.md +++ b/README.md @@ -168,11 +168,25 @@ answer is held in memory. There is no cursor: to read on in pieces, send the req the JSON read answers empty. A response the server could not finish is a 502 rather than fewer points, and is safe to send again. +`time_series.stream_datapoints_binary(&filters)` is the same read handed over as it arrives: a +`Stream` of `BinaryDatapoints`, one per frame, decoded on blocking threads while the rest of the +response downloads, so nothing waits for the whole body and memory holds what the consumer keeps. +A series longer than one frame comes as consecutive items. A refusal is the `Err` before there is +a stream; a response that breaks off is an error item **after** the items that arrived, so a +consumer acting on items as they come has to treat what it took as incomplete. The blocking +client has it as an iterator. `retrieve_datapoints_binary` collects this stream, which is where +all-or-nothing lives. + In Python, `client.timeseries.retrieve_datapoints_binary(filter_or_list)` on the sync client returns objects that implement the Arrow PyCapsule stream protocol, so `polars.DataFrame(series)`, `pyarrow.table(series)` and `pandas.DataFrame.from_arrow(series)` take them without copying; `series.to_polars()` and `series.to_pandas()` do the same. The package adds no dependency for this — bring polars, or pyarrow for pandas. +`retrieve_datapoints_binary_stream(filter_or_list)` returns a single-use reader: iterate it for one +`BinaryDatapoints` per frame, or hand it to `polars.DataFrame(reader)` (or `reader.to_polars()`) +for one long table of `timeseries_id`, `timestamp` and the value columns, with +`reader.external_ids` mapping the ids. Reading 100M float rows into polars that way costs about +as long as the download itself. ## Python bindings diff --git a/datahub_python_bindings/Cargo.toml b/datahub_python_bindings/Cargo.toml index cae7243..b6cbb3a 100644 --- a/datahub_python_bindings/Cargo.toml +++ b/datahub_python_bindings/Cargo.toml @@ -16,6 +16,7 @@ crate-type = ["cdylib"] pyo3 = { version = "0.29", features = ["extension-module", "chrono","chrono-tz","uuid","experimental-inspect", "abi3-py39","multiple-pymethods"] } intellistream-datahub-sdk = { path = ".." } tokio = { version = "1", features = ["full"] } +futures = "0.3" chrono = { version = "0.4", features = ["serde"] } serde = { version = "1", features = ["derive"] } serde_json = "1" diff --git a/datahub_python_bindings/python/intellistream_datahub_sdk/__init__.pyi b/datahub_python_bindings/python/intellistream_datahub_sdk/__init__.pyi index 0359bba..1bf38b0 100644 --- a/datahub_python_bindings/python/intellistream_datahub_sdk/__init__.pyi +++ b/datahub_python_bindings/python/intellistream_datahub_sdk/__init__.pyi @@ -590,6 +590,40 @@ class DatapointsCollectionDatapoints: def id(self) -> int | None: ... +class BinaryDatapointReader: + """A binary read as it arrives, from ``TimeSeriesServiceSync.retrieve_datapoints_binary_stream``. + + Single use, in one of two ways; a second use raises ``RuntimeError``. + + **Iterating** yields one :class:`BinaryDatapoints` per frame, as the frames arrive. A series + longer than one frame comes as several consecutive items, oldest first, so an item is a chunk + of a series rather than all of it; series of different value types mix freely. + + **As an Arrow stream** (``polars.DataFrame(reader)``, :meth:`to_polars`, + ``pyarrow.RecordBatchReader.from_stream(reader)``) it is one long table: ``timeseries_id``, + ``timestamp`` and the value columns, a batch per frame. Every series in the read has to share + those columns, so a read mixing value types (or values and aggregates) raises at the first + batch that differs; iterate such a read instead. A read with no rows has no schema and raises + ``ValueError``. The network is waited on with the GIL held, as the C stream protocol gives no + way to release it. + + A response that breaks off raises ``DataHubException`` (502, "cut short") from ``next()`` or + from the Arrow stream, **after** the rows that did arrive were handed out: treat what was + already taken as incomplete. ``retrieve_datapoints_binary`` is the all-or-nothing form. + """ + + def __iter__(self) -> Iterator[BinaryDatapoints]: ... + def __next__(self) -> BinaryDatapoints: ... + @property + def external_ids(self) -> dict[int, str]: + """``{timeseries_id: external_id}`` for the series read so far, in the order they came: + the key to the long table's ``timeseries_id``. Complete once the reader is exhausted.""" + def __arrow_c_stream__(self, requested_schema: object | None = None) -> object: + """The Arrow PyCapsule stream protocol, as the long table. ``requested_schema`` is ignored.""" + def to_polars(self) -> Any: + """The long table as a ``polars.DataFrame``. Needs polars; raises ``ImportError`` without it.""" + + class BinaryDatapoints: """One series' rows from ``TimeSeriesServiceSync.retrieve_datapoints_binary``, as an Arrow table. @@ -816,7 +850,37 @@ class TimeSeriesServiceSync: Notes ----- - Sync client only, like the binary insert. + Sync client only, like the binary insert. The frames are decoded while the rest of the + response downloads; for rows as they arrive, or a window too large to hold, use + :meth:`retrieve_datapoints_binary_stream`. + """ + def retrieve_datapoints_binary_stream( + self, input: RetrieveFilter | list[RetrieveFilter] + ) -> BinaryDatapointReader: + """The read :meth:`retrieve_datapoints_binary` makes, handed over as it arrives. + + Parameters + ---------- + input + As for :meth:`retrieve_datapoints_binary`. + + Returns + ------- + BinaryDatapointReader + Single use: iterate it for one :class:`BinaryDatapoints` per frame, or read it as an + Arrow stream of one long table. + + Raises + ------ + DataHubException + A refusal (400 for an unknown series, a ``cursor``, aggregates on a text series) + raises here, before there is a reader. A response that breaks off raises from the + reader instead, after the rows that did arrive: see :class:`BinaryDatapointReader`. + + Notes + ----- + Sync client only. Holding the reader holds the connection and the server's query open, + so read it through rather than leaving it half-consumed. """ def delete_datapoints(self, input: list[DeleteFilter]) -> None: ... def retrieve_latest_datapoints( diff --git a/datahub_python_bindings/src/timeseries/binary_read.rs b/datahub_python_bindings/src/timeseries/binary_read.rs index 8a044f6..f41efbd 100644 --- a/datahub_python_bindings/src/timeseries/binary_read.rs +++ b/datahub_python_bindings/src/timeseries/binary_read.rs @@ -1,15 +1,21 @@ use std::sync::Arc; use arrow_array::ffi_stream::FFI_ArrowArrayStream; +use std::sync::Mutex; + use arrow_array::{ - ArrayRef, Decimal128Array, Float32Array, Float64Array, Int64Array, RecordBatch, - RecordBatchIterator, StringArray, TimestampMillisecondArray, + ArrayRef, Decimal128Array, Float32Array, Float64Array, Int64Array, + RecordBatch, RecordBatchIterator, RecordBatchReader, StringArray, TimestampMillisecondArray, +}; +use arrow_schema::{ArrowError, DataType, Field, Schema, SchemaRef, TimeUnit}; +use futures::StreamExt; +use intellistream_datahub_sdk::timeseries::{ + BinaryDatapointStream, BinaryDatapoints, BinaryValues, DatapointValueType, }; -use arrow_schema::{ArrowError, DataType, Field, Schema, TimeUnit}; -use intellistream_datahub_sdk::timeseries::{BinaryDatapoints, BinaryValues, DatapointValueType}; -use pyo3::exceptions::{PyImportError, PyValueError}; +use tokio::runtime::Runtime; +use pyo3::exceptions::{PyImportError, PyRuntimeError, PyStopIteration, PyValueError}; use pyo3::prelude::*; -use pyo3::types::PyCapsule; +use pyo3::types::{PyCapsule, PyDict}; /// One series' rows from `retrieve_datapoints_binary`, as an Arrow table: `timestamp` and the /// value columns of its type. Read it with anything that takes the Arrow PyCapsule protocol. @@ -22,49 +28,59 @@ pub struct PyBinaryDatapoints { batch: RecordBatch, } +/// A series' `timestamp` and value columns, as Arrow arrays that take over its vectors. +fn columns( + value_type: DatapointValueType, + timestamps: Vec, + values: BinaryValues, +) -> Result<(Vec, Vec), ArrowError> { + let timestamp = Field::new( + "timestamp", + DataType::Timestamp(TimeUnit::Millisecond, Some("UTC".into())), + false, + ); + let mut fields = vec![timestamp]; + let mut columns: Vec = + vec![Arc::new(TimestampMillisecondArray::from(timestamps).with_timezone("UTC"))]; + let mut push = |name: &str, nullable: bool, array: ArrayRef| { + fields.push(Field::new(name, array.data_type().clone(), nullable)); + columns.push(array); + }; + match values { + BinaryValues::Bigint(v) => push("value", false, Arc::new(Int64Array::from(v))), + BinaryValues::Float(v) => push("value", false, Arc::new(Float64Array::from(v))), + BinaryValues::Float32(v) => push("value", false, Arc::new(Float32Array::from(v))), + BinaryValues::Decimal { unscaled, scale } => { + let precision = if value_type == DatapointValueType::Decimal32 { 9 } else { 18 }; + let array = Decimal128Array::from(unscaled).with_precision_and_scale(precision, scale)?; + push("value", false, Arc::new(array)) + } + BinaryValues::Text(v) => push("value", false, Arc::new(StringArray::from(v))), + BinaryValues::Mixed { numeric, text } => { + push("value_numeric", true, Arc::new(Float64Array::from(numeric))); + push("value_text", true, Arc::new(StringArray::from(text))); + } + BinaryValues::Aggregates { avg, min, max, sum } => { + push("avg", false, Arc::new(Float64Array::from(avg))); + push("min", false, Arc::new(Float64Array::from(min))); + push("max", false, Arc::new(Float64Array::from(max))); + push("sum", false, Arc::new(Float64Array::from(sum))); + } + other => { + return Err(ArrowError::NotYetImplemented(format!( + "binary values {other:?} have no Arrow mapping" + ))) + } + } + Ok((fields, columns)) +} + impl TryFrom for PyBinaryDatapoints { type Error = ArrowError; fn try_from(series: BinaryDatapoints) -> Result { let aggregates = series.is_aggregates(); - let timestamp = Field::new( - "timestamp", - DataType::Timestamp(TimeUnit::Millisecond, Some("UTC".into())), - false, - ); - let mut fields = vec![timestamp]; - let mut columns: Vec = - vec![Arc::new(TimestampMillisecondArray::from(series.timestamps).with_timezone("UTC"))]; - let mut push = |name: &str, nullable: bool, array: ArrayRef| { - fields.push(Field::new(name, array.data_type().clone(), nullable)); - columns.push(array); - }; - match series.values { - BinaryValues::Bigint(v) => push("value", false, Arc::new(Int64Array::from(v))), - BinaryValues::Float(v) => push("value", false, Arc::new(Float64Array::from(v))), - BinaryValues::Float32(v) => push("value", false, Arc::new(Float32Array::from(v))), - BinaryValues::Decimal { unscaled, scale } => { - let precision = if series.value_type == DatapointValueType::Decimal32 { 9 } else { 18 }; - let array = Decimal128Array::from(unscaled).with_precision_and_scale(precision, scale)?; - push("value", false, Arc::new(array)) - } - BinaryValues::Text(v) => push("value", false, Arc::new(StringArray::from(v))), - BinaryValues::Mixed { numeric, text } => { - push("value_numeric", true, Arc::new(Float64Array::from(numeric))); - push("value_text", true, Arc::new(StringArray::from(text))); - } - BinaryValues::Aggregates { avg, min, max, sum } => { - push("avg", false, Arc::new(Float64Array::from(avg))); - push("min", false, Arc::new(Float64Array::from(min))); - push("max", false, Arc::new(Float64Array::from(max))); - push("sum", false, Arc::new(Float64Array::from(sum))); - } - other => { - return Err(ArrowError::NotYetImplemented(format!( - "binary values {other:?} have no Arrow mapping" - ))) - } - } + let (fields, columns) = columns(series.value_type, series.timestamps, series.values)?; let batch = RecordBatch::try_new(Arc::new(Schema::new(fields)), columns)?; Ok(Self { id: series.id, @@ -76,6 +92,19 @@ impl TryFrom for PyBinaryDatapoints { } } +/// One frame's rows as a batch of the long table: `timeseries_id` ahead of the series' own +/// columns. No external id column: as a dictionary column polars spent 0.8 s per 100M rows making +/// it categorical, more than the rest of the conversion; `external_ids` maps the ids instead. +fn long_batch(series: BinaryDatapoints) -> Result { + let id = Arc::new(Int64Array::from(vec![series.id as i64; series.len()])) as ArrayRef; + let (fields, columns) = columns(series.value_type, series.timestamps, series.values)?; + let mut all_fields = vec![Field::new("timeseries_id", DataType::Int64, false)]; + all_fields.extend(fields); + let mut all_columns = vec![id]; + all_columns.extend(columns); + RecordBatch::try_new(Arc::new(Schema::new(all_fields)), all_columns) +} + #[pymethods] impl PyBinaryDatapoints { #[getter] @@ -149,6 +178,179 @@ impl PyBinaryDatapoints { } } +/// A binary read as it arrives, from `retrieve_datapoints_binary_stream`. Single use: read it +/// either by iterating, one `BinaryDatapoints` per frame, or as an Arrow stream of the long table. +#[pyclass(module = "intellistream_datahub_sdk", name = "BinaryDatapointReader", frozen)] +pub struct PyBinaryDatapointReader { + runtime: Arc, + state: Mutex, + seen: Seen, +} + +/// The series read so far, by id, in the order they first came. +type Seen = Arc>>; + +fn note(seen: &Seen, series: &BinaryDatapoints) { + let mut seen = seen.lock().unwrap(); + if !seen.iter().any(|(id, _)| *id == series.id) { + seen.push((series.id, series.external_id.clone())); + } +} + +enum ReaderState { + Unread(BinaryDatapointStream), + Iterating(BinaryDatapointStream), + Done, +} + +impl PyBinaryDatapointReader { + pub(crate) fn new(runtime: Arc, stream: BinaryDatapointStream) -> Self { + Self { + runtime, + state: Mutex::new(ReaderState::Unread(stream)), + seen: Arc::default(), + } + } +} + +#[pymethods] +impl PyBinaryDatapointReader { + fn __iter__(slf: PyRef<'_, Self>) -> PyRef<'_, Self> { + slf + } + + /// The next frame's rows: a chunk of one series, which may continue in the next item. + fn __next__(&self, py: Python<'_>) -> PyResult { + let mut state = self.state.lock().unwrap(); + let mut stream = match std::mem::replace(&mut *state, ReaderState::Done) { + ReaderState::Unread(stream) | ReaderState::Iterating(stream) => stream, + ReaderState::Done => return Err(PyStopIteration::new_err(())), + }; + let item = py.detach(|| self.runtime.block_on(stream.next())); + match item { + Some(Ok(part)) => { + *state = ReaderState::Iterating(stream); + note(&self.seen, &part); + PyBinaryDatapoints::try_from(part).map_err(|e| PyValueError::new_err(e.to_string())) + } + Some(Err(error)) => Err(crate::datahub_err(error)), + None => Err(PyStopIteration::new_err(())), + } + } + + /// `{timeseries_id: external_id}` for the series read so far, in the order they came: the key + /// for the long table's `timeseries_id`. Complete once the reader is exhausted. + #[getter] + fn external_ids<'py>(&self, py: Python<'py>) -> PyResult> { + let ids = PyDict::new(py); + for (id, external_id) in self.seen.lock().unwrap().iter() { + ids.set_item(id, external_id)?; + } + Ok(ids) + } + + /// The Arrow PyCapsule stream protocol, as one long table: `timeseries_id`, `timestamp` and + /// the value columns, a batch per frame. Every series in the read has to + /// share those columns; one that does not is an error at its batch. A read with no rows has + /// no schema to declare and raises. + #[pyo3(signature = (requested_schema=None))] + fn __arrow_c_stream__<'py>( + &self, + py: Python<'py>, + requested_schema: Option>, + ) -> PyResult> { + let _ = requested_schema; + let mut stream = { + let mut state = self.state.lock().unwrap(); + match std::mem::replace(&mut *state, ReaderState::Done) { + ReaderState::Unread(stream) => stream, + other => { + *state = other; + return Err(PyRuntimeError::new_err( + "this reader has already been read; a binary read streams once", + )); + } + } + }; + let first = match py.detach(|| self.runtime.block_on(stream.next())) { + Some(Ok(part)) => { + note(&self.seen, &part); + long_batch(part).map_err(|e| PyValueError::new_err(e.to_string()))? + } + Some(Err(error)) => return Err(crate::datahub_err(error)), + None => { + return Err(PyValueError::new_err( + "the read returned no rows, so there is no schema for an Arrow stream", + )) + } + }; + let reader = LongTable { + schema: first.schema(), + first: Some(first), + stream, + runtime: self.runtime.clone(), + seen: self.seen.clone(), + }; + let ffi = FFI_ArrowArrayStream::new(Box::new(reader)); + PyCapsule::new_with_value(py, ffi, c"arrow_array_stream") + } + + /// A `polars.DataFrame` of the long table. Needs polars. + fn to_polars<'py>(slf: &Bound<'py, Self>) -> PyResult> { + let polars = import(slf.py(), "polars", "to_polars")?; + polars.call_method1("DataFrame", (slf,)) + } +} + +/// The long table behind `__arrow_c_stream__`: the frame peeked for its schema, then the rest as +/// the consumer asks. Waits for the network with the GIL held, as the C stream gives no way to +/// release it; the reader belongs to the sync client. +struct LongTable { + schema: SchemaRef, + first: Option, + stream: BinaryDatapointStream, + runtime: Arc, + seen: Seen, +} + +impl Iterator for LongTable { + type Item = Result; + + fn next(&mut self) -> Option { + if let Some(first) = self.first.take() { + return Some(Ok(first)); + } + let part = match self.runtime.block_on(self.stream.next())? { + Ok(part) => part, + Err(error) => { + let message = format!("{}: {}", error.get_status(), error.get_message()); + return Some(Err(ArrowError::ExternalError(message.into()))); + } + }; + note(&self.seen, &part); + let external_id = part.external_id.clone(); + let batch = match long_batch(part) { + Ok(batch) => batch, + Err(error) => return Some(Err(error)), + }; + if batch.schema().fields() != self.schema.fields() { + return Some(Err(ArrowError::SchemaError(format!( + "series {external_id} has columns {:?}, the stream's are {:?}; read series of \ + different value types by iterating instead", + batch.schema().fields().iter().map(|f| f.name()).collect::>(), + self.schema.fields().iter().map(|f| f.name()).collect::>(), + )))); + } + Some(Ok(batch)) + } +} + +impl RecordBatchReader for LongTable { + fn schema(&self) -> SchemaRef { + self.schema.clone() + } +} + fn import<'py>(py: Python<'py>, module: &str, method: &str) -> PyResult> { py.import(module).map_err(|_| { PyImportError::new_err(format!( diff --git a/datahub_python_bindings/src/timeseries/mod.rs b/datahub_python_bindings/src/timeseries/mod.rs index 53422a7..f0a3979 100644 --- a/datahub_python_bindings/src/timeseries/mod.rs +++ b/datahub_python_bindings/src/timeseries/mod.rs @@ -471,6 +471,7 @@ pub fn register(m: &Bound<'_, PyModule>) -> PyResult<()> { m.add_class::()?; m.add_class::()?; m.add_class::()?; + m.add_class::()?; m.add_class::()?; m.add_class::()?; m.add_class::()?; diff --git a/datahub_python_bindings/src/timeseries/sync_service.rs b/datahub_python_bindings/src/timeseries/sync_service.rs index 4303ea2..67ced00 100644 --- a/datahub_python_bindings/src/timeseries/sync_service.rs +++ b/datahub_python_bindings/src/timeseries/sync_service.rs @@ -4,7 +4,7 @@ use crate::timeseries::datapoints::{ }; use crate::{DatahubIdentity, Identifiable}; use crate::{PyIdCollection, PyRetrieveFilter}; -use crate::timeseries::binary_read::{self, PyBinaryDatapoints}; +use crate::timeseries::binary_read::{self, PyBinaryDatapointReader, PyBinaryDatapoints}; use intellistream_datahub_sdk::generic::{DataWrapper, IdAndExtId}; use intellistream_datahub_sdk::{ApiService, TimeSeriesUpdateCollection}; use pyo3_async_runtimes::tokio::future_into_py; @@ -382,6 +382,26 @@ impl PyTimeSeriesServiceSync { })?; binary_read::to_py(series) } + /// The read `retrieve_datapoints_binary` makes, handed over as it arrives: a single-use + /// `BinaryDatapointReader`, iterated frame by frame or read as an Arrow stream. + fn retrieve_datapoints_binary_stream<'py>( + &self, + py: Python<'py>, + input: RetrieveFilters, + ) -> PyResult { + let service = self.api_service.clone(); + let filters: Vec = match input { + RetrieveFilters::One(f) => vec![f.into()], + RetrieveFilters::Many(fs) => fs.into_iter().map(Into::into).collect(), + }; + let wrapper = DataWrapper::::from_vec(filters); + let stream = py.detach(|| { + self.runtime + .block_on(service.time_series.stream_datapoints_binary(&wrapper)) + .map_err(crate::datahub_err) + })?; + Ok(PyBinaryDatapointReader::new(self.runtime.clone(), stream)) + } fn delete_datapoints<'py>(&self, py: Python<'py>, input: Vec) -> PyResult<()> { let service = self.api_service.clone(); let wrapper = diff --git a/python_tests/test_timeseries_binary_read.py b/python_tests/test_timeseries_binary_read.py index 8e9c03d..782ea5e 100644 --- a/python_tests/test_timeseries_binary_read.py +++ b/python_tests/test_timeseries_binary_read.py @@ -9,6 +9,7 @@ import decimal import http.server import pathlib +import struct import threading import pytest @@ -141,6 +142,149 @@ def test_a_refusal_is_the_problem_the_server_sent(stand_in): assert excinfo.value.status_code == 400 +# --- streaming, offline ----------------------------------------------------------------------- + + +def _frames(body): + """The frames of a response body, each as its own bytes, without the end marker.""" + frames, pos = [], 0 + while body[pos:pos + 4] == b"DHDP": + directory, payload = struct.unpack_from(", + ) -> Result { + let stream = self + .rt + .block_on(self.api.time_series.stream_datapoints_binary(json))?; + Ok(BinaryDatapointIter { + stream, + rt: self.rt.clone(), + }) + } +} + +/// A binary read frame by frame; see +/// [`BinaryDatapointStream`](crate::timeseries::BinaryDatapointStream) for what an item is and how +/// a response that breaks off shows up. +pub struct BinaryDatapointIter { + stream: BinaryDatapointStream, + rt: Arc, +} + +impl Iterator for BinaryDatapointIter { + type Item = Result; + + fn next(&mut self) -> Option { + self.rt.block_on(futures::StreamExt::next(&mut self.stream)) + } } /// Blocking counterpart of [`crate::ResourceService`]. diff --git a/src/generic.rs b/src/generic.rs index 0220427..697eb6f 100644 --- a/src/generic.rs +++ b/src/generic.rs @@ -883,18 +883,19 @@ pub trait ApiServiceProvider { } /// `POST` a JSON body to an endpoint that answers with bytes of type `accept` rather than JSON - /// (currently only `/timeseries/data/list/binary`), and return the whole body. + /// (currently only `/timeseries/data/list/binary`), returning the response to read as it + /// arrives. /// /// `application/problem+json` is offered beside `accept` so a refusal reaches the caller as the /// problem document it is, rather than as Spring's 406 for a type the handler does not produce. - /// A body that breaks off while it is being read is a 502: the server answered 200 and then - /// failed to finish, which is not the caller's fault and is safe to send again. - async fn execute_post_request_for_bytes( + /// The status is checked here, before any of the body is read, so a refusal is always an + /// `Err` and never a stream. + async fn execute_post_request_for_stream( &self, path: &str, json: &J, accept: &str, - ) -> Result, ResponseError> { + ) -> Result { let token = self.get_token().await?; let response = self .get_api_service() @@ -907,30 +908,23 @@ pub trait ApiServiceProvider { .await .map_err(ResponseError::from_err)?; let status = response.status(); - if !status.is_success() { - let content_type = response - .headers() - .get(reqwest::header::CONTENT_TYPE) - .and_then(|value| value.to_str().ok()) - .map(str::to_string); - let error = ResponseError { - status, - message: response - .text() - .await - .unwrap_or_else(|_| "Failed to read response body".to_string()), - content_type, - }; - return Err(self.on_request_error(error, &token).await); - } - match response.bytes().await { - Ok(body) => Ok(body.into()), - Err(error) => Err(ResponseError { - status: http::StatusCode::BAD_GATEWAY, - message: format!("the response was cut short while it was being read: {error}"), - content_type: None, - }), + if status.is_success() { + return Ok(response); } + let content_type = response + .headers() + .get(reqwest::header::CONTENT_TYPE) + .and_then(|value| value.to_str().ok()) + .map(str::to_string); + let error = ResponseError { + status, + message: response + .text() + .await + .unwrap_or_else(|_| "Failed to read response body".to_string()), + content_type, + }; + Err(self.on_request_error(error, &token).await) } /// `GET` an endpoint that answers with bytes rather than JSON (currently only diff --git a/src/timeseries/binary_read.rs b/src/timeseries/binary_read.rs index c57a76a..9b71658 100644 --- a/src/timeseries/binary_read.rs +++ b/src/timeseries/binary_read.rs @@ -5,13 +5,16 @@ //! the marker was cut short, whether inside a frame or between two, and is an error rather than a //! shorter answer. Codec 2 frames carry bucketed aggregates instead of values. //! -//! [`parse_read_response`] turns a body into one [`BinaryDatapoints`] per series, columnar, with -//! the frames of one series merged. [`TimeSeriesService::retrieve_datapoints_binary`] posts the -//! request and parses the answer. +//! [`TimeSeriesService::stream_datapoints_binary`] hands the frames over as they arrive, decoded +//! on blocking threads while the rest download; [`TimeSeriesService::retrieve_datapoints_binary`] +//! collects that stream into one [`BinaryDatapoints`] per series. [`parse_read_response`] does the +//! same for a body already in memory. use std::io::Cursor; +use std::pin::Pin; use std::sync::atomic::{AtomicUsize, Ordering}; use std::sync::Arc; +use std::task::{Context, Poll}; use arrow_array::{ Array, Decimal128Array, Float32Array, Float64Array, Int64Array, RecordBatch, StringArray, @@ -19,6 +22,9 @@ use arrow_array::{ }; use arrow_ipc::reader::StreamReader; use arrow_schema::{DataType, Field, Schema, SchemaRef, TimeUnit}; +use bytes::{Buf, Bytes, BytesMut}; +use futures::stream::{FuturesOrdered, Stream, StreamExt}; +use tokio::sync::mpsc; use oauth2::http::StatusCode; use crate::generic::{ApiServiceProvider, DataWrapper, RetrieveFilter}; @@ -182,43 +188,51 @@ fn read_varint(body: &[u8], pos: &mut usize) -> Option { /// Errors on anything short of a complete, well-formed answer: a missing end marker, bytes after /// it, a frame that breaks the envelope, or a payload outside its canon. pub fn parse_read_response(body: &[u8]) -> Result, String> { - let frames = scan_frames(body)?; + let mut assembler = FrameAssembler::new(); + assembler.push(body); + let mut frames = Vec::new(); + while let Some(frame) = assembler.next_frame()? { + frames.push(frame); + } + assembler.finish()?; let decoded = decode_frames(&frames)?; + merge_series(decoded.into_iter().flatten()) +} + +/// Merges consecutive parts of one series, the way they come off the frames, into one entry each. +fn merge_series( + parts: impl IntoIterator, +) -> Result, String> { // Grouped before merging so each series is allocated once at its final size: extending part // by part regrows and recopies a long series dozens of times. - let mut groups: Vec> = Vec::new(); - for (index, parts) in decoded.into_iter().enumerate() { - for part in parts { - let continues = groups.last().and_then(|g| g.last()).is_some_and(|(_, last)| { - last.id == part.id - && last.is_aggregates() == part.is_aggregates() - && last.timestamps.last() < part.timestamps.first() - }); - if continues { - groups.last_mut().unwrap().push((index, part)); - } else { - groups.push(vec![(index, part)]); - } + let mut groups: Vec> = Vec::new(); + for part in parts { + let continues = groups.last().and_then(|g| g.last()).is_some_and(|last| { + last.id == part.id + && last.is_aggregates() == part.is_aggregates() + && last.timestamps.last() < part.timestamps.first() + }); + if continues { + groups.last_mut().unwrap().push(part); + } else { + groups.push(vec![part]); } } groups.into_iter().map(merge).collect() } /// One series from its consecutive parts, in order. -fn merge(parts: Vec<(usize, BinaryDatapoints)>) -> Result { - let rows: usize = parts.iter().map(|(_, p)| p.len()).sum(); +fn merge(parts: Vec) -> Result { + let rows: usize = parts.iter().map(BinaryDatapoints::len).sum(); let mut parts = parts.into_iter(); - let (_, mut series) = parts.next().expect("a group has at least one part"); + let mut series = parts.next().expect("a group has at least one part"); series.timestamps.reserve_exact(rows - series.len()); series.values.reserve_exact(rows - series.len()); - for (index, part) in parts { + for part in parts { let BinaryDatapoints { timestamps, values, .. } = part; series.timestamps.extend(timestamps); if !series.values.append(values) { - return Err(format!( - "frame {index}: series {} changes shape between frames", - series.id - )); + return Err(format!("series {} changes shape between frames", series.id)); } } Ok(series) @@ -228,65 +242,80 @@ fn cut_short(index: usize) -> String { format!("the response ends without its end marker, at frame {index}; it was cut short") } -/// One frame's envelope, checked, with its payload still compressed. -struct FrameView<'a> { +/// Cuts a response into frames as its bytes arrive, in chunks of any size: a frame's header, its +/// directory and its payload may each be split anywhere. +struct FrameAssembler { + buf: BytesMut, index: usize, - value_type: DatapointValueType, - aggregates: bool, - rows: usize, - names: Vec<(i64, String)>, - payload: &'a [u8], - raw_len: usize, + ended: bool, } -/// Every frame up to the end marker, in order. -fn scan_frames(body: &[u8]) -> Result>, String> { - let mut frames = Vec::new(); - let mut pos = 0; - loop { - let rest = &body[pos..]; - if rest.starts_with(END_OF_RESPONSE) { - if rest.len() != END_OF_RESPONSE.len() { - return Err(format!( - "{} bytes after the end marker", - rest.len() - END_OF_RESPONSE.len() - )); +impl FrameAssembler { + fn new() -> Self { + FrameAssembler { + buf: BytesMut::new(), + index: 0, + ended: false, + } + } + + fn push(&mut self, chunk: &[u8]) { + self.buf.extend_from_slice(chunk); + } + + /// The next complete frame; `None` until more bytes arrive, and after the end marker. + fn next_frame(&mut self) -> Result, String> { + if !self.ended && self.buf.starts_with(END_OF_RESPONSE) { + self.ended = true; + self.buf.advance(END_OF_RESPONSE.len()); + } + if self.ended { + if !self.buf.is_empty() { + return Err(format!("{} bytes after the end marker", self.buf.len())); } - return Ok(frames); + return Ok(None); } - if rest.len() < HEADER_BYTES { - return Err(cut_short(frames.len())); + if self.buf.len() >= MAGIC.len() && !self.buf.starts_with(MAGIC) { + return Err(format!("frame {}: does not start with DHDP", self.index)); } - let (consumed, frame) = scan_frame(rest, frames.len())?; - frames.push(frame); - pos += consumed; + if self.buf.len() < HEADER_BYTES { + return Ok(None); + } + let total = check_header(&self.buf[..HEADER_BYTES], self.index)?; + if self.buf.len() < total { + return Ok(None); + } + let frame = FrameView::parse(self.buf.split_to(total).freeze(), self.index)?; + self.index += 1; + Ok(Some(frame)) + } + + /// After the last byte: the body has to have ended with its marker and nothing after it. + fn finish(&mut self) -> Result<(), String> { + if self.next_frame()?.is_some() || !self.ended { + return Err(cut_short(self.index)); + } + Ok(()) } } -/// One frame's envelope and directory: how many bytes it takes, and the view of it. -fn scan_frame(body: &[u8], index: usize) -> Result<(usize, FrameView<'_>), String> { +/// Checks a frame's 28-byte header and returns the frame's whole length. +fn check_header(header: &[u8], index: usize) -> Result { let fail = |message: String| format!("frame {index}: {message}"); - if &body[0..4] != MAGIC { - return Err(fail("does not start with DHDP".to_string())); + if header[4] != VERSION { + return Err(fail(format!("version {}", header[4]))); } - if body[4] != VERSION { - return Err(fail(format!("version {}", body[4]))); - } - let value_type = DatapointValueType::from_id(body[5]) - .ok_or_else(|| fail(format!("value type id {}", body[5])))?; - let aggregates = match body[6] { + let value_type = DatapointValueType::from_id(header[5]) + .ok_or_else(|| fail(format!("value type id {}", header[5])))?; + let aggregates = match header[6] { CODEC_ARROW_IPC => false, CODEC_ARROW_IPC_AGGREGATES => true, codec => return Err(fail(format!("codec {codec}"))), }; - if body[7] != COMPRESSION_ZSTD { - return Err(fail(format!("compression {}", body[7]))); - } - let rows = u32_at(body, 8); - let series_count = u32_at(body, 12); - let directory_len = u32_at(body, 16); - let payload_len = u32_at(body, 20); - let raw_len = u32_at(body, 24); + if header[7] != COMPRESSION_ZSTD { + return Err(fail(format!("compression {}", header[7]))); + } + let rows = u32_at(header, 8); let max_rows = if aggregates { MAX_AGGREGATE_ROWS } else { @@ -295,53 +324,72 @@ fn scan_frame(body: &[u8], index: usize) -> Result<(usize, FrameView<'_>), Strin if rows < 1 || rows > max_rows { return Err(fail(format!("{rows} rows, allowed 1 to {max_rows}"))); } + let raw_len = u32_at(header, 24); if raw_len > MAX_FRAME_RAW_BYTES { return Err(fail(format!( "payload is {raw_len} bytes raw, the cap is {MAX_FRAME_RAW_BYTES}" ))); } - let total = HEADER_BYTES + directory_len + payload_len; - if body.len() < total { - return Err(cut_short(index)); - } - - let directory = &body[HEADER_BYTES..HEADER_BYTES + directory_len]; - let mut names: Vec<(i64, String)> = Vec::with_capacity(series_count); - let mut at = 0; - for _ in 0..series_count { - let id = directory - .get(at..at + 8) - .map(|b| i64::from_le_bytes(b.try_into().unwrap())) - .ok_or_else(|| fail("directory is shorter than its series count".to_string()))?; - at += 8; - let len = read_varint(directory, &mut at) - .ok_or_else(|| fail("directory is shorter than its series count".to_string()))?; - let name = directory - .get(at..at + len) - .ok_or_else(|| fail("directory is shorter than its series count".to_string()))?; - at += len; - let name = String::from_utf8(name.to_vec()) - .map_err(|_| fail(format!("external id of series {id} is not UTF-8")))?; - names.push((id, name)); - } - if at != directory_len { - return Err(fail(format!("{} bytes after the directory", directory_len - at))); - } - let frame = FrameView { - index, - value_type, - aggregates, - rows, - names, - payload: &body[HEADER_BYTES + directory_len..total], - raw_len, - }; - Ok((total, frame)) + Ok(HEADER_BYTES + u32_at(header, 16) + u32_at(header, 20)) +} + +/// One complete frame whose header passed [`check_header`], its payload still compressed. Owns its +/// bytes, so it can be decoded on another thread. +struct FrameView { + index: usize, + value_type: DatapointValueType, + aggregates: bool, + rows: usize, + names: Vec<(i64, String)>, + bytes: Bytes, + payload_start: usize, + raw_len: usize, +} + +impl FrameView { + fn parse(bytes: Bytes, index: usize) -> Result { + let fail = |message: String| format!("frame {index}: {message}"); + let series_count = u32_at(&bytes, 12); + let directory_len = u32_at(&bytes, 16); + let directory = &bytes[HEADER_BYTES..HEADER_BYTES + directory_len]; + let mut names: Vec<(i64, String)> = Vec::with_capacity(series_count); + let mut at = 0; + for _ in 0..series_count { + let id = directory + .get(at..at + 8) + .map(|b| i64::from_le_bytes(b.try_into().unwrap())) + .ok_or_else(|| fail("directory is shorter than its series count".to_string()))?; + at += 8; + let len = read_varint(directory, &mut at) + .ok_or_else(|| fail("directory is shorter than its series count".to_string()))?; + let name = directory + .get(at..at + len) + .ok_or_else(|| fail("directory is shorter than its series count".to_string()))?; + at += len; + let name = String::from_utf8(name.to_vec()) + .map_err(|_| fail(format!("external id of series {id} is not UTF-8")))?; + names.push((id, name)); + } + if at != directory_len { + return Err(fail(format!("{} bytes after the directory", directory_len - at))); + } + let aggregates = bytes[6] == CODEC_ARROW_IPC_AGGREGATES; + Ok(FrameView { + index, + value_type: DatapointValueType::from_id(bytes[5]).expect("checked with the header"), + aggregates, + rows: u32_at(&bytes, 8), + names, + payload_start: HEADER_BYTES + directory_len, + raw_len: u32_at(&bytes, 24), + bytes, + }) + } } /// Decodes every frame, on as many threads as there are cores and frames, keeping their order. /// On failure, the error of the earliest frame that failed. -fn decode_frames(frames: &[FrameView<'_>]) -> Result>, String> { +fn decode_frames(frames: &[FrameView]) -> Result>, String> { let threads = std::thread::available_parallelism() .map_or(1, |n| n.get()) .min(frames.len()); @@ -349,32 +397,33 @@ fn decode_frames(frames: &[FrameView<'_>]) -> Result>, return frames.iter().map(decode_frame).collect(); } let next = AtomicUsize::new(0); - let mut decoded: Vec<(usize, Result, String>)> = std::thread::scope(|scope| { - let workers: Vec<_> = (0..threads) - .map(|_| { - scope.spawn(|| { - let mut done = Vec::new(); - loop { - let i = next.fetch_add(1, Ordering::Relaxed); - let Some(frame) = frames.get(i) else { return done }; - done.push((i, decode_frame(frame))); - } + let mut decoded: Vec<(usize, Result, String>)> = + std::thread::scope(|scope| { + let workers: Vec<_> = (0..threads) + .map(|_| { + scope.spawn(|| { + let mut done = Vec::new(); + loop { + let i = next.fetch_add(1, Ordering::Relaxed); + let Some(frame) = frames.get(i) else { return done }; + done.push((i, decode_frame(frame))); + } + }) }) - }) - .collect(); - workers - .into_iter() - .flat_map(|w| w.join().expect("a frame decoder panicked")) - .collect() - }); + .collect(); + workers + .into_iter() + .flat_map(|w| w.join().expect("a frame decoder panicked")) + .collect() + }); decoded.sort_unstable_by_key(|(i, _)| *i); decoded.into_iter().map(|(_, result)| result).collect() } /// One frame's rows, split into one part per series it names. -fn decode_frame(frame: &FrameView<'_>) -> Result, String> { +fn decode_frame(frame: &FrameView) -> Result, String> { let fail = |message: String| format!("frame {}: {message}", frame.index); - let raw = zstd::bulk::decompress(frame.payload, frame.raw_len) + let raw = zstd::bulk::decompress(&frame.bytes[frame.payload_start..], frame.raw_len) .map_err(|e| fail(format!("zstd: {e}")))?; if raw.len() != frame.raw_len { return Err(fail(format!( @@ -418,6 +467,122 @@ fn decode_frame(frame: &FrameView<'_>) -> Result, String> Ok(parts) } +type Item = Result; + +/// A binary read as it arrives: one [`BinaryDatapoints`] per frame, in the order the frames come, +/// which is the order of the request's items. Consecutive items may belong to the same series — +/// a series longer than one frame comes as several, oldest first — so an item is a *chunk* of a +/// series, not all of it. +/// +/// Frames are decoded on blocking threads, several at a time, while the rest are still +/// downloading. A refusal is not an item: it is the error [`TimeSeriesService::stream_datapoints_binary`] +/// returns before there is a stream. **A response that breaks off is an error item after the +/// items that did arrive**, so a consumer that acts on items as they come has to treat what it +/// already took as incomplete when one appears; the stream ends after it. Dropping the stream +/// closes the connection. +/// +/// Read it with [`futures::StreamExt::next`]. +pub struct BinaryDatapointStream { + items: mpsc::Receiver, + task: tokio::task::JoinHandle<()>, +} + +impl Stream for BinaryDatapointStream { + type Item = Item; + + fn poll_next(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll> { + self.items.poll_recv(cx) + } +} + +impl Drop for BinaryDatapointStream { + fn drop(&mut self) { + self.task.abort(); + } +} + +/// Frames decoded at once. Past a handful the decoders only compete for memory bandwidth. +fn decoders() -> usize { + std::thread::available_parallelism().map_or(1, |n| n.get()).clamp(1, 8) +} + +fn unreadable(path: &str, message: String) -> ResponseError { + ResponseError { + status: StatusCode::BAD_GATEWAY, + message: format!("POST {path} returned an unreadable answer: {message}"), + content_type: None, + } +} + +/// Reads the body, cuts it into frames, decodes up to [`decoders`] of them at once and sends their +/// parts on in frame order. Whatever fails, the parts of the frames before it are sent first. +async fn produce(response: reqwest::Response, path: String, items: mpsc::Sender) { + let limit = decoders(); + let mut body = response.bytes_stream(); + let mut assembler = FrameAssembler::new(); + let mut pending = FuturesOrdered::new(); + let mut failure: Option = None; + let mut eof = false; + loop { + while failure.is_none() && pending.len() < limit { + match assembler.next_frame() { + Ok(Some(frame)) => { + pending.push_back(tokio::task::spawn_blocking(move || decode_frame(&frame))) + } + Ok(None) => break, + Err(message) => failure = Some(unreadable(&path, message)), + } + } + let reading = !eof && failure.is_none() && pending.len() < limit; + if pending.is_empty() && !reading { + let last = match failure { + Some(error) => Some(error), + None => assembler.finish().err().map(|m| unreadable(&path, m)), + }; + if let Some(error) = last { + let _ = items.send(Err(error)).await; + } + return; + } + tokio::select! { + biased; + Some(decoded) = pending.next(), if !pending.is_empty() => match decoded { + Ok(Ok(parts)) => { + for part in parts { + if items.send(Ok(part)).await.is_err() { + return; + } + } + } + // The earliest frame still in flight, so its error comes before any other. + Ok(Err(message)) => { + failure = Some(unreadable(&path, message)); + pending = FuturesOrdered::new(); + } + Err(join) => { + failure = Some(ResponseError { + status: StatusCode::INTERNAL_SERVER_ERROR, + message: format!("decoding a frame failed: {join}"), + content_type: None, + }); + pending = FuturesOrdered::new(); + } + }, + chunk = body.next(), if reading => match chunk { + Some(Ok(bytes)) => assembler.push(&bytes), + Some(Err(error)) => { + failure = Some(ResponseError { + status: StatusCode::BAD_GATEWAY, + message: format!("the response was cut short while it was being read: {error}"), + content_type: None, + }) + } + None => eof = true, + }, + } + } +} + /// Appends a batch's rows to `parts`, one part per run of a series id. fn split_batch( batch: &RecordBatch, @@ -554,26 +719,49 @@ impl TimeSeriesService { /// read is a 400 for the whole request, not an empty result. A response the server could not /// finish is a 502 whose message says it was cut short, never fewer points; sending the /// request again is safe. + /// + /// The frames are decoded as they arrive, so this costs little more than the download; for + /// rows as they come, or a window too large to hold, use + /// [`stream_datapoints_binary`](Self::stream_datapoints_binary). pub async fn retrieve_datapoints_binary( &self, json: &DataWrapper, ) -> Result, ResponseError> { let path = format!("{}/data/list/binary", self.base_url); - let body = self - .execute_post_request_for_bytes(&path, json, MEDIA_TYPE) - .await?; - let parsed = tokio::task::spawn_blocking(move || parse_read_response(&body)) + let mut stream = self.stream_datapoints_binary(json).await?; + let mut parts = Vec::new(); + while let Some(part) = stream.next().await { + parts.push(part?); + } + tokio::task::spawn_blocking(move || merge_series(parts)) .await .map_err(|e| ResponseError { status: StatusCode::INTERNAL_SERVER_ERROR, - message: format!("decoding the binary read failed: {e}"), + message: format!("merging the binary read failed: {e}"), content_type: None, - })?; - parsed.map_err(|message| ResponseError { - status: StatusCode::BAD_GATEWAY, - message: format!("POST {path} returned an unreadable answer: {message}"), - content_type: None, - }) + })? + .map_err(|message| unreadable(&path, message)) + } + + /// The same read as [`retrieve_datapoints_binary`](Self::retrieve_datapoints_binary), handed + /// over frame by frame as the response arrives instead of collected: see + /// [`BinaryDatapointStream`] for what an item is and how a broken-off response shows up. + /// + /// The request is checked before there is a stream, so a refusal (a 400 for an unknown + /// series, a `cursor`, aggregates on a text series) is the `Err` here rather than an item. + /// Holding the stream holds the connection, and the server's query, open: a consumer that + /// stops reading for long can run into the server's timeouts. + pub async fn stream_datapoints_binary( + &self, + json: &DataWrapper, + ) -> Result { + let path = format!("{}/data/list/binary", self.base_url); + let response = self + .execute_post_request_for_stream(&path, json, MEDIA_TYPE) + .await?; + let (tx, items) = mpsc::channel(2 * decoders()); + let task = tokio::spawn(produce(response, path, tx)); + Ok(BinaryDatapointStream { items, task }) } } @@ -739,6 +927,129 @@ mod tests { assert!(error.starts_with("frame 17:"), "{error}"); } + fn assemble_in_chunks(body: &[u8], chunk: usize) -> Result, String> { + let mut assembler = FrameAssembler::new(); + let mut frames = Vec::new(); + for piece in body.chunks(chunk) { + assembler.push(piece); + while let Some(frame) = assembler.next_frame()? { + frames.push(frame); + } + } + assembler.finish()?; + merge_series(decode_frames(&frames)?.into_iter().flatten()) + } + + #[test] + fn a_frame_may_be_split_anywhere() { + for body in [JAVA_VALUES, JAVA_AGGREGATES] { + let whole = parse_read_response(body).unwrap(); + for chunk in [1, 3, 7, 28, 1000, 65536] { + assert_eq!(assemble_in_chunks(body, chunk).unwrap(), whole, "chunks of {chunk}"); + } + } + // Cut inside the marker itself: "DH" then nothing. + let mut body = response(&[float_frame(1, "a", 0..2)]); + body.truncate(body.len() - 2); + assert!(assemble_in_chunks(&body, 1).unwrap_err().contains("cut short")); + } + + /// A local HTTP server answering with `chunks` as a chunked body, each written and flushed on + /// its own; the chunk at `gate_after` waits for a permit on `permits`. + async fn serve(chunks: Vec>, permits: tokio::sync::mpsc::Receiver<()>, gate_after: usize) -> String { + use tokio::io::{AsyncReadExt, AsyncWriteExt}; + let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); + let address = listener.local_addr().unwrap(); + tokio::spawn(async move { + let mut permits = permits; + let (mut socket, _) = listener.accept().await.unwrap(); + let mut request = vec![0u8; 4096]; + let _ = socket.read(&mut request).await; + socket + .write_all(b"HTTP/1.1 200 OK\r\nContent-Type: application/vnd.intellistream.datapoint-block\r\nTransfer-Encoding: chunked\r\n\r\n") + .await + .unwrap(); + for (i, chunk) in chunks.iter().enumerate() { + if i == gate_after { + permits.recv().await; + } + socket.write_all(format!("{:x}\r\n", chunk.len()).as_bytes()).await.unwrap(); + socket.write_all(chunk).await.unwrap(); + socket.write_all(b"\r\n").await.unwrap(); + socket.flush().await.unwrap(); + } + let _ = socket.write_all(b"0\r\n\r\n").await; + let _ = socket.shutdown().await; + }); + format!("http://{address}/") + } + + async fn stream_of(url: &str) -> BinaryDatapointStream { + let response = reqwest::get(url).await.unwrap(); + let (tx, items) = mpsc::channel(4); + let task = tokio::spawn(produce(response, "test".to_string(), tx)); + BinaryDatapointStream { items, task } + } + + #[tokio::test] + async fn items_arrive_while_the_body_is_still_coming() { + let first = float_frame(1, "a", 0..3); + let rest = [float_frame(1, "a", 3..5), float_frame(2, "b", 0..2), END_OF_RESPONSE.to_vec()].concat(); + let (release, permits) = tokio::sync::mpsc::channel(1); + let url = serve(vec![first, rest], permits, 1).await; + let mut stream = stream_of(&url).await; + + // The server sends nothing more until this item is in hand. + let item = tokio::time::timeout(std::time::Duration::from_secs(5), stream.next()) + .await + .expect("the first frame did not arrive before the rest of the body") + .unwrap() + .unwrap(); + assert_eq!((item.id, item.timestamps.clone()), (1, vec![0, 1, 2])); + release.send(()).await.unwrap(); + + let rest: Vec = stream.map(|i| i.unwrap()).collect().await; + assert_eq!(rest.iter().map(|s| s.id).collect::>(), [1, 2], "one item per frame"); + let merged = merge_series(std::iter::once(item).chain(rest)).unwrap(); + assert_eq!(merged.len(), 2); + assert_eq!(merged[0].timestamps, [0, 1, 2, 3, 4]); + } + + #[tokio::test] + async fn a_stream_cut_short_ends_with_an_error_after_what_arrived() { + let frames = vec![float_frame(1, "a", 0..2), float_frame(2, "b", 0..2)]; + // Every frame intact and the chunked body properly ended, but no end marker. + let (_release, permits) = tokio::sync::mpsc::channel(1); + let url = serve(frames, permits, usize::MAX).await; + let items: Vec = stream_of(&url).await.collect().await; + assert_eq!(items.len(), 3); + assert!(items[0].is_ok() && items[1].is_ok()); + let error = items[2].as_ref().unwrap_err(); + assert_eq!(error.get_status(), StatusCode::BAD_GATEWAY); + assert!(error.get_message().contains("cut short"), "{}", error.get_message()); + } + + #[tokio::test] + async fn a_bad_frame_ends_the_stream_after_the_frames_before_it() { + let batch = RecordBatch::try_new( + DatapointValueType::Float.schema(), + vec![ + Arc::new(Int64Array::from(vec![1])), + Arc::new(TimestampMillisecondArray::from(vec![0]).with_timezone("UTC")), + Arc::new(Float64Array::from(vec![1.0])), + ], + ) + .unwrap(); + let bad = frame_around(DatapointValueType::Float, CODEC_ARROW_IPC_AGGREGATES, 2, "b", &batch); + let body = response(&[float_frame(1, "a", 0..2), bad, float_frame(3, "c", 0..2)]); + let (_release, permits) = tokio::sync::mpsc::channel(1); + let url = serve(body.chunks(5).map(<[u8]>::to_vec).collect(), permits, usize::MAX).await; + let items: Vec = stream_of(&url).await.collect().await; + assert_eq!(items.len(), 2, "the good frame, then the error; nothing after it"); + assert_eq!(items[0].as_ref().unwrap().id, 1); + assert!(items[1].as_ref().unwrap_err().get_message().contains("frame 1:")); + } + /// The same series asked for twice, the second window starting before the first ended: two /// entries, since merging them would put the timestamps out of order. #[test] diff --git a/src/timeseries/mod.rs b/src/timeseries/mod.rs index f8a64dc..df2035a 100644 --- a/src/timeseries/mod.rs +++ b/src/timeseries/mod.rs @@ -4,7 +4,7 @@ pub mod datapoint_listen; mod test; pub use binary::{BinaryIngestOptions, DatapointValueType, Frame, FrameWriter, ResolvedSeries}; -pub use binary_read::{BinaryDatapoints, BinaryValues}; +pub use binary_read::{BinaryDatapointStream, BinaryDatapoints, BinaryValues}; pub use datapoint_listen::{DatapointListener, LiveDatapoint}; use crate::buffer::DurableSpool; diff --git a/src/timeseries/test.rs b/src/timeseries/test.rs index 18fba9e..bfcecff 100644 --- a/src/timeseries/test.rs +++ b/src/timeseries/test.rs @@ -779,8 +779,9 @@ mod tests { whole_window.set_external_id(&float_ext_id); whole_window.set_start(start); whole_window.set_end(start + Duration::seconds(count)); + let request = DataWrapper::from_vec(vec![whole_window, item(&text_ext_id, text_count as u64)]); let first = api_service.time_series - .retrieve_datapoints_binary(&DataWrapper::from_vec(vec![whole_window, item(&text_ext_id, text_count as u64)])) + .retrieve_datapoints_binary(&request) .await .unwrap_or_else(|e| panic!("binary read failed with {}: {}", e.get_status(), e.get_message())); assert_eq!(first.len(), 2, "one entry per series, the frames of one merged"); @@ -792,6 +793,19 @@ mod tests { let want_values: Vec = (0..count).map(value).collect(); assert_eq!(float.values, BinaryValues::Float(want_values)); + println!("The same read, streamed frame by frame..."); + let mut stream = api_service.time_series.stream_datapoints_binary(&request).await?; + let (mut frames, mut streamed) = (Vec::new(), Vec::new()); + while let Some(chunk) = futures::StreamExt::next(&mut stream).await { + let chunk = chunk?; + if chunk.id == float.id { + streamed.extend_from_slice(&chunk.timestamps); + } + frames.push((chunk.len(), chunk.external_id)); + } + assert_eq!(frames.len(), 5, "two float frames and three text frames, one item each: {frames:?}"); + assert_eq!(streamed, want_timestamps); + let texts = &first[1]; assert_eq!(texts.external_id, text_ext_id); assert_eq!(texts.timestamps, want_timestamps[..text_count as usize]); From 2bebeaffe9742aebad2ebd979686009f3a5da8d9 Mon Sep 17 00:00:00 2001 From: jgjesdal Date: Mon, 12 Oct 2026 15:35:22 +0200 Subject: [PATCH 3/8] Give buffered and streamed binary reads one table, long or wide retrieve_datapoints_binary returns a BinaryReadResult: a sequence of the per-series BinaryDatapoints it returned before, plus the external_ids, long-table Arrow stream and to_polars the streamed reader has. Both call one pure-Python helper (_tables.py), so the tables cannot drift between them. to_polars(wide=True) gives one row per timestamp and one column per series, named by external id ({external_id}.{column} for mixed and aggregate series). Series on identical timestamps are put side by side for free; one value type is pivoted on a numeric key (1.7s per 100M rows, against 3.9s pivoting on names and 5.1s aligning); otherwise the frames are aligned, so different value types sit side by side. 100M float rows, 10 series: buffered and streamed, long or wide, all land within 0.07-0.50s of the server's 6.5s. Co-Authored-By: Claude Opus 5.5 Signed-off-by: jgjesdal --- AGENTS.md | 2 +- README.md | 19 ++- .../intellistream_datahub_sdk/__init__.pyi | 83 ++++++++++- .../intellistream_datahub_sdk/_tables.py | 60 ++++++++ .../src/timeseries/binary_read.rs | 134 ++++++++++++++++-- datahub_python_bindings/src/timeseries/mod.rs | 1 + .../src/timeseries/sync_service.rs | 6 +- python_tests/test_timeseries_binary_read.py | 93 +++++++++++- 8 files changed, 369 insertions(+), 29 deletions(-) create mode 100644 datahub_python_bindings/python/intellistream_datahub_sdk/_tables.py diff --git a/AGENTS.md b/AGENTS.md index 6636093..7205115 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -203,7 +203,7 @@ the generic default with `recognized: false`, not an error. `TimeSeriesService::insert_datapoints_binary` is the second ingest path, to `POST /timeseries/data/binary`: the same `DatapointsCollection` input, resolved through `/timeseries/byids` (cached per service instance; needs read access on the dataset), checked against each series' value type locally, cut into Arrow IPC frames at the contract's caps, zstd-compressed per frame (mandatory: level 1, 3 or 9, default 9) and posted through `execute_post_bytes_request`. Every refusal on this path is a problem document typed for its status — `invalid-frame` (400), `unknown-timeseries` (404), `request-too-large` (413), `unsupported-media-type` (415), `value-type-mismatch` and `external-id-mismatch` (422), `too-many-in-flight` (429) — with the kebab-case sub-case in a `reason` extension beside it. It was one type, `datapoint-block-rejected`, answering with all six statuses until the api split it (platform #120); the SDK matches the new slugs only, and nothing here recognises the old one — the binary path has never been in a release, so no caller can be on the other side of that change. `is_stale_series_rejection` reads the slug to decide the one re-resolve-and-retry, and reads it from the problem rather than the body text because the SDK's own pre-flight 404 names the series it could not resolve — an external id spelling `unknown-timeseries` used to trigger the retry. `FrameWriter` builds one frame and is public; `cut_into_writers` and `pack_requests` hold the caps. The byte layout is the platform's `binary_datapoints_format.md`. The arrow-rs crates (`arrow-array`, `arrow-schema`, `arrow-ipc`) serve both directions. In Python it is `insert_datapoints_binary` and `insert_from_lists_binary` on the **sync** client only, each taking an optional `zstd_level`; the async client has no binary methods. The ignored `timeseries::tests::test_datapoints_binary` is the live twin of `test_datapoints` and needs a backend that serves the endpoint; the writer's own tests in `binary.rs` run offline. `python_tests/test_timeseries_binary.py` round-trips the Python binding through the JSON read path. -The read is `TimeSeriesService::retrieve_datapoints_binary`, to `POST /timeseries/data/list/binary`: the JSON retrieve body in, the same frames out, one series per frame, then the 4-byte marker `DHDE`. A body without the marker was cut short (the server streams, so a failure after the first bytes can only end the body early) and is a **502** saying so, never fewer points; bytes after it are refused too. Codec 2 frames carry bucketed aggregates in their own schema (`avg`/`min`/`max`/`sum`, Float64, 50 000 buckets per frame) while the envelope's type byte stays the series' own. `parse_read_response` returns one columnar `BinaryDatapoints` per series, merging consecutive frames of one series; decimals stay exact as unscaled `i128` plus scale. `TimeSeriesService::stream_datapoints_binary` is the primitive: a `FrameAssembler` cuts the body into frames as chunks arrive (a frame may be split anywhere), each frame is an owned `Bytes` decoded on `spawn_blocking`, at most `min(cores, 8)` in flight through a `FuturesOrdered` so items stay in frame order, and a producer task feeds a bounded channel behind the `BinaryDatapointStream` (dropping it aborts the task and closes the connection). One item per frame, so a long series is several consecutive items. A refusal is the `Err` before there is a stream; a response that breaks off is an error item after the items that arrived, and that is the semantic difference stream consumers must handle. `retrieve_datapoints_binary` collects the stream and merges consecutive parts (each series allocated once at its final size); `parse_read_response` does the same for a body in memory, decoding on scoped threads. Measured on 100M float rows, 10 series: the server takes ~6.5 s; the buffered call adds ~0.24 s (the merge copy) and the stream into polars ~0.02 s. Single-threaded zstd used to add 1.3 s on its own. Differences from the JSON read worth knowing: no `cursor` (a 400; advance `start` past the last timestamp), a series with no points is absent rather than empty, and an unknown or unreadable series is a 400 for the whole request — `DataWrapper` serialises only `items`, so the request-level `ignoreUnknownIds` and `limit` are not reachable. `src/timeseries/fixtures/read_*.bin` are response bodies written by the platform's own `DatapointFrameWriter.wrap`, so the offline tests in `binary_read.rs` prove arrow-rs reads what the Java writer emits; regenerate them when the platform's writer changes. The ignored `timeseries::tests::test_datapoints_binary_read` is the live check against the JSON read. In Python it is `retrieve_datapoints_binary` on the sync client, plus `retrieve_datapoints_binary_stream`, whose single-use `BinaryDatapointReader` iterates per frame or is an Arrow stream of one long table (`timeseries_id`, `timestamp`, values; one schema, so a read mixing value types must be iterated). The long table carries no external id column on purpose: as a dictionary column, polars spent ~0.8 s per 100M rows making it categorical, more than everything else; `reader.external_ids` maps the ids instead. `retrieve_datapoints_binary` returns `BinaryDatapoints` objects that implement the Arrow PyCapsule stream protocol (`__arrow_c_stream__`, via `arrow-array`'s `ffi` feature enabled in the bindings crate only), so polars, pyarrow and pandas 3 read them zero-copy; the package still declares no runtime dependency, and `to_polars`/`to_pandas` import theirs lazily. `python_tests/test_timeseries_binary_read.py` covers it offline against a stand-in server and live when the backend serves the endpoint. +The read is `TimeSeriesService::retrieve_datapoints_binary`, to `POST /timeseries/data/list/binary`: the JSON retrieve body in, the same frames out, one series per frame, then the 4-byte marker `DHDE`. A body without the marker was cut short (the server streams, so a failure after the first bytes can only end the body early) and is a **502** saying so, never fewer points; bytes after it are refused too. Codec 2 frames carry bucketed aggregates in their own schema (`avg`/`min`/`max`/`sum`, Float64, 50 000 buckets per frame) while the envelope's type byte stays the series' own. `parse_read_response` returns one columnar `BinaryDatapoints` per series, merging consecutive frames of one series; decimals stay exact as unscaled `i128` plus scale. `TimeSeriesService::stream_datapoints_binary` is the primitive: a `FrameAssembler` cuts the body into frames as chunks arrive (a frame may be split anywhere), each frame is an owned `Bytes` decoded on `spawn_blocking`, at most `min(cores, 8)` in flight through a `FuturesOrdered` so items stay in frame order, and a producer task feeds a bounded channel behind the `BinaryDatapointStream` (dropping it aborts the task and closes the connection). One item per frame, so a long series is several consecutive items. A refusal is the `Err` before there is a stream; a response that breaks off is an error item after the items that arrived, and that is the semantic difference stream consumers must handle. `retrieve_datapoints_binary` collects the stream and merges consecutive parts (each series allocated once at its final size); `parse_read_response` does the same for a body in memory, decoding on scoped threads. Measured on 100M float rows, 10 series: the server takes ~6.5 s; the buffered call adds ~0.24 s (the merge copy) and the stream into polars ~0.02 s. Single-threaded zstd used to add 1.3 s on its own. Differences from the JSON read worth knowing: no `cursor` (a 400; advance `start` past the last timestamp), a series with no points is absent rather than empty, and an unknown or unreadable series is a 400 for the whole request — `DataWrapper` serialises only `items`, so the request-level `ignoreUnknownIds` and `limit` are not reachable. `src/timeseries/fixtures/read_*.bin` are response bodies written by the platform's own `DatapointFrameWriter.wrap`, so the offline tests in `binary_read.rs` prove arrow-rs reads what the Java writer emits; regenerate them when the platform's writer changes. The ignored `timeseries::tests::test_datapoints_binary_read` is the live check against the JSON read. In Python it is `retrieve_datapoints_binary` on the sync client, plus `retrieve_datapoints_binary_stream`, whose single-use `BinaryDatapointReader` iterates per frame or is an Arrow stream of one long table (`timeseries_id`, `timestamp`, values; one schema, so a read mixing value types must be iterated). The long table carries no external id column on purpose: as a dictionary column, polars spent ~0.8 s per 100M rows making it categorical, more than everything else; `reader.external_ids` maps the ids instead. The buffered `retrieve_datapoints_binary` returns a `BinaryReadResult`, a sequence of per-series `BinaryDatapoints` with the same `external_ids`, long-table Arrow stream and `to_polars(wide=False)` as the reader: both call `python/intellistream_datahub_sdk/_tables.py`, so the two tables cannot drift. `wide=True` puts series side by side when every one has the same timestamps (free), pivots on a numeric key when they share one value column type (~1.7 s per 100M rows; joining names first doubled it, `concat(how="align")` tripled it), and aligns otherwise, which is how mixed value types sit side by side. Each per-series `BinaryDatapoints` implements the Arrow PyCapsule stream protocol (`__arrow_c_stream__`, via `arrow-array`'s `ffi` feature enabled in the bindings crate only), so polars, pyarrow and pandas 3 read them zero-copy; the package still declares no runtime dependency, and `to_polars`/`to_pandas` import theirs lazily. `python_tests/test_timeseries_binary_read.py` covers it offline against a stand-in server and live when the backend serves the endpoint. ### The `ApiServiceProvider` trait (`src/generic.rs`) diff --git a/README.md b/README.md index 4e9f04d..0b1c6d7 100644 --- a/README.md +++ b/README.md @@ -182,11 +182,20 @@ returns objects that implement the Arrow PyCapsule stream protocol, so `polars.DataFrame(series)`, `pyarrow.table(series)` and `pandas.DataFrame.from_arrow(series)` take them without copying; `series.to_polars()` and `series.to_pandas()` do the same. The package adds no dependency for this — bring polars, or pyarrow for pandas. -`retrieve_datapoints_binary_stream(filter_or_list)` returns a single-use reader: iterate it for one -`BinaryDatapoints` per frame, or hand it to `polars.DataFrame(reader)` (or `reader.to_polars()`) -for one long table of `timeseries_id`, `timestamp` and the value columns, with -`reader.external_ids` mapping the ids. Reading 100M float rows into polars that way costs about -as long as the download itself. +`retrieve_datapoints_binary_stream(filter_or_list)` returns a single-use reader of the same read +as it arrives: iterate it for one `BinaryDatapoints` per frame. Both the buffered result and the +reader turn into one table the same way: + +```python +read = client.timeseries.retrieve_datapoints_binary(filters) # or ..._binary_stream(filters) +long = read.to_polars() # timeseries_id, timestamp, value: one row per point +wide = read.to_polars(wide=True) # timestamp, then one column per series named by external id +read.external_ids # {timeseries_id: external_id} +``` + +The wide table matches series on `timestamp`, leaving nulls where one has no point, and holds +series of different value types side by side. Reading 100M float rows into either costs within +half a second of the download itself. ## Python bindings diff --git a/datahub_python_bindings/python/intellistream_datahub_sdk/__init__.pyi b/datahub_python_bindings/python/intellistream_datahub_sdk/__init__.pyi index 1bf38b0..d2e39e6 100644 --- a/datahub_python_bindings/python/intellistream_datahub_sdk/__init__.pyi +++ b/datahub_python_bindings/python/intellistream_datahub_sdk/__init__.pyi @@ -603,7 +603,8 @@ class BinaryDatapointReader: ``pyarrow.RecordBatchReader.from_stream(reader)``) it is one long table: ``timeseries_id``, ``timestamp`` and the value columns, a batch per frame. Every series in the read has to share those columns, so a read mixing value types (or values and aggregates) raises at the first - batch that differs; iterate such a read instead. A read with no rows has no schema and raises + batch that differs; read such a read with ``to_polars(wide=True)`` or by iterating. A read + with no rows has no schema and raises ``ValueError``. The network is waited on with the GIL held, as the C stream protocol gives no way to release it. @@ -620,8 +621,75 @@ class BinaryDatapointReader: the key to the long table's ``timeseries_id``. Complete once the reader is exhausted.""" def __arrow_c_stream__(self, requested_schema: object | None = None) -> object: """The Arrow PyCapsule stream protocol, as the long table. ``requested_schema`` is ignored.""" - def to_polars(self) -> Any: - """The long table as a ``polars.DataFrame``. Needs polars; raises ``ImportError`` without it.""" + def to_polars(self, wide: bool = False) -> Any: + """A ``polars.DataFrame`` of the read; needs polars, and raises ``ImportError`` without it. + + Parameters + ---------- + wide + ``False`` (default): the long table, ``timeseries_id``, ``timestamp`` and the value + columns, one row per point. Every series has to share those columns. + ``True``: one row per timestamp and one column per series, named after its external + id (``{external_id}.{column}`` for a series with several value columns: ``mixed``, or + aggregates), matched on ``timestamp`` with nulls where a series has no point. Series + of different value types sit side by side, each keeping its type. + + Raises + ------ + ValueError + The read returned no rows; or, for the long table, its series do not share their + columns; or, for the wide one, a series has repeated timestamps. + + Notes + ----- + The same table whether the read was buffered or streamed. The wide table costs nothing + beyond the long one when every series has the same timestamps, and a pivot (about 1.7 s + per 100M rows) when they do not. + """ + + +class BinaryReadResult(Sequence[BinaryDatapoints]): + """A buffered binary read, from ``TimeSeriesServiceSync.retrieve_datapoints_binary``. + + A sequence of :class:`BinaryDatapoints`, one per series that had points, in the order of the + request; and the same tables a :class:`BinaryDatapointReader` gives, through + :meth:`to_polars` and the Arrow stream of the long table. + """ + + def __len__(self) -> int: ... + def __getitem__(self, index: int) -> BinaryDatapoints: ... # type: ignore[override] + def __iter__(self) -> Iterator[BinaryDatapoints]: ... + @property + def external_ids(self) -> dict[int, str]: + """``{timeseries_id: external_id}`` for the series in the read: the key to the long + table's ``timeseries_id``.""" + def __arrow_c_stream__(self, requested_schema: object | None = None) -> object: + """The Arrow PyCapsule stream protocol, as the long table. ``requested_schema`` is ignored.""" + def to_polars(self, wide: bool = False) -> Any: + """A ``polars.DataFrame`` of the read; needs polars, and raises ``ImportError`` without it. + + Parameters + ---------- + wide + ``False`` (default): the long table, ``timeseries_id``, ``timestamp`` and the value + columns, one row per point. Every series has to share those columns. + ``True``: one row per timestamp and one column per series, named after its external + id (``{external_id}.{column}`` for a series with several value columns: ``mixed``, or + aggregates), matched on ``timestamp`` with nulls where a series has no point. Series + of different value types sit side by side, each keeping its type. + + Raises + ------ + ValueError + The read returned no rows; or, for the long table, its series do not share their + columns; or, for the wide one, a series has repeated timestamps. + + Notes + ----- + The same table whether the read was buffered or streamed. The wide table costs nothing + beyond the long one when every series has the same timestamps, and a pivot (about 1.7 s + per 100M rows) when they do not. + """ class BinaryDatapoints: @@ -821,7 +889,7 @@ class TimeSeriesServiceSync: def retrieve_datapoints(self, input: RetrieveFilter) -> list[DatapointsCollectionDatapoints]: ... def retrieve_datapoints_binary( self, input: RetrieveFilter | list[RetrieveFilter] - ) -> list[BinaryDatapoints]: + ) -> BinaryReadResult: """Read datapoints through ``POST /timeseries/data/list/binary``: the request ``retrieve_datapoints`` takes, answered as compressed Arrow frames instead of JSON. @@ -836,9 +904,10 @@ class TimeSeriesServiceSync: Returns ------- - list[BinaryDatapoints] - One per series that has points in its window, in the order of ``input``; a series - with none is left out rather than returned empty. + BinaryReadResult + A sequence of :class:`BinaryDatapoints`, one per series that has points in its window, + in the order of ``input`` (a series with none is left out rather than returned empty), + with ``to_polars(wide=False)`` for one table of them all. Raises ------ diff --git a/datahub_python_bindings/python/intellistream_datahub_sdk/_tables.py b/datahub_python_bindings/python/intellistream_datahub_sdk/_tables.py new file mode 100644 index 0000000..81fba15 --- /dev/null +++ b/datahub_python_bindings/python/intellistream_datahub_sdk/_tables.py @@ -0,0 +1,60 @@ +"""Polars tables from a binary read, shared by the buffered result and the streamed reader so the +two cannot drift: the long table, and its wide form with one column per series.""" + + +def _polars(method): + try: + import polars + except ImportError: + raise ImportError(f"{method}() needs polars; install it with `pip install polars`") from None + return polars + + +def to_polars(read, wide): + """`read` is a `BinaryReadResult` or a `BinaryDatapointReader`: an Arrow stream of the long + table, and an iterable of `BinaryDatapoints` (whole series, or a reader's per-frame chunks).""" + pl = _polars("to_polars") + if not wide: + return pl.DataFrame(read) + return _wide(pl, read) + + +def _wide(pl, parts): + # Chunks of one series arrive consecutively; a dict keyed by id keeps the read's order. + series = {} + for part in parts: + external_id, frames = series.setdefault(part.id, (part.external_id, [])) + frames.append(part.to_polars()) + if not series: + raise ValueError("the read returned no rows, so there are no columns for a wide table") + + columns = [] + for external_id, frames in series.values(): + frame = pl.concat(frames, rechunk=False) if len(frames) > 1 else frames[0] + if not frame["timestamp"].is_sorted() or frame["timestamp"].n_unique() != frame.height: + raise ValueError( + f"series {external_id} has repeated timestamps, as when one series is asked for " + f"twice with overlapping windows; a wide table needs one value per timestamp" + ) + values = [c for c in frame.columns if c != "timestamp"] + names = {values[0]: external_id} if len(values) == 1 else {c: f"{external_id}.{c}" for c in values} + columns.append(frame.rename(names)) + + first = columns[0]["timestamp"] + if all(c.height == first.len() and c["timestamp"].equals(first) for c in columns[1:]): + # Every series on the same timestamps: the columns side by side, nothing to match up. + return columns[0].hstack([s for c in columns[1:] for s in c.drop("timestamp").get_columns()]) + + single_values = [c.columns[1:] for c in columns] + if all(len(v) == 1 for v in single_values) and len({c.dtypes[1] for c in columns}) == 1: + # One value column of one type: a pivot on a numeric key is the fastest way to match rows. + names = [v[0] for v in single_values] + long = pl.concat( + [c.rename({n: "value"}).with_columns(pl.lit(i, pl.Int32).alias("series")) for i, (c, n) in enumerate(zip(columns, names))], + rechunk=False, + ) + wide = long.pivot(on="series", index="timestamp", values="value").sort("timestamp") + return wide.rename({str(i): n for i, n in enumerate(names)}).select(["timestamp", *names]) + + # Different value types or several value columns: matched on timestamp, each keeping its type. + return pl.concat(columns, how="align") diff --git a/datahub_python_bindings/src/timeseries/binary_read.rs b/datahub_python_bindings/src/timeseries/binary_read.rs index f41efbd..e4aefd5 100644 --- a/datahub_python_bindings/src/timeseries/binary_read.rs +++ b/datahub_python_bindings/src/timeseries/binary_read.rs @@ -15,7 +15,7 @@ use intellistream_datahub_sdk::timeseries::{ use tokio::runtime::Runtime; use pyo3::exceptions::{PyImportError, PyRuntimeError, PyStopIteration, PyValueError}; use pyo3::prelude::*; -use pyo3::types::{PyCapsule, PyDict}; +use pyo3::types::{PyCapsule, PyDict, PyList}; /// One series' rows from `retrieve_datapoints_binary`, as an Arrow table: `timestamp` and the /// value columns of its type. Read it with anything that takes the Arrow PyCapsule protocol. @@ -295,10 +295,125 @@ impl PyBinaryDatapointReader { PyCapsule::new_with_value(py, ffi, c"arrow_array_stream") } - /// A `polars.DataFrame` of the long table. Needs polars. - fn to_polars<'py>(slf: &Bound<'py, Self>) -> PyResult> { - let polars = import(slf.py(), "polars", "to_polars")?; - polars.call_method1("DataFrame", (slf,)) + /// A `polars.DataFrame`: the long table, or with `wide` one column per series, matched on + /// timestamp. Needs polars. Reads the reader, like any other use of it. + #[pyo3(signature = (wide=false))] + fn to_polars<'py>(slf: &Bound<'py, Self>, wide: bool) -> PyResult> { + tables_to_polars(slf.as_any(), wide) + } +} + +fn tables_to_polars<'py>(read: &Bound<'py, PyAny>, wide: bool) -> PyResult> { + read.py() + .import("intellistream_datahub_sdk._tables")? + .call_method1("to_polars", (read, wide)) +} + +/// A buffered binary read, from `retrieve_datapoints_binary`: the series it returned, one +/// `BinaryDatapoints` each, which it behaves as a list of, and the same tables the streamed +/// reader gives. +#[pyclass(module = "intellistream_datahub_sdk", name = "BinaryReadResult", frozen, sequence)] +pub struct PyBinaryReadResult { + series: Vec>, +} + +impl PyBinaryReadResult { + pub(crate) fn new(py: Python<'_>, series: Vec) -> PyResult { + let series = series + .into_iter() + .map(|s| { + let s = PyBinaryDatapoints::try_from(s).map_err(|e| PyValueError::new_err(e.to_string()))?; + Py::new(py, s) + }) + .collect::>()?; + Ok(Self { series }) + } +} + +#[pymethods] +impl PyBinaryReadResult { + fn __len__(&self) -> usize { + self.series.len() + } + + fn __getitem__(&self, py: Python<'_>, index: isize) -> PyResult> { + let len = self.series.len() as isize; + let at = if index < 0 { index + len } else { index }; + if at < 0 || at >= len { + return Err(pyo3::exceptions::PyIndexError::new_err("series index out of range")); + } + Ok(self.series[at as usize].clone_ref(py)) + } + + fn __iter__<'py>(&self, py: Python<'py>) -> PyResult> { + Ok(PyList::new(py, &self.series)?.into_any().try_iter()?.into_any()) + } + + fn __repr__(&self) -> String { + let rows: usize = self.series.iter().map(|s| s.get().batch.num_rows()).sum(); + format!("BinaryReadResult(series={}, rows={rows})", self.series.len()) + } + + /// `{timeseries_id: external_id}` for the series in the read, in order: the key to the long + /// table's `timeseries_id`. + #[getter] + fn external_ids<'py>(&self, py: Python<'py>) -> PyResult> { + let ids = PyDict::new(py); + for s in &self.series { + ids.set_item(s.get().id, &s.get().external_id)?; + } + Ok(ids) + } + + /// The Arrow PyCapsule stream protocol, as the long table: `timeseries_id`, `timestamp` and + /// the value columns, a batch per series, which all have to share those columns. A read with + /// no rows has no schema and raises. + #[pyo3(signature = (requested_schema=None))] + fn __arrow_c_stream__<'py>( + &self, + py: Python<'py>, + requested_schema: Option>, + ) -> PyResult> { + let _ = requested_schema; + let mut batches = Vec::with_capacity(self.series.len()); + for s in &self.series { + let s = s.get(); + let id = Arc::new(Int64Array::from(vec![s.id as i64; s.batch.num_rows()])) as ArrayRef; + let mut fields = vec![Arc::new(Field::new("timeseries_id", DataType::Int64, false))]; + fields.extend(s.batch.schema().fields().iter().cloned()); + let mut columns = vec![id]; + columns.extend(s.batch.columns().iter().cloned()); + let batch = RecordBatch::try_new(Arc::new(Schema::new(fields)), columns) + .map_err(|e| PyValueError::new_err(e.to_string()))?; + if let Some(first) = batches.first() { + let first: &RecordBatch = first; + if first.schema().fields() != batch.schema().fields() { + return Err(PyValueError::new_err(format!( + "series {} has columns {:?}, the table's are {:?}; series of different value \ + types do not share one long table, but to_polars(wide=True) holds them", + s.external_id, + batch.schema().fields().iter().map(|f| f.name()).collect::>(), + first.schema().fields().iter().map(|f| f.name()).collect::>(), + ))); + } + } + batches.push(batch); + } + let Some(first) = batches.first() else { + return Err(PyValueError::new_err( + "the read returned no rows, so there is no schema for an Arrow stream", + )); + }; + let schema = first.schema(); + let reader = RecordBatchIterator::new(batches.into_iter().map(Ok), schema); + PyCapsule::new_with_value(py, FFI_ArrowArrayStream::new(Box::new(reader)), c"arrow_array_stream") + } + + /// A `polars.DataFrame`: the long table, or with `wide` one column per series, matched on + /// timestamp. Needs polars. + #[pyo3(signature = (wide=false))] + fn to_polars<'py>(slf: &Bound<'py, Self>, wide: bool) -> PyResult> { + tables_to_polars(slf.as_any(), wide) } } @@ -336,7 +451,7 @@ impl Iterator for LongTable { if batch.schema().fields() != self.schema.fields() { return Some(Err(ArrowError::SchemaError(format!( "series {external_id} has columns {:?}, the stream's are {:?}; read series of \ - different value types by iterating instead", + different value types with to_polars(wide=True), or by iterating", batch.schema().fields().iter().map(|f| f.name()).collect::>(), self.schema.fields().iter().map(|f| f.name()).collect::>(), )))); @@ -359,9 +474,4 @@ fn import<'py>(py: Python<'py>, module: &str, method: &str) -> PyResult) -> PyResult> { - series - .into_iter() - .map(|s| PyBinaryDatapoints::try_from(s).map_err(|e| PyValueError::new_err(e.to_string()))) - .collect() -} + diff --git a/datahub_python_bindings/src/timeseries/mod.rs b/datahub_python_bindings/src/timeseries/mod.rs index f0a3979..6f15aff 100644 --- a/datahub_python_bindings/src/timeseries/mod.rs +++ b/datahub_python_bindings/src/timeseries/mod.rs @@ -472,6 +472,7 @@ pub fn register(m: &Bound<'_, PyModule>) -> PyResult<()> { m.add_class::()?; m.add_class::()?; m.add_class::()?; + m.add_class::()?; m.add_class::()?; m.add_class::()?; m.add_class::()?; diff --git a/datahub_python_bindings/src/timeseries/sync_service.rs b/datahub_python_bindings/src/timeseries/sync_service.rs index 67ced00..0d6f05a 100644 --- a/datahub_python_bindings/src/timeseries/sync_service.rs +++ b/datahub_python_bindings/src/timeseries/sync_service.rs @@ -4,7 +4,7 @@ use crate::timeseries::datapoints::{ }; use crate::{DatahubIdentity, Identifiable}; use crate::{PyIdCollection, PyRetrieveFilter}; -use crate::timeseries::binary_read::{self, PyBinaryDatapointReader, PyBinaryDatapoints}; +use crate::timeseries::binary_read::{PyBinaryDatapointReader, PyBinaryReadResult}; use intellistream_datahub_sdk::generic::{DataWrapper, IdAndExtId}; use intellistream_datahub_sdk::{ApiService, TimeSeriesUpdateCollection}; use pyo3_async_runtimes::tokio::future_into_py; @@ -368,7 +368,7 @@ impl PyTimeSeriesServiceSync { &self, py: Python<'py>, input: RetrieveFilters, - ) -> PyResult> { + ) -> PyResult { let service = self.api_service.clone(); let filters: Vec = match input { RetrieveFilters::One(f) => vec![f.into()], @@ -380,7 +380,7 @@ impl PyTimeSeriesServiceSync { .block_on(service.time_series.retrieve_datapoints_binary(&wrapper)) .map_err(crate::datahub_err) })?; - binary_read::to_py(series) + PyBinaryReadResult::new(py, series) } /// The read `retrieve_datapoints_binary` makes, handed over as it arrives: a single-use /// `BinaryDatapointReader`, iterated frame by frame or read as an Arrow stream. diff --git a/python_tests/test_timeseries_binary_read.py b/python_tests/test_timeseries_binary_read.py index 782ea5e..669331b 100644 --- a/python_tests/test_timeseries_binary_read.py +++ b/python_tests/test_timeseries_binary_read.py @@ -118,7 +118,7 @@ def test_aggregates_convert_to_pandas(stand_in): def test_no_points_is_no_series(stand_in): client, server = stand_in server.body = b"DHDE" - assert client.timeseries.retrieve_datapoints_binary(_filter()) == [] + assert len(client.timeseries.retrieve_datapoints_binary(_filter())) == 0 def test_a_response_cut_short_raises_rather_than_returning_fewer_points(stand_in): @@ -285,6 +285,73 @@ def test_a_refusal_is_raised_before_there_is_a_reader(chunked): assert excinfo.value.status_code == 400 +# --- one table, buffered or streamed -------------------------------------------------------- + + +def _both(client, server, frames): + """The same body read buffered and streamed.""" + server.chunks = frames + [b"DHDE"] + buffered = client.timeseries.retrieve_datapoints_binary(_filter()) + return buffered, lambda: client.timeseries.retrieve_datapoints_binary_stream(_filter()) + + +def test_buffered_and_streamed_give_the_same_tables(chunked): + pl = pytest.importorskip("polars") + client, server = chunked + buffered, streamed = _both(client, server, MIXED) + + assert len(buffered) == 2 and buffered[-1].external_id == "rd_mixed_numbers" + assert buffered.external_ids == {17: "rd_mixed", 18: "rd_mixed_numbers"} + long = buffered.to_polars() + assert long.columns == ["timeseries_id", "timestamp", "value_numeric", "value_text"] + assert long.equals(streamed().to_polars()) + assert long.equals(pl.DataFrame(buffered)) + assert buffered.to_polars(wide=True).equals(streamed().to_polars(wide=True)) + + +def test_wide_names_several_value_columns_after_their_series(chunked): + pl = pytest.importorskip("polars") + client, server = chunked + buffered, _ = _both(client, server, MIXED) + + wide = buffered.to_polars(wide=True) + + # rd_mixed has three rows, rd_mixed_numbers two: matched on timestamp, the gap left null. + assert wide.columns == [ + "timestamp", + "rd_mixed.value_numeric", "rd_mixed.value_text", + "rd_mixed_numbers.value_numeric", "rd_mixed_numbers.value_text", + ] + assert wide["timestamp"].to_list() == [T0 + datetime.timedelta(seconds=s) for s in range(3)] + assert wide["rd_mixed.value_text"].to_list() == [None, "fault", None] + assert wide["rd_mixed_numbers.value_numeric"].to_list() == [1.0, 2.0, None] + + +def test_wide_holds_series_of_different_value_types(chunked): + pl = pytest.importorskip("polars") + client, server = chunked + buffered, streamed = _both(client, server, VALUES[:2] + [VALUES[5]]) # bigint, float, text + + with pytest.raises(ValueError, match="do not share one long table"): + buffered.to_polars() + wide = buffered.to_polars(wide=True) + assert wide.columns == ["timestamp", "rd_bigint", "rd_float", "rd_text"] + assert wide.schema["rd_bigint"] == pl.Int64 and wide.schema["rd_text"] == pl.String + assert wide["rd_bigint"].to_list() == [1, 42, -9_000_000_000] + assert wide["rd_float"].to_list() == [1.5, -2.25, None] + assert wide["rd_text"].to_list() == ["open", "blåbær", None] + assert wide.equals(streamed().to_polars(wide=True)) + + +def test_wide_of_no_rows_raises_like_the_long_table(chunked): + pytest.importorskip("polars") + client, server = chunked + buffered, streamed = _both(client, server, []) + for read in (buffered, streamed()): + with pytest.raises(ValueError, match="no rows"): + read.to_polars(wide=True) + + # --- live --------------------------------------------------------------------------------------- START = datetime.datetime(2024, 3, 1, tzinfo=datetime.timezone.utc) @@ -372,3 +439,27 @@ def test_streaming_read_matches_the_buffered_read(sync_client, make_ts, binary_r assert pl.concat([c.to_polars() for c in chunks]).equals(buffered.to_polars()) assert long.select("timestamp", "value").equals(buffered.to_polars()) assert reader.external_ids == {buffered.id: ts.external_id} + + +def test_wide_from_series_on_the_same_and_on_different_timestamps(sync_client, make_ts, binary_read): + pl = pytest.importorskip("polars") + a, b, c = (make_ts(value_type="float") for _ in range(3)) + sync_client.timeseries.insert_from_lists_binary(TIMESTAMPS, [float(i) for i in range(len(TIMESTAMPS))], a) + sync_client.timeseries.insert_from_lists_binary(TIMESTAMPS, [-float(i) for i in range(len(TIMESTAMPS))], b) + every_other = TIMESTAMPS[1::2] + sync_client.timeseries.insert_from_lists_binary(every_other, [0.5] * len(every_other), c) + for ts, n in ((a, len(TIMESTAMPS)), (b, len(TIMESTAMPS)), (c, len(every_other))): + _wait_for(sync_client, ts, n) + rf = lambda *series: [intellistream_datahub_sdk.RetrieveFilter(ts=s, start=START, limit=10_000) for s in series] + + # Same timestamps: the columns side by side. + same = sync_client.timeseries.retrieve_datapoints_binary(rf(a, b)).to_polars(wide=True) + assert same.columns == ["timestamp", a.external_id, b.external_id] + assert same["timestamp"].to_list() == TIMESTAMPS + assert same[b.external_id].to_list() == [-float(i) for i in range(len(TIMESTAMPS))] + + # Different timestamps: matched up, the gaps null; streamed gives the same table. + mixed = sync_client.timeseries.retrieve_datapoints_binary(rf(a, c)).to_polars(wide=True) + assert mixed["timestamp"].to_list() == TIMESTAMPS + assert mixed[c.external_id].to_list() == [None, 0.5] * (len(TIMESTAMPS) // 2) + assert mixed.equals(sync_client.timeseries.retrieve_datapoints_binary_stream(rf(a, c)).to_polars(wide=True)) From 0d621964f61d33726c9ea90b89f327cd2065cce1 Mon Sep 17 00:00:00 2001 From: jgjesdal Date: Mon, 12 Oct 2026 18:31:31 +0200 Subject: [PATCH 4/8] Insert a dataframe through the binary endpoint, straight from Arrow TimeSeriesService::insert_table_binary takes wide Arrow RecordBatches: a timestamp column and one column per series, named by external id. Columns already in their series' type are sliced rather than copied, nulls are left out as "no point", unsorted rows are sorted with the last of a repeated timestamp kept, and frames are compressed on scoped threads. Values are stored as the string path stores them; the offline tests decode both paths' frames and compare. A mixed series takes a column as typed rather than classifying each value. Series resolution, request packing and the stale-series retry are now shared with the string path rather than duplicated. In Python, insert_dataframe_binary takes a polars or pandas DataFrame or a pyarrow table through the Arrow C stream, finding the timestamp in the pandas index, a timestamp column or the only timestamp-typed column. What to_polars(wide=True) returns from a read goes back in unchanged. 100M float points from a polars frame: 3.1s, against about 75s through Python lists. Co-Authored-By: Claude Opus 5.5 Signed-off-by: jgjesdal --- AGENTS.md | 2 + Cargo.toml | 1 + README.md | 17 + .../intellistream_datahub_sdk/__init__.pyi | 51 ++ .../src/timeseries/binary_read.rs | 91 ++- .../src/timeseries/sync_service.rs | 27 +- .../test_timeseries_binary_dataframe.py | 193 +++++ src/timeseries/binary.rs | 174 +++-- src/timeseries/binary_table.rs | 709 ++++++++++++++++++ src/timeseries/mod.rs | 2 + 10 files changed, 1193 insertions(+), 74 deletions(-) create mode 100644 python_tests/test_timeseries_binary_dataframe.py create mode 100644 src/timeseries/binary_table.rs diff --git a/AGENTS.md b/AGENTS.md index 7205115..11c928f 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -205,6 +205,8 @@ the generic default with `recognized: false`, not an error. The read is `TimeSeriesService::retrieve_datapoints_binary`, to `POST /timeseries/data/list/binary`: the JSON retrieve body in, the same frames out, one series per frame, then the 4-byte marker `DHDE`. A body without the marker was cut short (the server streams, so a failure after the first bytes can only end the body early) and is a **502** saying so, never fewer points; bytes after it are refused too. Codec 2 frames carry bucketed aggregates in their own schema (`avg`/`min`/`max`/`sum`, Float64, 50 000 buckets per frame) while the envelope's type byte stays the series' own. `parse_read_response` returns one columnar `BinaryDatapoints` per series, merging consecutive frames of one series; decimals stay exact as unscaled `i128` plus scale. `TimeSeriesService::stream_datapoints_binary` is the primitive: a `FrameAssembler` cuts the body into frames as chunks arrive (a frame may be split anywhere), each frame is an owned `Bytes` decoded on `spawn_blocking`, at most `min(cores, 8)` in flight through a `FuturesOrdered` so items stay in frame order, and a producer task feeds a bounded channel behind the `BinaryDatapointStream` (dropping it aborts the task and closes the connection). One item per frame, so a long series is several consecutive items. A refusal is the `Err` before there is a stream; a response that breaks off is an error item after the items that arrived, and that is the semantic difference stream consumers must handle. `retrieve_datapoints_binary` collects the stream and merges consecutive parts (each series allocated once at its final size); `parse_read_response` does the same for a body in memory, decoding on scoped threads. Measured on 100M float rows, 10 series: the server takes ~6.5 s; the buffered call adds ~0.24 s (the merge copy) and the stream into polars ~0.02 s. Single-threaded zstd used to add 1.3 s on its own. Differences from the JSON read worth knowing: no `cursor` (a 400; advance `start` past the last timestamp), a series with no points is absent rather than empty, and an unknown or unreadable series is a 400 for the whole request — `DataWrapper` serialises only `items`, so the request-level `ignoreUnknownIds` and `limit` are not reachable. `src/timeseries/fixtures/read_*.bin` are response bodies written by the platform's own `DatapointFrameWriter.wrap`, so the offline tests in `binary_read.rs` prove arrow-rs reads what the Java writer emits; regenerate them when the platform's writer changes. The ignored `timeseries::tests::test_datapoints_binary_read` is the live check against the JSON read. In Python it is `retrieve_datapoints_binary` on the sync client, plus `retrieve_datapoints_binary_stream`, whose single-use `BinaryDatapointReader` iterates per frame or is an Arrow stream of one long table (`timeseries_id`, `timestamp`, values; one schema, so a read mixing value types must be iterated). The long table carries no external id column on purpose: as a dictionary column, polars spent ~0.8 s per 100M rows making it categorical, more than everything else; `reader.external_ids` maps the ids instead. The buffered `retrieve_datapoints_binary` returns a `BinaryReadResult`, a sequence of per-series `BinaryDatapoints` with the same `external_ids`, long-table Arrow stream and `to_polars(wide=False)` as the reader: both call `python/intellistream_datahub_sdk/_tables.py`, so the two tables cannot drift. `wide=True` puts series side by side when every one has the same timestamps (free), pivots on a numeric key when they share one value column type (~1.7 s per 100M rows; joining names first doubled it, `concat(how="align")` tripled it), and aligns otherwise, which is how mixed value types sit side by side. Each per-series `BinaryDatapoints` implements the Arrow PyCapsule stream protocol (`__arrow_c_stream__`, via `arrow-array`'s `ffi` feature enabled in the bindings crate only), so polars, pyarrow and pandas 3 read them zero-copy; the package still declares no runtime dependency, and `to_polars`/`to_pandas` import theirs lazily. `python_tests/test_timeseries_binary_read.py` covers it offline against a stand-in server and live when the backend serves the endpoint. +`TimeSeriesService::insert_table_binary` (`src/timeseries/binary_table.rs`) is the third ingest entry point, Arrow in: a wide `RecordBatch` (a timestamp column plus one column per series, named by external id). It shares series resolution and its cache, request packing and the stale-series retry with the string path (`resolve_series`, `post_frame_requests`, `assemble_frame` and `ipc_stream` in `binary.rs`), and stores what the string path stores: its offline tests decode both paths' frames with the read parser and compare. Columns already in the canon's type are sliced, not copied; nulls are dropped as "no point"; unsorted rows are sorted and a repeated timestamp keeps its last value; timestamps must carry a time zone and whole milliseconds. Frames hold one value type, series in ascending id order, short series sharing a frame, and are compressed on scoped threads. A `mixed` series takes a column as typed (numeric → `value_numeric`, string → `value_text`), unlike the string path's per-value classification. In Python it is `insert_dataframe_binary` (sync client), importing anything with `__arrow_c_stream__` zero-copy; with no `timestamp=` it takes the pandas index if that is a timestamp (pandas records it in the schema metadata, and other index columns are dropped as not series), then a `timestamp` column, then the only timestamp-typed column. 100M float points from polars took ~3 s with the client using ~3.6 cores, so the api, not the client, sets that pace; `python_tests/test_timeseries_binary_dataframe.py` round-trips offline through a stand-in that replays the posted frames, and live through `to_polars(wide=True)`. + ### The `ApiServiceProvider` trait (`src/generic.rs`) Every subservice implements `ApiServiceProvider`, which owns the HTTP plumbing: token acquisition, `execute_get_request`, `execute_post_request`, `execute_file_upload_request`, `execute_get_stream_request`. Subservice methods should go through these helpers rather than calling `reqwest` directly — a few early methods (e.g. `TimeSeriesService::list`) still bypass the trait and should be migrated when touched. diff --git a/Cargo.toml b/Cargo.toml index fc2916f..8b1dd02 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -46,6 +46,7 @@ base64 = "0.22" arrow-array = "59.3" arrow-schema = "59.3" arrow-ipc = "59.3" +arrow-select = "59.3" #[lib] diff --git a/README.md b/README.md index 0b1c6d7..7982698 100644 --- a/README.md +++ b/README.md @@ -139,6 +139,23 @@ and sends again, once. Resolving by external id goes through The durable spool does not cover this path. `binary::FrameWriter` is public for producers that build frames themselves. +### From a dataframe + +`time_series.insert_table_binary(&batches, "timestamp", &options)` takes Arrow `RecordBatch`es in +the wide shape: a timestamp column and one column per series, named by its external id. In Python +that is `insert_dataframe_binary`, which takes a polars or pandas DataFrame or a pyarrow table as +it is, through the Arrow C stream: + +```python +df = pl.DataFrame({"timestamp": stamps, "pump.temp": temps, "pump.count": counts}) +client.timeseries.insert_dataframe_binary(df) # or a pandas frame with a timestamp index +``` + +Columns are used without copying where they already have their series' type; nulls are "no +point here", so a ragged table works; values are stored as the string path stores them. 100M float +points from a polars frame took about 3 s, against about 75 s through Python lists. What +`to_polars(wide=True)` returns from a read goes back in unchanged. + ## Binary datapoint read `time_series.retrieve_datapoints_binary(&filters)` takes the request `retrieve_datapoints` diff --git a/datahub_python_bindings/python/intellistream_datahub_sdk/__init__.pyi b/datahub_python_bindings/python/intellistream_datahub_sdk/__init__.pyi index d2e39e6..50b7aea 100644 --- a/datahub_python_bindings/python/intellistream_datahub_sdk/__init__.pyi +++ b/datahub_python_bindings/python/intellistream_datahub_sdk/__init__.pyi @@ -873,6 +873,57 @@ class TimeSeriesServiceSync: input: list[DatapointsCollectionString], zstd_level: int | None = None, ) -> list[str]: ... + def insert_dataframe_binary( + self, + table: Any, + timestamp: str | None = None, + zstd_level: int | None = None, + ) -> int: + """Insert a wide table through ``POST /timeseries/data/binary``, straight from Arrow. + + Parameters + ---------- + table + A polars or pandas DataFrame (pandas 2.2 or later, which exports through pyarrow), a + pyarrow Table or RecordBatchReader: anything with ``__arrow_c_stream__``. Every column + but the timestamp is a series, named by its external id. + timestamp + The timestamp column. If not given: the pandas index when it is a timestamp, else a + column named ``timestamp``, else the only timestamp-typed column. It has to carry a time + zone and whole milliseconds, or be ``Int64`` epoch milliseconds. + zstd_level + 1, 3 or 9, as for :meth:`insert_datapoints_binary`. + + Returns + ------- + int + The points sent. + + Raises + ------ + DataHubException + 404 naming a column that is not a series you can read, before anything is sent; 422 + for a naive or sub-millisecond timestamp, or a column its series cannot hold (a float + column for a ``bigint`` series, say). + ValueError + No timestamp column, or several and none named. + + Notes + ----- + The table is read without copying it; timestamps get one pass to milliseconds and the + frames are compressed on several threads, so no object or string is made per value. + 100M float points from a polars frame took about 3 s, against about 75 s through + :meth:`insert_from_lists_binary`. + + A null is "no point at this timestamp", so a ragged table works (pandas exports NaN as + null; polars keeps NaN, which is then stored as a value). Rows need not be sorted, and a + timestamp repeated within a series keeps its last value. Values are stored as the string + path stores them, decimals rounded half-up and ``decimal32`` clamped, with one deliberate + difference: a ``mixed`` series takes a numeric column as numbers and a string column as + text, where the string path classifies each value. ``to_polars(wide=True)`` on a read + gives back exactly this shape. A table larger than one request is not atomic: a refusal + can leave the earlier requests stored. + """ def insert_from_lists_binary( self, timestamps: list[datetime.datetime], diff --git a/datahub_python_bindings/src/timeseries/binary_read.rs b/datahub_python_bindings/src/timeseries/binary_read.rs index e4aefd5..a30b78c 100644 --- a/datahub_python_bindings/src/timeseries/binary_read.rs +++ b/datahub_python_bindings/src/timeseries/binary_read.rs @@ -1,6 +1,6 @@ use std::sync::Arc; -use arrow_array::ffi_stream::FFI_ArrowArrayStream; +use arrow_array::ffi_stream::{ArrowArrayStreamReader, FFI_ArrowArrayStream}; use std::sync::Mutex; use arrow_array::{ @@ -475,3 +475,92 @@ fn import<'py>(py: Python<'py>, module: &str, method: &str) -> PyResult) -> PyResult<(Vec, Vec)> { + if !table.hasattr("__arrow_c_stream__")? { + return Err(pyo3::exceptions::PyTypeError::new_err( + "expected a table with __arrow_c_stream__: a polars or pandas DataFrame (pandas 2.2 or \ + later, with pyarrow), or a pyarrow Table or RecordBatchReader", + )); + } + let capsule = table.call_method0("__arrow_c_stream__")?; + let capsule = capsule.cast::()?; + // Checks the capsule's name is arrow_array_stream before handing out the pointer. + let pointer = capsule.pointer_checked(Some(c"arrow_array_stream"))?.cast::(); + // Moves the stream out of the capsule and leaves it released, as the protocol requires. + let reader = unsafe { ArrowArrayStreamReader::from_raw(pointer.as_ptr()) } + .map_err(|e| PyValueError::new_err(e.to_string()))?; + let schema = reader.schema(); + let batches = reader + .collect::, _>>() + .map_err(|e| PyValueError::new_err(e.to_string()))?; + let index = schema + .metadata() + .get("pandas") + .and_then(|m| serde_json::from_str::(m).ok()) + .and_then(|m| m.get("index_columns").cloned()) + .and_then(|c| c.as_array().cloned()) + .map(|c| c.iter().filter_map(|v| v.as_str().map(str::to_string)).collect()) + .unwrap_or_default(); + Ok((batches, index)) +} + +/// The timestamp column: as named, else the pandas index if it is a timestamp, else `timestamp`, +/// else the one timestamp column there is. +pub(crate) fn timestamp_column(schema: &Schema, named: Option, index: &[String]) -> PyResult { + if let Some(name) = named { + return Ok(name); + } + let is_time = |name: &str| { + schema + .field_with_name(name) + .is_ok_and(|f| matches!(f.data_type(), DataType::Timestamp(_, _))) + }; + if let [only] = index { + if is_time(only) { + return Ok(only.clone()); + } + } + if schema.field_with_name("timestamp").is_ok() { + return Ok("timestamp".to_string()); + } + let times: Vec<&str> = schema + .fields() + .iter() + .filter(|f| matches!(f.data_type(), DataType::Timestamp(_, _))) + .map(|f| f.name().as_str()) + .collect(); + match times[..] { + [only] => Ok(only.to_string()), + [] => Err(PyValueError::new_err( + "the table has no timestamp column; name one with timestamp=, or give it a \ + timestamp index", + )), + _ => Err(PyValueError::new_err(format!( + "the table has several timestamp columns ({}); name one with timestamp=", + times.join(", ") + ))), + } +} + +/// The batches without the pandas index columns that are not the timestamp: an index is not a +/// series. +pub(crate) fn without_index(batches: Vec, index: &[String], timestamp: &str) -> PyResult> { + let Some(first) = batches.first() else { return Ok(batches) }; + let schema = first.schema(); + let keep: Vec = (0..schema.fields().len()) + .filter(|&i| { + let name = schema.field(i).name(); + name == timestamp || !index.iter().any(|x| x == name) + }) + .collect(); + if keep.len() == schema.fields().len() { + return Ok(batches); + } + batches + .iter() + .map(|b| b.project(&keep).map_err(|e| PyValueError::new_err(e.to_string()))) + .collect() +} diff --git a/datahub_python_bindings/src/timeseries/sync_service.rs b/datahub_python_bindings/src/timeseries/sync_service.rs index 0d6f05a..d5a1f53 100644 --- a/datahub_python_bindings/src/timeseries/sync_service.rs +++ b/datahub_python_bindings/src/timeseries/sync_service.rs @@ -4,7 +4,7 @@ use crate::timeseries::datapoints::{ }; use crate::{DatahubIdentity, Identifiable}; use crate::{PyIdCollection, PyRetrieveFilter}; -use crate::timeseries::binary_read::{PyBinaryDatapointReader, PyBinaryReadResult}; +use crate::timeseries::binary_read::{self, PyBinaryDatapointReader, PyBinaryReadResult}; use intellistream_datahub_sdk::generic::{DataWrapper, IdAndExtId}; use intellistream_datahub_sdk::{ApiService, TimeSeriesUpdateCollection}; use pyo3_async_runtimes::tokio::future_into_py; @@ -298,6 +298,31 @@ impl PyTimeSeriesServiceSync { }) } + /// `POST /timeseries/data/binary` from a table: a polars or pandas DataFrame, or anything else + /// with `__arrow_c_stream__`, one column per series named by its external id beside a + /// timestamp column (or index). Returns the number of points sent. + #[pyo3(signature = (table, timestamp=None, zstd_level=None))] + fn insert_dataframe_binary<'py>( + &self, + py: Python<'py>, + table: Bound<'py, PyAny>, + timestamp: Option, + zstd_level: Option, + ) -> PyResult { + let (batches, index) = binary_read::import_table(&table)?; + let Some(first) = batches.first() else { return Ok(0) }; + let timestamp = binary_read::timestamp_column(&first.schema(), timestamp, &index)?; + let batches = binary_read::without_index(batches, &index, ×tamp)?; + let service = self.api_service.clone(); + let options = binary_options(zstd_level); + let sent = py.detach(|| { + self.runtime + .block_on(service.time_series.insert_table_binary(&batches, ×tamp, &options)) + .map_err(crate::datahub_err) + })?; + Ok(sent.points) + } + /// The binary twin of `insert_from_lists`: parallel timestamp and value sequences for one /// series, which is the shape a DataFrame column pair arrives in. #[pyo3(signature = (timestamps, values, ts, zstd_level=None))] diff --git a/python_tests/test_timeseries_binary_dataframe.py b/python_tests/test_timeseries_binary_dataframe.py new file mode 100644 index 0000000..d022bb8 --- /dev/null +++ b/python_tests/test_timeseries_binary_dataframe.py @@ -0,0 +1,193 @@ +"""Binary ingest from a dataframe (`insert_dataframe_binary`): a polars, pandas or pyarrow table, +one column per series named by its external id, handed over through the Arrow C stream. + +Offline, a stand-in server answers the series lookup and keeps what is posted; the posted frames +are then served back through the binary read, so a round trip needs no backend. The live test +round-trips a wide table through a real one. +""" +import datetime +import http.server +import json +import threading + +import pytest + +import intellistream_datahub_sdk +from intellistream_datahub_sdk import DataHubException +from python_tests.fixtures import * # noqa: F401,F403 + +pl = pytest.importorskip("polars") +UTC = datetime.timezone.utc +T0 = datetime.datetime(2025, 1, 1, tzinfo=UTC) +SERIES = {"pump.temp": (11, "float"), "pump.count": (12, "bigint"), "pump.state": (13, "text")} + + +class _Api(http.server.BaseHTTPRequestHandler): + """Answers /timeseries/byids from SERIES, keeps posted frames, and serves them back on a read.""" + protocol_version = "HTTP/1.1" + posted = [] + + def _reply(self, status, content_type, body): + self.send_response(status) + self.send_header("Content-Type", content_type) + self.send_header("Content-Length", str(len(body))) + self.end_headers() + self.wfile.write(body) + + def do_POST(self): + body = self.rfile.read(int(self.headers.get("Content-Length", 0))) + if self.path == "/timeseries/byids": + wanted = [i.get("externalId") for i in json.loads(body)["items"]] + items = [ + {"id": str(SERIES[e][0]), "externalId": e, "name": e, "valueType": SERIES[e][1]} + for e in wanted if e in SERIES + ] + self._reply(200, "application/json", json.dumps({"items": items}).encode()) + elif self.path == "/timeseries/data/binary": + type(self).posted.append(body) + self._reply(204, "application/json", b"") + elif self.path == "/timeseries/data/list/binary": + self._reply(200, "application/vnd.intellistream.datapoint-block", b"".join(type(self).posted) + b"DHDE") + else: + self._reply(404, "text/plain", b"") + + def log_message(self, *args): + pass + + +@pytest.fixture +def api(): + _Api.posted = [] + server = http.server.ThreadingHTTPServer(("127.0.0.1", 0), _Api) + threading.Thread(target=server.serve_forever, daemon=True).start() + client = intellistream_datahub_sdk.DataHubClient( + base_url=f"http://127.0.0.1:{server.server_address[1]}", token="test-token" + ) + yield client, _Api + server.shutdown() + + +def _read_back(client): + """What the posted frames read back as: the wide table, by external id.""" + return client.timeseries.retrieve_datapoints_binary( + intellistream_datahub_sdk.RetrieveFilter(ts="pump.temp", start=T0) + ).to_polars(wide=True) + + +def test_a_polars_frame_round_trips(api): + client, server = api + df = pl.DataFrame({ + "timestamp": [T0 + datetime.timedelta(seconds=s) for s in (2, 0, 1)], # unsorted on purpose + "pump.temp": [2.5, 0.5, None], # a gap + "pump.count": [3, 1, 2], + "pump.state": ["off", "on", "on"], + }) + + assert client.timeseries.insert_dataframe_binary(df) == 8 + assert len(server.posted) == 1 + + # The stand-in replays the posted frames, which are grouped by value type; a real read follows + # the request's items. So compare by name. + back = _read_back(client) + assert sorted(back.columns) == ["pump.count", "pump.state", "pump.temp", "timestamp"] + assert back["timestamp"].to_list() == [T0 + datetime.timedelta(seconds=s) for s in range(3)] + assert back["pump.temp"].to_list() == [0.5, None, 2.5] + assert back["pump.count"].to_list() == [1, 2, 3] + assert back["pump.state"].to_list() == ["on", "on", "off"] + + +def test_a_pandas_frame_uses_its_timestamp_index(api): + pd = pytest.importorskip("pandas") + pytest.importorskip("pyarrow") + client, _ = api + index = pd.date_range("2025-01-01", periods=3, freq="s", tz="UTC", name="time") + df = pd.DataFrame({"pump.temp": [1.0, float("nan"), 3.0]}, index=index) + + # pandas exports NaN as null, so the middle row is no point at all. + assert client.timeseries.insert_dataframe_binary(df) == 2 + assert _read_back(client)["pump.temp"].to_list() == [1.0, 3.0] + + +def test_a_pyarrow_table_with_a_named_timestamp_column(api): + pa = pytest.importorskip("pyarrow") + client, _ = api + table = pa.table({ + "at": pa.array([T0, T0 + datetime.timedelta(milliseconds=5)], pa.timestamp("ns", "UTC")), + "pump.temp": [1.0, 2.0], + }) + assert client.timeseries.insert_dataframe_binary(table, timestamp="at") == 2 + assert _read_back(client)["timestamp"].to_list() == [T0, T0 + datetime.timedelta(milliseconds=5)] + + +def test_an_unknown_column_is_not_found_before_anything_is_sent(api): + client, server = api + df = pl.DataFrame({"timestamp": [T0], "pump.temp": [1.0], "no.such.series": [2.0]}) + with pytest.raises(DataHubException, match="no.such.series") as excinfo: + client.timeseries.insert_dataframe_binary(df) + assert excinfo.value.status_code == 404 + assert server.posted == [] + + +@pytest.mark.parametrize("timestamps, message", [ + ([datetime.datetime(2025, 1, 1)], "no time zone"), + ([T0 + datetime.timedelta(microseconds=1500)], "finer than a millisecond"), +]) +def test_timestamps_that_are_not_an_instant_in_milliseconds_are_refused(api, timestamps, message): + client, server = api + df = pl.DataFrame({"timestamp": timestamps, "pump.temp": [1.0]}) + with pytest.raises(DataHubException, match=message) as excinfo: + client.timeseries.insert_dataframe_binary(df) + assert excinfo.value.status_code == 422 + assert server.posted == [] + + +def test_a_column_its_series_cannot_hold_is_refused(api): + client, server = api + df = pl.DataFrame({"timestamp": [T0], "pump.count": [1.5]}) + with pytest.raises(DataHubException, match="cannot hold"): + client.timeseries.insert_dataframe_binary(df) + assert server.posted == [] + + +def test_the_timestamp_column_has_to_be_found(api): + client, _ = api + with pytest.raises(ValueError, match="no timestamp column"): + client.timeseries.insert_dataframe_binary(pl.DataFrame({"pump.temp": [1.0]})) + two = pl.DataFrame({"a": [T0], "b": [T0], "pump.temp": [1.0]}) + with pytest.raises(ValueError, match="several timestamp columns"): + client.timeseries.insert_dataframe_binary(two) + with pytest.raises(TypeError, match="__arrow_c_stream__"): + client.timeseries.insert_dataframe_binary([1, 2, 3]) + + +# --- live --------------------------------------------------------------------------------------- + +START = datetime.datetime(2024, 3, 1, tzinfo=UTC) + + +def test_a_wide_table_round_trips_through_the_api(sync_client, make_ts): + temp, count = make_ts(value_type="float"), make_ts(value_type="bigint") + n = 150_000 # past one frame's 100 000 rows + stamps = pl.datetime_range(START, START + datetime.timedelta(seconds=n - 1), "1s", time_zone="UTC", eager=True) + df = pl.DataFrame({ + "timestamp": stamps, + temp.external_id: pl.Series([i * 0.5 for i in range(n)]), + # Every third point missing: a ragged column, its nulls left out. + count.external_id: pl.Series([None if i % 3 == 0 else i for i in range(n)], dtype=pl.Int64), + }) + + sent = sync_client.timeseries.insert_dataframe_binary(df) + assert sent == n + n - (n + 2) // 3 + + end = START + datetime.timedelta(seconds=n) + filters = [intellistream_datahub_sdk.RetrieveFilter(ts=s, start=START, end=end) for s in (temp, count)] + + from polling import poll_until + back = poll_until( + lambda: sync_client.timeseries.retrieve_datapoints_binary(filters), + lambda read: sum(len(s) for s in read) == sent, + ).to_polars(wide=True) + assert back.equals(df) + + # What came back goes back in as it is. + assert sync_client.timeseries.insert_dataframe_binary(back) == sent diff --git a/src/timeseries/binary.rs b/src/timeseries/binary.rs index f156c22..13fde08 100644 --- a/src/timeseries/binary.rs +++ b/src/timeseries/binary.rs @@ -50,8 +50,8 @@ pub const NUMERIC_ROWS_PER_FRAME: usize = 100_000; /// Rows per frame for `text` and `mixed`. pub const TEXT_ROWS_PER_FRAME: usize = 10_000; -const NUMERIC_UNSCALED_MAX: i128 = 999_999_999_999_999_999; -const DECIMAL32_UNSCALED_MAX: i128 = 999_999_999; +pub(crate) const NUMERIC_UNSCALED_MAX: i128 = 999_999_999_999_999_999; +pub(crate) const DECIMAL32_UNSCALED_MAX: i128 = 999_999_999; /// Headroom under the raw cap: arrow-rs pads every buffer to 64 bytes, which the estimate ignores. const RAW_BYTES_MARGIN: usize = 64 * 1024; @@ -191,7 +191,7 @@ impl BinaryIngestOptions { self } - fn validate(&self) -> Result<(), ResponseError> { + pub(crate) fn validate(&self) -> Result<(), ResponseError> { if ![1, 3, 9].contains(&self.zstd_level) { return Err(ResponseError::bad_request(format!( "zstd level {} is not one of 1, 3 or 9", @@ -456,41 +456,9 @@ impl FrameWriter { } let raw = self.write_ipc(&kept)?; - if raw.len() > MAX_FRAME_RAW_BYTES { - return Err(format!( - "frame payload is {} bytes, the cap is {MAX_FRAME_RAW_BYTES}; split it", - raw.len() - )); - } - let compressed = - zstd::bulk::compress(&raw, zstd_level).map_err(|e| format!("zstd failed: {e}"))?; - - let directory_len: usize = directory - .iter() - .map(|(_, external_id)| 8 + varint_size(external_id.len()) + external_id.len()) - .sum(); - let mut frame = Vec::with_capacity(HEADER_BYTES + directory_len + compressed.len()); - frame.extend_from_slice(MAGIC); - frame.push(VERSION); - frame.push(self.value_type.id()); - frame.push(CODEC_ARROW_IPC); - frame.push(COMPRESSION_ZSTD); - for n in [rows, directory.len(), directory_len, compressed.len(), raw.len()] { - frame.extend_from_slice(&(n as u32).to_le_bytes()); - } - for (id, external_id) in &directory { - frame.extend_from_slice(&id.to_le_bytes()); - write_varint(&mut frame, external_id.len()); - frame.extend_from_slice(external_id.as_bytes()); - } - frame.extend_from_slice(&compressed); - Ok(Frame { - bytes: frame, - rows, - series: directory.len(), - raw_len: raw.len(), - clamped: self.clamped, - }) + let mut frame = assemble_frame(self.value_type, rows, &directory, &raw, zstd_level)?; + frame.clamped = self.clamped; + Ok(frame) } fn write_ipc(&self, kept: &[usize]) -> Result, String> { @@ -575,20 +543,70 @@ impl FrameWriter { columns.push(Arc::new(StringArray::from(text))); } } - let schema = self.value_type.schema(); - let batch = RecordBatch::try_new(schema.clone(), columns).map_err(|e| e.to_string())?; - let mut raw = Vec::with_capacity(self.estimated_raw_bytes()); - let mut writer = - StreamWriter::try_new(&mut raw, schema.as_ref()).map_err(|e| e.to_string())?; - writer.write(&batch).map_err(|e| e.to_string())?; - writer.finish().map_err(|e| e.to_string())?; - drop(writer); - Ok(raw) + let batch = + RecordBatch::try_new(self.value_type.schema(), columns).map_err(|e| e.to_string())?; + ipc_stream(&batch, self.estimated_raw_bytes()) + } +} + +/// A batch in the canon as an uncompressed Arrow IPC stream: a frame's payload before zstd. +pub(crate) fn ipc_stream(batch: &RecordBatch, capacity: usize) -> Result, String> { + let mut raw = Vec::with_capacity(capacity); + let mut writer = + StreamWriter::try_new(&mut raw, batch.schema().as_ref()).map_err(|e| e.to_string())?; + writer.write(batch).map_err(|e| e.to_string())?; + writer.finish().map_err(|e| e.to_string())?; + drop(writer); + Ok(raw) +} + +/// The envelope around a payload: header, series directory (ascending ids, as the rows are +/// ordered) and the zstd-compressed stream. +pub(crate) fn assemble_frame( + value_type: DatapointValueType, + rows: usize, + directory: &[(i64, &str)], + raw: &[u8], + zstd_level: i32, +) -> Result { + if raw.len() > MAX_FRAME_RAW_BYTES { + return Err(format!( + "frame payload is {} bytes, the cap is {MAX_FRAME_RAW_BYTES}; split it", + raw.len() + )); } + let compressed = + zstd::bulk::compress(raw, zstd_level).map_err(|e| format!("zstd failed: {e}"))?; + let directory_len: usize = directory + .iter() + .map(|(_, external_id)| 8 + varint_size(external_id.len()) + external_id.len()) + .sum(); + let mut frame = Vec::with_capacity(HEADER_BYTES + directory_len + compressed.len()); + frame.extend_from_slice(MAGIC); + frame.push(VERSION); + frame.push(value_type.id()); + frame.push(CODEC_ARROW_IPC); + frame.push(COMPRESSION_ZSTD); + for n in [rows, directory.len(), directory_len, compressed.len(), raw.len()] { + frame.extend_from_slice(&(n as u32).to_le_bytes()); + } + for (id, external_id) in directory { + frame.extend_from_slice(&id.to_le_bytes()); + write_varint(&mut frame, external_id.len()); + frame.extend_from_slice(external_id.as_bytes()); + } + frame.extend_from_slice(&compressed); + Ok(Frame { + bytes: frame, + rows, + series: directory.len(), + raw_len: raw.len(), + clamped: 0, + }) } /// The server counts the character limit in UTF-16 units, so this does too. -fn checked_text(value: &str) -> Result { +pub(crate) fn checked_text(value: &str) -> Result { if value.is_empty() { return Err("text value is empty".to_string()); } @@ -634,7 +652,7 @@ fn decimal_parts(value: &str) -> Option<(bool, String, i32)> { } /// Parses a decimal literal to its unscaled value at `scale` decimals, rounded half-up. -fn parse_scaled(value: &str, scale: u32) -> Option { +pub(crate) fn parse_scaled(value: &str, scale: u32) -> Option { let (negative, digits, point) = decimal_parts(value)?; let total = i32::try_from(digits.len()).ok()?; // Digits to keep: everything before the point plus `scale` after it. @@ -747,7 +765,7 @@ pub(crate) fn pack_requests(frames: Vec) -> Vec> { requests } -fn unprocessable(message: String) -> ResponseError { +pub(crate) fn unprocessable(message: String) -> ResponseError { ResponseError { status: StatusCode::UNPROCESSABLE_ENTITY, message, @@ -761,7 +779,7 @@ fn unprocessable(message: String) -> ResponseError { /// Read from the problem document, not the body text: the SDK's own pre-flight 404 names the /// series it could not resolve, so an external id spelling one of these slugs used to earn a /// pointless second attempt. -fn is_stale_series_rejection(error: &ResponseError) -> bool { +pub(crate) fn is_stale_series_rejection(error: &ResponseError) -> bool { let Some(problem) = error.problem() else { return false; }; @@ -814,7 +832,11 @@ impl TimeSeriesService { options: &BinaryIngestOptions, ) -> Result, ResponseError> { let collections = json.get_items(); - let resolved = self.resolve_series(collections).await?; + let wanted: Vec<(Option, Option<&str>)> = collections + .iter() + .map(|c| (c.id, c.external_id.as_deref())) + .collect(); + let resolved = self.resolve_series(&wanted).await?; let items: Vec<(ResolvedSeries, &[DatapointString])> = resolved .into_iter() .zip(collections.iter()) @@ -833,12 +855,23 @@ impl TimeSeriesService { .map_err(unprocessable)?; frames.push(frame); } + self.post_frame_requests(frames, options).await?; + let mut result = DataWrapper::new(); + result.set_http_status_code(204); + Ok(result) + } - // Concurrently, not one after another. A call of up to 3.2M points packs into a single - // request and this changes nothing, but a larger one becomes several and the api - // validates them independently, so there is no reason to serialise them. Bounded by - // `request_concurrency` so a very large call cannot open an unbounded number of - // connections. + /// Packs frames into requests under the caps and posts them. + /// + /// Concurrently, not one after another. A call of up to 3.2M points packs into a single + /// request and this changes nothing, but a larger one becomes several and the api validates + /// them independently, so there is no reason to serialise them. Bounded by + /// `request_concurrency` so a very large call cannot open an unbounded number of connections. + pub(crate) async fn post_frame_requests( + &self, + frames: Vec, + options: &BinaryIngestOptions, + ) -> Result<(), ResponseError> { let path = format!("{}/data/binary", self.base_url); let bodies = pack_requests(frames); let limit = options.request_concurrency.max(1); @@ -850,9 +883,7 @@ impl TimeSeriesService { outcome?; } } - let mut result = DataWrapper::new(); - result.set_http_status_code(204); - Ok(result) + Ok(()) } async fn post_frames( @@ -881,17 +912,18 @@ impl TimeSeriesService { } } - /// One [`ResolvedSeries`] per collection, in order, from the cache or `/timeseries/byids`. - async fn resolve_series( + /// One [`ResolvedSeries`] per `(id, external id)` asked for, in order, from the cache or + /// `/timeseries/byids`; the id wins when both are given. + pub(crate) async fn resolve_series( &self, - collections: &[DatapointsCollection], + wanted: &[(Option, Option<&str>)], ) -> Result, ResponseError> { let mut lookups: Vec = Vec::new(); let mut seen: HashSet = HashSet::new(); { let cache = self.binary_series.lock().unwrap(); - for collection in collections { - match (collection.id, &collection.external_id) { + for &(id, external_id) in wanted { + match (id, external_id) { (Some(id), _) => { if !cache.by_id.contains_key(&id) && seen.insert(format!("id:{id}")) { lookups.push(IdAndExtId::from_id(id)); @@ -930,10 +962,10 @@ impl TimeSeriesService { } let cache = self.binary_series.lock().unwrap(); - let mut resolved = Vec::with_capacity(collections.len()); + let mut resolved = Vec::with_capacity(wanted.len()); let mut missing = Vec::new(); - for collection in collections { - let hit = match (collection.id, &collection.external_id) { + for &(id, external_id) in wanted { + let hit = match (id, external_id) { (Some(id), _) => cache.by_id.get(&id), (None, Some(external_id)) => cache.by_external_id.get(external_id), (None, None) => None, @@ -941,10 +973,8 @@ impl TimeSeriesService { match hit { Some(series) => resolved.push(series.clone()), None => missing.push( - collection - .id - .map(|id| id.to_string()) - .or_else(|| collection.external_id.clone()) + id.map(|id| id.to_string()) + .or_else(|| external_id.map(str::to_string)) .unwrap_or_default(), ), } diff --git a/src/timeseries/binary_table.rs b/src/timeseries/binary_table.rs new file mode 100644 index 0000000..80280a8 --- /dev/null +++ b/src/timeseries/binary_table.rs @@ -0,0 +1,709 @@ +//! Binary ingest straight from Arrow: a wide table, one timestamp column and one column per +//! series named by its external id, as polars, pyarrow and pandas all hand it over. +//! +//! The columns are used as they are wherever they already have the canon's type (a `Float64` +//! column for a `float` series is sliced, never copied); timestamps get one pass to epoch +//! milliseconds, the per-row `timeseries_id` column is materialised per frame, and the IPC +//! encoding and zstd are the serialisation. What this path does not do is make an object, or a +//! string, per value, which is what the string path costs. +//! +//! Values reach the series as the string path would store them: decimals round half-up at the +//! series' scale and `decimal32` clamps; text is held to the same limits. Two differences are +//! deliberate. A null is "no point at this timestamp", which is what makes a ragged wide table +//! work. And a `mixed` series takes a typed column as it is typed: a numeric column is +//! `value_numeric`, a string column `value_text`, where the string path classifies each string. + +use std::sync::atomic::{AtomicUsize, Ordering}; +use std::sync::Arc; + +use arrow_array::cast::AsArray; +use arrow_array::types::{ + Decimal128Type, Float32Type, Float64Type, Int16Type, Int32Type, Int64Type, Int8Type, + TimestampMicrosecondType, TimestampMillisecondType, TimestampNanosecondType, + TimestampSecondType, UInt16Type, UInt32Type, UInt64Type, UInt8Type, +}; +use arrow_array::{ + Array, ArrayRef, BooleanArray, Decimal128Array, Float32Array, Float64Array, Int64Array, + RecordBatch, StringArray, TimestampMillisecondArray, UInt32Array, +}; +use arrow_schema::{DataType, TimeUnit}; +use oauth2::http::StatusCode; + +use crate::http::ResponseError; +use crate::timeseries::binary::{ + assemble_frame, checked_text, ipc_stream, is_stale_series_rejection, parse_scaled, + unprocessable, BinaryIngestOptions, DatapointValueType, Frame, ResolvedSeries, + DECIMAL32_UNSCALED_MAX, MAX_SERIES_PER_FRAME, NUMERIC_UNSCALED_MAX, +}; +use crate::timeseries::TimeSeriesService; + +/// What [`TimeSeriesService::insert_table_binary`] sent. +#[derive(Debug, Clone, Default, PartialEq, Eq)] +pub struct TableIngest { + /// Points sent: non-null cells, after a repeated timestamp in one series kept its last value. + pub points: usize, + pub series: usize, + pub frames: usize, + /// `decimal32` values clamped to the type's range. + pub clamped: usize, +} + +/// One series' points from one batch, in the canon: timestamps ascending and unique, values in the +/// column type(s) the series' frames carry. +struct Points { + series: usize, + timestamps: ArrayRef, + values: Vec, +} + +impl Points { + fn len(&self) -> usize { + self.timestamps.len() + } +} + +/// Epoch milliseconds from a timestamp column. Copy-free when it is already in milliseconds. +fn timestamps_ms(name: &str, column: &ArrayRef) -> Result { + if column.null_count() > 0 { + return Err(format!("timestamp column {name} has nulls")); + } + let exact = |values: &[i64], per_ms: i64| -> Result { + let mut out = Vec::with_capacity(values.len()); + for &v in values { + if v % per_ms != 0 { + return Err(format!( + "timestamp column {name} has a value finer than a millisecond; \ + the series store milliseconds, so round it first" + )); + } + out.push(v / per_ms); + } + Ok(Arc::new(TimestampMillisecondArray::from(out).with_timezone("UTC"))) + }; + match column.data_type() { + DataType::Timestamp(_, None) => Err(format!( + "timestamp column {name} has no time zone; give it one (e.g. UTC) rather than leave \ + the instant to be guessed" + )), + DataType::Timestamp(TimeUnit::Millisecond, Some(_)) => Ok(Arc::new( + TimestampMillisecondArray::new(column.as_primitive::().values().clone(), None) + .with_timezone("UTC"), + )), + DataType::Timestamp(TimeUnit::Microsecond, Some(_)) => { + exact(column.as_primitive::().values(), 1_000) + } + DataType::Timestamp(TimeUnit::Nanosecond, Some(_)) => { + exact(column.as_primitive::().values(), 1_000_000) + } + DataType::Timestamp(TimeUnit::Second, Some(_)) => Ok(Arc::new( + TimestampMillisecondArray::from_iter_values( + column.as_primitive::().values().iter().map(|s| s * 1_000), + ) + .with_timezone("UTC"), + )), + DataType::Int64 => Ok(Arc::new( + TimestampMillisecondArray::new(column.as_primitive::().values().clone(), None) + .with_timezone("UTC"), + )), + other => Err(format!( + "timestamp column {name} is {other}; it has to be a timestamp with a time zone, or \ + Int64 epoch milliseconds" + )), + } +} + +/// A numeric column as `f64`, for `float` and `mixed`. `None` when the column is not numeric. +fn as_f64(column: &ArrayRef) -> Option { + macro_rules! widen { + ($t:ty) => { + Float64Array::from_iter_values(column.as_primitive::<$t>().values().iter().map(|&v| v as f64)) + }; + } + Some(match column.data_type() { + DataType::Float64 => column.as_primitive::().clone(), + DataType::Float32 => widen!(Float32Type), + DataType::Int8 => widen!(Int8Type), + DataType::Int16 => widen!(Int16Type), + DataType::Int32 => widen!(Int32Type), + DataType::Int64 => widen!(Int64Type), + DataType::UInt8 => widen!(UInt8Type), + DataType::UInt16 => widen!(UInt16Type), + DataType::UInt32 => widen!(UInt32Type), + DataType::UInt64 => widen!(UInt64Type), + DataType::Decimal128(_, scale) => { + let divisor = 10f64.powi(*scale as i32); + Float64Array::from_iter_values( + column.as_primitive::().values().iter().map(|&v| v as f64 / divisor), + ) + } + _ => return None, + }) +} + +/// A string column of any of the three string layouts, each value held to the text limits. +fn as_text(name: &str, column: &ArrayRef) -> Option> { + let checked = |values: &mut dyn Iterator>| -> Result { + values + .map(|v| checked_text(v.unwrap_or_default()).map_err(|e| format!("column {name}: {e}"))) + .collect::, String>>() + .map(StringArray::from) + }; + Some(match column.data_type() { + DataType::Utf8 => checked(&mut column.as_string::().iter()), + DataType::LargeUtf8 => checked(&mut column.as_string::().iter()), + DataType::Utf8View => checked(&mut column.as_string_view().iter()), + _ => return None, + }) +} + +/// Unscaled values at `scale` decimals, rounded half-up like the string path's `parse_scaled`. +fn as_unscaled(name: &str, column: &ArrayRef, scale: i8) -> Result, String> { + let rescale = |v: i128, from: i8| -> i128 { + if from <= scale { + v * 10i128.pow((scale - from) as u32) + } else { + let divisor = 10i128.pow((from - scale) as u32); + let (quotient, remainder) = (v / divisor, v % divisor); + if remainder.abs() * 2 >= divisor { + quotient + v.signum() + } else { + quotient + } + } + }; + macro_rules! ints { + ($t:ty) => { + column.as_primitive::<$t>().values().iter().map(|&v| rescale(v as i128, 0)).collect() + }; + } + let floats = |values: &mut dyn Iterator| -> Result, String> { + values + .map(|v| { + parse_scaled(&v.to_string(), scale as u32) + .ok_or_else(|| format!("column {name}: {v} is not a decimal number")) + }) + .collect() + }; + Ok(match column.data_type() { + DataType::Decimal128(_, from) => { + let from = *from; + column.as_primitive::().values().iter().map(|&v| rescale(v, from)).collect() + } + DataType::Int8 => ints!(Int8Type), + DataType::Int16 => ints!(Int16Type), + DataType::Int32 => ints!(Int32Type), + DataType::Int64 => ints!(Int64Type), + DataType::UInt8 => ints!(UInt8Type), + DataType::UInt16 => ints!(UInt16Type), + DataType::UInt32 => ints!(UInt32Type), + DataType::UInt64 => ints!(UInt64Type), + DataType::Float64 => floats(&mut column.as_primitive::().values().iter().copied())?, + DataType::Float32 => floats(&mut column.as_primitive::().values().iter().map(|&v| v as f64))?, + other => return Err(format!("column {name} is {other}, which is not a number")), + }) +} + +/// A column with no nulls, in the canon's value type(s) for `value_type`. +fn to_canon( + name: &str, + value_type: DatapointValueType, + column: &ArrayRef, + clamped: &mut usize, +) -> Result, String> { + let wrong = || format!("column {name} is {}, which a {value_type:?} series cannot hold", column.data_type()); + Ok(match value_type { + DatapointValueType::Float => vec![Arc::new(as_f64(column).ok_or_else(wrong)?) as ArrayRef], + DatapointValueType::Float32 => { + let values: Float32Array = match column.data_type() { + DataType::Float32 => column.as_primitive::().clone(), + _ => { + let wide = as_f64(column).ok_or_else(wrong)?; + Float32Array::from_iter_values(wide.values().iter().map(|&v| v as f32)) + } + }; + vec![Arc::new(values)] + } + DatapointValueType::Bigint => { + macro_rules! widen { + ($t:ty) => { + Int64Array::from_iter_values(column.as_primitive::<$t>().values().iter().map(|&v| v as i64)) + }; + } + let values = match column.data_type() { + DataType::Int64 => column.as_primitive::().clone(), + DataType::Int8 => widen!(Int8Type), + DataType::Int16 => widen!(Int16Type), + DataType::Int32 => widen!(Int32Type), + DataType::UInt8 => widen!(UInt8Type), + DataType::UInt16 => widen!(UInt16Type), + DataType::UInt32 => widen!(UInt32Type), + DataType::UInt64 => { + let values = column.as_primitive::().values(); + if values.iter().any(|&v| v > i64::MAX as u64) { + return Err(format!("column {name} has a value past the bigint range")); + } + widen!(UInt64Type) + } + _ => return Err(wrong()), + }; + vec![Arc::new(values)] + } + DatapointValueType::Numeric | DatapointValueType::Decimal32 => { + let (precision, scale) = if value_type == DatapointValueType::Numeric { (18, 6) } else { (9, 4) }; + let mut unscaled = as_unscaled(name, column, scale)?; + if value_type == DatapointValueType::Numeric { + if let Some(v) = unscaled.iter().find(|v| v.abs() > NUMERIC_UNSCALED_MAX) { + return Err(format!( + "column {name}: {} does not fit Decimal(18, 6)", + *v as f64 / 1e6 + )); + } + } else { + for v in unscaled.iter_mut() { + let bounded = (*v).clamp(-DECIMAL32_UNSCALED_MAX, DECIMAL32_UNSCALED_MAX); + if bounded != *v { + *clamped += 1; + *v = bounded; + } + } + } + let array = Decimal128Array::from(unscaled) + .with_precision_and_scale(precision, scale) + .map_err(|e| e.to_string())?; + vec![Arc::new(array)] + } + DatapointValueType::Text => vec![Arc::new(as_text(name, column).ok_or_else(wrong)??)], + DatapointValueType::Mixed => { + let rows = column.len(); + if let Some(numeric) = as_f64(column) { + vec![Arc::new(numeric), Arc::new(StringArray::new_null(rows))] + } else if let Some(text) = as_text(name, column) { + vec![Arc::new(Float64Array::new_null(rows)), Arc::new(text?)] + } else { + return Err(wrong()); + } + } + }) +} + +/// One series' column of a batch as its points: nulls dropped, values in the canon, timestamps +/// ascending with a repeated one keeping its last value, as the string path keeps it. +fn points( + index: usize, + series: &ResolvedSeries, + name: &str, + timestamps: &ArrayRef, + column: &ArrayRef, + clamped: &mut usize, +) -> Result { + let (timestamps, column) = if column.null_count() == 0 { + (timestamps.clone(), column.clone()) + } else { + let present = BooleanArray::from_iter((0..column.len()).map(|i| Some(column.is_valid(i)))); + let filter = |a: &ArrayRef| arrow_select::filter::filter(a.as_ref(), &present).map_err(|e| e.to_string()); + (filter(timestamps)?, filter(column)?) + }; + let mut values = to_canon(name, series.value_type, &column, clamped)?; + let mut timestamps = timestamps; + + let ts = timestamps.as_primitive::().values(); + if !ts.windows(2).all(|w| w[0] < w[1]) { + let mut order: Vec = (0..ts.len() as u32).collect(); + order.sort_by_key(|&i| ts[i as usize]); + // Stable, so equal timestamps stay in input order and the last of each run wins. + let kept: Vec = order + .iter() + .enumerate() + .filter(|(at, &i)| order.get(at + 1).is_none_or(|&next| ts[next as usize] != ts[i as usize])) + .map(|(_, &i)| i) + .collect(); + let indices = UInt32Array::from(kept); + let take = |a: &ArrayRef| arrow_select::take::take(a.as_ref(), &indices, None).map_err(|e| e.to_string()); + timestamps = take(×tamps)?; + values = values.iter().map(take).collect::>()?; + } + Ok(Points { series: index, timestamps, values }) +} + +/// One frame's worth of points: slices of series of one value type, in ascending id order. +struct FrameSpec { + value_type: DatapointValueType, + pieces: Vec<(usize, usize, usize)>, +} + +/// Cuts the points into frames: each value type on its own, series in ascending id order, a frame +/// filled to its row cap, and a series longer than the room left continued in the next frame. +fn plan_frames(points: &[Points], series: &[ResolvedSeries]) -> Vec { + let mut order: Vec = (0..points.len()).filter(|&p| points[p].len() > 0).collect(); + order.sort_by_key(|&p| (series[points[p].series].value_type.id(), series[points[p].series].id)); + let mut specs: Vec = Vec::new(); + for p in order { + let value_type = series[points[p].series].value_type; + let max_rows = value_type.max_rows(); + let mut start = 0; + while start < points[p].len() { + let open = specs.last().is_some_and(|f| { + f.value_type == value_type + && f.pieces.iter().map(|(_, _, n)| n).sum::() < max_rows + && f.pieces.len() < MAX_SERIES_PER_FRAME + }); + if !open { + specs.push(FrameSpec { value_type, pieces: Vec::new() }); + } + let frame = specs.last_mut().unwrap(); + let room = max_rows - frame.pieces.iter().map(|(_, _, n)| n).sum::(); + let take = room.min(points[p].len() - start); + frame.pieces.push((p, start, take)); + start += take; + } + } + specs +} + +fn build_frame( + spec: &FrameSpec, + points: &[Points], + series: &[ResolvedSeries], + zstd_level: i32, +) -> Result { + let rows: usize = spec.pieces.iter().map(|(_, _, n)| n).sum(); + let mut ids: Vec = Vec::with_capacity(rows); + let mut directory: Vec<(i64, &str)> = Vec::with_capacity(spec.pieces.len()); + for &(p, _, n) in &spec.pieces { + let s = &series[points[p].series]; + ids.extend(std::iter::repeat_n(s.id as i64, n)); + directory.push((s.id as i64, s.external_id.as_str())); + } + // A frame of one slice is the series' own arrays, sliced; several are concatenated. + let column = |pick: &dyn Fn(&Points) -> &ArrayRef| -> Result { + if let [(p, start, n)] = spec.pieces[..] { + return Ok(pick(&points[p]).slice(start, n)); + } + let slices: Vec = spec.pieces.iter().map(|&(p, start, n)| pick(&points[p]).slice(start, n)).collect(); + let refs: Vec<&dyn Array> = slices.iter().map(|a| a.as_ref()).collect(); + arrow_select::concat::concat(&refs).map_err(|e| e.to_string()) + }; + let mut columns: Vec = vec![Arc::new(Int64Array::from(ids)), column(&|p| &p.timestamps)?]; + let width = points[spec.pieces[0].0].values.len(); + for v in 0..width { + columns.push(column(&|p| &p.values[v])?); + } + let batch = RecordBatch::try_new(spec.value_type.schema(), columns).map_err(|e| e.to_string())?; + let raw = ipc_stream(&batch, rows * 32 + 2048)?; + assemble_frame(spec.value_type, rows, &directory, &raw, zstd_level) +} + +/// A batch's frames, compressed on as many threads as there are cores and frames. +fn table_frames( + batch: &RecordBatch, + timestamp: &str, + series: &[ResolvedSeries], + zstd_level: i32, +) -> Result<(Vec, TableIngest), String> { + let schema = batch.schema(); + let ts_index = schema.index_of(timestamp).map_err(|_| format!("no column named {timestamp}"))?; + let timestamps = timestamps_ms(timestamp, batch.column(ts_index))?; + let mut clamped = 0; + let mut all = Vec::with_capacity(series.len()); + let mut s = 0; + for (c, field) in schema.fields().iter().enumerate() { + if c == ts_index { + continue; + } + all.push(points(s, &series[s], field.name(), ×tamps, batch.column(c), &mut clamped)?); + s += 1; + } + let specs = plan_frames(&all, series); + let next = AtomicUsize::new(0); + let threads = std::thread::available_parallelism().map_or(1, |n| n.get()).min(specs.len()).max(1); + let mut built: Vec<(usize, Result)> = std::thread::scope(|scope| { + let workers: Vec<_> = (0..threads) + .map(|_| { + scope.spawn(|| { + let mut done = Vec::new(); + loop { + let i = next.fetch_add(1, Ordering::Relaxed); + let Some(spec) = specs.get(i) else { return done }; + done.push((i, build_frame(spec, &all, series, zstd_level))); + } + }) + }) + .collect(); + workers.into_iter().flat_map(|w| w.join().expect("a frame builder panicked")).collect() + }); + built.sort_unstable_by_key(|(i, _)| *i); + let frames = built.into_iter().map(|(_, f)| f).collect::, _>>()?; + let summary = TableIngest { + points: all.iter().map(Points::len).sum(), + series: all.iter().filter(|p| p.len() > 0).count(), + frames: frames.len(), + clamped, + }; + Ok((frames, summary)) +} + +impl TimeSeriesService { + /// Inserts a wide Arrow table through `POST /timeseries/data/binary`: the column named + /// `timestamp` holds the timestamps, and every other column is a series, named by its + /// external id. The batches are taken as they are, one after another; polars, pyarrow and + /// pandas hand a table over this way through the Arrow C stream. + /// + /// The timestamp column is a timestamp with a time zone (any unit, whole milliseconds), or + /// `Int64` epoch milliseconds; a naive one, or one with nulls, is refused. In a series column + /// a null is "no point at this timestamp", so a ragged table works; NaN is a value. A column + /// is converted to its series' value type as the string path would store it (see the module + /// documentation); one that cannot be, such as a float column for a `bigint` series, is a + /// 422 before anything is sent. Rows need not be sorted, and a timestamp repeated within a + /// series keeps its last value. + /// + /// Series are resolved by external id as on the string path: a name that is not a series, + /// or one you cannot read, is a 404 naming it before anything is sent. Each batch is posted + /// before the next is converted, so memory holds one batch's frames. A table larger than one + /// request is not atomic: a refusal can leave the earlier requests stored. + pub async fn insert_table_binary( + &self, + batches: &[RecordBatch], + timestamp: &str, + options: &BinaryIngestOptions, + ) -> Result { + options.validate()?; + let Some(first) = batches.first() else { + return Ok(TableIngest::default()); + }; + let names: Vec = first + .schema() + .fields() + .iter() + .map(|f| f.name().clone()) + .filter(|n| n != timestamp) + .collect(); + if first.schema().index_of(timestamp).is_err() { + return Err(ResponseError::bad_request(format!( + "no column named {timestamp}; the columns are {names:?}" + ))); + } + let mut total = TableIngest::default(); + for batch in batches { + if batch.schema().fields() != first.schema().fields() { + return Err(ResponseError::bad_request( + "the batches do not share one schema".to_string(), + )); + } + let sent = match self.insert_table_batch(batch, timestamp, &names, options).await { + Err(error) if is_stale_series_rejection(&error) => { + self.evict_binary_series_cache(); + self.insert_table_batch(batch, timestamp, &names, options).await + } + other => other, + }?; + total.points += sent.points; + total.series = total.series.max(sent.series); + total.frames += sent.frames; + total.clamped += sent.clamped; + } + Ok(total) + } + + async fn insert_table_batch( + &self, + batch: &RecordBatch, + timestamp: &str, + names: &[String], + options: &BinaryIngestOptions, + ) -> Result { + let wanted: Vec<(Option, Option<&str>)> = names.iter().map(|n| (None, Some(n.as_str()))).collect(); + let series = self.resolve_series(&wanted).await?; + let (batch, timestamp, level) = (batch.clone(), timestamp.to_string(), options.zstd_level); + let (frames, summary) = + tokio::task::spawn_blocking(move || table_frames(&batch, ×tamp, &series, level)) + .await + .map_err(|e| ResponseError { + status: StatusCode::INTERNAL_SERVER_ERROR, + message: format!("building frames failed: {e}"), + content_type: None, + })? + .map_err(unprocessable)?; + if !frames.is_empty() { + self.post_frame_requests(frames, options).await?; + } + Ok(summary) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::timeseries::binary::FrameWriter; + use crate::timeseries::binary_read::{parse_read_response, BinaryDatapoints, BinaryValues, END_OF_RESPONSE}; + use arrow_array::{LargeStringArray, StringViewArray, TimestampMicrosecondArray}; + use arrow_schema::{Field, Schema}; + + fn series(id: u64, external_id: &str, value_type: DatapointValueType) -> ResolvedSeries { + ResolvedSeries { id, external_id: external_id.to_string(), value_type } + } + + fn table(timestamps: ArrayRef, columns: Vec<(&str, ArrayRef)>) -> RecordBatch { + let mut fields = vec![Field::new("timestamp", timestamps.data_type().clone(), false)]; + let mut arrays = vec![timestamps]; + for (name, array) in columns { + fields.push(Field::new(name, array.data_type().clone(), true)); + arrays.push(array); + } + RecordBatch::try_new(Arc::new(Schema::new(fields)), arrays).unwrap() + } + + fn ms(values: &[i64]) -> ArrayRef { + Arc::new(TimestampMillisecondArray::from(values.to_vec()).with_timezone("UTC")) + } + + /// What a body of these frames reads back as. + fn decode(frames: &[Frame]) -> Vec { + let mut body: Vec = frames.iter().flat_map(|f| f.bytes.clone()).collect(); + body.extend_from_slice(END_OF_RESPONSE); + parse_read_response(&body).unwrap() + } + + /// The same points through the string path's `FrameWriter`. + fn through_strings(s: &ResolvedSeries, points: &[(i64, &str)]) -> Vec { + let mut writer = FrameWriter::new(s.value_type); + writer.series(s.id, &s.external_id).unwrap(); + for (t, v) in points { + writer.add(s.id, *t, v).unwrap(); + } + decode(&[writer.build(1).unwrap()]) + } + + fn through_table(batch: &RecordBatch, resolved: &[ResolvedSeries]) -> Vec { + decode(&table_frames(batch, "timestamp", resolved, 1).unwrap().0) + } + + #[test] + fn a_float_column_stores_what_the_string_path_stores() { + let s = [series(7, "pump", DatapointValueType::Float)]; + // Unsorted, a repeated timestamp (the last value wins), a null (no point), microseconds. + let ts = Arc::new( + TimestampMicrosecondArray::from(vec![3_000_000, 1_000_000, 2_000_000, 1_000_000, 4_000_000]) + .with_timezone("UTC"), + ); + let values = Arc::new(Float64Array::from(vec![Some(3.5), Some(1.0), None, Some(1.25), Some(f64::NAN)])); + let got = through_table(&table(ts, vec![("pump", values)]), &s); + let want = through_strings(&s[0], &[(3_000, "3.5"), (1_000, "1.0"), (1_000, "1.25"), (4_000, "NaN")]); + assert_eq!(got[0].timestamps, want[0].timestamps); + assert_eq!(got[0].timestamps, [1_000, 3_000, 4_000]); + assert_eq!(format!("{:?}", got[0].values), format!("{:?}", want[0].values)); + } + + #[test] + fn decimals_round_and_clamp_as_the_string_path_does() { + let s = [ + series(1, "numeric_from_float", DatapointValueType::Numeric), + series(2, "decimal32_from_decimal", DatapointValueType::Decimal32), + series(3, "numeric_from_int", DatapointValueType::Numeric), + ]; + let floats = Arc::new(Float64Array::from(vec![0.0000005, -1.2345675, 123.456789])); + let decimals = Arc::new( + Decimal128Array::from(vec![123_456_789_i128, -12_345_678_950, 99_999_999_995]) + .with_precision_and_scale(20, 6) + .unwrap(), + ); + let ints = Arc::new(arrow_array::Int32Array::from(vec![7, -3, 0])); + let batch = table(ms(&[1, 2, 3]), vec![("numeric_from_float", floats), ("decimal32_from_decimal", decimals), ("numeric_from_int", ints)]); + let (frames, summary) = table_frames(&batch, "timestamp", &s, 1).unwrap(); + let got = decode(&frames); + + let strings: [&[&str]; 3] = [ + &["0.0000005", "-1.2345675", "123.456789"], + &["123.456789", "-12345.67895", "99999.999995"], + &["7", "-3", "0"], + ]; + for (i, values) in strings.iter().enumerate() { + let points: Vec<(i64, &str)> = values.iter().enumerate().map(|(t, v)| (t as i64 + 1, *v)).collect(); + let want = through_strings(&s[i], &points); + let got = got.iter().find(|g| g.id == s[i].id).unwrap(); + assert_eq!(got.values, want[0].values, "{}", s[i].external_id); + } + // 99999.999995 rounds past decimal32's range and clamps; the string path counts the same. + assert_eq!(summary.clamped, 1); + } + + #[test] + fn text_comes_from_every_string_layout_and_mixed_from_typed_columns() { + let s = [ + series(1, "a", DatapointValueType::Text), + series(2, "b", DatapointValueType::Text), + series(3, "c", DatapointValueType::Mixed), + series(4, "d", DatapointValueType::Mixed), + ]; + let batch = table( + ms(&[1, 2]), + vec![ + ("a", Arc::new(StringViewArray::from(vec!["open", "blåbær"]))), + ("b", Arc::new(LargeStringArray::from(vec![Some("x"), None]))), + ("c", Arc::new(Float64Array::from(vec![1.5, 2.5]))), + ("d", Arc::new(StringArray::from(vec!["3.5", "fault"]))), + ], + ); + let got = through_table(&batch, &s); + assert_eq!(got[0].values, BinaryValues::Text(vec!["open".into(), "blåbær".into()])); + assert_eq!(got[1].values, BinaryValues::Text(vec!["x".into()])); + assert_eq!( + got[2].values, + BinaryValues::Mixed { numeric: vec![Some(1.5), Some(2.5)], text: vec![None, None] } + ); + // A string column of a mixed series is text, "3.5" included: the column is typed. + assert_eq!( + got[3].values, + BinaryValues::Mixed { numeric: vec![None, None], text: vec![Some("3.5".into()), Some("fault".into())] } + ); + } + + #[test] + fn long_series_are_cut_at_the_row_cap_and_short_ones_share_a_frame() { + let s = [ + series(9, "long", DatapointValueType::Float), + series(2, "short_b", DatapointValueType::Float), + series(1, "short_a", DatapointValueType::Float), + ]; + let n = 250_000; + let ts: Vec = (0..n as i64).collect(); + let long = Arc::new(Float64Array::from_iter_values((0..n).map(|i| i as f64))); + let sparse = |every: usize| -> ArrayRef { + Arc::new(Float64Array::from_iter((0..n).map(|i| (i % every == 0).then_some(1.0)))) + }; + let batch = table(ms(&ts), vec![("long", long), ("short_b", sparse(100_000)), ("short_a", sparse(50_000))]); + let (frames, summary) = table_frames(&batch, "timestamp", &s, 1).unwrap(); + + assert_eq!(summary.points, n + 3 + 5); + // Ascending ids: the two short series (5 + 3 rows) open the first frame, and the long one + // fills the rest of it and runs on over two more. + let rows: Vec = frames.iter().map(|f| f.rows).collect(); + assert_eq!(rows, [100_000, 100_000, 50_008]); + assert_eq!(frames[0].series, 3, "short_a, short_b and the start of long"); + let read = decode(&frames); + assert_eq!(read.iter().map(|s| (s.id, s.len())).collect::>(), [(1, 5), (2, 3), (9, n)]); + } + + #[test] + fn what_the_table_cannot_say_is_refused() { + let float = [series(1, "v", DatapointValueType::Float)]; + let naive = Arc::new(TimestampMillisecondArray::from(vec![1])); + assert!(table_frames(&table(naive, vec![("v", Arc::new(Float64Array::from(vec![1.0])))]), "timestamp", &float, 1) + .unwrap_err() + .contains("no time zone")); + + let sub_ms = Arc::new(TimestampMicrosecondArray::from(vec![1_500]).with_timezone("UTC")); + assert!(table_frames(&table(sub_ms, vec![("v", Arc::new(Float64Array::from(vec![1.0])))]), "timestamp", &float, 1) + .unwrap_err() + .contains("finer than a millisecond")); + + let bigint = [series(1, "v", DatapointValueType::Bigint)]; + assert!(table_frames(&table(ms(&[1]), vec![("v", Arc::new(Float64Array::from(vec![1.0])))]), "timestamp", &bigint, 1) + .unwrap_err() + .contains("cannot hold")); + + let text = [series(1, "v", DatapointValueType::Text)]; + assert!(table_frames(&table(ms(&[1]), vec![("v", Arc::new(StringArray::from(vec!["x".repeat(65)])))]), "timestamp", &text, 1) + .unwrap_err() + .contains("characters")); + } +} diff --git a/src/timeseries/mod.rs b/src/timeseries/mod.rs index df2035a..1327c3e 100644 --- a/src/timeseries/mod.rs +++ b/src/timeseries/mod.rs @@ -1,10 +1,12 @@ pub mod binary; pub mod binary_read; +pub mod binary_table; pub mod datapoint_listen; mod test; pub use binary::{BinaryIngestOptions, DatapointValueType, Frame, FrameWriter, ResolvedSeries}; pub use binary_read::{BinaryDatapointStream, BinaryDatapoints, BinaryValues}; +pub use binary_table::TableIngest; pub use datapoint_listen::{DatapointListener, LiveDatapoint}; use crate::buffer::DurableSpool; From 2592a6064e7aec8dff93472c289bd3f6a29e2fd4 Mon Sep 17 00:00:00 2001 From: jgjesdal Date: Wed, 14 Oct 2026 09:22:31 +0200 Subject: [PATCH 5/8] Read compression 2 frames, the api's relayed ClickHouse batches A binary read now answers with compression 2: the payload is an Arrow IPC stream whose record batches zstd-compress each buffer in-band, relayed from ClickHouse without decoding. arrow-ipc's zstd feature decodes it; compression 1 is still read. At compression 2 the envelope's raw length is the buffers uncompressed, which arrow-rs does not report, so only its cap is checked. The read fixtures are regenerated with the platform's relayStream, the old compression 1 bodies kept beside them, plus read_batches.bin: several compressed batches to a frame and a series the relay cut across two. Co-Authored-By: Claude Opus 5.5 Signed-off-by: jgjesdal --- AGENTS.md | 2 +- Cargo.toml | 2 +- python_tests/test_timeseries_binary_read.py | 7 +- src/timeseries/binary.rs | 4 + src/timeseries/binary_read.rs | 89 ++++++++++++++---- .../fixtures/GenerateReadFixtures.java | 68 +++++++++++-- src/timeseries/fixtures/read_aggregates.bin | Bin 401 -> 1062 bytes .../fixtures/read_aggregates_zstd.bin | Bin 0 -> 401 bytes src/timeseries/fixtures/read_batches.bin | Bin 0 -> 69537 bytes src/timeseries/fixtures/read_values.bin | Bin 2594 -> 5740 bytes src/timeseries/fixtures/read_values_zstd.bin | Bin 0 -> 2594 bytes 11 files changed, 139 insertions(+), 33 deletions(-) create mode 100644 src/timeseries/fixtures/read_aggregates_zstd.bin create mode 100644 src/timeseries/fixtures/read_batches.bin create mode 100644 src/timeseries/fixtures/read_values_zstd.bin diff --git a/AGENTS.md b/AGENTS.md index 11c928f..0d180a2 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -203,7 +203,7 @@ the generic default with `recognized: false`, not an error. `TimeSeriesService::insert_datapoints_binary` is the second ingest path, to `POST /timeseries/data/binary`: the same `DatapointsCollection` input, resolved through `/timeseries/byids` (cached per service instance; needs read access on the dataset), checked against each series' value type locally, cut into Arrow IPC frames at the contract's caps, zstd-compressed per frame (mandatory: level 1, 3 or 9, default 9) and posted through `execute_post_bytes_request`. Every refusal on this path is a problem document typed for its status — `invalid-frame` (400), `unknown-timeseries` (404), `request-too-large` (413), `unsupported-media-type` (415), `value-type-mismatch` and `external-id-mismatch` (422), `too-many-in-flight` (429) — with the kebab-case sub-case in a `reason` extension beside it. It was one type, `datapoint-block-rejected`, answering with all six statuses until the api split it (platform #120); the SDK matches the new slugs only, and nothing here recognises the old one — the binary path has never been in a release, so no caller can be on the other side of that change. `is_stale_series_rejection` reads the slug to decide the one re-resolve-and-retry, and reads it from the problem rather than the body text because the SDK's own pre-flight 404 names the series it could not resolve — an external id spelling `unknown-timeseries` used to trigger the retry. `FrameWriter` builds one frame and is public; `cut_into_writers` and `pack_requests` hold the caps. The byte layout is the platform's `binary_datapoints_format.md`. The arrow-rs crates (`arrow-array`, `arrow-schema`, `arrow-ipc`) serve both directions. In Python it is `insert_datapoints_binary` and `insert_from_lists_binary` on the **sync** client only, each taking an optional `zstd_level`; the async client has no binary methods. The ignored `timeseries::tests::test_datapoints_binary` is the live twin of `test_datapoints` and needs a backend that serves the endpoint; the writer's own tests in `binary.rs` run offline. `python_tests/test_timeseries_binary.py` round-trips the Python binding through the JSON read path. -The read is `TimeSeriesService::retrieve_datapoints_binary`, to `POST /timeseries/data/list/binary`: the JSON retrieve body in, the same frames out, one series per frame, then the 4-byte marker `DHDE`. A body without the marker was cut short (the server streams, so a failure after the first bytes can only end the body early) and is a **502** saying so, never fewer points; bytes after it are refused too. Codec 2 frames carry bucketed aggregates in their own schema (`avg`/`min`/`max`/`sum`, Float64, 50 000 buckets per frame) while the envelope's type byte stays the series' own. `parse_read_response` returns one columnar `BinaryDatapoints` per series, merging consecutive frames of one series; decimals stay exact as unscaled `i128` plus scale. `TimeSeriesService::stream_datapoints_binary` is the primitive: a `FrameAssembler` cuts the body into frames as chunks arrive (a frame may be split anywhere), each frame is an owned `Bytes` decoded on `spawn_blocking`, at most `min(cores, 8)` in flight through a `FuturesOrdered` so items stay in frame order, and a producer task feeds a bounded channel behind the `BinaryDatapointStream` (dropping it aborts the task and closes the connection). One item per frame, so a long series is several consecutive items. A refusal is the `Err` before there is a stream; a response that breaks off is an error item after the items that arrived, and that is the semantic difference stream consumers must handle. `retrieve_datapoints_binary` collects the stream and merges consecutive parts (each series allocated once at its final size); `parse_read_response` does the same for a body in memory, decoding on scoped threads. Measured on 100M float rows, 10 series: the server takes ~6.5 s; the buffered call adds ~0.24 s (the merge copy) and the stream into polars ~0.02 s. Single-threaded zstd used to add 1.3 s on its own. Differences from the JSON read worth knowing: no `cursor` (a 400; advance `start` past the last timestamp), a series with no points is absent rather than empty, and an unknown or unreadable series is a 400 for the whole request — `DataWrapper` serialises only `items`, so the request-level `ignoreUnknownIds` and `limit` are not reachable. `src/timeseries/fixtures/read_*.bin` are response bodies written by the platform's own `DatapointFrameWriter.wrap`, so the offline tests in `binary_read.rs` prove arrow-rs reads what the Java writer emits; regenerate them when the platform's writer changes. The ignored `timeseries::tests::test_datapoints_binary_read` is the live check against the JSON read. In Python it is `retrieve_datapoints_binary` on the sync client, plus `retrieve_datapoints_binary_stream`, whose single-use `BinaryDatapointReader` iterates per frame or is an Arrow stream of one long table (`timeseries_id`, `timestamp`, values; one schema, so a read mixing value types must be iterated). The long table carries no external id column on purpose: as a dictionary column, polars spent ~0.8 s per 100M rows making it categorical, more than everything else; `reader.external_ids` maps the ids instead. The buffered `retrieve_datapoints_binary` returns a `BinaryReadResult`, a sequence of per-series `BinaryDatapoints` with the same `external_ids`, long-table Arrow stream and `to_polars(wide=False)` as the reader: both call `python/intellistream_datahub_sdk/_tables.py`, so the two tables cannot drift. `wide=True` puts series side by side when every one has the same timestamps (free), pivots on a numeric key when they share one value column type (~1.7 s per 100M rows; joining names first doubled it, `concat(how="align")` tripled it), and aligns otherwise, which is how mixed value types sit side by side. Each per-series `BinaryDatapoints` implements the Arrow PyCapsule stream protocol (`__arrow_c_stream__`, via `arrow-array`'s `ffi` feature enabled in the bindings crate only), so polars, pyarrow and pandas 3 read them zero-copy; the package still declares no runtime dependency, and `to_polars`/`to_pandas` import theirs lazily. `python_tests/test_timeseries_binary_read.py` covers it offline against a stand-in server and live when the backend serves the endpoint. +The read is `TimeSeriesService::retrieve_datapoints_binary`, to `POST /timeseries/data/list/binary`: the JSON retrieve body in, the same frames out, one series per frame, then the 4-byte marker `DHDE`. Read frames are **compression 2**: the payload is an Arrow IPC stream whose record batches zstd-compress each buffer in-band (Arrow's body compression), because the api relays ClickHouse's compressed batches without decoding them; `arrow-ipc`'s `zstd` feature is what decodes that, and a frame holds whole batches up to the row cap, so most hold fewer rows than the cap. Compression 1 (one zstd stream around the payload, what ingest sends) is still read. At compression 2 the envelope's raw length is the batches' buffers uncompressed, which arrow-rs does not report, so only its cap is checked. A body without the marker was cut short (the server streams, so a failure after the first bytes can only end the body early) and is a **502** saying so, never fewer points; bytes after it are refused too. Codec 2 frames carry bucketed aggregates in their own schema (`avg`/`min`/`max`/`sum`, Float64, 50 000 buckets per frame) while the envelope's type byte stays the series' own. `parse_read_response` returns one columnar `BinaryDatapoints` per series, merging consecutive frames of one series; decimals stay exact as unscaled `i128` plus scale. `TimeSeriesService::stream_datapoints_binary` is the primitive: a `FrameAssembler` cuts the body into frames as chunks arrive (a frame may be split anywhere), each frame is an owned `Bytes` decoded on `spawn_blocking`, at most `min(cores, 8)` in flight through a `FuturesOrdered` so items stay in frame order, and a producer task feeds a bounded channel behind the `BinaryDatapointStream` (dropping it aborts the task and closes the connection). One item per frame, so a long series is several consecutive items. A refusal is the `Err` before there is a stream; a response that breaks off is an error item after the items that arrived, and that is the semantic difference stream consumers must handle. `retrieve_datapoints_binary` collects the stream and merges consecutive parts (each series allocated once at its final size); `parse_read_response` does the same for a body in memory, decoding on scoped threads. Measured on 100M float rows, 10 series: the server takes ~6.5 s; the buffered call adds ~0.24 s (the merge copy) and the stream into polars ~0.02 s. Single-threaded zstd used to add 1.3 s on its own. Differences from the JSON read worth knowing: no `cursor` (a 400; advance `start` past the last timestamp), a series with no points is absent rather than empty, and an unknown or unreadable series is a 400 for the whole request — `DataWrapper` serialises only `items`, so the request-level `ignoreUnknownIds` and `limit` are not reachable. `src/timeseries/fixtures/read_*.bin` are response bodies written by the platform's own `DatapointFrameWriter.relayStream` (`read_*_zstd.bin` are the compression 1 bodies its former `wrap` wrote), so the offline tests in `binary_read.rs` prove arrow-rs reads what the Java writer emits; regenerate them when the platform's writer changes. The ignored `timeseries::tests::test_datapoints_binary_read` is the live check against the JSON read. In Python it is `retrieve_datapoints_binary` on the sync client, plus `retrieve_datapoints_binary_stream`, whose single-use `BinaryDatapointReader` iterates per frame or is an Arrow stream of one long table (`timeseries_id`, `timestamp`, values; one schema, so a read mixing value types must be iterated). The long table carries no external id column on purpose: as a dictionary column, polars spent ~0.8 s per 100M rows making it categorical, more than everything else; `reader.external_ids` maps the ids instead. The buffered `retrieve_datapoints_binary` returns a `BinaryReadResult`, a sequence of per-series `BinaryDatapoints` with the same `external_ids`, long-table Arrow stream and `to_polars(wide=False)` as the reader: both call `python/intellistream_datahub_sdk/_tables.py`, so the two tables cannot drift. `wide=True` puts series side by side when every one has the same timestamps (free), pivots on a numeric key when they share one value column type (~1.7 s per 100M rows; joining names first doubled it, `concat(how="align")` tripled it), and aligns otherwise, which is how mixed value types sit side by side. Each per-series `BinaryDatapoints` implements the Arrow PyCapsule stream protocol (`__arrow_c_stream__`, via `arrow-array`'s `ffi` feature enabled in the bindings crate only), so polars, pyarrow and pandas 3 read them zero-copy; the package still declares no runtime dependency, and `to_polars`/`to_pandas` import theirs lazily. `python_tests/test_timeseries_binary_read.py` covers it offline against a stand-in server and live when the backend serves the endpoint. `TimeSeriesService::insert_table_binary` (`src/timeseries/binary_table.rs`) is the third ingest entry point, Arrow in: a wide `RecordBatch` (a timestamp column plus one column per series, named by external id). It shares series resolution and its cache, request packing and the stale-series retry with the string path (`resolve_series`, `post_frame_requests`, `assemble_frame` and `ipc_stream` in `binary.rs`), and stores what the string path stores: its offline tests decode both paths' frames with the read parser and compare. Columns already in the canon's type are sliced, not copied; nulls are dropped as "no point"; unsorted rows are sorted and a repeated timestamp keeps its last value; timestamps must carry a time zone and whole milliseconds. Frames hold one value type, series in ascending id order, short series sharing a frame, and are compressed on scoped threads. A `mixed` series takes a column as typed (numeric → `value_numeric`, string → `value_text`), unlike the string path's per-value classification. In Python it is `insert_dataframe_binary` (sync client), importing anything with `__arrow_c_stream__` zero-copy; with no `timestamp=` it takes the pandas index if that is a timestamp (pandas records it in the schema metadata, and other index columns are dropped as not series), then a `timestamp` column, then the only timestamp-typed column. 100M float points from polars took ~3 s with the client using ~3.6 cores, so the api, not the client, sets that pace; `python_tests/test_timeseries_binary_dataframe.py` round-trips offline through a stand-in that replays the posted frames, and live through `to_polars(wide=True)`. diff --git a/Cargo.toml b/Cargo.toml index 8b1dd02..3e27035 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -45,7 +45,7 @@ geojson = "1" base64 = "0.22" arrow-array = "59.3" arrow-schema = "59.3" -arrow-ipc = "59.3" +arrow-ipc = { version = "59.3", features = ["zstd"] } arrow-select = "59.3" diff --git a/python_tests/test_timeseries_binary_read.py b/python_tests/test_timeseries_binary_read.py index 669331b..60f9de9 100644 --- a/python_tests/test_timeseries_binary_read.py +++ b/python_tests/test_timeseries_binary_read.py @@ -59,10 +59,13 @@ def _filter(ts="rd_float"): return intellistream_datahub_sdk.RetrieveFilter(ts=ts, start=T0, limit=1000) -def test_every_value_type_converts_through_the_arrow_stream(stand_in): +# read_values.bin is compression 2, record batches compressing their own buffers, as a read answers +# now; read_values_zstd.bin is the same series as compression 1, one zstd stream per frame. +@pytest.mark.parametrize("fixture", ["read_values.bin", "read_values_zstd.bin"]) +def test_every_value_type_converts_through_the_arrow_stream(stand_in, fixture): pl = pytest.importorskip("polars") client, server = stand_in - server.body = (FIXTURES / "read_values.bin").read_bytes() + server.body = (FIXTURES / fixture).read_bytes() series = client.timeseries.retrieve_datapoints_binary([_filter("rd_bigint"), _filter("rd_float")]) diff --git a/src/timeseries/binary.rs b/src/timeseries/binary.rs index 13fde08..c4a4f34 100644 --- a/src/timeseries/binary.rs +++ b/src/timeseries/binary.rs @@ -32,7 +32,11 @@ pub const MEDIA_TYPE: &str = "application/vnd.intellistream.datapoint-block"; pub const MAGIC: &[u8; 4] = b"DHDP"; pub const VERSION: u8 = 1; pub const CODEC_ARROW_IPC: u8 = 1; +/// The payload is one zstd stream around an Arrow IPC stream with uncompressed buffers. pub const COMPRESSION_ZSTD: u8 = 1; +/// The payload is an Arrow IPC stream whose record batches compress each buffer with zstd +/// (Arrow's body compression): what a binary read answers with, ClickHouse's batches relayed. +pub const COMPRESSION_ARROW_ZSTD: u8 = 2; /// Bytes before the series directory. pub const HEADER_BYTES: usize = 28; diff --git a/src/timeseries/binary_read.rs b/src/timeseries/binary_read.rs index 9b71658..ac1c55e 100644 --- a/src/timeseries/binary_read.rs +++ b/src/timeseries/binary_read.rs @@ -3,7 +3,8 @@ //! The request is the JSON retrieve body; the answer is the frames `POST /timeseries/data/binary` //! takes, one series per frame, concatenated, and ended by the 4-byte marker `DHDE`. A body without //! the marker was cut short, whether inside a frame or between two, and is an error rather than a -//! shorter answer. Codec 2 frames carry bucketed aggregates instead of values. +//! shorter answer. Codec 2 frames carry bucketed aggregates instead of values. A read's frames are +//! compression 2, each record batch compressing its own buffers, but compression 1 is read too. //! //! [`TimeSeriesService::stream_datapoints_binary`] hands the frames over as they arrive, decoded //! on blocking threads while the rest download; [`TimeSeriesService::retrieve_datapoints_binary`] @@ -30,8 +31,8 @@ use oauth2::http::StatusCode; use crate::generic::{ApiServiceProvider, DataWrapper, RetrieveFilter}; use crate::http::ResponseError; use crate::timeseries::binary::{ - DatapointValueType, CODEC_ARROW_IPC, COMPRESSION_ZSTD, HEADER_BYTES, MAGIC, MAX_FRAME_RAW_BYTES, - MEDIA_TYPE, VERSION, + DatapointValueType, CODEC_ARROW_IPC, COMPRESSION_ARROW_ZSTD, COMPRESSION_ZSTD, HEADER_BYTES, + MAGIC, MAX_FRAME_RAW_BYTES, MEDIA_TYPE, VERSION, }; use crate::timeseries::TimeSeriesService; @@ -312,7 +313,7 @@ fn check_header(header: &[u8], index: usize) -> Result { CODEC_ARROW_IPC_AGGREGATES => true, codec => return Err(fail(format!("codec {codec}"))), }; - if header[7] != COMPRESSION_ZSTD { + if header[7] != COMPRESSION_ZSTD && header[7] != COMPRESSION_ARROW_ZSTD { return Err(fail(format!("compression {}", header[7]))); } let rows = u32_at(header, 8); @@ -339,10 +340,14 @@ struct FrameView { index: usize, value_type: DatapointValueType, aggregates: bool, + compression: u8, rows: usize, names: Vec<(i64, String)>, bytes: Bytes, payload_start: usize, + /// The payload decompressed at compression 1. At compression 2 it is the batches' buffers + /// uncompressed, which arrow-rs does not report, so only its cap is checked; each buffer's own + /// zstd frame and the row count still are. raw_len: usize, } @@ -378,6 +383,7 @@ impl FrameView { index, value_type: DatapointValueType::from_id(bytes[5]).expect("checked with the header"), aggregates, + compression: bytes[7], rows: u32_at(&bytes, 8), names, payload_start: HEADER_BYTES + directory_len, @@ -423,17 +429,23 @@ fn decode_frames(frames: &[FrameView]) -> Result>, Str /// One frame's rows, split into one part per series it names. fn decode_frame(frame: &FrameView) -> Result, String> { let fail = |message: String| format!("frame {}: {message}", frame.index); - let raw = zstd::bulk::decompress(&frame.bytes[frame.payload_start..], frame.raw_len) - .map_err(|e| fail(format!("zstd: {e}")))?; - if raw.len() != frame.raw_len { - return Err(fail(format!( - "payload decompressed to {} bytes, the envelope says {}", - raw.len(), - frame.raw_len - ))); - } - let reader = - StreamReader::try_new(Cursor::new(raw), None).map_err(|e| fail(format!("arrow: {e}")))?; + let payload = frame.bytes.slice(frame.payload_start..); + let payload = if frame.compression == COMPRESSION_ZSTD { + let raw = zstd::bulk::decompress(&payload, frame.raw_len) + .map_err(|e| fail(format!("zstd: {e}")))?; + if raw.len() != frame.raw_len { + return Err(fail(format!( + "payload decompressed to {} bytes, the envelope says {}", + raw.len(), + frame.raw_len + ))); + } + Bytes::from(raw) + } else { + payload + }; + let reader = StreamReader::try_new(Cursor::new(payload), None) + .map_err(|e| fail(format!("arrow: {e}")))?; let expected = if frame.aggregates { aggregates_schema() } else { @@ -771,12 +783,18 @@ mod tests { use crate::timeseries::binary::FrameWriter; use arrow_ipc::writer::StreamWriter; - /// Written by the platform's own frame writer (`DatapointFrameWriter.wrap`, the code that + /// Written by the platform's own frame writer (`DatapointFrameWriter.relayStream`, the code that /// answers a binary read), from `feat/binary-datapoint-read`: one series per value type, then - /// the end marker. What the Rust reader must accept from the Java writer. + /// the end marker, every frame compression 2. What the Rust reader must accept from the Java writer. const JAVA_VALUES: &[u8] = include_bytes!("fixtures/read_values.bin"); + /// The same series as compression 1 frames, which a read answered with before the relay. + const JAVA_VALUES_ZSTD: &[u8] = include_bytes!("fixtures/read_values_zstd.bin"); /// One codec-2 frame of three hourly buckets for series 21, `rd_aggregates`. const JAVA_AGGREGATES: &[u8] = include_bytes!("fixtures/read_aggregates.bin"); + const JAVA_AGGREGATES_ZSTD: &[u8] = include_bytes!("fixtures/read_aggregates_zstd.bin"); + /// Compression 2 with buffers zstd actually shrinks: two record batches of 3000 floats in one + /// frame for series 31, then 12 000 texts for series 32 that the relay cut into two frames. + const JAVA_BATCHES: &[u8] = include_bytes!("fixtures/read_batches.bin"); const T0: i64 = 1_700_000_000_000; @@ -818,7 +836,15 @@ mod tests { #[test] fn reads_every_value_type_the_java_writer_answers_with() { - let series = parse_read_response(JAVA_VALUES).unwrap(); + assert_eq!(JAVA_VALUES[7], COMPRESSION_ARROW_ZSTD); + assert_eq!(JAVA_VALUES_ZSTD[7], COMPRESSION_ZSTD); + for body in [JAVA_VALUES, JAVA_VALUES_ZSTD] { + reads_every_value_type(body); + } + } + + fn reads_every_value_type(body: &[u8]) { + let series = parse_read_response(body).unwrap(); let names: Vec<&str> = series.iter().map(|s| s.external_id.as_str()).collect(); assert_eq!( names, @@ -860,7 +886,13 @@ mod tests { #[test] fn reads_an_aggregate_frame_the_java_writer_answers_with() { - let series = parse_read_response(JAVA_AGGREGATES).unwrap(); + for body in [JAVA_AGGREGATES, JAVA_AGGREGATES_ZSTD] { + reads_an_aggregate_frame(body); + } + } + + fn reads_an_aggregate_frame(body: &[u8]) { + let series = parse_read_response(body).unwrap(); assert_eq!(series.len(), 1); let s = &series[0]; assert_eq!((s.id, s.external_id.as_str()), (21, "rd_aggregates")); @@ -878,6 +910,23 @@ mod tests { ); } + #[test] + fn reads_compressed_batches_several_to_a_frame_and_a_series_cut_across_frames() { + let series = parse_read_response(JAVA_BATCHES).unwrap(); + assert_eq!(series.len(), 2); + let floats = &series[0]; + assert_eq!((floats.id, floats.external_id.as_str()), (31, "rd_float_batches")); + assert_eq!(floats.timestamps, (0..6000).map(|i| T0 + i * 1000).collect::>()); + assert_eq!(floats.values, BinaryValues::Float((0..6000).map(|i| i as f64 * 0.5).collect())); + let texts = &series[1]; + assert_eq!((texts.id, texts.external_id.as_str()), (32, "rd_text_cut")); + assert_eq!(texts.timestamps, (0..12_000).map(|i| T0 + i * 1000).collect::>()); + assert_eq!( + texts.values, + BinaryValues::Text((0..12_000).map(|i| format!("v{}", i % 10)).collect()) + ); + } + #[test] fn consecutive_frames_of_one_series_merge_in_order() { let body = response(&[ @@ -942,7 +991,7 @@ mod tests { #[test] fn a_frame_may_be_split_anywhere() { - for body in [JAVA_VALUES, JAVA_AGGREGATES] { + for body in [JAVA_VALUES, JAVA_VALUES_ZSTD, JAVA_AGGREGATES, JAVA_BATCHES] { let whole = parse_read_response(body).unwrap(); for chunk in [1, 3, 7, 28, 1000, 65536] { assert_eq!(assemble_in_chunks(body, chunk).unwrap(), whole, "chunks of {chunk}"); diff --git a/src/timeseries/fixtures/GenerateReadFixtures.java b/src/timeseries/fixtures/GenerateReadFixtures.java index aab854a..4c31343 100644 --- a/src/timeseries/fixtures/GenerateReadFixtures.java +++ b/src/timeseries/fixtures/GenerateReadFixtures.java @@ -1,15 +1,18 @@ -// Regenerates read_values.bin and read_aggregates.bin with the platform's own frame writer, the -// code that answers POST /timeseries/data/list/binary. Run from a platform checkout that has built -// datahub-api-model, with arrow-format, flatbuffers-java, zero-allocation-hashing, zstd-jni and -// slf4j-api on the classpath: +// Regenerates read_values.bin, read_aggregates.bin and read_batches.bin with the platform's own +// DatapointFrameWriter.relayStream, the code that answers POST /timeseries/data/list/binary: Arrow +// streams compressed in-band, as ClickHouse answers, relayed into compression 2 frames. Run from a +// platform checkout that has built datahub-api-model, with arrow-format, flatbuffers-java, +// zero-allocation-hashing, zstd-jni and slf4j-api on the classpath: // // java -cp ":" \ // GenerateReadFixtures.java src/timeseries/fixtures // -// The Rust tests in binary_read.rs assert on the exact values written here. +// The Rust tests in binary_read.rs assert on the exact values written here. read_*_zstd.bin are +// the same series as compression 1 frames, written by the writer's former `wrap`, which is gone. import ai.intellistream.datahub.api.binary.*; import ai.intellistream.datahub.api.binary.ArrowIpc.*; +import java.io.ByteArrayInputStream; import java.io.ByteArrayOutputStream; import java.math.BigDecimal; import java.nio.ByteBuffer; @@ -24,8 +27,8 @@ public class GenerateReadFixtures { static final long T0 = 1_700_000_000_000L; static final PayloadCodec CODEC = new ZstdPayloadCodec(3); - /** An ingest frame's batches, rewrapped the way the API answers a read. */ - static byte[] response(DatapointValueType type, long id, String ext, DatapointFrameWriter w) { + /** An ingest frame's batches as ClickHouse would answer them, relayed the way the API answers a read. */ + static byte[] response(DatapointValueType type, long id, String ext, DatapointFrameWriter w) throws Exception { byte[] ingest = w.build(CODEC); int dirLen = ByteBuffer.wrap(ingest).order(ByteOrder.LITTLE_ENDIAN).getInt(16); int payLen = ByteBuffer.wrap(ingest).order(ByteOrder.LITTLE_ENDIAN).getInt(20); @@ -36,7 +39,28 @@ static byte[] response(DatapointValueType type, long id, String ext, DatapointFr for (Batch b : ArrowIpc.readStream(ArrowSchemaCanon.of(type), raw, 0, 100_000)) { data.add(b.toData()); } - return DatapointFrameWriter.wrap(type, false, id, ext, data, CODEC); + return relay(type, false, id, ext, ArrowIpc.writeStream(ArrowSchemaCanon.of(type), data, CODEC)); + } + + static byte[] relay(DatapointValueType type, boolean aggregates, long id, String ext, byte[] stream) throws Exception { + ByteArrayOutputStream out = new ByteArrayOutputStream(); + DatapointFrameWriter.relayStream(type, aggregates, id, ext, new ByteArrayInputStream(stream), CODEC, out); + return out.toByteArray(); + } + + /** {@code rows} rows of series {@code id} from {@code from}, a second apart, values from {@code values}. */ + static BatchData batch(long id, int from, int rows, java.util.function.IntFunction values) { + long[] ids = new long[rows]; + long[] stamps = new long[rows]; + for (int r = 0; r < rows; r++) { + ids[r] = id; + stamps[r] = T0 + (from + r) * 1000L; + } + return new BatchData(rows, new ColumnData[]{ + new ColumnData(null, 0, null, longs(ids)), + new ColumnData(null, 0, null, longs(stamps)), + values.apply(from), + }); } static ByteBuffer longs(long... v) { @@ -108,8 +132,34 @@ public static void main(String[] args) throws Exception { new ColumnData(null, 0, null, doubles(2, 3, 4)), new ColumnData(null, 0, null, doubles(6, 10, 14)), }); - aggregates.writeBytes(DatapointFrameWriter.wrap(DatapointValueType.FLOAT, true, 21, "rd_aggregates", List.of(buckets), CODEC)); + aggregates.writeBytes(relay(DatapointValueType.FLOAT, true, 21, "rd_aggregates", + ArrowIpc.writeStream(ArrowSchemaCanon.aggregates(), List.of(buckets), CODEC))); aggregates.writeBytes(FrameLimits.END_OF_RESPONSE); Files.write(out.resolve("read_aggregates.bin"), aggregates.toByteArray()); + + // Batches big enough that zstd shrinks their buffers, several to a frame, and one stream + // that came uncompressed and longer than a frame, which the relay cuts and compresses. + ByteArrayOutputStream batches = new ByteArrayOutputStream(); + int n = 3000; + List floats = List.of( + batch(31, 0, n, from -> new ColumnData(null, 0, null, doubles(java.util.stream.IntStream.range(from, from + n).mapToDouble(i -> i * 0.5).toArray()))), + batch(31, n, n, from -> new ColumnData(null, 0, null, doubles(java.util.stream.IntStream.range(from, from + n).mapToDouble(i -> i * 0.5).toArray())))); + batches.writeBytes(relay(DatapointValueType.FLOAT, false, 31, "rd_float_batches", + ArrowIpc.writeStream(ArrowSchemaCanon.of(DatapointValueType.FLOAT), floats, CODEC))); + int t = 12_000; + BatchData texts = batch(32, 0, t, from -> { + ByteBuffer offsets = ByteBuffer.allocate((t + 1) * 4).order(ByteOrder.LITTLE_ENDIAN); + ByteArrayOutputStream chars = new ByteArrayOutputStream(); + for (int i = 0; i < t; i++) { + offsets.putInt(chars.size()); + chars.writeBytes(("v" + (i % 10)).getBytes(java.nio.charset.StandardCharsets.UTF_8)); + } + offsets.putInt(chars.size()); + return new ColumnData(null, 0, offsets.flip(), ByteBuffer.wrap(chars.toByteArray())); + }); + batches.writeBytes(relay(DatapointValueType.TEXT, false, 32, "rd_text_cut", + ArrowIpc.writeStream(ArrowSchemaCanon.of(DatapointValueType.TEXT), List.of(texts)))); + batches.writeBytes(FrameLimits.END_OF_RESPONSE); + Files.write(out.resolve("read_batches.bin"), batches.toByteArray()); } } diff --git a/src/timeseries/fixtures/read_aggregates.bin b/src/timeseries/fixtures/read_aggregates.bin index adab85f38dd0ec815623a6129f14ca2bd4af309f..0f2d4584995b44784f364ca24c1ddcda71d56a1d 100644 GIT binary patch literal 1062 zcmb_bJ5Iwu5FI;)n8=Y_1e78|E>fo807zKEky0Y@sgQ!DSjL4BiUXodg(5+bBX9yv zK*0^5;s*37>B5`!4Eadu7-{xp=DppWopGXGbYj`IT_GY1z6YNnm;!gE5uTVo3oN( zmK6PLfS3(s&8O%s?`QLL{ zXJK?@b11bh{a~m9jS2ulPAxHTYtk6xSicJtN&Wq+D$#iCK3(;1O=W+dTYj{$~@XBizK5mzefr4Ndr7=0jb z-Mw*pegZ#s?e4CD4ps#zs(^oDQ%;NIh$1}^sf|@dxcLD#P*`A}7uOlWf`U#0jV0iR z=8K9P1;+~Of0khU?e+P0SHk=aKj-f}lYZ#ipXdK>%$-k$e!HV*oZa15s9{-TGhqsv z!ZHT%0-2d94-!BlF(eidC5nvg{~L_1HS3W}gWNCXeVGVOQy+$HMi{rY-}ZdOXdk|U zYYQ%8e(BG<8NGf~!V*^U8}cT1tFTZRN4qcTbYG>UBg3@xIrOA$)ioTkr*K}d#0}#O v>M*nJQ6L%4B_x9@Z*S~->O;vYS)2n}i%)xZ8LiK!|E>c5QGXhyL`XzMbOxL{ zXJK?@b11bh{a~m9jS2ulPAxHTYtk6xSicJtN&Wq+D$#iCK3(;1O=W+dTYj{$~@XBizK5mzefr4Ndr7=0jb z-Mw*pegZ#s?e4CD4ps#zs(^oDQ%;NIh$1}^sf|@dxcLD#P*`A}7uOlWf`U#0jV0iR z=8K9P1;+~Of0khU?e+P0SHk=aKj-f}lYZ#ipXdK>%$-k$e!HV*oZa15s9{-TGhqsv z!ZHT%0-2d94-!BlF(eidC5nvg{~L_1HS3W}gWNCXeVGVOQy+$HMi{rY-}ZdOXdk|U zYYQ%8e(BG<8NGf~!V*^U8}cT1tFTZRN4qcTbYG>UBg3@xIrOA$)ioTkr*K}d#0}#O v>M*nJQ6L%4B_x9@Z*S~->O;vYS)2n}i%)xZ8LiK!|E>c5QGXhyL`XzMbOxGspGg-`Nv1F0O%*>3oEbr01cmA38b|>Ec z6MyW)#?*=KuB@!A&dTmS=XB@SpSZNRG6*OLs3R5t00IC2aDa=l6aWyw3JM?s*53=V zvzeiV-A7|rLla|HQ!8_q-@pP5?2Y_~8WsQ#fCNAQpaHP&G6*y*vpL=|t1~^dvzCU#}F#rs3>uPOp?&4}}?+C0T11;Y@gZjfuu{r54U zfxX`Q0t<*RaD@O|{us;L+1lL2(Ao?rznAxTyhrcd{=L8hu>Xn({2zM(fPn+>f$eo9 zf2jXh!2<|^`+kqlpMBzit?xcy1D7_qKf3(2dXERxU-1Bd{27;j)c?Is`d5|u?@fQy z{w7mn{;5#@Yx}ouf0MQUVZZP3`>%fd&vLK+pCeK?GyO8^)e8Xth>3t_5g4F)V+<0o z2V(Soh5^fe>(^Wd=*TN*+G@5&u`Tc_#+h&S2{7Pd0 zp$}mQQF+_mJx?WjujVYuQk}9cT~DSn<10@`s83PGPvrG|4664@iHIFU*q61jTU6;I$wb!oNVa zr`}kRw5_7?-9bP2Hg|06-qGV}Oxb+>`k_$&v*?+6lH5qr@ry+<)op062pMsbQe{~4 z-BiNPy!|!%?`SSca4~nD=5mcyn(L&Dh!}FF2Dlf$a6Val>KKmDHBt?ctTw4*;x1?l zY%v#6*arAbB}rxPwg4gcNn1=xvbdbVd7}%a^rh(Wnv{`ML*FWKHFFxZ`O|L)=%$5q z8wvMPekkEqfpAo$;{c-U)&nkg}zu9^fc@I>Yu%07}2p`b{wfKAq~ zI+_nM9b<68b_ZjDcwuv;W6i?a1uq45F2CwuJlU#fE! z!=kFk@C2Gy8Oz&BIX0RUzlnk3#YD)9oe*PAk4r6nSnE4_<<8AgM({kuFZyiV*{fqg z_n7?pZF66a6qhC>UucnlF*Y5NV9%i-=?$DamRO!;vN!Y!9Onzp_Lk-yY%tiUk(DJI zhqgC7B_~v)|3TNIULJ#r#tW=|bD^``P@$C)7nML#E4VB!QAU)iIDLU;{CU}C(B4Ze z6_u+Dr)5>Dwf6FO*Q%DYTiPuTN?Nj%GASK$t5kMSi_`eDu?MD6WPLCV=|iuB2h%9V zDRf)V_rvNCsL`fI#(*DR^dZTzVpay6jXK{hQ122kBWtVI`2;1_xTy7~K7umEB+E-% zlX1@2N)xJq^}qOVUnxq5MZ0vUEHqhev#?=z%GH>Ey;VUeNLH98uR_t3ehu1m+MBk0 zMs;OLJdAKP6X`0uOTVie+P)a?lg0 zIa7YA`h)s6%sIOSD?>@*veXqRj*0>_*8Oh}$Mi5CC7#N)hQPJ<=*dF^cSb|ZdRZC> zw+2S1xIu9%;!jBWVP8R}u#MB{XVL8W$U$NsP5Z^Nm_p4M7Ym8+>y=S-$!V1G$FcA1FP? z;|2xHl)tm_p9|ueHWeb0=H$N ztXRGC73GVn0!-LFfoEjDnAtjx^u=6%-zN)R_w4n_oLVC^WF~nU)`I2l{s4o5;{~OCRbX6sK-a z3)wTjitx&K(Zu3LbCi(&r!!urIJhqy!>sxxmCNmE$@rD216;oH1|`g>C&y~Vgf@HI z!Y)L3j9(jaI%jk*={laQK2~^C2+|;?4&#YAs8sbTyv~53M4E-N zsZli%PZ+(~fw#9JJ*O8$Opciy*Qcc8(Z5ODbvT{0bkScgw2g_Uq?$-{=b9;UP>P?{2e=4$-)x?m+tr&d-X<5?Cy-j9-hO zE%^)7@aBUDg(ymKm`jjXK{Y@sonRU67W`)*lY$;*zE3ElZbJL;t}%PvT3XNe5pgvN zmu?A{_i3}avRjPpk2HQ+67q(b3~7#e#Lpm;K6V1_IXP2UQm!8=MZSMA)nj(T(iyjx zDBCRvP#7jJO;CgWxwk>GS8lz{(gPO0GyL?}@nRa-9OMcxY2?B}arZ&T+g@4tw9Pje_Y%(Jf)r#m_)CnGNgc z_E8?8h_c!=2!-bs%|O;IZ&-0x?B##=AkL5ITjb7~+_a|nER{L_47=U$SJ*E}N`$Sj z2dBzJSP60#_Rd36?D{Y(viHS7kK+c%YXU*?)17WF9T^6i>}s&WEp9@{ywEkl*1b(I z#mO%D0@?+1P`U)47~v(lAhb!4DoAAurX$oKAMVDvV(He4Nq$|z)OpIGiG6p&zUBih z{sPxQ46IN>&Xjy*i3aMcm4O-A7|g}ZlT`3H|8eJ-(?&YN#-_O|%NyLN`GkpvOi7*M zCdF++CN$#y&uiCUu2CE+_!NmOP)3%hI)QyWcX-1r>H?pDhHBf(K<*ex%x%9?wmUv6 zF26>`g%$mejB}MvT??8AlOe^U%4OLd72V(;GD%iqZpD8VFe$>TJ&Nos@-gOcBVHV8 z6cyc=^)eS_wNrHWt9llUDx8wHB?t`y=T_{ws64(1AwNZR&H`2c>6twoQx=vwxH+(j zSfdDT9qgVDu!&P{*Toi9(y!IgnwDMF5;Fdyj$gx=c6Ode_Sd}oXST^yJDHcFoEfU9 zkTL9Ejr|U}&m{-zHJig_Ad}H16HGK%oa4F@sj7T^X+Goz3Cz$Zd-DaGR+hcY7MSjl zU4w$VKZ%{GS@P|xz@J1WBx^UIT))Es;$9Lm+RNO}{m;z0H$#rTree{#N>uV5sj1pi zYdLJ18II9hBnV0s7TzM`wn94p9CWa$tcF=ims^uiyrdRHH3@G3t2A!jRt+baMsR@e0{sXeXI=$i&M6sN?rcq`|57^O z?W@08?T86;W>!*aqtI7$sN`P8Kd0!QG}vwyJZ4M675jVoI!CH??G!a5y2gTwW!0JpK=as;)+TJvS$nW{BT*q_O%Qh6 z!#Sog9bmn{uK;??l*|al;L9ce1IvbnABaqkVV=FBhQ&2^sk~Jo5RI81R!VPG_{wWB zIa1hpt#kAx>*cXJObOzTz)o>4(p^-!?XtLH>`F0%{9p9>8456*$6mzEXGsI2CPeIr z+G7buq{O0CT7=cVP_CogKxGcZh#>s}N5;oVH;fE;FaTj?F=QKf@}D5Xge3FN=~C0P zXW(8uA47cTIq4)M_LA)tHHO$yzRDq35&aWcj&jd-<=qC7B5|9Z`zvxOE8@(#&ZF>ECKQGKac-7!6YEU}+9q3mL8Bdc`e>-Q)O2{Pwa)`l+?^X%qWTUMh0tv1F6!xC+uVWEqRwP;!nr zi$`9Bjq*^Yx{_v%_qrU+I393(CXn{pLv%;!s4y^O*T5B7@#;hNh1?1AXcnpE=7(fE zsCE+|ouWD*KY|o9i!$RC!)i6R2Jq(-2843`9P8?=!=^be{@~r*Zlv2t|5LrkM(H5~ zRVpQKTG5trNtHgr&y{J`+z>i?M2Ir`(0EHT_I%Oq`Z4QfFxXI<(Gs&+mJY(5m8m{% zU;L3c4@pJJ6D+5`-%)#z<`}gV`d82qzyy0u9_9or^+G=hY2Ml+fT`DeqCcmJ@2k$z z>2(cHtM$S2<>?2Csg&Yr<*MvXnjY~F9V92QU8L93*p`s8KkFQ?2r?82pg!F#d02_E znx)SvY|I)2R!k_LQ+g&6{0a@X{07FcgzJFQng8u)uws80fG-+X9*zbAvwuxkL%C`P zLJ))r5HO8Xao+VIs=42&qdhH$uC@9552J`LV`}9UZplAsF2yZ77*3L2;aPFb8iuDM4{N? z+yP3KYfR_7;WGwtWI`|w=|!)N2g4}34YWtl*IqefTw<7PD}b71HS4njqXpdfiMwGQ z5fZUNH{y+gsB-UGcq^`z zYEW#EXKSA80vCHyei$*S^WrQ_LT#Mds)N>P#sk`Slm8xn6O!80$GApXrH*_P**Q!H z%wb3Knlu=36v7g8L0Asp2WjMPK{tNoZo0*cqICkd(7fqvIaFt*@rm|Va;u24slAP? zwfijJc(e@;(>;`DfT&<`?p!QySqNBb#@w=nGYemOZB%(l(`QBPyf%3ga@PdD@V9|0 zfKxR41hzGt#*nQjp^ASm&nvtEwjyy~Qs9qfXrqsqMtpZXjRGs5&)t+_qa^V!A1%8q;31X5`un zwKNuDP|KPU@fIRHh>hY{!mP9SGOpe{hU`|NrtUe#P^Q5Ea|cM~fdTTL4?j zI9hhRfW#;yhb1jTEdhiCd#%u z)$s->X03oOf4?>V`L+`A^^4epTUypglhvz3N!6_jaRo~O@!jL`m2 zNZ_a-Rkl}I3D`)+#2~DBKm_N?QOSCws%lrRsko_T*iZ9II=4s}#q&7&16Z@KQQJ$b0##h&*O$d`D_K^n z^o7X{RKtRbb>;hcI*=KsaL-bHt-ETHv}U;MS89G#W9Wu~3X$e%)c}$S1UrcKRflad zVn2YaWqHHH6ayw)(aEJj2cth+Hb+0Jc1eZShb!ur4;wJw@2x+oawkPAYBS6aU+3Yp z-0fcp9tDxL(9GpqXs}$SrNiYM)D$Y+C<#-Np*Bk620z@K#($h=bHx4{Qbx)y63Z=A zj;{mnxJKXS7vFfX6()Qy3v!%kn9=TSC-u8i@%MpeVB4^c#g}=Es?(e#nS}z!_V3>Y z&^1WOa$?maE=af?5Y3*x?YQ$B65Le#1l4HIw3+QPH?4I`jO?m2kELhIswNZM=Ca1E z4%+(QY(_akMC_6mgj(_S;#$CQhT!*KZtD_LQX}MpFMU)jJjFn!l?%7x^k#WK zke}id`9f~qPwE*pDq=&_78^HEgfdL5n@y_*{Db~HFwr+0-+F;2sBEU}NBw!h1!B-9 z*TVhk2a6IsJm(v$XGz<#&PVzc^=jC>pC$Y%1yqT15i-^DlAj4CBhSS=*BOuR$S!Ga z^4%7>o)o%5C0&wcw67rar07dCC5`7aBcd0@uJ4|%or5{0agN~JB%(kmS;Ff0_HiHK z3?W!VPXPLLl#U34;8x%O2g@Iq zJzH=NZXJ&C$U$je^1U?pF0-)_Hx3nx@@`CpnN6`i$k~MC+zKXS%u8RB8iqfSt&8k( zTbHw}V%9~j1PhSNR-(r*bR)w=SPq)D2QCfy(8 z5piP@)P*VWbkvVxSM4{)wP|1-WO$7V<8|j}5(c*PZ9hsp6#FUm6OzJVi!VTq((RX7 zZ?X8Im_oD)(F47-xZ1FGVO@jQ0;dj_dcjOHPavf8>cc}>%T(OOO5;AU z3+F*Gmp-|K_&HtnQ#@lp!lgCsN(9c=M3m*oO_QF@UAR#nyGX9NB?t;8{-OCMPmwUC zKZcoBTLpG|16}#<{~P4LJtr2>0qFEY)2hTPR}zYYK#hj8m<|VmnGJF~21H{k#RIkh zI4p?x#AL&L-}lEJUX(3vA&({Ab~~Ok^x6f9WHrBvF*RhVAqyqZEggwvh=2zCX&Z+lW8#+fCP9{X<-?g-omoQeT`mL0Nyz$twNGwf{z{{=$4wnRa5)ve@;W z!AOR?!jc6~=TNqX$5~ztTg^2&9*(V3_o*L|U)z?PsLwNNCY|AbFWIlCD+qXHQt(6yfD2u^t+$`> zyfIe&XvF6m?L^}g7}+vc_fIv2@=PPyz-5H1o5_}=^rdm z|G^^cA1%oL!D4Ft9^XGr{g-wu!TttxR_@+aL#5DoeNb0t{ojY;L-~iHq*niKq5qE- zjsIwY{0|nImoJSy=dVEm{)=fjOUrkbzANoe-fv%+_=qmvHC=yd=Kj#|{HdXOdet^` z@9E1GIvby|Jb&+|_h+}L$9GNcpBnQ&H28mN;(q@0>wD`NS>E~t_xL2x^z{nk{E9F7 zmv5Po|JKki-#p;P@cgP1{O3=^)vYJ!$ES^mbv5O^W5Mpf4xd4aM_$WI&sk^`!Ye2w5J3Qz=40=6kQ$-7W+QS?Ix?)B$> z26|pKc~Xw=V#EKeT{>vd!R*vgx1p7IL9^kNqpZMBC}bi3{DbN~ASZ5sE4OsqF`ejV zNzY6Ru6^#e25!iKw==i)>eQhcjg(ATmABx`sYDrCsbVYmi2OGg0i&b6>zJ|=R(#3O zU#eSHx-p#J&9w1@jDJ6LtZR&$XjaK8lu-MNDp-CaPa#uf4;5Gb9ZXnv=kz>0=Zb?E zw951l)3RW7C!adRg8gV%J8s`h@7`QFuNHYrHSd}#ugXIzVJZ#&pmYqvM-=vvUmz-ZC8QmM&Z~cs$@Sl8#jQq3jkb-|=(E85| zE}$)AQ4#MTze#GFiU(zrB(e-c_aPew=-p!yVeY`j!7@ug4So7-`Z;Ym{#w#X+?9lt zxD5##iTj+9N$-?_i@_BWFLf0oGo^hjI+9<4k|Gv02vjJL=S9Ilev^Wc!XXhe@ez6) zTGvd%jGz#;5K;z2#y{WZnUIR`7%?f9O$sK=Gsvvy3rnHM6!JH0(_o`;CSgXeD5NOI z5GjcC!k}HgK9)W?J82IxcG3=H9OQMVXehU@^IWSMMU!mWm?K8Z{@skN3k}N)m8u_T z#`2Wsv{_{i!qH<$Q09dSpw4~pdv4f)*VB5P;VGVd)y!cAeUvHl`B#IqH5=`78}*6` z^aG{x6Z(u|XYt74c<9r-X)t#|#4V3Z#sTBc|H>JQ_bBIEWY4k6pJeU?ANp(Q%vqNq zO23+xBc<>ah9id}Ni9}-1{+o67n)b^)W{B=(wsTByv@HbrTKqlN&)}MlyLu*DRuk{ z0k>HG<9h|vHw>Qw{}3TDreQc7l1Gf(G#ceINe`g?m_d-fAA=CXC=xO93VZ~dsh(sf zW;*;>)J)ish?&rRS>K>-Ue`hAn4XiqnU0;#B?Sk=Gg4j_p8^uX57gy0t1G=rPEU57 zoPqoTB^9M*G;)+zoI)HS5j62l_~lwak$;+ilHgyvB*fg)uoxdwvoLw15acdkC&%>6 zB_(o6lTf81L;~}AvtKEhD6de{P`PB_WW0mTgBcn~8xU1sRDy~FiiMN~ed5rNT*G8W zGKzu+xg?SBY5vfvwOhnOOey@^|qF)kL(x(Wyh^*p}Vu60veoSo??XdFT zauHSGSt4qpo{cUJ1rYcD~&^1HriYCjAU+Ec-U9)!+kDK z7SFV{SE zwH!U*iSYf8Ffjo2l?xEd2JpgSr3mE(#-qKu1>rgy{MY8B56%!<)PvGRVA9_{RvI8z7trGdWRH46W>BI)-$ z2Kur?2e%(qOCL|v(;)2k9ST5h{{4oam&Pl0>vP#F%q#d8Tw|$h4LM6^jG=+5Vzu62gvmD{1dlwmVLILKE<=G=Q-thSZi-TzPNhv)aZkV-u|*n zIZ^tKThWJbln?Xs&(ez#0*l4zOV?Lr_>%G|^0nB)nQSJ;LsXQQJdQ0)rzi!aGhK7p ziF;jf7mdV^nlB%B5sop08+2ovZWPG_37xsdGfBInamYabQT^zNAqSW@vXiLcRjMe~ zQFbijNwSAYYZCi`8xbihMqPjrzeY7Bw)W!e%$i+;F#q z1O-*Vx7^YjjMEa^J!1TVS`ckFoE7}kcw^fnoM9UEEhsxs34n_Ym5vE0p3*YEV)4{M zI6o{oyR{2x9MzQ(jYii~6g*sCVnAB+)6veH&X>It^gl;U7N ze4Pc}5VdXyTZqqWyV>aZ%#no6c@4Ew$usg~G*taAH(_Mn=#udV=9R2;NU$@fN>Y=g5eXY| z7F4cj>&+yxVKV(R`WZ}`IQ{3hYrcIuL2$b0{1f6|%V4!saOG6bSsA$#Y?!dX%u%PB z9s@9Qv_57KlD8`7TOh7fLU}*S`z-{Ws}RFi%rgfG{0GiM`lneQ`s}or>{3RTsF@Rf z&I!?)GIVwOwB8f>K}jc~pRq_`(nZmMEn4br)Fx>6;Y6e#@9=x!_cD1%LV8&}&_*Fo zL0Z89KiEEMhewmULjp4@9Y#RV(@%^k8_fq_CpGJ$?8Vh;^tq-Yp`!>N636N1k^Q7y z$pV5lLaA;m6t=XDtv<&EN-?9feKf|pA`3Q&;lpwec|L8YIDa%*#L47$E z3D_OA2(^a5^dT5E^xb~EXGao;%S!mBXj)nKh}C5~Y(Cd-dQsEEhoY_xi_q-l%rTU9 zEJLK~aD){YHq6VWd>XcJx0!G>@94nUah3t2n|NuqKz{)%m!a+rlY1%A!*e6m4mMOj z>u5U2V1+>>fq1kf+*&`mA8%o_vGw~{%yBPlSqn2A6=Pu5D)OXUfJ8~@(db28e#i5 z%TBDL45!)t#K?Whq_!>j*S8UIaaBx_qeY_2SZpYTy^&G}w2fMX#ktyZc z4YYe`gCJ#)3U^dxB5y+sglKdc$UhobXXnJSE9NhzBXvS6oxCxQXB*S58#nsikv-jI?oLN z##N@s=TOg|KgSPcULd-`y+e=Mx5e`@t>%oNtX$ama2)K29zoDNQ`)C>s_2*0eWZ$h zlsK!rp|x2LnyJc?)w^Gjz9{Q|f|@n`s05wlI;Ckp*HM?z-hATdzI3aNQkJZwOi72@ zD$OtT`X#KpBJ1_VNhZgH7`WrA&^T8*|rnFO|tP zsZWU-R)1MntEbVK|DEh|%b{gmtRQtVLF#3$txT}x?aFVE!;E-xs8*C~J<@A*$k>9p zKBjmOM9N8%lrO14!Wf?pm9w`rYWa*s9K{miNlMWd=^->qV1w`k(ft`J6mj&5Ki{hw zupE?^vBGCyhxPai^;1flvW_i%t9n+%aJ*> zO_HN*M>S4++_nk&;kVB&D~&!?)Fo;&6t<_R5h|-ALN55Zkl3bn#+AU-5#=U|bNK2& zx5)SH>JT((Xeuxc5nBKua}>bNvaNw&MM@rC<5ASDx@BGahAwkKDaBt1wn<9SB1gmI zWGR7vDcnJuX#W??OT5PvuTAzM!WRoc)40CAbAuj+-IPRFSabVb%7~b0QNj4GuqBXH zY|C^eRZP3cr(pR)OegTwu|mgPwc9{C6GN#aw+utToxs*@CzE3A{ba@8{I3tMn6&BHK&r}t&|at0tR^MF6+ zMvTjvQP*>%uK^QBYd;#Dwca4rW47>9R_V)HQ|T7&FQhH^DPZ9ef~JIM^DvS0EMrYO z`xefv1X)eeXW=@`yLVM13Z@nO5?{lKKp}Cw(|J|#&ZD0~CJGgu@}}iHOrbxcD{*qz z(e-Yh_r6_&c>~c$>j4^^VoBA*dd4?QUP~h>OScWentM4Bv z7)gOD(#KCsItQK+Hrkui>v^I6C@YGa#cRLrgcRgJrJKD@x-1 z>~}WG#Y(sRVm>CTU;l}Twq1?j^s)HkEptfnsB~FQN4Ws_(qy2WP+Km++j02E6-YNt zHV3@9ne*?7x;GzssmRdKWORaay!}V{Z1A}d^h1wG^+bkq98Y!6AX_K#&*7g}6?dYD zIjJ%7ZRkL_!~&jiy3fhRqdW=jISvo2(JhU;TfbDkJy8=RFVReh>+n)k#OX*n7PiL= zK>u!hdbtgEm*qLma*K(gq!bH#F>q!igwryQHsO^?sgv3yagOg8sTTXXlR*OCz<;3D zzyK1*2CX}|;trn&p(6SM%x#c5!VOxpLpE*EYaYf04oDhN*0N*io~2N9R$e)%bh$&ams)h6+Ebc$&W2~n=S9?BP>~1tH01kFs}g5V*xftqNoh7)2PWUT~>nu zASN^A6D_&KXI{RF3>~?;;+KR;Xr(5+AJ7p2V}e$>3$b)meqyyRR=2DM@XzM^CMGgf zbt(o_4T(QS1;nKUh<0Q)xfL)Zb^D#!tkZf&1O|^p)|E|!8NQ&dL*0Pl_A$jHU1I?P zrXs_(o&JQS#Ds49!?llRkm@%wR(M$Ynxcbs9WZlWv9rA?43-w?{_uS&}oNzlP zu#)CHF(+7SN|V*h)4D3;izPo2{Xj&Lk`9V?YEf2iqBcUigXROjG(NgEM_w9(p$<+7 z%p&F#MtY_M?0V zoy<0=J8~X%LkfwGF@!$UjTF(dLGGbke&FP=S!ul*_Rq5oioHtfje1wjP5tmldbdzj zf#%F@xjS;;t+AEiJpt<3m~eTXdYm^^uGqkfo%2!_$Xddb*&O}(57~qTuWvc05?$@r zQ0;qz1+!MBy$lu@oUrSIZ;$N}_7qU;109EoK$sQ0-)Fr^G=K~^c3#G~su_JfSNeL@ zqTAZ66Ox8UQsq=_tc`v8oZecBqnH0Z3o2Sl$h62AQJPY8Sk2wu{>N8bZ<-U=rdbrg$EtI>Cln=ITth86GhVv!OM63LI>+ z#!04xYk$Fbf^PxTm@6~DXHh_wEw5N{v=z<~OMl+pUSme&)y;RRL;Utkw%^Nsl+UA3 zQGJ0nZ=!UD8wxrW%2Uc6SpzwU{y4!wgPS^mC;6!QvAyB|}-Riu_4sB3$tv?jV?-xbA!UNK?|7@m80u z89PS?U-aA19;yjRRkn&6RW<7KDE}bh=cncGSD{v^%+t;(F5lX~1^kfPp}v9)GV3r? z6eBG|FoCuU=eTzkoJ{eX@31&8!^j^j9DcT)Y_QXO-pD;=qllyxD#+51XDs1HwSU4o z2=9W2#Wi(Nf^Br&)O7gJ*^1*E4Mlk_YQs}q{$Z7hISD61A3Sd@(1(8z-yA`0__SXi zWn{+jHv`s=Pym4>!eK}W2+ z#$KKnp9)!8we+nO*D$A@pQj!Bh5oUS#aEW+f*|lNd^Gukh~4XD!cOMYuTuwpjvXB5 z2^h(XS)0J>i4W&g+(`o8EXC1PK}9^j@T}w3!> zLjE`+fEG~ged(*3SS4WOJZoX%-*WP88q_kUb6xNnqreC%{3YyDP;^0yquex!2R!IzQTI_;ASFs=05?U?(bujt#w~JOmyUV?-+%J9$v=M40BzQayW?DIP zRcEt@o16TS17Y|?vxNqHQVPz{Fx4)!OTZqhM3%OrA3!(EjmGF}y0OWbA(pdaFh}Eo z(hphu`GW$*bPY5>C@XDc)X}8#?FIGj7WlG9-8!F!#BU*5r~|&h3`x1t;(L_wV+C@o zN~jJ0dqIQTX+)YMpQ0M;nO?I)<`yiU@z{eZQc0SeeAz{^%Xn_+jlHwcj~-~jEUBe> zvJDiQDv&BsTv@SwjmYn~BRDm1dk}7WjX(>ofuxI(9%ItFM668P>VvUJ{h3ch>JDR3 zph7WO^71qkoU^u6OqEC*ZXLu-h)?m?GE%;4oOOIz^}5*(SGU$E)yW^Gsp!zKWb}e_ ze+oGh3PNd_CL4E1Yt}h2Wxr~4ZrFQ4DT#Rm7CTBR<2At=HD?4cU;+GF*nYa#YgH8W zEUtHB?6H6GKdO4AiJylaJ5RPSVdzfWm*ObmOK=z^LkA`OneC~@SB?KX*)=Q~l(aJB zV<^r@f!Q>MJLHB+R}l9~!gJ~yaO5$-{sG(WW$c@o{x0DIa&DL-Sj0|KzN{5dFPsIq zI|#qM63Dg-B`sZC!GMMlV=Xt9{@M7Lp{IsXX4@Wo7E~Hx->r zS7#Yy!uB~}F%m)*MQQU}kg_dXOvn4S&h2{GO)^yBf0!TannYAhtMnxKL@0@nlG*%_ zR(c$^9b-PdO`TKmpzKF63!e+>C>R3hrc*28Trd^z;lYLYB*uip=!q4j}K1D^U?f}tu(=`(6#Os~?J*K>rKPXnauT9^C zp(#a(B5XG+Kt__RG#(v#d~cFusoZ>>(G}Am@)}%b7rnsm9ei{M#|gm?1Rwnz1S<)H z)sneljr!EFymV*ihZSyHY(6ZjTVJ*Lfwcm6enVzyUmun#A%04-vTzYu-U`y}sqgOm zQx6Ymb|ONi1;bus#Mfy(!Bn3IRbe+hlTsQKOkdwoOW!@t2W*2VWgy^*y&%>~te5Z* zffI7QUvMXgn3WnWA94}o5(ZGurn4fHigwr6Yf!{E2LwF%{t0+;Q0cgEg0*d!m8ptd z#gSwmZz1O2eDU!Z&tOSi2vgx;x1AM%Y( zDC96wn#85LdI!tK8X5M0y;;nIy>Q0?!NMvt)Udon>WNjIyt+zy19Sy@bvQb@%8f$2>cur91l>R}+dI2^`v-?d$0w&}=NFe(*EhF!_YaRx z&o4iIzW#dq4Ji0e4?y96dH{<4(*scapB{jc|MUQq{-+0^?Em8clnMWV0(x@*dB3v& z62uT+6CLLa_jBvBQ^F08^>Z7rOa8H)RA5 z8BG(q2ltI!YrU~PjTd3`lj`PFsC}rWexB>97`~@-lgT4?hfeOA3oDssa{21Owz%Pv45mZikTU3Fv7n$fEA!Q%U)2ghEK}e>zG@bXYx%Fm!=?N zpV1k`K>vB;ZO7lGhy~@!R>|_c*i?&|35y3#{~)(t;3hp-dX&@>c|ocQtcT@$x$!F| z!K-|@;S0!v2D}57)b)lzFOn;GFBo9h!J=gzm+r!slBp0x7HEM^Q^d}-$u8u z!s0`7IZ~y|57vkjS()-8w*+r$%6*U(`lY$*?Q9i}m0}v*LkV-Hdr#Ayau1b$%HqVi zaNgoyAb06b^Bg_VZ@aoAh14?nB#VeOq0681jKsDPI>7S-6VQzOO4Q!m*J67yn zM!I;xC1))e7g`&(Jq^^yzp5cUc%pKoq)p4vk*BF-k76}I-3Sp7A%E8ziB9S;Ug)yi zXJJq8gex0rL{%io%a&Iow@2X~Juh0k8B*MX8^WXr&(son5TrpEO&_^L(;}&Om3RqaHSjrmAby_zvj7Q!PT8b*{W0YWa`Pjj!F)$ zF(Qw~dQEWT3{*1V@sy0Oo3t@)rXs`197Gp#TBr0%9+Eu7Z516o+l8}^VI4`8qvbkiWBtyjZ= zeQ2_AW*SQ3C+|ejoW}u$it(2hq#R9cO!;aVw2j{ZZxo{P`Gi=%m(T-d6dW+U;b6bc@lgjl#>}1ak5{w} z?`nQ5)-IU()`+Vg(74XSjvb&|-lm+V37zU8&n#yKs-`a7)!gFibgwyD^RpFT-;TFS z49%(r7ELJFQ2dsrx&=Y;PUca@tAc-$=;|-3lXFDLfpQ%1=qrvFHNxwHF#~G~@CF>U z0bjb8+$>{TaER-Bfd5kZwZ`Eri=u~CyP$kg)uDVjY01oVH@QRFs7NKjnwntn>i{ zn!q=yNYOErjrvH&818!pJB(LxE>Lh5XJ)oy z3@w;iFi#NQZN^?0=xtB9=7gmwYuct%oNPNr4Y;3Qd25l=ErCP!C(Zeu*wNoz@rIlsWx2`w91L)lNa3@CBQ;>yIfNL*5O z1LJkZj+pweb%WIX_wf!!OkL<^(5^wcAS0e&{ebL!cfV#G?|~_cPQk~B2O~o!6-{fn zoaP>9?V)lX8eR0?^Y)-DG077^;wFuE;wJIm?LQqo`S}X)ZztO&CFT+X2PX7w7=Fug z2~nG9Q;K|359b@^>#|zfX5q!kkGu)yA%*C3_Fx;uu!4C4iwz7TrWXc%D=FOBI^3LcbVw*q%uT( z-=I|{l*w>fjR-N6yOk5_z;TM`5-lK}{oXFsJvSS;Ghu(q{#%~tE)bY6wQNqNL$AUmPoNV|(>AGIm`C)klEc7JTNqr1AucpsR>$F9T2MmO6$_8aYX zc!?HNm!P5UF2 z>5E})e)W1E@^R!4m36+%hg*)ZP^YIvWl_}ZmMklACn3UUnZcX>z$rC<;+&-hUpsOyJU9`Mxz|NqufZOZo(BH03NQI%ZA1d`y z_*Qrfq(5U!(xgci=jFw!2-6W|&VNrf*7{DBMcR#_Gc*>~q&?qj+R^gME&;(kxIl1^ zkQN~mVmpv>&9ATeS~-|}6xk9=UCb6FmszMB=M{XTw^bqFxq3a+M7N@SX4k?X!#8?~ zfH|JfB||V}XzGeqyHsYQfLkfI^ct1tB)FK>2CXl|LZmtGd)eO0X}FUtS9#_`G#@48 zUi-36TZ#khvzeI*sZ3OzhyhUp;*Vj4Ah>Q!66s8Cp$DStkm^LMYp_NNzZ&yMYGbzMA`~BaHdS8ggz$>+~{PKC-cO(ffh zjiIakWz!@+z*m+_K-jYa9$3sYG0r8HJI)fUhGFUmiBDEIl_N;*Q$MTVTqC)c0)a z-!fz_5=6NiRKO}F6e%jxmT{u#S*NP;{cw=FZ#KzKlz8}3#Yd$ffX9cxH5UQ$HCk7m zo(6++Y)xE46t6wPQMilH7;$!(M$C7nCO~C^{1kahZ0lMdnI;)|8qyYYNjO>|;J1a# z-q$L~%_6or#kl$hIQh~Oy;cT?)F)aqi9%kviUJ*_y3!X_ffK3I7(t1`0%znOmIr&g z5ogcbk9a+$Vql3l1OE$}9&F)EwIi)0&7H`ns8TW$>KKLdMd20Y|03?Lg5q4;ZDHdY z+#7dyXq@2g?(W{WCb+x1ySuwXfZ*=#K?4Mm{5|KlYOl3-{pahjtM01j;63Xa;~K=H z6QVOJS|Zf7sN1mjkjjJoxPg9GtwC$!wluA&1Uc8tG{_|ZJH5OHMs;ks7nd4k`F7Gn zg;q))<<33T=gUlCIkVDcjA|S1W7O!s-olp~F0QKD(T5 zWM0uXZ(-%$yoJv*^N#aC0m^Gz)~RFES@EwXc|O{`2$7UOi4oQ_#8ch=CDKmXU(o}- zNN|wpxX4wTr5*bb239@}s=d|3XpfU0BjrkApP4JyiE-LF!S6Wvx=K zgk#aYc@%g>OHv7QQi7lfX(F;d`^TvwkrKES+@~jjgQCq~NN6aJAcx%Ky5Uf7E>1I`{S5YByJm zS56XA@iX#NyxWfZ?+ajLgCOyAjFnLO(pv36i-Bd z3mP@(H{tI1AK|YdbP7`q`ipihSIQusYpWGCy84O}d4C4U{`Qci`CGm<8i z%dWjkoDkWl7z=dXimPR>Hw6;yJ zJuWWJdhBDQI*W^BFehvGfOhd&n1mv}84h6XaZnwp{EFd;uv@eQp@&bKiKYg}6x1QDVPl-bmpsjmOBeVfv56 zxhudy44AtKS3k-_oJcfLVzEnaM`E({RAYUU@l9flJTF-e+SRe{WHZAR4s#6tx&uE? z$ew8k2)b#2$|`~1UVw1f>2)aLR@;12FkGbwiJBd{H2SJN@VtO&npQWjQA4k-@iFDx z%y}=DM>W6DIVqTmX$ILBa4ZN`2T&_~9fnhx+={C+eug&8SfQNoT*wJi9wt7(Pn(Y& ztVWu9TaUXH9_?A>u*Cw82R%#Xfik*Xz(-FdpgbnhW|LX!mD5kNF=Whu6; z%7qlnezUU}(82rp298d5sOT|*qtm(5InB}_6=GVpC7r3%RDLtOz6c93uELy?CvSV+ z2n}VR-DY~w!iIdn7pA;vJV^&N2GO`$@oiFew1koNAxn4GK^#h?6bV4^&7EB$^E{Rl zf&nzEs7;_d&HhKZlYP_B@~MqSi}Q=QF*6jSE!Wzmq3->)_C}?56$|#c?XrcD9Bosg zk(4V%_qid-#sZ;c`qguYyQ}O4LJQVfMP+?3}X9g>Lbx18)OW+9R(ty9d1QRt5*YxJ7 zCMMy%xC2?QbOv-Q`!t(zDWeWHDHl`Xt;xzR))Xh!4kIE%W!4U%&aV zZXWRU8U_vc=T7>BVg@dCf-1+=tSXnWYA*Vyl#9uC;&>(Vi(Js(&W=kCZLbDFR^WO^ zLV3#DaHOFeb7h)G^gET+VbVGk-eTN-yegM%Df!kRtR>cySI9oEH^DGcY3Nc=ccBmj zR%QldWlwCi~Ycl6T+HHp4aThs7 zvk-KZLk+yq8DHD4ttYMm0#$`K*qvrGP}QiaKb9ZjXK>UoC{}2Os0thjc^5MhE$NhZ zp%6qnfO{C06%CsQ24c;?*+K4|=54)<*`37k>pQ;a82FiZvewtMz|H}Tvuf2e9IALy zucKDo%@65(l7^*9itFq>M#A$c$KN1qyof_K6(vcF}L0rY=mq*rteTqZgt(^G1R6Gw7Fa#-eBakY*f4 zu3+kPw$I#pw>=4DX2{mKR{5^=lfl;Gm?av`N6eFK&eM0$BiFo|o)8%D zZIbThWOvx~zA{vGlBNQ4MMePT5Bp;!$e++~{6)+_wOB;9mV6tv3C?+_e30B7We}zW zQavp2iLmEpj7GhrWM__dO8|G3sxd{&t&Jp|`}=csLUqP3v8a4G#q%pb7J@hyc?Oyu5*OyqGo@u=_12b%D?v9imdO(dY&rMr>vPrtR=rj<11d&U zsp(RevZv~>%hga zm#c_CZRSmKXVDC-)2Uw4e3kwDfPoQ|Dl|`2gQ6|_+^CE&cGYd~Q83dM`E&r9A!TlFMtchh+D9X4v1<5LM z&-r)hQ}Ls&8yDAJo&x~ptgIBul4nrcjMgPRsT}J76@@O6=!8cE<2aYUP0v0eql?2?I-JuGPzFO`L*< zlIu*bOOyG<*X#tf49TZ}-w06NK zMrIq+8OBcXHR0*Svq0hs?QugAWFZns->2ny?pWFP^VHfR)usIji{+Qrt)hR$^rAU4 zUDB)wQr)3-S&E!0O_lmj-RG&05Pfap8`dWINv`uUH!~n}hN%4Z;ro{pPDwh;C>mW^ zZ5hE|#6VP|k{HNte9cVSMGU6M3~=$X-bHzz#QT#Nj73n21jLGd4N5;yxu)p1mEnf9 z0+H4qF`v55Z9Ndvk7=W^lXz_Wgggb_w$c6{lpAFjJ7K({`2{X$K4+yR!?s(4wpN^N znP}y1Z_SaKa`crc9#MW&G>1X!!kzKAaJ=TBrp+?yjnqaM&G5b>aY0wT2&S|+xw-s> z#6zY|j}0{ew!b$z6eP4T#Y5`)3=Er?KYt4!)Ay(nQ7@r;RPT-&ds0gV*H~{Qx5*P! z5F)SLPE~JOUH5q&vjMZLOAFs)ywkN++`n;04(V_MPps_AV*X2yi2A4^TnkY!CEKsF_>E|JgE9B1x0j02^|nYZrthb%rRVlq@EoJo3>I56*7FUwwE@<`6yD68qe z7n*Cav1IVZG?*zLDD%^hB{xM9P(2S_dOJ^Y+TeR3qdqBwlD-{0X?${om(KL7+b2DT z5k`j?b+QC5Z!*jV;IckWUYi-{oFQ*>?{e6dHLhzpGi+cpU&4)n9yr3%WMnHSJyiV= zXtTC*E!}I;`yB zry@@q?m^sB#OIK`LfY3)k)$xGd-INrzzjX2S)LZhFSRwICX|k+azcz(c_*5!{uzg2 zKKWu`;n)o<99x2gW8{_KKLvAVBb2N3M}FR)Q&f_}xIQ71 z!ZySmlM8pljj)LMG|LUYA^bSQyWcUpL!5)W3#kHqAsRj1TEkXAJGwPN6}_tDbjeFuU_ng|EU3j?iVxfb32QB@*PrKVdTfkVv6{ym7EncR>K{>F8F zT)(#axXlgv={-}cZo>5YG+BuG>dRQugGyK#m^h}@R0mgBxx}jp0wj0ld1OA+A4KnK zJ+^t92ph(wMyA|N2U$vRm*n`v0vjv2R{eOlbu2gdbPCO+wX9pZ%)oAq~n^0_4OYcKGpT$;(c?&sisiQE74G4rsZ8S zs7ik#^>`7dYPTdyHGJ;07o4c^B+M`8gpMW{$kbP3VoKwQ-VW7rB|V7r78xe7h+hh4 zC<%aU$W&dX>W{oV*djBcqc22Xg>wpC55zO^%XPaF)^Y4rw62;Z@v|quSKRC7)!(hX z!GBkGS3Rkm4+V z(xk*J4Br|Pv7%|~#Kd}Zt7*1X?5lECW-mqbl>NXVO=F3+aoiDMzj|Kx;>|-vUAIy& z&z2o)G~Q+Az|Ed!4s$zXModVQP$IKS0Kz+%r>hRUGWkX~gwhp11v=@lc4C_$^as@i zy6r%yK$CW5eXbkdZQ$G}PuLmlG%;#W!Lz-}E$g?_9jZK4^Qin=<@}2+Ds}q5n(8FS zf7b)N^XE|Mto_aFWfvMPcG+CPI(a}Crx2Z~EDfKmri{R_fK^lZy}3JH0n)t$9~5kp zs|)8SPA|+mr1d;G1ad3^ROYUl<8_-?_NO>2PsXEIpOIl>ss^kKy_z`Ko@~vP3gIe} zuXz%kt)cx_lxuMqZ>zYYzuo-?1t^KJls3Z^AG4Fj=8c_L3~(Fd=)&AxSyU5RB}~ZN z68I+u7Oef*BycT}T1Od%pV6J~@UG*f&-p7&j%Oh0;pSZZ1`Oi*e^_sWcQRyRKY9LC z^Jr_|*L|!3S5H?I1ByqLPiU~0@THW zI{d6eT_|~{!Xt6_gP{hKOx8G>a#t`$9HdoIA7ncd72>_Bb0Vg-OxxHXMAqRygMFyb z>v0$}`3xpqUV#olOWU^&x!lTnudxQ66zWlJL;FT8^_!~|E2aT;qZ(!4agJY8JJ{wA zsSdK+svQV|T$mh?WdZAg4aB;rzfY}3%8atmsrzy(e9O2^>D+W_-=uye{)m_?84X(1 zG4EtC!}$}c7#wlV9|({_J`H_>{I)MihF8c4(>Vaz5_BNv=Gfd&&|C$CM9dD@8GY7% zDPRJzXcs-mO~)N6k+iek%Mw#1E@VzJr6QUA?)5ts;K~y~JsVfY$|s-5HPK*k#O<7B zD)W5Ea;GxJV3)aoUTNBOu^wd;L?%3@=|=lt^Q8%IHK*(~)Dm&yfvkt#_dsq*T2->W zZ1E(Bb)eCLfA9ZQ|07+dCN!*FPra+$O!ZI7Y?SpSLr}7m^l9-Ey8JmT)l1RiHY3l3 zx;W6nH(hmM71@7I+wM{NLQ$ByILQ^^m-)%TVWj;l{!iroVsXf1eDO?THI&*gui%eB zD;L`RPu|!tp?i~#w7@hyLKG(;V@=%eZs>O#cI^vQ-K*>RXiMd3)G;BC>_Pb>DlE0> zNW~}IWY~o-PQpYa4NCYC?rznp-&@U1*f=xzzjZ8D_$y1$P-C`E0!E)PuI_jo0D{+g zf%`VYAB$+UFJ_4X{KGea`uoHPh>2JbxjV}ab=*3eAGJn5Q=ZKV*cPk@* zO?tA);tN zflj)vk~ZApYvgaKT5>LrdFzkVmt&$!v26Zox_)lx|$*jG8Uo z>Qaqtzl3r(B`8i%vWVPu{!RL1=`iSO24u(cRZ;pe31cMHP=&b$$4l;Gl(Q+tr5Ot< z{s>iF!M5>WOAO8y>jeQdbb@?FM2nZY5_J>yE(pBl12Oor+7_@jYD>+Px{*Bt=4l`M z9gK*ywUJ}FyE!z=D%Mq5>9Vl$F4{-?c1z}|G4kpFm6oPLk9yzVTzqp_<7~>YL%!&C zbn2LalLa@StQYZt5>jMV30&~@M``2(h1%QnotXKN^Uz%fy@hHv>r>yG?W64{M*bfq&yq#=-1m*H;~Fj6fSjp#sD3iF~M5)J0%t0dS& zY7E^F?c1k3hwUd3dD5~IZ>U=`uwmycm;sJQcnpXbQL?mX0b)F!tCN)%N^Uc+j)#h< zAB0G&oMI51&P2H1U!DX=i7}Km;1pZ3L1UxFAGiX@4U4jnd9m^{l;DcU~w!rhBsgse8^BSyRs9LP6AaEbI)?CTx-KV5YdI7lt$`u5lC0xRjG zS$@mHCQ#-Z_iF^urnN5>}91&>(oCmvJg;?e-2&TIfZh{;L{>=OX`DrDSnka{-65ZDI)tc z0$r~lydV?0-kkN17Q3zYn=!Kf(P@ABdQ7#S>+&$wr)4m3Nk;R3uFBC zioqSZS7iUn6p1wrPZxp{q)(=WKIqrd0dgSbX4_n!(MSiFNAwL?7;)8axi$Cl8Awjk z(sMfr4Ba_YefeBEcP>wb%K%&SWmV`lFH?!FF{|D!KtEvhMb1LASWLdI$UwP?=Id1X zD5Xc7h-3-5o64_|DaF%4Vuk5~%-cWhgxtj>ai#pLtJtS+ z-6%v_b46PF^2U1gZ4qiJCe{(GBDPVII@AOgtdhX~F$v@xP)>hFd-&)BnNe%(VaoxES;dRW*X4LA3Q_Uy0*AOh5n^Igg)Ir$6>ZSp zvi6qkzifW{DtQ47;nmCY-e0!)8Sd^Wl|e^>-Pc21|Nt2umlCYx?93Ppt+ zvm;?8g(-qaq6XhylVP3Kxmp%XF)_-vrZDk56uEBeWZSY=-AI@=?6QMjp zf0UjM8M|LiG@oa=#UVmUkiZZAr%lP+inR~V83GvMnU6tu?T^}$bD(SE1`{A!r*RwH z@T&_Z^$R)8i&=)x(;W2t{X(GZUikwm%*81g#bw-J*x4_aLZr$Kk7F1+sdn?6SNM1e zGh^0Gb$9eTScy}SqfE(grUAYD$KZCbz_FgUiB0gw99s37Szr&J*v{YJFJL)f zyrpi{!1kVVU`A;?#$#B-_y|m@%YjMt_GI;i@>^9Rx&-yis9JY9HR86IeR0krz`S)& zzBA5QD#Nsu;tmJ` zeO3AUc#m

M_q-L#VCjKc3Jua>tT~J3q}k=AX=An$#1271t2bGSXuu%m{iZgMY-}Et9noh zE0Iy5rR7%FM@;7usKyJB==%*7XRkonvO3pbB^)~Me!Bz&?t+xjbZAknMK+hY=5Uo{_6CthzZ!EDxqQL~- z#Mxif2Q5k2(6n$7=wIW|ycc`zbav_))z0HoT?&@P+ldb4o+@~h{rxFjz(~QM&ow{Y z?R&WLz3Jc%Xb6~}p&S25b~0`1)i|bePw_+a8l}X9BKOPX(;%$-^g7mCEOjsM{$GxJ zS)-T>;V5D+q#wlJApxg-(wZ!<%h;jHA(FPQtXML(Yy4%rR;cZa z>CR!zIc+}%MPxE`wrSnITqN-;ED*}5kPtd>SIY6U%^iSzV@X3>60VEgG4*# zhRH&nkSn1<=7`2S;ZTx(Z|aU~jMOAS5B1#W|!2YL;ioXXQN_40Tz=xK^Mm&RF;d86opqI%MM{W8-!f=r!C~8LJlEheg0PN+N{^RA@ zBWg=r4>>oo-_^%J{8SB^DudP``8c8Z+P%pf=Xc7CoRl;=XJXf&h385&HXgHs6oWuq z)ySqXTLXbMxbc4Z*ddoHuV&Nh>3Xo9{MAt=S2Y8Hi3^G2cd6gH04p&8cn(2pB=3Ga z)Z~9G5LG37PzX()i7%_}idfgO8Q_ADjEDccM$AF8$L0}`-_!i3^W4s$A(4~!R5{p5!U$?(zUbqC{-hTdS*MyVJX(i5LM6L%5uIwT_~Hju^T z41F-*u{~^E#?G3nO}FL~5QtXuHK6J6CeBs;P1~HNUmeU0VuYF&4-rJaA*ArAp0{U!i44$BwKuTHfL9 zAfb=R41Y@UG~w>WJw|*A>4D4`atS0VQ{9xoC}Xd`6SK3Cx;`uvxW zj@flPyT7+{!uo<*t8|TNo^E5i-Tgt`dC=>q$cFKjd$@ z-pl`)2rUSc72ms%iOcAFv6pXrV86Zj zBkL{>2U7MZbMWK7BLZt8wi1F}2oTc2q@27E>^43B)x`R-NMP46vCR*6YO9aeu`&x~ z99OHLVM`BOzLH&)Fz%)YCG|=bQ7q2$$^2e=2)du~xa9q-X#7}&F_mgM&r*ZCCMP)d zxk*01lv0%Z$mUi?5NYB@Ur6U;3>a(nRh=q3S9zgzN{OH^m5|~mtcunXzbkxG zo;^Fe@D%1d%yC7DwxT`?j{$ia)kgqq~*LH+5KrRxVI$cNLz@!l$+LgVsiE1PKUFjJmOU zIkIU|TP;$0OPNARoJojK8>H|<+(FeU%B_?p=$sM!BOUX!{%8_#)KIO1NR)wB%&Vsj z8_V9U_25NGE-O^|uT8y12Cm>uEg`oWK|U(Uacf!}C+49aAC;b8i;d2)sD9DE6nbg# zv*dQhDVrA@nDeudWw6KSQl|~``(#W?+faC+&qY*F_KME&npQGxV!K4?O57ay5s$1z zVNAy}=`ekG~D zr2kxi3X=h{C}35vK2In9t9i?Ttu~y5H0x=Wk?Aa6w85gR6XiDg59!H}QP8T2c`J(v z&Leo=3T;8aA0UBz0`oL9Jt&EecgF&?zW3UWN+{zPYXy`~YZ~7*ePGRA%Zqu;uXUl( z&TE#djyq7|G|>N*!e5#I-pFAJMbi7->31lQA5R?7bFmy~JHqYprv;f11TX-GcdTg`sce(*D%{66Jijrj@wpd@>NX> zyH?E{o9l*dgM$1slE2i``8#S%d}uT$_SDy&3^8fb5>=$^DLPb`Cf}RhTm%h?+2+p2 zHB$0TCwfgRSlrXMka))YFgeJRawV6^>dA8;^hhYT2|Dr%*@pztU4shW_8_DP$m_52S6WI52Ya=xxt1 zwj|tlL&Ghavn*9KufG{mf8(kWz&@K=!ZxCy%}dmfa-`^-XR7@9_~s7Ybz@h$fYWTr z2ThEc2;%7@cPuJG7Q`wjP}yPtC$Gd;MfW;wU5eHyU}Uk*D!$kV;g4*uVDyLd?+cJ& z(k(!U^wez_*f(+gov|8E@fZ^^En#)j_Khif9!}O@sJd0(rbE-1jGA%RQloB*-C4Yum9Z(yR+^=ncTRX2(*K#F0258mI_{`DYUPy^NI+741!YH2m#Mx;%ano_We|$$ zny`%uw*M0kHpF1{vF>=jR`wMXX}?lMqE5q}2c-qt!-;IqTDLUZ+Y)mnZe-3dr6ZXK z^z|I;bJYppo=vOc2Cs(sV8|Re3(`#Gl)biBQK}>S}u_)%pTG0Z!#u&|e5T zGIi7#98x=?YD0mpSOI*aq8i}iVY!4GG&*^`y=`fddSr1$O}bEg^nF_Vj&v zVhn5T9(2v)uNVPG#Tm}tQ>v%K_yhY5arRhgI4M8Tg~}_pSgr)+0GCrI=arH_G9e_h zs5Veehqx2@5IR3lkp|*r)fT!TZb`w09!zEGE8t4Kcl&@kd-Y7gRA!Z&&`Nx;=xpw# z!k*gK0qJ6W3b8Hj#BbxiA9hOzF5N(r?1fl*n%~1tK24Jv=X4!dPElS|RANwkewBQB zgpILpl6{92=hVU&#SyO1zNTgSVv&TeFu)0TcXQrJ#}BKvC#$@^cK_J=A0`H@%Gx$D zxgPy$x^0#FYHpRAOMA=g0jZ-hmXsO`c9X=|U{U^Ba9fEM-k-CceWBE1oy{3f7uoRy z$VX>HKtn#I9?9P>U;-*vjo6dazRyFKP9;-8wTJ5vV;T}@ByfQ-gj7)oT*#?n?i*L~ zE1H@)x9DJZ%6wrwmOSLQuWHiMZmfRO5W1Y`TY`^CpT-Ml?Pe+NdonPrJTd; zTcx%-oDKMI$bBbs!Q2{RYo>T^u8qdYA0xwtCXFgsRFe7(wts3B0IbcZ;{qrn@&+`Fm{~ekiE2Eb zwLRV_smCp8v5lC!oIfUeJSmJSvA}%(OQx3^KT~c;Ty8kfE#Y9$+hCZTCwdqJmXLmh#Nx1Pg(~m~L^hB8TJ zz9NeF^>l#N1?^0^nm4KxxK~j_zRvdA8SHuK?lILfSf(xmdh*QH^_yuyGQ^Z@3p$h7 zslQBLdc7?8xpKEqD#it{;>m|{4VIZ4adf7yON}0qo+ymb7o@A8`8MhA{^uX8;K<-FdSF*EfZ6~O80BWJV z5Ay#$!3>~B&Rf&0CrWu}x{0#CWC==*W5~C2qd-O}Y?>%8n5}~q zGW|l@ZS+n!qu_anF3G_+#hq2NkgC~*ORK%xiGDp8two#qm7eCqjfQ5`do@G4`K8LK z-c%(s^1j$(3HKsD^p}YvtV8mvT#yD&9pH6FupI8O$DO$V2PF3fW7$jriWwE{Me~El z69Aa-wzCZ_!@T#AS|p6SXzx>bqYlF!2c-o1LfP~TZCKg2wW8H=z9&bG$(dF*cx3o7 zmYV>xE_VVZ}sM7ZbeFd)9>bgZm@ zY5VeZPNGLJe3g+cBi_x&`TJo5zZO|_Qz`+&^QfIK>r{%3j2ER+k~JNX#2>!5O7Bg6 zm)s4-y3>*ii+(n<3^ims%e+J3s4Q^BNY9C zdm)Pr+91p+6U3#HvHrkMMXaKjk&zDHMHsd&gVubM>iz(QX04^L)xU^#)an>8fUx&QjfNB4B}B-~(Kw~;LelR0 z6Se2*ZZUHazKeGT^}493khh}j!?XnH4NeRTbo_`}nI`mT-@I+={5g1SWT0&g-puU> zZ{}96t2xtcVAWlWkE%XUgvH57R*-wByu+Vz9#viW^ZlOfkdjbO79FWIG+=JR`8*ar z{EY=sCSGiYeFx00qVSdKZ!>ga?M9-*OdZS=YS~Z^qMpJ&2N@R{Bi{L7wWY1i+cI)x z=;qGgwM5(-6SJbP>3oY#>o(Kut2kEWs_emn_e&qn@OSTKC~gmfo^_}@+q-ZV<~7UV zK(VvJIrG7tHK~P|qY^M(Gcp z3xJkj>^%V$z_RVau2|*6*u3Re7$~L3f+dK|d;?#!p!lrzhE1 zP^f@Fd z3bx@JBsfKC3;kQj`HCJ1JIw&8;#>oK(G#2%Z^;2!=CQl1zkWMTJ2bQJYqHR8t=2!4 z@d2$zg(}NmZV3nP5UB2a5#9OTA>FIKD0)!qxXsl>z&Z{$(&er@$WVg4B-otv(zDVN}-8 zqET)ATDoA&7mtoZHJ@@Z)#D1cl<2B580l=nmFPc3O6Sm2Kj|L}J+=65bG+lE%`*;c z&R_*67|XN7Ouz9&DJ+WJk`kezN$Y^Y^=>e@PP>o68JRt@HqY#jH40A+jXk6{$PGQZ z+Y2NltXP$V&WW2=Fu!2?x%N$dTym}364cVckSeQYUF&IQy1w|anjl<~f6hpS#PGY& zZ&iRL4?o`7yyn2Vo1H8(DJF}$ZdlGIZ&Lo8(hFP(JfmC|-Q~5eWYfcSiSCkMKKy#V z8zFi*F-@jBD**s+_jmO zYQZa&Z)8JAWl?LOk`8I-Co`D-KxRpZzpIYWO>xUJVEkMQjGvQ8{q448$v6|&J@&6x zJ4weT`krU^YJ8a@EJIPkfuU!EbNnmnttdJJ&^KUq?e z7Dk;+b6K6Wy<NywH_x-a-nQe#6u1ULx!-<$?sPL3-q zaWMVzO;Y|ftr!+_$c>nmC^biZnUD@2YhJH9@yhxu8ZAtw_*0Ol4tFQ+A>yB)oj}_2 zPk|~hv3oO)^uY8@LPaMj#-0RF8$46#ltG2IWgBO&UAn#6XN{L9TTIrp6crgpN*+~% zN#y3gF2ckljdS=D?kUx#-+Rq1*tjzEk*&vaFonspa%Gk&9MRV&YD@a>ZJ+M!$%D%f zpY%U>8pRHIwZq&N>O+v=BW5Z?l+ZBZqT$RIC41 z4_JnmJJzrgW2{RykiMt*I_cz77!Ug#Tf;}IsS9*w$S7KPNA4IK!BitJ$yL;#uEX%2 z{3toScXKB!_;fGX1tr;(>>@r&+za;(RWUCLK^Tju09}OzdEMca6B}3VxtGiN9W^28 z8Bw8Z%b>Bqa^=x_MKz#oRHckoV+ri@V~an;I!JD-@F09}KCs`HdaUy{5bB}?oYIVh z-FJuT$uOQ_Kg)iDk>bFs3i)XI@hXudMNrJFHP4Ol^-subko7r3Af^=jY48(-{JtZU z$gPxhPXlO4(1x6oV?$Q~XZ2UG*KCKK-e)bZ=PFF)v=S8+Mp_Y)7~) z+JZ2`r_V%RgmVa93p79~9N5|J{Go~LytrXoGqK5Ef%^ZfE;p<3qne9bz_cIg&#dKO zLK@iuTHY=_ijKo7GPCWB&hMh=Q)k!locxWjTvm^5;72jPbN-JrC;wi$e;@s>5dF`i zy^H^ziPWYNErB1sQ_~y&=TWY-e(!(I)OHnAe~}qNfCrGOTCq64$V!{Ek6JK{g2xuk^7H!1ooJ~9+HVfj^r}b zT-Q`EuJEAN779L5h`GzB_#t*)xQYG${~zm#96uF`V}(-a%)e7juX9FCx>jk7W1r(x z;8x+1&fU8xMY!?iwj2jAkScU;RpHL&OYXX1SW2X+TE7tLLWS=+9_U{aXyat zBv5p$xJ54c5ds12B(KQ$8+&{L$TxPxJeT!aPdm=b zHtxe79|O{h9Wl$rIjY2`9pj}N=VOnL0?EaSo8@90MdH(r@{*0evBgJ#_+v**b5V|L z@M%YQ@yEN^;=@1;vErt=h(|K`w8OmU<34QhAs|5Ph)FK&5gk755HG?wA6tA72q{+F zBp31s0iSk|7h*h%@Qep_o7d^6F|GuNy2)#Glo&?|L|x}KI&z6C;YMBM)jCp%qvS?i z=2bdki!0$mUF4NILXV^5LY?OoKDvu7;Y6L|1s=`EQgX&S=kD_Mf;^ATo$CF~PjVRc zRNkzndDQx&9#B%9^Zk#Gb7=P1-mIp0U#-S?82V-J zQKFrd{LK$@F!vH)twwn$`VsF@BAtc(j}CHB_gr4BMtA`IQ+Fug&Mf}s2RTT4Dz8?< zJed7acPOFGB>qSHIdFSyuU11mNd2;RD8bIC{^t8RPZvoIR zaa%kNdzAqpKjSudEcPM;pnt}#@fhsc1cW@tt?+2<$pk<@$1U+F?9l~;JjE^WNbDg5 zKtIKO=MmgH_YZlDo8{r!oAf_=FMmP#k+Z|o?d;@cNyxaRu@OEERBumeR!ns)ba=EN zq}}4$2%iGVx2rZQCOalNxLFWVZeeePPXfi;F`5+<9RnR6%?U}jp4P)BfCB9cO^Wf3 zE)H(ygoInm>*3=-&i14x#aKrZhetC)ysg&t@G&4`yK0kSw4;iHn;9Y2R_c2AD3G!p zqe(H+QN-cVln`ypeLZ{xNZ7v6s2J|Z=HO;Zh`gn-9zG1jYENoZ40R-Pcr+n|-{M*i z9|9t`t2QbIJEA+dnGiy6VXub|0-@V68WjT_Asje6W}OK;9k<&}x1<~iTODWG$+x5& z2pb)H+YdIS?7?}r+V-MNDLcYS$J}<)O(|Q#QpebK@=Yll!a_&?_Ja*6YrBib2=-G`D7bk?U-#eW?M~34m@*uxosZTe05MbvD^P)ga{dwmZkixw-3$K(n#-76w>H z0Q|My&hasB=6X}mY_z?GLCPTjdo8zf96bG0PY#-mw3jfzIs~Av`FDmP=l{gYB^mQuYBbYeb#ngWRz7 z58Mr)#eT9LqX+I9&|u$Lzt9bL1*oyFtWWBO zy989&XV$BB!(9MM>?7+jy5Y_N1@_+c3tez$fE;_9`lPOLFU22#UG5%x_qE*9X)dku zDh`x1+j0d;%WS$Ok*w1xE~Ro?4wMwzbcGU&Y^o))tkX#@sd9Rb{|{wAn!gQ^X@*QA zWSSt;0GZ~;G(M*3F%6Gtc1)vVnjF*MnC8YbHm0dD4UK7LOe15O7}LO*=EXEFrfD$^ zi)mI&qhgvA)1a8<#55+RDKQO+X+}&VVww=sfSBgPG#;kuFb#)kHcX>onheumnC8MX z7N)5%4TWhYOe0~M2-85A=D{=$rfDz@gJ~8_qhOi@(;%4Uz%&M?DKHIzX$DLqV448a z0GMWOn%^{V)3{B;HjUaeXw#TYLpF`rG+@(sO~W;f)-+htSWQDUjnp(y(>P7TG>y_U zNYfZiLo|)hG(gk%Ov5vc&NMjF*i1t+jm$K@X?oM}rrAxSo4N9gm1n9vL*=&dGd^tXPP|2zc_zp+K%V*WjE`q}Jj3Ie z9na`^CdV^4p1JXijb~~+L*tnl&&YTt#xpRUdGU;kXIebN;+YlCsCXvDGbo-p@r;RQ zN<2g2nGw&3cqYU%AfEZ~jE84BJj3Cc4bNzJCc`rrp1JUhg=Z=}L*bbT&q#PC!ZQ$_ zdGL&bXBs@i;F$%_D0n8pGYFnJ@Qi_H3OqyLnE}rTcqYIz0G^q9=JyQTGj7kYJ)`yv z+B0U)kUb;z4A?VX&u~4X^$gZCR?kp9BlQf_GfvMiJ)`st(lbWS5IrOG4A3(^&+t5> z^9;^2HqX#JBlC>!ncg$JXLir%p1I7GWvnbyWf>~VOj$y%zu^f-pcF*}aYaZHY5a2#{v7#qjbIEKbCGmepQOpIe-9P{EB z7ss?XhQ%=}j!|(;iepe5bK)2i$CNmR#4#g|5phh2V?Z49;TR9cbU22?F&mE2a7>0{ zFdTE?7z@W#IEKP86ONH^OoU?~9P{892gfuxhQTolj!|$-f@2UIbKn>Q#}qh*z%c`k z5pYa^V*ng8cg*h?xMSRoVLL|c7_?)|jv+fn>=>|PypG{IM(Y@?W2}y$I!5XksAHUt zVLC?X7^Gv2jv+cm=op}5e2(EcM&}ruV{DG0IY#Cf-!Z*oc*pFH(H(Q&m3>zBRoO>n z-;{k)_C?tTW#5y1PWCm~$7J78*-;jMm_66AoWZ#c{KKAw4 z$7A1)eLD8#*oR}^jeR!u)!0X4-;8}S_Qlu-W8aH?F7~z9$70`#eJb{)*oR`@iG3#a zmDopO--vx8_J!C7V&8{-9`<$E$6?=weH!*<*oR@?g?$$GRoF*i--LY<_C?qSVc&y& z4)!(J$6()reG2v^*oR=>fqe${71&2$-++Ar_667nVBg%n-#)l~Zu{8wsqI7CXSR=Q zpV&UIeO~*x_G#_I+Gn+oYM<0TsC`cRnD!~{L)vGwk7%FJKA?R*`*`;0?8Dh-vyWz< z%s$_~-ag*G-98=6m0_$5Q)L(`!%P`Q$}mxefildKVVn%pWEdvHEEz`0FiD0%GR%=- zj0{s`7$UeSyM#nHYhQTq+jbUsIQ)3t!!^{{)#xOC4 zficXBVO$K;Vi*>~tQbbcFe!#XG0cf!ObkF9&EEq>w8-{HdwPDbPF&l<#7_ni%hVdGPYZ$Fzu!gZ3hH4n8 zVW5U_8ir{YrD2eUF&c(w7@=W+hVdDOXBeGfaE7rNhGrOLm3vd}Nx2v09+Z1e?m4;FK9*}!K z?)kXa;~tNDJMQVYm*XCedpGXcxL4yIje9fh$+#Ee9*lc0?zyS^;U0&38}4bim*F0Udl&9mxL4sGg?kh3Nw^o` z9)x=j?m4*E;2wi}3+^enm*5_Pdk5|rxL4pFfqMh)3Ah*F9)Npu_kQ=_?z!D#yQg*! z?Vi~^vU_6p!0vh7nu_jdPmFIRrC@=KLpsQfbJ7b(9)`31@^PkwRoOOs!i{IcX1CBG#3 z1<5Z*elhY(kza`XGUOK_zXbUO$S*&B@$pNKUwHho;};#jKCeCq<(?=#pxHOUzC19`o-uMqF;o50s6(~7oJ~qe!=<0<`UW<7w=B=2g zVqS`QDCV7*XJTH7c_ikIm?vUhh$d~HIHka);z3vR`aOlNzH?r=QNLLp3*#|c}DYy<_XOMn&&f* zXP(YHoOw3$Xy(bx^UdqcJ%lwG9k5@i=CyFA&&$u3QHVY17T zU6kySWEUj69NERlE=6`BvdfTNgzOSz7a+U**u}>#J$B)-%Z^=i?2=;_9J}1u#l|i* zcA>G$j9p~x5@Qz_yS&)N#V##&VX@1KT~zFnViy#D#gk2=;5@8nzyFA#%!7dGUVX(`BT@>t+ zU>5|t9N5LcE(LZWu*-m51nd%E7XZ7=?eg0NZWp&**mhCd1#K6zUC4G3+XZYFuU)uy z(b@%T7pq;Uc9GfzY8R(nn08Ux1!)(fU5IuO+68DApIvx%(b)xO7n@yZc9Ge|w@Ysq z-Y&acbh~_a<(-vxRo+o~H|3p_cTwI!dH3X-XVB*;GKbY1>O;OH{hLscLCl3csKX% z_YUrz+dH;*YVXkAnY|->C-x5Po!2|AcUteT-dVk)dMEV`>YdX&rguv3klq=+BYG$F z4(OfFJDzts?{MDPyrX$1^Un9K_m1~&_fFSxTRY2V~ukbw1YhSjS`Cj&(ZLof zbuQMmSjS@Bigha1rC5hz-HCN3)|FUCV%>;!BG!di2V&iabspArSjS=AhIJa&Wmtz{ z-Gy}))>T+XVcmpv64pgn2VvcVbq>}wSjS-9f^`bkC0K`G-GOxm))iPsVBLUq0@ejs z2VmXYy5Bmub#Cj})~T&STW7Y8Y@OITuytPRxYlW{!&+yxj%uCMI;eF{>zLLltwUO8 zw2o+<&^n-XKI?ea>8!(9XS0rGoyKW`Qxwi&W-*(E zY!sac?Aahio`7NuE`W-*$DXcnPafM)TTg=ZF>S#V~t znT2K+nOS_Z^k(7BvYSOW%U`*bS2>kWxs*pals~zXH#w6pxsoS2k{`K|7dep+xsV4r zkpH-k_c)L5xQ^#Ij^DVA*Eo&OxQxd*jK8>xw>XQhxQeGZil4ZNmpF-!xQK^1h<~_; zcQ}V{xQ1sqhF`daS2%@FxP(VIgg>~0H#mbYxPm7*f*-hn7dU|rxPS*ZfY1GRxVxS0 zYDc@-$u4%Vd!6fA$GX+2E_J9oo#{$Py3vU)bfEj3=Q_u^&1o)k-+SkMcijg~JU8jw zgmaV4O*A*j+~m8dcN6cX-A%fi&U9raD>GG@iONhejG17}^kOC#GqsqB#Y`(^QZZACnNZAhVkQ$am6(adOe1CzF;j?{K+N=E zCJ!@pn2Ezo8)nilQ-+x^%yeNU3o})iiNZ`1W|A;dgqa}B^k60jGc}lr!AuKgQZQ43 znGno$U?u}I6_|;@Oao>TFjIh;0L(Nu({CoYncQY#n@MdZw3*CiBAZEUCa{^jX5yMj zYbLCjtY)H`NoppjnVe=~nn`IUq?wFnBAQ8PCZL&oX5yJiXC|DPY-XaFNoFSBOudA20cMsmDt^UfS`Jj+b)0gyW?fFWGph#!EC_n(>m1mtwpG2yaeE-xtD$~!M)`665C5^FQL6;_7d4kVlRQcN>x^( zveJ~5q^uNWB`7OBS;@&tO;%#E(vp>wtdwLWBr6?R$;e7YRwA;}kd=h26l5hJEB#o> z$4Wg`;<3_>m2|9>VSJU0BJ&N)=Y3u+oH; zB&-x+B?v1$SjoXk4OU{X(t?!~tdw9S1S=g_$-qhlRwA&{fRzNS6ksI)E6uI+TM2F@ zx0TpdQdtRuWqYY$dOixK`3y32P;*m8e#dS_x_;rC?`QV>B&h> zPHJ)zlarR5q~xR|Cm}iM$Vo;{DsmE$lZKom5{;8)oFwC<7$?Cv>BUJdPHJ%yi<4HIq~fF$C!sj$#7QPjDsd8tlSZ5* z;-nBKfjH^INght>a1w`;Hk_p4qzorvIO)Pk7EY>g5`~i{oFw6-2q!@}>A^`3PHJ!x zgOe7Vq~N3kCm}fLz)1#9DsU2klLnk5;G_U20XS*yq~A$!C%K)(c9PmjXeXJSM0S$c zNnj^=oy2vL)=5|=S)D|6lGI61Cpn$Obdu6ZNGBPcM0AqSNkAw0oWygI&Pg~Y*_=dk zlFUiIlX@rdPTHNMJLyPQMzS(em6527G-V_yBSjer%1BQ}axzkrk(i9MWF#dcB^e3H zNJmC8GE$L|h>SF3Bq1XO841WpKSuH~Qjd{%jI?7U9V6u!3CBn`MzS$djge@KG-D(g zBgGgA#z-$laxqeikywnhVk8wKr5FjtNGC=zF;a<i_$b9kC_Xyzk%^B=d_>}-5g&>8D8xq~KKk&HhmSgZ#Nne2A8Gg~!$%lCy6};O zk1Bjb;iCy3N%$zjM-V=G@R5U$8hpgyqXi!+_$a|g2tGRSk%5m2d_>@*0UrtYD8NSm zKAQXJ_YvGjZXdCIr1lZoM`j<9eI)h~*hgL;aebuq5!OdmA5ncI^%2xZP9HISr1TNe zM@AnJeI)b|&__NW@qDE75za?8AJKdy^O5hP-bcKTb|2|Jy3v)5tZYVfh-f3Bjes`t*@$N& zosDodve}4cBbkkS8}&BgZM54+S9IkfD;HI{h{{D%E|PLll#8HT^yDHZ7d5$v$wf;p zQgTs}i;!G&msa+tS+LuNa`Y}i<~ZEx=86Fq>GF$BDzTEBA|1UOk^_=%|tR2`6lX3#G7b0k#6Y9LslND@(`7W zraUC&p(qbQdFaVQP9AFV5R-?NJf!5IBo85Z=*UAx9xCz>k%xvnB;=tW4*_}T$3s3I z>hTbdhju)qhKVUhc-N<;h_uv2=)gk;9xCtB;cU{4*_^+?xEj9a1XgX z#P*QdLue0~Jw*19*h63sc|FASkk&(34_Q4#^^nvq)kj_Im57|6K^N`F#zK41b@gCYeqR~E9eP?d$KEHq^yDGNne2+BfF7ILys zlZBWpv}7SA3nf_y$wEgKGO|#Sg@`OPWFa971z8BlLO&Mru~3hNcr3JIAsq|lSO~{L zHx{z7P>qFXEHq;w84JZ&2*yG$7ILvri-lM$v|=F@3#C{H#X=_*GO7J^#HX(6VClomo-$Y>#=g@hIYTF7T1o`rN4!db{>A)19` z7V<6BTZp&NZXsRJm4mDtROKKl2TeIh%0W>Mf^yK4gPa`HNXJ1r4#IKJje~3)RO28T2hBK0#z8R- zf^pD`gIpZc;vg0WtvE=x;2;JEEjUQQK?x2*aL|E+3>;M8 zAOZ&sI7q-j0S*Fi(A+`4gWwKwJBaNdwS&+OGCPRuAhCnM4)QvP>maRzunw|1i0UAz zgP;y_I*92YrGt zWuPhpQ5k5;KvD*ZG7yx3o($wn zDh5h15Q>3L3}j-U5(AMKXv9Dw1`06{h=D!~J(Ks*EK41_a~%|J8*$qeKhs5cOApxr>apDX`Z`KQW1RQ{RrkCcC+`~&5m zC;vG4r^!D|{#o*ml7Eu?gXEth{}}nF$Uj8>8S;;ie}eo2(#p9}w3_@}}@6#kj;kA#0B`~%^i2md(u zr@=oA{#o#kf`1bHgW#V7{}}kEz&`~38SsyQe**ji;GemFe*eJzHKT7`~{bTeG(LX}}0R7|h56?e3|KR*% z^AF8GGXMDg>HWj|XZMfppZi?d$I3oc_Mx)RlzpV^6J;ML`#jml$v#c?VY1JXeU$8z zWFI8^9NEXnK1KE+vd@rxgzOV!A0Yeu*vH2{J@(@ z9rod{&xU<8?2}<14EtQz$HG1p_Mx!PgncCJ6JZ|+`#jjk!9ET4VX)7FeH84IU>^kg z9N5ReJ_Yt6u+M;f1nd)F9{~Hz?ep6QZXdUO*!EG|2W=m-eaQ9^+XrkPuYI`o(b@-V zAFF++_L15LY9FV4nD$ZH2WcOpeTeoE+6QPKpM7}t(b)%QADew>_L14gw@+^$-afm1 zbUjz@v2st9d#KzqbM7ambJx}g&a!-?cnB24E9wqlAxd+KTNA59lPmz0w+%x1J zA@>Bi2gp4??(uO?k9&CBv*R8e_vE+-$2~Xhv2jn0duZG<;~p9J#JC5>JumKYaZihT zSlqMX9u@bbxCg~OC+;zEPlhkH2Ov*8{M_hh&S!#x-7 zv2ag?dnnv9;T{S1M7RgSJrC}2a8HAK7~Heq9tHO#xCg;K2ktR&Pl0;~+%w=F0rv#B z2f#gZ_x$dGyT|PwwtLj>LA%H79mIIqwC=&W$Lb!cd!+7xy2t4rrhAm` zLAuB29-@1M?g6^T=N_JWbnd~q$L1cIdt~nM-P5~=chBw~-96{IGLMycs?0-Wo+s0^N5%y#5^G8`7n=%c{kAry{%)?-w1@kDFC&4@j<~cBrfq4qdLtvf(^9YzHz&rrvnVaV~58OO%^RUgM zHV@i7X7iBEBQ_7%JYMr~&7(CB);w18P|YJX57azP^Dxb$G!N1|M)MHOBQy`tJU;XA z%%d|8&OA2r(99z3}XU01+-ih%JjCWqVr`2X$~sflk+M#db)c;CWF05#G+BqqI!o43vQCn9kgRiL9V6=$ zS%=6vL)H(p3>#yT_Bk+DvUbzrRX zVjUOjv{;A5IxE&uu}+G0P^@!e9TV%6Sck+qBi0eIPKb3ttn*D>*3qy| zhIKHkb736|>r_~W!a5Vyk+4pLbs((sU>yhRG+2khIt$iOuug(?5Ug`x9RuqWSckwm z1J)6+PJnd)tTVUHZymUG+}2@RM{OOnb_T1RLdpmluK;aNv#9h`M+)}dKPW*y%;y>)o&?AFocTsg08w_qRx53y3QyUCzFtfqP1``_$Y%s6ExCYZ2 z3~Ml}!Ken48VqVMr@@#8QyL6uFr&eU1``?#XfU6_cm~rM3}-N#!Dt4P8O%3WZ!q3q zyTNpW{p-qKR{pB;7nQ%J{3YeDD1Slu>&ahE{%Z0UlfRbyrR1+9e&0I# z{%Y|Ti@#R%m_R{%Y_SgTEI1rQojwewfd z^q10KNPijqMf8`@UqFBP{KfN^&R;lx+5AQGm&{+jzj}Z1{@VSe`|Dm;_Oh~9mA$C! zHDxa;dqvp`%3e?QaCKlbvm zSC74T?6qSr9ed^23&&nJ_Oh{8jlF2>HDfOsd&Srb#$GS>avg4@e& zFSfnZ_CnjsY%j9C#P$N)%WE&Ly|nhi+RJJ$s=cK4g4)YzFQ&be_CngrXfL9@g!Tg3 z%V#g1y>#}%*~?}xn!RN9^6k~zi?`QqFJ0G_yR6(*-E*5vKxJ$)dDegjX*NMAK+*RT(5_gTbOT=9v z?gDYwhr2x7)!{AMp6fpzd7nHf4%;jXRCUY^FYsp+n=1MXblDUq| zWn``*a}k+q$Xr6^3Njavxqi&$W3C=^@tAAJTsr2;F&B=xZp>w4t{QXEm}|ydGUkdg z7mT@H%;jRP7IU$fYsFkD=1MUain&hAWn!)pbCH;9#9Sig3NaUmxjxM0VXh8yahPkv zTpH%eFc*foF3e?Nt_pKem}|mZ66T697lgSU%;jLN26Hi(Yr$L!=1MRZg1HXNWniuX za}k(pz+3|63NROdx#s5j%>_4?+gxmOsm+Bpm)TrobBWCbHka32Tyts7g*BJeTvT&O z%>^}=(_Bn*Db0m6m(g5Aa|z7_G?&j@Jag&Hg)^7UTr_jZ%;lS_Hy3ZN-CVk@D{onO ztIAtc-kS23l((Y11?8>z3Z@GA@#ak@iTJe^O zw^F=?;;j>JnRu(jTO{5Z@s^0ULc9gytq*T`c&o!(9NyaSmWH=7yoKSd3vXF?tHN6p z-kR{1gtsER1>vm+Z#j6Y!CMU8TJV;Fw-UUC;H?918F;I}TLj)3@RoqL0=xy_t+}^; zZ^6Cg_7>Y)YHy*vW%d@?TVij4z2)^5*IQa|VZCMb7S&r)Z$Z7~^cK@wN^c>(W%L%& zTS9LEz2);3&s#cg;k;$@7R_5SZ~5Npy~TTL_m=LhYh78(%34*{qO#VMwWO>SWi2Rc zJz2}iT20nsveuHdl&qCxEhK9lS;Mb;v+){wP?tQBM}AZz_t%g0(h*5a|&jLmYh75&!dey9qOjJ4wIr++VJ!%2Jy^@Z zS`F4>u-1aL6s(nCEd*;FSj)g#1=b?4)_}DHtQBA_0Bg;y^;-*WEw{DU)>2ywZ7s94 z$kq~D3v4Z~wYb*OS_^9}tF@@sl3ELDEvL1Z)>2vvX)U9*h}IHX3urB$wRqOjSqo<^ zo3&`xl3B~QR&OodTD!G$Sy#@oa#oeIsGK$BEGcJ2ISa~JPtJ03R+F=soVDaEC1)i$ z3&~kW&N6aVk+X=LHRLQIX9YP6$XP$m@^My=vv{1f<18I#MW|Wq|Smm%jqnpvy{$4I?L!RqO*j~0y@j*ES|G;&cZp%<}8}CWX|%P)jNxK*6u9b zS;x9EmX)!pj74RvDPu_)E6P|<#(FZAld+nN#bm4{V<{Oc$yi9nIx?1#v5JgEWUL`$ z2^lNMSU|@5F_w?9dW^+mtQ}+N7%RtEIL5j$mW{D$j74Lt8Dq&9E5=wb#(FW9i?Lda z#bT@#W2qP`#aJlDIx&`su}X|ZVyqEki5M%ySRlswFqVh0I*i3(tPNvn7%RhA7{5W08#|HWt`eUSn~Mr8O4TSXN_EjU_b})L2eqF^#1( z7SdQoV-bxdG#1cUK4bBWr85@JST-)}ys>s;>8`GPW#y|XUs3sL%2!gp zit-hdubzD6rR;;R;4vG{7mS1P_r@fC`%PJCtJ zs}f(4_-e#gBEAao6^O4seC6S*4qtKjYQt9=zRK_whOaJsW#Ov|Us3pK!dDW$itrVL zuO58m;Hw5-G5Bi1R|>vL@D+lu4t!I$X?=zDmDN{NUrBuh^_A0COkXK|h4hutS43Y4eFgND&sRKO z>3oIrmCaW)U&(ys`>OX9@2lNcy030^Wh*ONRoRNlR#Uc;vQ?C=pltPID<@ku*^0?l zOSV$7Rg$fcY;|NSBU=^OipW+&wi2>ckgb4h^*Jt#E90V=Eh5 z)!2&0Rx`Geu~m$%U~KhbD;Ha}*owtgE4EUxRf?@pY;|HQ6I+$oio{kUwi2;bh^;_u z^7! zs{~vX;3@!D&0Y1o3hpYmtJtnmy9(_pv#ZFi61xiQDzB@!uF|>+>nf|OsIHQ_3hFAS ztC+4*x(ewkqpOIn61ocLDxa%(uF|;*=PH}4Xs(jE%6C=oD&AGQt8`bL>dI7Brm8X( zm8qsoC1t88Q$d;P$y83JYBCj*sg_KoWU3@nA(`sPR7R#MG8K`jhD;@7svuJVnd-+> zKBnq16_2TQOr>M098=+#>c&(yrm8U&jj3i#C1a`>Q^A<(#Z)e)YB3dysa8y-VyYBV zp_uB#R3@e>F%^laMocAQst{9wnCin+9;WIr6^E%dOr>F}3{zp4>cUhOrm8R%g{dY? zC1I)vQ$d*O!Bh^WYA_XpsTNG7V5$UDA(-mGR0gIhFcpER223SjssK{~m}+jS-&Al@ zxlP43mD*HjQ<+UgHkH^^U{iTb#Wj`IR9I74O+_`8)KpMYIZeehmC{s5QyEP~G?ma) zKvVfl#WR)8R5(-FOhq%5%v8RqdQZ3JVoTGAx{Z;D#%knp8D~WkEeP(#p9_RPw99n$5S|- zy782ar)oS!$#^QpQ!t)-@sx|FT0F(#sTEJDcq+wHD4sg;l!>QGJVoNE5l@MD zD#TMDp8D{Vho?F`#o?(9Pic56!&4ZZy6}{Rrz$)};i(BvNq8#4QxKkd@RWn68a&0| zsRd6dcq+kD2%b9dl!2!TJVoHC0Z$2dD!@|!o|=2=_Y~YyZcnj2rS=rsQ)W+*Jtg)O z*i&9laXqE=6xLH#Pf+T)RU#0EY)NwCQB_@O36}5mO`@B zk)@0*Rb(k5OAT2{$WlR;0?O003QaP5wvDA&FY%En{DH==7SW3oH zF_wa{)QhEDEY)Hu7E7&IO2twsmO`=AiKR>|RbnX;OO04c#8M%a0-(3Ub=ifk#drNEZ*T8e8at);M*vRaC2 zDXFEPmU3E(X(^?pkd`u9ifAdJrGS?5S&C;VouzP=vRR5|DVe2wOZArGEwx)p7j@+* zD@Rp1ipo(_j*@azl%t>=_2eigM>RQ$$x%y=QgT$1qmUeRRN#!BGp2QgBp)qYxZ*;3xw}6*!8(Q3H+=a8!V!030=U z)bA*`quh>SJ4)>+w4=<9B0Ea#D6pfvj^a8>>nNuKiO6DlvQN5#hN9~T%9d)QHLs=QB%1~5>nlhA>p`r{0 zWvC}ZIT@e_~hT1Wd zj-hf4g=452L)jRr#!xhdnlY4&p<)aLW2hHHxfrU&P%MU8F_em-QVfM+s1rk(7^=ij zB!(I>l!&2134tmhT1TchM_VHg<+@*Ls=NA!cY{3nlO}vp&|?gVWjLm>@iG!)TLLPG%!Bvt;ek$@4k)MYAB;=hTkgpLYDDBLVaek$=3iJwONB;uzKKY{q^!%rT5 z>hKeXpEmrY;in8gVfg97PZoZv@DqifCj2DfrwBhm`02q<4t{Fz6N8@?{G{Ng1V178 z>A+70ek$-2fu9EaB;cn2KLPk@?x){Ra6h^I#P*ZgPiQ}x{Y3VY*iT?TdHuxolh#jI zKUw`m^^??3P(L~S#PpNWPe?x*{Y3PW&`&@=`TWH5lg>{#KiT|5^OMX^zMpzO@qXI< zr2FYkS9Y?pQ@;O3DLX~k3Cd1Sc5<>)lbx9Cv}7kGJ0;l($xcUhGO|;Vorvr- zWG5jz1=$J6PCs_?u~Uzoc@;I189T+;3C2z@c5<;( zi=9~Pv|=X}JEhnO#ZD)7GO<&Mok;97VkZ$hh1dziP9Jviuv3SfIPA1xCk;Dg*a^c< z7k09+Q-z%<>@;B~2|GpD3Bpbfc5<*&gPj=cv|uL%J0;i&!A=KuGO$yDoe1nSU?%}P z1=tC|PIEi`c7ofGROKcrH%+-o z%1u#jf^yT7o1EO#n5$6ux_%tiRvb)o1kuTx{2u~rJIm$GP;TACZU^vZt^+L z89hhzoX~SX&-pyZ^PJ9eIM3NUNAsM_bH3+#&+(qyJ*Ru_a#xnKvRswrs4O>SIVsCU zSq{o_PnL7CT$AOPEVpDiCCepQ4#{#ymNT+kk>!XiH)J^>%LQ2u$Z|iH^RZly<#;T& zV>unm8lIT_2vSPsT=FP3w$T#MycEVp7g70abq4#jdOmNT(j ziRDNvH)1&v%Y|4D#Bv{&^RQfp5~TFz)WqUD5^16s~!IiBTo zmcv=jW;vSWWR~+S*ISOa+-^DDa{pcVXXRg&e^mZW`6uOHlz&kEJ^AP4Uz2}K{w?{Z zDEh z_$T9EjDIlxz4+(iUyFY%{;l|@;$Mn?DE^)JXX0Oper_$T3CgntnJJ^1I~UxR-P{w?^Y;9r7&2>u=TXW(Cf ze+2#w_$T0BfPVn~&HelRgZt<9kL{n@KeT^l|H%G{{R8{w^^fbH)<3L&R{yB}N&SQR z=k$;1pVB|1e@6d^{t5j9`seeH=bz3$oPRd|X#UCk^Zo1nFta@>*Qj2u_wI3mXlIZnuNL5>53y?H#8 z-TyzXh+>dLGh>pn6l0qi##mBOgrOpgC1Dub*dN5DJ5D%4;HqRcgYOQTT*S!t(J*$s;p!Cg@Yysz7UsN& zInbAhvFiyxN)Zm9P2rosCKoVLeZd$zb~uE>4x7Eh*MvDQV8r`uF?QYIGL)pS*(AZ< z&*Z$o5senct}A>W#XM~GHXj6Yp2zI&lf>9{hVP;Xht1B0Kj+U=B8@2@G0ps!6qGS# z4a4AfQnrjJA22jNox(DrtYYf;6_iLL%6rTMel`VVL|MU9@)Iaqh61aKyZkT;%aF2+ zDdETUeWCCVw;J1BpIvAK`T4eKurEPE2E*)08Aw?E!adMCpEfl%0unM1W=*m|!g3cV zAU~hBV{92nNPn0aNdgJWUWf<1!?!82H^3o%VMe4O1dOr}0`kMRLD{3=5Kb79l!1Ve z7d$}kyxZj23~)$qmB2}0+8{r#HgWbPu;Ab7fIx8(mb{<{dgs|D%0_@gy28XsHoCCH1u2l9XWMSJ3^=4S zY&S_l7nZQFO!^@yW>7X0Mr7AXrl8AsVLt3MNilTUWSBcUKr%(YY&`5T+e%VQziccF z%T||6(JLDbJImfDDW+HUBFu#S9+ZMA8Vz8fZF6ByTj9@Lc4&j4m*c6M z)ciO10nxXu$DbSS&;~-W@ot;c+&8%ZX{lBDIb?^{A9^-kVUwEuCJ_)_YCZCNXPed+ zY7#%QL8ZKj07(2+h3C9&8YfgQp1MILzwrh{`K^bZKiQ`BhHA#UZBWzS*aM`+R@vtz z+q9lgm3V~>YU&#^KyqZ z(T}cT%#i1N*5vU9;PyYKxV<4-(Rjv8`rPHULT`hE?RTfRJt3;m%8Z%RxwC8X-Uj>I zZ%uL8A%~;+^cm8e-ddrT0jT}T6t_D>F1m_7lRT%gChujixBcW4w<|<48c&}|oIAKy z=xHF*et3%88L}r@nLd*+C$c8*X&}_j{rGi8GQ^;DZYtVgMl#rdHP;_4J0nRncs$n@ zJvJ>FWKci%FgkBqGSHxQt~lCZS~9?(YA!Wec3RTkpnNVSdW-=fu9&pnYG489N&8R3>UQ#{+UgjQ`;eIhv!Y#D-zaLbuV zokcI9kGAf4BH15&Dr76%Eo&xbwuO&A-1@dnvM<;mgca_VIpaTz=A$94y={`5V4aZ2 zaJP&Z+-%DtTCVj$n`CeB@eowFTiVR|S@a@Wru9ynWKZyskgYJclo`xy%K}=m)wfNO z9egN+73Owl=EN*|0llyFT$^Nfuyjadm|M~ec(!F8y{A>RO|mOkECdzic6(;;EP5Wj zt94JCWM^<>>(7uR!EB*hW?qGuw{zFf%$c4L;dbr^bmL4@2>U5_75!+YIwa{S_dU91 zCO^ddDR%{ZeS_l8C^Q#6S6fo(VlsI<`a4@*smQ)E#N&Gw(xCH zXIYhK4H7RBZX&690!&YiPm3x`y+mt(c%ER4q~`F_hb??sjUDth!>YUg8OYF@l=L zvmUnaZjoW76>Ienl?Xp{sVO|8VUAaeBrBj;OW<*U(5_3p!_yeH@M_t|vMScVy!OXUV^bM^)^p@*ut}A7i(X!Rwq%2K&5^(P?#l#uo&DLw8Auz zz{=sOp%tcx9;|R~jlRMJ(TR15tER8;ifG3|aBK7w{vukiWVmX23NMMKtc~Fsl)?xR z%^Dq6Ls3IpUK2Zb(X97`AJYYs4INHz^)(*b#@Y@BUGhlUoX(pp>zG*5S9`1#Yda8R zqh1WecB}5 zgLNZwRljbs!XxrV=6n4Ili41q8<{Kmm6Hh`Td|q%^zTlFd9Y$Lm-R~~;~szU;Q!fj z)Arl+cw+;>w^^h9l1uPlpuJND6PiDMuVLAzS*;%75SbJl z`vc9KB$&|b@%V;ie6wQx1}?ZS(8y_s0i}$GG!XF3(E3qaFeeb{lp$FEANOcj_HLH1 zXW)W+12vp%7|`@_y9R=H^MU#tTyRg|F((NIGbaqR|z zSF?EiC0uZK-~p!$Iy8A)v0>S>S+pL33+@UOce0^F6UU_*2%gQm>t%4koq@ZZB5D*<`u7baRw*}7uYRCm*^Sl;4Q-wad^&zF`mG+k*3mF?Fu=hsFTyo{ ztfK7elHT!JeZ$Ow02@1p2-n=P?6T-3J>^JnSBAq zc4OhLl(Dd~Xuh68?Kmcr6M(YI6L^jr^DO(isCTHg8I#!?0Jn1pcTFF=SQfpgCtF*9 z$?OSGwv!EaO&vR3_H{u|y7n3-lO3R7Hx}kf8q+O{UeJ@Mb;M+L2gusxg}Ek=DV2Sl z*AuJN$7FT|NZ2`qxh9V7FN>bn6RwrRWOfD!+sTHxCX5M{p|4N>t=Bd-Ylkl#x8`{3p zD;p!)RT%9s1b1bIzqutou=q7lT3xqc4f7ybHEX^OiPZsrUI$mW6p$+(BE!>c$ zxudycl24Oz6$C=-_djc)fGo`(O(cu@G##nh0n_^YO)O>*rIgVKG6~fIz=6$_@Z z{b3d}x}~I1B$?#ZBvIu8rgi%tw4myiCXXH?i+VMQRT+Y5UH%{oH{H_2(F0_XXOnOh z1WfDn7qL*#Eln8RO&0ZRdNLYc)s?x7hR*m0R#j#a&`_SgchzmdZwWf-?^@-RNzjLm z``cHY$z0Zhj`>?x9nB=@K}Y?~s&-{Aqo6PRjj9&N1Qe9(kF46U_>jp_2sS7UX)3?6 zGd$?$Xj-+&%YRWFC;t|8qWt8}@PMDKDSnfe`yxNCa0#VSet2iN-|w8M@+L3)MRJ_{ z5=yE3=k{=)-$_&c29NS0Dz1=^f|bv04|DwVO{+F|!`s81etS%n*LevqMB?PGKA~D)Oqn|T0DVNUUi6#F{s66^9=~Wa9s3UYfU1A-&@}HmXcblaqS(~oJLo;C z>P4!l?03)#s{BQa>DUhF9g6zG-!yM$_?ye$sMmh&FQUrl)&>bq=z1okfLlXU^JSf& zp5d(x5bV%FOh_KLkXYlxQlTE@t@RTu&~PRshf5-=`LL9zyLfAT1XJ_}1|*9cO{~GQ zVAO@FH4Xudu4h0pxdfsbo&~0MO|A73w9!EfNCwx5SmVt)NUfY&>mjJ2;S5L`*Mg|# z&Du}BJ+;OrD55{mAt_umvBrx9qIylObra;#^>oM`t{PFzi?x?}W@@dAAcYR1Lz1}i z#2QbQ2=(aHS|>pi4W~nHb47`2o-85it|`GC?+qc0`XrdsQZh~OrkuY|&iuig1x?7|dxZ`HOI#3JJDoDtSpbP2C}KSS2F_43#Ii7YdA%pK)(c zH_$V|HSqR`l7gW}?iO#E>P1#hGzWbzYTt|~$r~zl=PWT*i`1WJ4)~teHjXIC8A^Az zSYjS6+V@1W-}kikkMNSLp&RZTKJ#$V`!>x!UyOEpcuD3^pt}X12`L(A)8zQ-YTpbm z$ry5V=PWYiit5@ldwox68;6&q4OzQeEHY(^(%Lk8e3i6+gq5TW8M$*7n36>SZJKOf zuy%V`$(C$~)P#||tbsV} z1RklF_pun;=VK^%H{@jw5V1~pBs7m(jOF+s)RGXqi~)D-gf~(?k5-KB6>L?QBX|N^ zc&w8*@<3i@F}BC&sK8)?moi|2o$x|R$p6OoL8PPMI4auAX>Kv!dxk^_--0~KnKrQ`tQgMn-{R4LgXSvioPwpBv*L*5+- zQ)3Ct*pVdzad}_V__i%ntO2jD(S-1pp^_$?_sYK8q1U?8R#++H8Q+`|L~zi;M#TgIi(`{+Ob~1~|3P)MR;qA(a&# zLuBKjibLyQQlGX7VSyo@bt#6}jX$86u?|k|Q#4tgXNYDYVu)RMaYdVTaAKd731ObG zJ4+^p*ooh*D6tMs5NM45I4SzAcm_|*syj*gTFk@yWTl-H{aQSUch3qqN%~Sej=!8` zbyD<8@faSPrGAq1xp)+RHf!HW(a*&%@FrRBO-NhCT)bY^fXVXBhHv1%83IpFecI8! z2zZ903whAnK|YU(%kQnw{OW@^o}uqT9`Lr2cVOajd$Tp8eGtkSaxUb4Z*zHBCN8@- zLGvpfaU|mlj@;*MEI-D;QF_BPqwxrZjBy;9Wg8UdAN9xtpjP^oEWH{o;-QKeD zd30QIuaf3hPlQ;8K91bwEg|nf$0hde*NpZ=2xrLQ$erH8^0IVXLa&e}x^*fP(bhXF zk9WfcBbs`L<(1vALkW=|x)gXCa-4jVCA;SZnwxW3CRBl&3sL)4OCFw9^xZm@fs4{XY zyDJ$a@1v`f^bu#MS%(h?gW$Oz=C%%ebFk!6uHX>B#+kx3lvg^dp)&8 z@rbGPT`&>h?+a?m{C4_4mndmRd;3?w|O#e`+P$w?aY{(Yn!yXk;`h zS}$5Z8XavAjfpngC3K){=D4ho(0>K=tH#;$XD{v&*|qznz{&M40e?^s=tqTwR_cX> zlmzKtBoX0rHOe<8>MH(P)bDx%!TrBW#DzdYqC$Iw_6mvr;&+J%cp(9|S8x>h_b4PJ zEa1q3p!YjhNGSWCzW0Cy{I7OI1mPEn{Qd|D+7l9r_99;M6~rC?$ck$U(!a*`KeBs& z_xw*!zefHm|5rbbm(7Lt3F4@LVBg3nui!92dcUCNuMvs;9^L+b$1b#6MUeOZZga0d z>X)p5cAXX+Bn9DjTfPwizL8f0d<6V2`ZW*x1@rK${x1Rvf&ZNe(C=P^ghhm4g8bAY zzl(n#KthUwzJJZjKQa}9++Sn(H4ne$|NoGvKt@c^-f!CVzl#2k(C?JsRQ;wC|Hl6l z{SW^}>Ytpw(*INPzj^FGCsGLa?>M#pA7a1e^M8-)|DR7V{ilK7*I~lRTY{@dNk~a> zYJ%hM2q$a^(f@7#|0Ne3E%X$aH&og`C9XIxW54&@`J)GAcMt9yj4X`MMGU_MhiD_R zl%@{m2^R?$2shpPaoy+${OQJrGz-T+iVX^yU!AafkhB{O$@_)Q3(qbTE5kQ>4fS7* z(KI{gvk!H4_4eGncdGi!1E0k1cBP!woPQ&Nd0aPU+Ei3h)}T##GpF-XF}y$@8uY-; zaor?~`L-jkNWJN-HS)-V4BT1mO3eO`?+QN%U_=&S((in#9_l>Pj*k9zC15kCVbg|_*?+H5bXqK z=UnUQ^u#~5KNJ{rcl=qR3-{rg{gV@aJG7Tq6bYh``Appp&{F+`*Tfm`$>jy8yR-0Y)IgWA&OeI3HEh?CM1;1AV7fy%#neX-z4lj@wpY^fx zRv1W&In%9JEj4Cglyqx>nR@A^YKY8FcHf!dd3c%ZE3xvfjMLf|#X2WHQ=r9JTx*zq z%j=IiQ}@Do*R+Qs5Yq~y%vZ_J&Uzo!SpljBf0{f#9BU^j7CP2- z+s^{23T_%IebBtyrlfqzCSvTSgw1Bkdk2{2drqpQYwxZ5dE2K`Lp<2AXM%soycT69 zP7B#pxa~@qun2gIisGd=sEUeFZ}OgQx+&83%ku`Y1*P7a6@!9vY8bH@*S``w<(77Cf`Z8bAluqso!-1>OoPo>sznJ}@r-Rqs_D=rHQeY~qhvBL*TB35Ot zj@}irA}d)NTxQ>>3{FxWmL%@k*_Cw~Cngl|57g?De(zsUtIlt}%pC}b#reVY#|^%u z=bjp;uD8{TbXQevo|C497>l)TDgJnO&BSm)AtJLxO^cl#{%4Qd3jBJ3;@ik$8XJ~& z#{#q8@W+qNJoKB_eJbynJ}Q8*|4EPVM|H|A@B5lGyLBM>uzF|pm_s1(6FMV&dg^3@ z@Q;|ZCvh}~!qwHsH!ciR2VQKewUdnrD7$QC2BD=*o6#$Fvb2gmYZ&J{J8g&6~umL+^rK_O{(Oxt)tU(-cA{?Ig|>+N{HpCHjTc zZtF=2)VvGiPgNDo4cuT{C|()*+bFcI$+V%hpzIO)PS&ab83tTba53iEt7q$#yykZi zrxeTgOS|;t=RNb9YztUKHWmM{dAI2$Q@?$pR@QOglmIIz>D9^g!*i0^oV``V+ch}b zrepKcC^2L6kggwQd#nA_&Ko-=7}xf_S^=Ub3U2lFcqvW|*l@oD{mJxk{-ObMbQTBc z-_X#nESj*=QYjuS>(|bg-W0lX>-@>(yLyyNeoNZI3aWLQ9as0Tw@LtV0$@EH;_Gc- z=lg_*5fdkTE3_x%KJF{-0d+?Y8N`6i#H&Td&OA*xIe#GatxyGRZp^4DtE71Nknx?m zXa8yZR*0=9-5I;95vFKafE*0SoUzt5F=oDja0{Rr_b3?zvGW^hQ6tG$k78koVL3u`bW~Pl>+qanFVvZO6(jqelf^ zc9BHqLDkr-?sF=~@-9MCzwkHHHeO!)WO#U_&*CzlTZVt7Q$Cb&uF>AIt31l)v%{r4wQkagBHgWO49Ak*OU%joIT!2@Q7}lEi{0~ons;| zJ$Nv6v9T5>d+UA8E9G06XV1NUX?T9m=N}z6mW;2#MxsMHubp*xa-(3sd5rWsQ8{+)x^?#9R}x_YoXD^X2^+44l6P8= zL3v%h1h1CIvtPsR=tG989URxzDCxeXV|VM^zN@5V0rzEdtXITt{OpelG-h@2hv@1V5GuDRxyy0*(J-!F>=&Q?4M>=sLr0= zoy+Imyfi*9dih(|x2Pl4DCPr&s4JsQ8Zd%gu-1-Tmi68{L~k{#OuD(d5<=RjFSzQ( zwnwU_NjfVTx_C3jpdTD4{x4AEMfuZPBDlxRW1jvfl(9?5a69{Ib?~Bf5W3K&@6pPR z+W47UcYCm}|5CTQ_4&}BsXk`clzDWcn^Vch6KovrLU zJ^h}9$CnQU4c(P{e&1!T?rGhH{Oy5IBSc+(e06J%0G2$teGWE7xjC5imkVz~vSY-b zsDFqbyRS9d8LOdwuqBPEf;N+WlJr@zTxyh59{%j2OWeWGeTHX0?t1sqR|rm)9TYo} zBzMM5=s&xbgT6MId@H?!2^j7eyV~$^F+Ip=;^U*o-|f<>O&+rHO5eNPP#wrxJr{Go zasLT_z+td5*2(xI3_LQ-amp*-=uh_S*CrIc5{LKRI-paTGaLFR5TE**kFnC{9JwmT z#AS7!i3+J8ZJ6;$7C40_aZlkbp&wFiA>2H9Q{|$PetU1CZmaWeSn^f+GbeA2(A9$v z;=fZpu*e5S2R=Q#*=1hkHd6WVY3)zlyCuzYOFnFeaq%{JMw(_l4lWrIflmyyrS4$n}_s3qAH`_z|!_^S;g4lO2uWw^IIcy1% zhZQG%TqLB?UfO?Rp?0tv+@g2VWB7BKz$1uUpjuMD?A7a{N&>|s1uuKc3wppb;&hVZs|X1 zZ&ysd-BtY7D>D<*wnesHPKs^scL{`V=YJ9rER*e!3O{5`urY+cC^_xxPRYm5A5cqu zl{r4|kvg-oxAh-5<@G;=$|f!c{Fm2`wQ!PU-?{dz^9rAqD=!`tJYJJ%%e}vL!SJ|_ z6g-Fi{=_Ywd85LM(U06({?2M@1VD|&?d|ti)xs1uii3Tl>^070HJqJR3d?dy z30AIM=-TU+t&Wwg+xs7T zo{lQBzjq>UocLxw;y+$LOj4Yc^Uplhb*?^+JD{C9l{pcHUpZ*H@4oQ!d6S9*+gb^e zrJA&O66!Na@_a)Tapo^wuWGHSgNYH_TdTG`2S5F3EGRIhX$10+XyPPgew)@Y?H)#RDg3D6_qI~pCZ`g6p-$~< zuCY4lEb6J;HS;%0GTDDeu3q{ZXfvS9){xkvQ5s+Z)<)~ZQ@el*#RJ6Y(4GC#=lhd} zy(Z54(zR#hHujbF9KL^Z8=VR^x#N(yI@`YMh*tcMGo{r7$PAG|Qd@!LXyn`x(!>mFQK&uQg{ zV1E%g;miu}3Yp6wy~Om}78emO_iBhj&-DH-JRA0Q@Y654yPG2n5jiAc8AN#ndthi5 z8MWL$e5phkX^E%w!0s6Cj{N+#f9O(?vbLogr3vP1*c}n|wtw(afwG3B1Em^v&M-0J z^HTr7r95RdOB+f)Ox4gVB5J9>|5A?fF-vnwGHj3G?ugI){=Q3D%8Hi8lqlF+Om}z` zzn^m{QyFTBqWHjiF^S=y7yElJWhl#A!YP+w4=`roQH%XOm(r9ESSnM_!tP*phkst^ zXJ1NDmaC2MR54~@QS<$s zmu@TXwiKp_!1iEvhkd5Z%0&AdzGV5AOjOj$NcP=IBl9TIGH3vjjuP=(;bv1S(dIB^83yA=b!bpOyaXoXTV7XGtW`K8QF{#Al^1JF8OB+=LVjdJi5% zB;!{&*_oA4b3GCPGyqOVh~QUxvok8?%{56*pgOQMBH4STCp)e3fVm3E0+a?8M~HZ@ zu(MMtrOaU@G$;T(sGIDy(w%*$Qr!F?NeyHLPS+LjTItG8suVQ`k>o+@U~AoE&y~*X z+m*Y`MM$EcePD525ziILCr63>(EFt7qbpCg@b79zr09lg0Q^(@X zfQ0Rxb^byvTRNp&@)R^433zTlS;sHbc1zzW-*@Uj{4wD4cI-NTzP3v`seI2V@%RIP z(zf9`e!jL-`gZxQQ@i7L147%er#^}OV8lN>@?Pxwj&bXjQSFvt(H16U%i#JJ+HXtW zc}vf73uUl{JidiEw52PyrSol5``xD24Zz72)e4QTBKq-X=2vT^A8hP>Z~ob$#(%MDqB zjRVIwWDafY7u%5jwl4KtCd^*zq(B9`mI2J5?z zukSjvE+n>o{rcKOolT0VV=-LU`D$Z_`?V*F5zTL{ha3yxn$FIR4epLl?nmgqwH|cL zho5l1&{*cK|HM0@dC7XfF&D1lY}rV5mwR$7LVwA+-!U7mV z^N~gq_iJs7Va*HHY)29t%5ia5^ z)(CQ!YdaREKX2XXm;e`Y{(JC>m-B?&HPtI;oNGT}c_oif=llXk^2_Ql4q-K~u}?|@ zQN4sfZhmQX6E$|&l41?SIq$%p z{M5Sr4oWqq*x2G#-CnOic0Q>N+*;)e7WdLkWMZM4Dog79ve~sq zE|}iCVX_f3y;03q?~|p}DqKL{3p5#xDcw-vt8-+@wTCWf-*Yu##JFx$FRJ&-rq{|| zP`hVsk`n{nP+3&(kxi|YzMy!|$Rs>wdcAr^Ak zK1~IzSaI!3RrPHA3WqfdT#m8y2v)=P#j3hCR|`8duemHT zn|!T?>;GxdC}2JQ1zPuN^2EYs9?@n$ypSPj_cs;byn7LqmPT#hmI ze60HIvsIOBP8G&!e!(p=n($VA_7qi^jX@z%a~yY{p@+BP*ppSkHadmwn$0+GMw7Qy zuYJ1eL7U@+Sj__5F@~PERgZnD>VBIeg(jNUaEtUNFDte^Nfl&ss8C?ChPzMK^Rnu; zPgdP)BV7p7)W><#n>?+$>=RW*Y{UvdnsT^fbUjb2PWuE^A)C+vEiapi1I;R0XGZTT z8(UtZtiZU2xseCo?Hw7VDjQe^Q>H}xoJJm%T{5IbsJ+!3gygf1``L|@mLUw?B5Ia2 z2OzmDWj~9N^fDPkg$T7J&3;HW>xiG}$c?fM%uILKG=R-HbM!1iW5iv0_G!H@Yp^a1p&hwaERn_a1{cL$8vCF1Q@TwaVIx} z@rV|0Y%eYyaS(r;gC!S$l@VxfY!5CKu^)egV?w?L=Ige2Vc9qm0)#)r(Ih*9t8~#` z*lt`hVlQ5r10(B$@wzRZ*e+b6;5k5y10u_Tm37gc*iKvmLI@u!0H4B7fc4cf&rIIc zF}`~3*-9LDCwEKkoy5o_Rmb3J@Uy8n+K&BJ!LkH*vP1`U)%RI{T;|S!ExBch;mIN$ zovT-#wZ#SR{Iyx|R$^$fKnH%+@!7*T+nwA^xwjI7lX*JFui8Ez)-z->?7?{k_ zQNDWaS!$f*&Vfz2C5isYES)1)PdK|2Xb;?uHy+f-{++ zbLgtpGh7^P+kT^9QKEM;O-J^s>a+84ncD|8*zMQ*db@k;W&DbrvWtBL}D`lzr!AIoFXa;T}Ez?8>&z1Fn0O!h=(ok-MmO)|; z56V&{z=!3FX*zBmEl-KzylLhGU-*!GAr0>4)bfaUiC4;0^@R`0=hKe6*|n4s5jIv_ZPoTBF zZJ}!gWlZO`RIH8^+^siqj%r+XA9#`;x#!&~w}4uYb|yAXxYs?gj=ZzHI$Tgvk978F z->43z-H!YTe`>ENjN};D;$9L?+eAwXdgZrqIR#+(Jbw~#Adc89?6%euuva5pu%b#LhY*NPp6F+uB``QR>UVyj5#-q%tckQn&7#e8rlHKA1~ zROt0<>P`0R27^`G8%eMK^9fi3R)G~@8CU`qfqCFHFbhlrQ@{l93iu0n35)>4z#z~M zaDX158|Va{0Uf|opbcmNngAw12O5C};1Tc;r~#^h`#=S74=4jlfnuN#$Om$PY=8og zfpj1hAOXohB9H*Y1989&AO?sAqJRh>3;Kp@}`5C9**8}I}?0C&I@zyVIcWxyV| z2-pEwz#2FYSO8}MGvG8}3YY*!00uwWGIPq6QrHe6(K%fmnOPUcu-V@yow_ScPmwjnN(4uCoAlC_IgO^J?li*7m zgcdZb2iZ^DZ#d&}lez3e8-V6Ds|Hz5Bcz3wM*T>`m*1W;DwMX-wERRN{;o1aDdoG_6@C z=-7l=!)@FT`mz^|4NYm541!J|8@zDsbb=SH8+xaCU(kVxV-06;H|fisv@U2;^PV8_ zi31HsamI9lC#@5DyLng8?up$EyKpyyLL}W-6A$$RBwgFvSu9pPtBOTs<*_nYNvxZ! zNLCOF&vIiquxwc7EMpdm1!pO<6j-t>36?PH2Xl-0fyrmiFvplfOg6Kf$zs+stC&<~ z9y5cP#JtIjWCk(uOgE+j(}rozG-jfhaHcX-fho(BUO{O760Z{Gvvd%LWWHB1PY=aZ9Kyv>$O~qO8u)CX%$gHe6D|Li+iP zaLUpQB_c(a*M>@pSlWK>BQ0eChC>mgw`iBi#Hjjmh-x)%E)tk!n+1#8*7%~{hHs7H0^ z2Ca!_Jv`0W^DXc>XEnRYKFs1pJXpUDCVlShQHt^$__N+%CxiUS+ZSF(`7~Jdn{ISJ z8e^xOV}1C+MLPvK>JPK+QE#}3avLofRl|+g4)I%j7Pe}iP1Istkc;JM6kK1~RzKUm z++Tx^klZ}5EvaRlFcU2|g4OEA_xqU6KrMR-<8oen9nJ~I?|>P1PV=bKK!1*+Aai%2 zl!UURT+$^|If9g)6zq(l9LM20=Yb@oRHgS~uCD$1%MIY>k59e4zu~3)L>}oiFU)@X z^7n_qNz*b>8b>jJ=}`I{GIP@pqAq}To>2`7)jZy zARXIXFn?cg!s30}S$3do0G23|UjTOz6ys#lAT0=#Z@tN8PX?CJiGrm*Kc{mNER{o8 z&o9&Ia%kH((4G)9?QywG#vWggE*Q~DSeD~BTqiC_f{fSloH^Fb90aLzEC;_@o)ZZ_ z?%XNlGhhvS&>^BH#TmYJU}ozo|G+o1@9u7 z1V7Atfu3Aee5EkzQL>^SBT2lFs#!4!tH&UXUQ|Ltzfd)&$D1s3ou>FuJnh!!R&eRae>M+B0+spfO?KI8E6tv%lT<~ zpdK|(b`%`sQSzifGf7B=p<&QrwI3U*Ze98CO0}xY4y;}o79A&#R-*axF zueZU()h)&bc=5g2KW(JC`yGGWF;>oL8-QW5C(Uxtxr27xFgOpBl4R+lp>C0c9ZC75 zOD@R~q;$ovBd&B-J4(_NZX|`lYSJ9rF&=TCGlT3Ko7wv6;NG{Z_wH^nCfX?-X6N(U zIqyAP1iT5!GKu$|VShJUW1}uUmJepS%Q=I20Nw+0a;V>TgGUDfmzRBaId@BiF6kMAcOQJ^2!%YHh=BKR)6B}Fv>@56Q$OrR2QIh7MIH>HCp z2@XlslP;MChx+OU{pBDD9+LcDI4!@2-441Aq*VD0%W)s9YlBgJEqdaUB;Z!1_x}6D zQT?-k8#YOc-kn4`JMJH*R%(m;u|0U#zITg=eOAZ!t$hM!UmEvnZ)Y$HI_qfhu|41E zMQ{f%#dX3{E~%}>$7A+twzbX9{$GSj3jggtZqqD6ftw#^(jpWnSg(3p)WYz-@>8Ib zlvldsk{m%wm!BXaX{;5mj(6JPdjywTgyp!aJLWZ0mma&-L`3C}vxFG-l literal 2594 zcmYM0dpy(oAIJCI3^Q_>+Yocf{mjgzRepu7A{z=F(q&_FUukkixtj_(xg=Q%$t6b_ zAxy{2y(FR1$(%WQNaa?XBZu_+PMt^Z$K(C@d>)@apO5GJ_5P4tNS**#FbD(!fUN?y z5_oZeLLf-+E-3_yej*?=IP~Nh(?+W=R2RTb79w;rvPcT&!imjh|TP_ZMNa>uSl|w1gd7KgVa=GGt zL+ZTf`T2QIyO64k6zDZN%;WJJmO<*PkScQH4u%lg$vt))!z`-4DDZV)ak^t01WWu* zLUT`P+5K5&!|jOMbc@Ee2@%{LBS=9Td?2LuT*~O6HlY6<%ZRx$8eW|RC zbIb{cL0l)qn`Pdqu@)GBB%ucd01(zDJ!SBOgY*dCgS-SCSh66! zpomknZ}13!NCq;DG&IY$cmlM%eb(rnTlj2tRc#rObZIn_xP6J@moWDpRyLIKW+OM^ z8@kF4hnbk>8mv7*2j8@}tch`tx~&vmaE439@W(~WYntZ6M*46ar?jK60qOKhC1gvr zbiCgdoDEG!Nt-Fw^fo-o=TL*W_weH^S7vg`H&&OkUDf%k+ zVHjyep3I}|Xq&uxi_Rb|0d#dR;_Cg^=pZyS*rcl>0O5nFps}~tDb*hyh%ay-my4>W z4zGV zBqM1Q*lS>8Kw=8uDJi-Ch}qybg~I?@GLZU57_%g|VJ#cX#hn~$n1%mT^UU^|vqigp z8y!N+*>HwQSt#-0{WaU){uiH1l6C;(Uutgy}j! znk_U*fuwTFxy85WQB6)SvNj}fb`(@_u3B?-hqLXvs&g8YKdjmmq zHXE?Ul%yzY--k&7wtZb_*NIG~O{RL^gEWn4;&cb@^!Z%Z!#K?9d9$+$A&c|-EIFgL zoH=H7md~4sdSc6`tW}itqn>DP+0vnq*%a9a7jTs^PwJo=sS^{G&ayfmay@8EHM8Y+ zq?fKbLpR-m{cuY{3JD^UOy-tNPTxqKjAk-ep@H8=Q$ZGD;AXfvnj=Xt*TSrW*Yn>U zEz5MV)E5Tp#bzi*Lxl7hG-6KM?|Pr;VLv#U;f64$GO^(#ZK-u;F_KW&;A`C1a_dla z9Ok;9sfB-z*qvQ!j*!DVD)J(slUqXYsujaWKRME93d$}zmBtvX>Sdj?j*m_CE+(+m zrF#i)-ddhpvozC2Fv~|WMIyK@h6mLG^}0r?;4CD6qVx>ho_)kC1dwmtpwMZzX&@lB zw?t_y9J1({pDJJP%3IiHQ8RfyVDVH~OOJWi)`G^i*CvU3BB&b8INg%wuDTa{d&V16 z6;}QluArX#{Kg0r2_k;IOc&1WWH2CMJtq9naHlxw3T+( z(AH6e7GG|BqLw$9;(^c36u)og^NFL>WCMh$V`J}r*L~im8c6SSEJ6^`MdvJXi9hj36Tz(~40%9ImW*maR1G}mBcHEz zWWL4U(VC&5(zGWbM~C-nUHA7Sx0U@x1ufhcwJ5|UEU6!wDS?1cpk|&REXiF`G9auo zjF@i;<_S<_AuObkxg`E?)Zl7PJLeg=c%F)#yFSp{^SLLm))gnMbtffkdBbJ4mihCM zJTUzWI%YhvJJk<#adMKxzCIxl4p($-p=Qh1Mh@)LplgrqoS)z6t!3-A%?9D%lvbaz zACW=D=4A!7dfO{j@dPD7Ioc$2v9To>Wa0S^?Yoeis4cNK7^ey{>OGQ}-bWttkpxNl zbEg2#xGV1=a0ap~JI+986pLhC3LAp1^;=pS4O|$2)nTlq6n%Fy`~Qj!h842@yXH)j+AokKwCpIG1`!-qB3-x6nX z(0*Tip65d@6gTNFb0U1EM;7g#LAhxyLHAaF$2qZ2zWybT(nJQnqHOjmg%QPO3nuo; zdbOH5x&_-*%Adl+hU7>?FwOp$PSN94_<94Cy^@%%PPtO}S#>MR&~s0kDur_8&AOq9 zC2sxK1e?h4^(}Z*xhs^~J?60i?APy`{fB;}D>NNtTpgpJ_eCT!UDrtLt!)==tQ=ym z6@K}JSeq{rxmTS&$Rg5I{Lb4ALAj;xl`(gh+-c-Nplo$?wV`Q|m$+DS%lgdn(Z7Qu z=1DyfH@sl)zqrRw&~%dLa%O<2?s2byKC7xvlDgDR%1elKj3|7=8`!V1c0wT;Zvy`~Ibj=gsa_Z!BEIMy}d{A#Z zDzkR?pr+d8{{eTYF+^09{rp5j|Ac=9xJ+UfKZ;NDhi1TZbOj!st!eDs-@b@G{5%DmHZWI+@H> zGrMu9KEvYiQIwKa#xuFw@oepD{zX$WBk!J99jweAHhf7af;h}}E7yM|Od+)?|I6Cn5uJ#R%NUot3mZ9V6k99%ja-Vq zbb4vS@v9&7T6590e~y$17zTT~LzMGcHR5=xbMM$2NX9cw`K#U9T^+UMHx-%A5e3n` V7xw2RkCR+JDds7XCI6ef{|B(lG0y-1 diff --git a/src/timeseries/fixtures/read_values_zstd.bin b/src/timeseries/fixtures/read_values_zstd.bin new file mode 100644 index 0000000000000000000000000000000000000000..a61144c3d9efaf800def7136dd4beb6fbc3e36b9 GIT binary patch literal 2594 zcmYM0dpy(oAIJCI3^Q_>+Yocf{mjgzRepu7A{z=F(q&_FUukkixtj_(xg=Q%$t6b_ zAxy{2y(FR1$(%WQNaa?XBZu_+PMt^Z$K(C@d>)@apO5GJ_5P4tNS**#FbD(!fUN?y z5_oZeLLf-+E-3_yej*?=IP~Nh(?+W=R2RTb79w;rvPcT&!imjh|TP_ZMNa>uSl|w1gd7KgVa=GGt zL+ZTf`T2QIyO64k6zDZN%;WJJmO<*PkScQH4u%lg$vt))!z`-4DDZV)ak^t01WWu* zLUT`P+5K5&!|jOMbc@Ee2@%{LBS=9Td?2LuT*~O6HlY6<%ZRx$8eW|RC zbIb{cL0l)qn`Pdqu@)GBB%ucd01(zDJ!SBOgY*dCgS-SCSh66! zpomknZ}13!NCq;DG&IY$cmlM%eb(rnTlj2tRc#rObZIn_xP6J@moWDpRyLIKW+OM^ z8@kF4hnbk>8mv7*2j8@}tch`tx~&vmaE439@W(~WYntZ6M*46ar?jK60qOKhC1gvr zbiCgdoDEG!Nt-Fw^fo-o=TL*W_weH^S7vg`H&&OkUDf%k+ zVHjyep3I}|Xq&uxi_Rb|0d#dR;_Cg^=pZyS*rcl>0O5nFps}~tDb*hyh%ay-my4>W z4zGV zBqM1Q*lS>8Kw=8uDJi-Ch}qybg~I?@GLZU57_%g|VJ#cX#hn~$n1%mT^UU^|vqigp z8y!N+*>HwQSt#-0{WaU){uiH1l6C;(Uutgy}j! znk_U*fuwTFxy85WQB6)SvNj}fb`(@_u3B?-hqLXvs&g8YKdjmmq zHXE?Ul%yzY--k&7wtZb_*NIG~O{RL^gEWn4;&cb@^!Z%Z!#K?9d9$+$A&c|-EIFgL zoH=H7md~4sdSc6`tW}itqn>DP+0vnq*%a9a7jTs^PwJo=sS^{G&ayfmay@8EHM8Y+ zq?fKbLpR-m{cuY{3JD^UOy-tNPTxqKjAk-ep@H8=Q$ZGD;AXfvnj=Xt*TSrW*Yn>U zEz5MV)E5Tp#bzi*Lxl7hG-6KM?|Pr;VLv#U;f64$GO^(#ZK-u;F_KW&;A`C1a_dla z9Ok;9sfB-z*qvQ!j*!DVD)J(slUqXYsujaWKRME93d$}zmBtvX>Sdj?j*m_CE+(+m zrF#i)-ddhpvozC2Fv~|WMIyK@h6mLG^}0r?;4CD6qVx>ho_)kC1dwmtpwMZzX&@lB zw?t_y9J1({pDJJP%3IiHQ8RfyVDVH~OOJWi)`G^i*CvU3BB&b8INg%wuDTa{d&V16 z6;}QluArX#{Kg0r2_k;IOc&1WWH2CMJtq9naHlxw3T+( z(AH6e7GG|BqLw$9;(^c36u)og^NFL>WCMh$V`J}r*L~im8c6SSEJ6^`MdvJXi9hj36Tz(~40%9ImW*maR1G}mBcHEz zWWL4U(VC&5(zGWbM~C-nUHA7Sx0U@x1ufhcwJ5|UEU6!wDS?1cpk|&REXiF`G9auo zjF@i;<_S<_AuObkxg`E?)Zl7PJLeg=c%F)#yFSp{^SLLm))gnMbtffkdBbJ4mihCM zJTUzWI%YhvJJk<#adMKxzCIxl4p($-p=Qh1Mh@)LplgrqoS)z6t!3-A%?9D%lvbaz zACW=D=4A!7dfO{j@dPD7Ioc$2v9To>Wa0S^?Yoeis4cNK7^ey{>OGQ}-bWttkpxNl zbEg2#xGV1=a0ap~JI+986pLhC3LAp1^;=pS4O|$2)nTlq6n%Fy`~Qj!h842@yXH)j+AokKwCpIG1`!-qB3-x6nX z(0*Tip65d@6gTNFb0U1EM;7g#LAhxyLHAaF$2qZ2zWybT(nJQnqHOjmg%QPO3nuo; zdbOH5x&_-*%Adl+hU7>?FwOp$PSN94_<94Cy^@%%PPtO}S#>MR&~s0kDur_8&AOq9 zC2sxK1e?h4^(}Z*xhs^~J?60i?APy`{fB;}D>NNtTpgpJ_eCT!UDrtLt!)==tQ=ym z6@K}JSeq{rxmTS&$Rg5I{Lb4ALAj;xl`(gh+-c-Nplo$?wV`Q|m$+DS%lgdn(Z7Qu z=1DyfH@sl)zqrRw&~%dLa%O<2?s2byKC7xvlDgDR%1elKj3|7=8`!V1c0wT;Zvy`~Ibj=gsa_Z!BEIMy}d{A#Z zDzkR?pr+d8{{eTYF+^09{rp5j|Ac=9xJ+UfKZ;NDhi1TZbOj!st!eDs-@b@G{5%DmHZWI+@H> zGrMu9KEvYiQIwKa#xuFw@oepD{zX$WBk!J99jweAHhf7af;h}}E7yM|Od+)?|I6Cn5uJ#R%NUot3mZ9V6k99%ja-Vq zbb4vS@v9&7T6590e~y$17zTT~LzMGcHR5=xbMM$2NX9cw`K#U9T^+UMHx-%A5e3n` V7xw2RkCR+JDds7XCI6ef{|B(lG0y-1 literal 0 HcmV?d00001 From 242ca16922a203589425aab35e20d18cb0a6cb69 Mon Sep 17 00:00:00 2001 From: jgjesdal Date: Wed, 14 Oct 2026 10:30:13 +0200 Subject: [PATCH 6/8] Send binary ingest frames compressed in-band, as compression 2 BinaryIngestOptions::compression picks where the zstd goes: Payload, one stream around the Arrow payload (compression 1, still the default), or Buffers, arrow-rs compressing each buffer in-band (compression 2, the form a read answers with). Both the string and the dataframe path take it; in Python it is compression="payload" or "buffers". The server recounts a compression 2 frame's raw length from its buffers and refuses a mismatch, so in_band_raw_len reads it back from the written stream instead of predicting which buffers arrow-rs writes. Frames of every value type from both paths pass the platform's own DatapointFrame decoder. Adds a live pandas round trip: a timestamp index, NaN and pd.NA as gaps. Co-Authored-By: Claude Opus 5.5 Signed-off-by: jgjesdal --- AGENTS.md | 2 +- .../intellistream_datahub_sdk/__init__.pyi | 9 +- datahub_python_bindings/src/timeseries/mod.rs | 26 ++- .../src/timeseries/sync_service.rs | 18 +- python_tests/test_timeseries_binary.py | 6 +- .../test_timeseries_binary_dataframe.py | 53 +++++- src/timeseries/binary.rs | 179 +++++++++++++++--- src/timeseries/binary_table.rs | 34 ++-- src/timeseries/mod.rs | 4 +- 9 files changed, 271 insertions(+), 60 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index 0d180a2..d211978 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -201,7 +201,7 @@ the generic default with `recognized: false`, not an error. ### Binary datapoints (`src/timeseries/binary.rs`, `src/timeseries/binary_read.rs`) -`TimeSeriesService::insert_datapoints_binary` is the second ingest path, to `POST /timeseries/data/binary`: the same `DatapointsCollection` input, resolved through `/timeseries/byids` (cached per service instance; needs read access on the dataset), checked against each series' value type locally, cut into Arrow IPC frames at the contract's caps, zstd-compressed per frame (mandatory: level 1, 3 or 9, default 9) and posted through `execute_post_bytes_request`. Every refusal on this path is a problem document typed for its status — `invalid-frame` (400), `unknown-timeseries` (404), `request-too-large` (413), `unsupported-media-type` (415), `value-type-mismatch` and `external-id-mismatch` (422), `too-many-in-flight` (429) — with the kebab-case sub-case in a `reason` extension beside it. It was one type, `datapoint-block-rejected`, answering with all six statuses until the api split it (platform #120); the SDK matches the new slugs only, and nothing here recognises the old one — the binary path has never been in a release, so no caller can be on the other side of that change. `is_stale_series_rejection` reads the slug to decide the one re-resolve-and-retry, and reads it from the problem rather than the body text because the SDK's own pre-flight 404 names the series it could not resolve — an external id spelling `unknown-timeseries` used to trigger the retry. `FrameWriter` builds one frame and is public; `cut_into_writers` and `pack_requests` hold the caps. The byte layout is the platform's `binary_datapoints_format.md`. The arrow-rs crates (`arrow-array`, `arrow-schema`, `arrow-ipc`) serve both directions. In Python it is `insert_datapoints_binary` and `insert_from_lists_binary` on the **sync** client only, each taking an optional `zstd_level`; the async client has no binary methods. The ignored `timeseries::tests::test_datapoints_binary` is the live twin of `test_datapoints` and needs a backend that serves the endpoint; the writer's own tests in `binary.rs` run offline. `python_tests/test_timeseries_binary.py` round-trips the Python binding through the JSON read path. +`TimeSeriesService::insert_datapoints_binary` is the second ingest path, to `POST /timeseries/data/binary`: the same `DatapointsCollection` input, resolved through `/timeseries/byids` (cached per service instance; needs read access on the dataset), checked against each series' value type locally, cut into Arrow IPC frames at the contract's caps, zstd-compressed per frame (mandatory: level 1, 3 or 9, default 9) and posted through `execute_post_bytes_request`. `BinaryIngestOptions::compression` picks where the zstd goes: `FrameCompression::Payload` (compression 1, the default: one stream around the Arrow payload) or `FrameCompression::Buffers` (compression 2: arrow-rs compresses each buffer in-band, the form a read answers with). At compression 2 the envelope's raw length is what the server recounts — every record batch buffer uncompressed, padded to 8 — so `in_band_raw_len` reads it back from the written stream rather than predicting which buffers arrow-rs writes; a mismatch is a 400. The api only takes compression 2 on ingest from platform `357794cce` on. Every refusal on this path is a problem document typed for its status — `invalid-frame` (400), `unknown-timeseries` (404), `request-too-large` (413), `unsupported-media-type` (415), `value-type-mismatch` and `external-id-mismatch` (422), `too-many-in-flight` (429) — with the kebab-case sub-case in a `reason` extension beside it. It was one type, `datapoint-block-rejected`, answering with all six statuses until the api split it (platform #120); the SDK matches the new slugs only, and nothing here recognises the old one — the binary path has never been in a release, so no caller can be on the other side of that change. `is_stale_series_rejection` reads the slug to decide the one re-resolve-and-retry, and reads it from the problem rather than the body text because the SDK's own pre-flight 404 names the series it could not resolve — an external id spelling `unknown-timeseries` used to trigger the retry. `FrameWriter` builds one frame and is public; `cut_into_writers` and `pack_requests` hold the caps. The byte layout is the platform's `binary_datapoints_format.md`. The arrow-rs crates (`arrow-array`, `arrow-schema`, `arrow-ipc`) serve both directions. In Python it is `insert_datapoints_binary` and `insert_from_lists_binary` on the **sync** client only, each taking an optional `zstd_level` and `compression` (`"payload"` or `"buffers"`); the async client has no binary methods. The ignored `timeseries::tests::test_datapoints_binary` is the live twin of `test_datapoints` and needs a backend that serves the endpoint; the writer's own tests in `binary.rs` run offline. `python_tests/test_timeseries_binary.py` round-trips the Python binding through the JSON read path. The read is `TimeSeriesService::retrieve_datapoints_binary`, to `POST /timeseries/data/list/binary`: the JSON retrieve body in, the same frames out, one series per frame, then the 4-byte marker `DHDE`. Read frames are **compression 2**: the payload is an Arrow IPC stream whose record batches zstd-compress each buffer in-band (Arrow's body compression), because the api relays ClickHouse's compressed batches without decoding them; `arrow-ipc`'s `zstd` feature is what decodes that, and a frame holds whole batches up to the row cap, so most hold fewer rows than the cap. Compression 1 (one zstd stream around the payload, what ingest sends) is still read. At compression 2 the envelope's raw length is the batches' buffers uncompressed, which arrow-rs does not report, so only its cap is checked. A body without the marker was cut short (the server streams, so a failure after the first bytes can only end the body early) and is a **502** saying so, never fewer points; bytes after it are refused too. Codec 2 frames carry bucketed aggregates in their own schema (`avg`/`min`/`max`/`sum`, Float64, 50 000 buckets per frame) while the envelope's type byte stays the series' own. `parse_read_response` returns one columnar `BinaryDatapoints` per series, merging consecutive frames of one series; decimals stay exact as unscaled `i128` plus scale. `TimeSeriesService::stream_datapoints_binary` is the primitive: a `FrameAssembler` cuts the body into frames as chunks arrive (a frame may be split anywhere), each frame is an owned `Bytes` decoded on `spawn_blocking`, at most `min(cores, 8)` in flight through a `FuturesOrdered` so items stay in frame order, and a producer task feeds a bounded channel behind the `BinaryDatapointStream` (dropping it aborts the task and closes the connection). One item per frame, so a long series is several consecutive items. A refusal is the `Err` before there is a stream; a response that breaks off is an error item after the items that arrived, and that is the semantic difference stream consumers must handle. `retrieve_datapoints_binary` collects the stream and merges consecutive parts (each series allocated once at its final size); `parse_read_response` does the same for a body in memory, decoding on scoped threads. Measured on 100M float rows, 10 series: the server takes ~6.5 s; the buffered call adds ~0.24 s (the merge copy) and the stream into polars ~0.02 s. Single-threaded zstd used to add 1.3 s on its own. Differences from the JSON read worth knowing: no `cursor` (a 400; advance `start` past the last timestamp), a series with no points is absent rather than empty, and an unknown or unreadable series is a 400 for the whole request — `DataWrapper` serialises only `items`, so the request-level `ignoreUnknownIds` and `limit` are not reachable. `src/timeseries/fixtures/read_*.bin` are response bodies written by the platform's own `DatapointFrameWriter.relayStream` (`read_*_zstd.bin` are the compression 1 bodies its former `wrap` wrote), so the offline tests in `binary_read.rs` prove arrow-rs reads what the Java writer emits; regenerate them when the platform's writer changes. The ignored `timeseries::tests::test_datapoints_binary_read` is the live check against the JSON read. In Python it is `retrieve_datapoints_binary` on the sync client, plus `retrieve_datapoints_binary_stream`, whose single-use `BinaryDatapointReader` iterates per frame or is an Arrow stream of one long table (`timeseries_id`, `timestamp`, values; one schema, so a read mixing value types must be iterated). The long table carries no external id column on purpose: as a dictionary column, polars spent ~0.8 s per 100M rows making it categorical, more than everything else; `reader.external_ids` maps the ids instead. The buffered `retrieve_datapoints_binary` returns a `BinaryReadResult`, a sequence of per-series `BinaryDatapoints` with the same `external_ids`, long-table Arrow stream and `to_polars(wide=False)` as the reader: both call `python/intellistream_datahub_sdk/_tables.py`, so the two tables cannot drift. `wide=True` puts series side by side when every one has the same timestamps (free), pivots on a numeric key when they share one value column type (~1.7 s per 100M rows; joining names first doubled it, `concat(how="align")` tripled it), and aligns otherwise, which is how mixed value types sit side by side. Each per-series `BinaryDatapoints` implements the Arrow PyCapsule stream protocol (`__arrow_c_stream__`, via `arrow-array`'s `ffi` feature enabled in the bindings crate only), so polars, pyarrow and pandas 3 read them zero-copy; the package still declares no runtime dependency, and `to_polars`/`to_pandas` import theirs lazily. `python_tests/test_timeseries_binary_read.py` covers it offline against a stand-in server and live when the backend serves the endpoint. diff --git a/datahub_python_bindings/python/intellistream_datahub_sdk/__init__.pyi b/datahub_python_bindings/python/intellistream_datahub_sdk/__init__.pyi index 50b7aea..15ec25f 100644 --- a/datahub_python_bindings/python/intellistream_datahub_sdk/__init__.pyi +++ b/datahub_python_bindings/python/intellistream_datahub_sdk/__init__.pyi @@ -8,7 +8,7 @@ registration in src/lib.rs::intellistream_datahub_sdk() also adds them. from __future__ import annotations import datetime -from typing import Any, Iterable, Iterator, Mapping, Optional, Sequence, Union +from typing import Any, Iterable, Iterator, Literal, Mapping, Optional, Sequence, Union from uuid import UUID @@ -872,12 +872,14 @@ class TimeSeriesServiceSync: self, input: list[DatapointsCollectionString], zstd_level: int | None = None, + compression: Literal["payload", "buffers"] | None = None, ) -> list[str]: ... def insert_dataframe_binary( self, table: Any, timestamp: str | None = None, zstd_level: int | None = None, + compression: Literal["payload", "buffers"] | None = None, ) -> int: """Insert a wide table through ``POST /timeseries/data/binary``, straight from Arrow. @@ -893,6 +895,10 @@ class TimeSeriesServiceSync: zone and whole milliseconds, or be ``Int64`` epoch milliseconds. zstd_level 1, 3 or 9, as for :meth:`insert_datapoints_binary`. + compression + ``"payload"`` (the default) zstd-compresses each frame's Arrow stream whole; + ``"buffers"`` compresses each Arrow buffer inside it, the form a binary read answers + with. The server stores the same points either way. Returns ------- @@ -930,6 +936,7 @@ class TimeSeriesServiceSync: values: list[float], ts: Identifiable, zstd_level: int | None = None, + compression: Literal["payload", "buffers"] | None = None, ) -> list[str]: ... def insert_from_lists( self, diff --git a/datahub_python_bindings/src/timeseries/mod.rs b/datahub_python_bindings/src/timeseries/mod.rs index 6f15aff..19c9685 100644 --- a/datahub_python_bindings/src/timeseries/mod.rs +++ b/datahub_python_bindings/src/timeseries/mod.rs @@ -482,15 +482,27 @@ pub fn register(m: &Bound<'_, PyModule>) -> PyResult<()> { } /// Options for the binary ingest bindings. zstd level 1, 3 or 9; 9 is the default, as in the -/// Rust and Java SDKs, because the client is what pays for it. +/// Rust and Java SDKs, because the client is what pays for it. `compression` is `"payload"` +/// (the default) or `"buffers"`. pub(crate) fn binary_options( zstd_level: Option, -) -> intellistream_datahub_sdk::timeseries::BinaryIngestOptions { - use intellistream_datahub_sdk::timeseries::BinaryIngestOptions; - match zstd_level { - Some(level) => BinaryIngestOptions::new().zstd_level(level), - None => BinaryIngestOptions::default(), - } + compression: Option<&str>, +) -> PyResult { + use intellistream_datahub_sdk::timeseries::{BinaryIngestOptions, FrameCompression}; + let mut options = BinaryIngestOptions::default(); + if let Some(level) = zstd_level { + options = options.zstd_level(level); + } + options.compression = match compression { + None | Some("payload") => FrameCompression::Payload, + Some("buffers") => FrameCompression::Buffers, + Some(other) => { + return Err(pyo3::exceptions::PyValueError::new_err(format!( + "compression {other:?} is not \"payload\" or \"buffers\"" + ))) + } + }; + Ok(options) } /// Parallel timestamp and value sequences for one series, which is the shape a DataFrame's diff --git a/datahub_python_bindings/src/timeseries/sync_service.rs b/datahub_python_bindings/src/timeseries/sync_service.rs index d5a1f53..e35a077 100644 --- a/datahub_python_bindings/src/timeseries/sync_service.rs +++ b/datahub_python_bindings/src/timeseries/sync_service.rs @@ -276,19 +276,21 @@ impl PyTimeSeriesServiceSync { }) } /// `POST /timeseries/data/binary`: the same collections as `insert_datapoints`, sent as - /// zstd-compressed Arrow frames. `zstd_level` is 1, 3 or 9 and defaults to 9. - #[pyo3(signature = (input, zstd_level=None))] + /// zstd-compressed Arrow frames. `zstd_level` is 1, 3 or 9 and defaults to 9; `compression` is + /// `"payload"` (default) or `"buffers"`. + #[pyo3(signature = (input, zstd_level=None, compression=None))] fn insert_datapoints_binary<'py>( &self, py: Python<'py>, input: Vec, zstd_level: Option, + compression: Option<&str>, ) -> PyResult> { let service = self.api_service.clone(); let vec: Vec> = input.into_iter().map(|item| item.into()).collect(); let wrapper = DataWrapper::>::from_vec(vec); - let options = binary_options(zstd_level); + let options = binary_options(zstd_level, compression)?; py.detach(|| { let result = self .runtime @@ -301,20 +303,21 @@ impl PyTimeSeriesServiceSync { /// `POST /timeseries/data/binary` from a table: a polars or pandas DataFrame, or anything else /// with `__arrow_c_stream__`, one column per series named by its external id beside a /// timestamp column (or index). Returns the number of points sent. - #[pyo3(signature = (table, timestamp=None, zstd_level=None))] + #[pyo3(signature = (table, timestamp=None, zstd_level=None, compression=None))] fn insert_dataframe_binary<'py>( &self, py: Python<'py>, table: Bound<'py, PyAny>, timestamp: Option, zstd_level: Option, + compression: Option<&str>, ) -> PyResult { let (batches, index) = binary_read::import_table(&table)?; let Some(first) = batches.first() else { return Ok(0) }; let timestamp = binary_read::timestamp_column(&first.schema(), timestamp, &index)?; let batches = binary_read::without_index(batches, &index, ×tamp)?; let service = self.api_service.clone(); - let options = binary_options(zstd_level); + let options = binary_options(zstd_level, compression)?; let sent = py.detach(|| { self.runtime .block_on(service.time_series.insert_table_binary(&batches, ×tamp, &options)) @@ -325,7 +328,7 @@ impl PyTimeSeriesServiceSync { /// The binary twin of `insert_from_lists`: parallel timestamp and value sequences for one /// series, which is the shape a DataFrame column pair arrives in. - #[pyo3(signature = (timestamps, values, ts, zstd_level=None))] + #[pyo3(signature = (timestamps, values, ts, zstd_level=None, compression=None))] fn insert_from_lists_binary<'py>( &self, py: Python<'py>, @@ -333,11 +336,12 @@ impl PyTimeSeriesServiceSync { values: Vec, ts: Identifiable, zstd_level: Option, + compression: Option<&str>, ) -> PyResult> { let service = self.api_service.clone(); let collection = lists_to_collection(timestamps, values, ts)?; let wrapper = DataWrapper::>::from_vec(vec![collection]); - let options = binary_options(zstd_level); + let options = binary_options(zstd_level, compression)?; py.detach(|| { let result = self .runtime diff --git a/python_tests/test_timeseries_binary.py b/python_tests/test_timeseries_binary.py index d07f5b4..38b3d29 100644 --- a/python_tests/test_timeseries_binary.py +++ b/python_tests/test_timeseries_binary.py @@ -31,15 +31,15 @@ def fetch(): return poll_until(fetch, lambda dps: len(dps) >= expected) -@pytest.mark.parametrize("zstd_level", [None, 1, 3, 9]) -def test_insert_from_lists_binary_round_trips_floats(sync_client, make_ts, zstd_level): +@pytest.mark.parametrize("zstd_level, compression", [(None, None), (1, None), (3, None), (9, None), (3, "buffers")]) +def test_insert_from_lists_binary_round_trips_floats(sync_client, make_ts, zstd_level, compression): ts = make_ts(value_type="float") # Out of order on purpose: the frame writer sorts, the read must still be in time order. values = [i * 1.25 - 20.0 for i in range(len(TIMESTAMPS))] shuffled = list(zip(TIMESTAMPS, values))[::-1] result = sync_client.timeseries.insert_from_lists_binary( - [t for t, _ in shuffled], [v for _, v in shuffled], ts, zstd_level=zstd_level + [t for t, _ in shuffled], [v for _, v in shuffled], ts, zstd_level=zstd_level, compression=compression ) assert result == [] diff --git a/python_tests/test_timeseries_binary_dataframe.py b/python_tests/test_timeseries_binary_dataframe.py index d022bb8..4c69445 100644 --- a/python_tests/test_timeseries_binary_dataframe.py +++ b/python_tests/test_timeseries_binary_dataframe.py @@ -96,6 +96,23 @@ def test_a_polars_frame_round_trips(api): assert back["pump.state"].to_list() == ["on", "on", "off"] +def test_buffers_compression_sends_compression_2_frames_that_read_back_the_same(api): + client, server = api + df = pl.DataFrame({ + "timestamp": [T0 + datetime.timedelta(seconds=s) for s in range(3)], + "pump.temp": [0.5, None, 2.5], + "pump.state": ["on", "on", "off"], + }) + assert client.timeseries.insert_dataframe_binary(df, compression="buffers") == 5 + assert [server.posted[0][7]] == [2] + back = _read_back(client) + assert back["pump.temp"].to_list() == [0.5, None, 2.5] + assert back["pump.state"].to_list() == ["on", "on", "off"] + + with pytest.raises(ValueError, match="compression"): + client.timeseries.insert_dataframe_binary(df, compression="lz4") + + def test_a_pandas_frame_uses_its_timestamp_index(api): pd = pytest.importorskip("pandas") pytest.importorskip("pyarrow") @@ -165,7 +182,8 @@ def test_the_timestamp_column_has_to_be_found(api): START = datetime.datetime(2024, 3, 1, tzinfo=UTC) -def test_a_wide_table_round_trips_through_the_api(sync_client, make_ts): +@pytest.mark.parametrize("compression", ["payload", "buffers"]) +def test_a_wide_table_round_trips_through_the_api(sync_client, make_ts, compression): temp, count = make_ts(value_type="float"), make_ts(value_type="bigint") n = 150_000 # past one frame's 100 000 rows stamps = pl.datetime_range(START, START + datetime.timedelta(seconds=n - 1), "1s", time_zone="UTC", eager=True) @@ -176,7 +194,7 @@ def test_a_wide_table_round_trips_through_the_api(sync_client, make_ts): count.external_id: pl.Series([None if i % 3 == 0 else i for i in range(n)], dtype=pl.Int64), }) - sent = sync_client.timeseries.insert_dataframe_binary(df) + sent = sync_client.timeseries.insert_dataframe_binary(df, compression=compression) assert sent == n + n - (n + 2) // 3 end = START + datetime.timedelta(seconds=n) @@ -191,3 +209,34 @@ def test_a_wide_table_round_trips_through_the_api(sync_client, make_ts): # What came back goes back in as it is. assert sync_client.timeseries.insert_dataframe_binary(back) == sent + + +def test_a_pandas_frame_round_trips_through_the_api(sync_client, make_ts): + pd = pytest.importorskip("pandas") + pytest.importorskip("pyarrow") + temp, count = make_ts(value_type="float"), make_ts(value_type="bigint") + n = 120_000 # past one frame's 100 000 rows + index = pd.date_range(START, periods=n, freq="s", tz="UTC", name="time") + df = pd.DataFrame({ + # NaN every fifth row: pandas exports it as null, so no point there. + temp.external_id: [float("nan") if i % 5 == 0 else i * 0.25 for i in range(n)], + count.external_id: pd.array([None if i % 3 == 0 else i for i in range(n)], dtype="Int64"), + }, index=index) + + sent = sync_client.timeseries.insert_dataframe_binary(df) + temps, counts = df[temp.external_id].dropna(), df[count.external_id].dropna() + assert sent == len(temps) + len(counts) + + end = START + datetime.timedelta(seconds=n) + filters = [intellistream_datahub_sdk.RetrieveFilter(ts=s, start=START, end=end) for s in (temp, count)] + + from polling import poll_until + read = poll_until( + lambda: sync_client.timeseries.retrieve_datapoints_binary(filters), + lambda read: sum(len(s) for s in read) == sent, + ) + by_id = {s.external_id: s.to_pandas() for s in read} + for column in (temps, counts): + back = by_id[column.name] + assert back["timestamp"].tolist() == column.index.tolist() + assert back["value"].tolist() == column.tolist() diff --git a/src/timeseries/binary.rs b/src/timeseries/binary.rs index c4a4f34..dac2460 100644 --- a/src/timeseries/binary.rs +++ b/src/timeseries/binary.rs @@ -16,7 +16,8 @@ use arrow_array::{ ArrayRef, Decimal128Array, Float32Array, Float64Array, Int64Array, RecordBatch, StringArray, TimestampMillisecondArray, }; -use arrow_ipc::writer::StreamWriter; +use arrow_ipc::writer::{IpcWriteOptions, StreamWriter}; +use arrow_ipc::CompressionType; use arrow_schema::{DataType, Field, Schema, SchemaRef, TimeUnit}; use futures::future::join_all; use oauth2::http::StatusCode; @@ -153,11 +154,35 @@ impl DatapointValueType { } } +/// Where a frame's zstd goes. Either way the frame is zstd at the chosen level and the server +/// stores the same points. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)] +pub enum FrameCompression { + /// Compression 1: one zstd stream around the whole Arrow IPC payload. + #[default] + Payload, + /// Compression 2: every record batch compresses each buffer in-band (Arrow's body + /// compression), the form a binary read answers with. + Buffers, +} + +impl FrameCompression { + /// The envelope's compression byte. + pub fn id(self) -> u8 { + match self { + FrameCompression::Payload => COMPRESSION_ZSTD, + FrameCompression::Buffers => COMPRESSION_ARROW_ZSTD, + } + } +} + /// How [`TimeSeriesService::insert_datapoints_binary`] compresses and retries. #[derive(Debug, Clone)] pub struct BinaryIngestOptions { /// zstd level per frame: 1, 3 or 9. Compression is mandatory, so there is no way to turn it off. pub zstd_level: i32, + /// Whether zstd wraps the whole payload or each Arrow buffer. + pub compression: FrameCompression, /// Retries of one request after a 429 or a 5xx, one second apart per attempt. pub max_retries: u32, /// Requests in flight at once when one call packs into more than one, which happens above @@ -169,6 +194,7 @@ impl Default for BinaryIngestOptions { fn default() -> Self { BinaryIngestOptions { zstd_level: 9, + compression: FrameCompression::Payload, max_retries: 3, request_concurrency: 4, } @@ -185,6 +211,11 @@ impl BinaryIngestOptions { self } + pub fn compression(mut self, compression: FrameCompression) -> Self { + self.compression = compression; + self + } + pub fn max_retries(mut self, retries: u32) -> Self { self.max_retries = retries; self @@ -421,6 +452,11 @@ impl FrameWriter { /// Sorts by (id, timestamp), keeps the last value of a repeated pair, checks the caps, writes /// the Arrow stream, compresses it and returns the complete frame. pub fn build(&self, zstd_level: i32) -> Result { + self.build_with(FrameCompression::Payload, zstd_level) + } + + /// [`FrameWriter::build`] with the zstd where `compression` puts it. + pub fn build_with(&self, compression: FrameCompression, zstd_level: i32) -> Result { if self.rows.is_empty() { return Err("frame has no rows".to_string()); } @@ -459,13 +495,21 @@ impl FrameWriter { )); } - let raw = self.write_ipc(&kept)?; - let mut frame = assemble_frame(self.value_type, rows, &directory, &raw, zstd_level)?; + let batch = self.record_batch(&kept)?; + let mut frame = assemble_frame( + self.value_type, + rows, + &directory, + &batch, + self.estimated_raw_bytes(), + compression, + zstd_level, + )?; frame.clamped = self.clamped; Ok(frame) } - fn write_ipc(&self, kept: &[usize]) -> Result, String> { + fn record_batch(&self, kept: &[usize]) -> Result { let ids: Vec = kept.iter().map(|&i| self.rows[i].id).collect(); let timestamps: Vec = kept.iter().map(|&i| self.rows[i].timestamp).collect(); let mut columns: Vec = vec![ @@ -547,40 +591,90 @@ impl FrameWriter { columns.push(Arc::new(StringArray::from(text))); } } - let batch = - RecordBatch::try_new(self.value_type.schema(), columns).map_err(|e| e.to_string())?; - ipc_stream(&batch, self.estimated_raw_bytes()) + RecordBatch::try_new(self.value_type.schema(), columns).map_err(|e| e.to_string()) } } -/// A batch in the canon as an uncompressed Arrow IPC stream: a frame's payload before zstd. -pub(crate) fn ipc_stream(batch: &RecordBatch, capacity: usize) -> Result, String> { +/// A batch in the canon as an Arrow IPC stream, its buffers compressed in-band when `in_band` +/// names a zstd level. +fn ipc_stream(batch: &RecordBatch, capacity: usize, in_band: Option) -> Result, String> { + let options = match in_band { + None => IpcWriteOptions::default(), + Some(level) => IpcWriteOptions::default() + .try_with_compression(Some(CompressionType::ZSTD)) + .and_then(|o| o.try_with_compression_level(Some(level))) + .map_err(|e| e.to_string())?, + }; let mut raw = Vec::with_capacity(capacity); - let mut writer = - StreamWriter::try_new(&mut raw, batch.schema().as_ref()).map_err(|e| e.to_string())?; + let mut writer = StreamWriter::try_new_with_options(&mut raw, batch.schema().as_ref(), options) + .map_err(|e| e.to_string())?; writer.write(batch).map_err(|e| e.to_string())?; writer.finish().map_err(|e| e.to_string())?; drop(writer); Ok(raw) } -/// The envelope around a payload: header, series directory (ascending ids, as the rows are -/// ordered) and the zstd-compressed stream. +/// A compression 2 frame's raw length as the server counts it: every record batch buffer +/// uncompressed, each padded to 8 bytes. Read back from the stream, so it holds whichever buffers +/// arrow-rs chose to write. +fn in_band_raw_len(stream: &[u8]) -> Result { + let fail = |at: usize| format!("unreadable IPC message at {at}"); + let mut total = 0; + let mut pos = 0; + loop { + let prefix = stream.get(pos..pos + 8).ok_or_else(|| fail(pos))?; + let meta_len = u32::from_le_bytes(prefix[4..8].try_into().unwrap()) as usize; + if meta_len == 0 { + return Ok(total); + } + let meta = stream.get(pos + 8..pos + 8 + meta_len).ok_or_else(|| fail(pos))?; + let message = arrow_ipc::root_as_message(meta).map_err(|_| fail(pos))?; + let body = pos + 8 + meta_len; + if let Some(batch) = message.header_as_record_batch() { + for buffer in batch.buffers().into_iter().flatten() { + let (offset, length) = (body + buffer.offset() as usize, buffer.length() as usize); + if length == 0 { + continue; + } + let declared = stream.get(offset..offset + 8).ok_or_else(|| fail(pos))?; + let declared = i64::from_le_bytes(declared.try_into().unwrap()); + let raw = if declared == -1 { length - 8 } else { declared as usize }; + total += raw.div_ceil(8) * 8; + } + } + pos = body + message.bodyLength() as usize; + } +} + +/// The envelope around a batch: header, series directory (ascending ids, as the rows are +/// ordered) and the batch as an Arrow IPC stream compressed as `compression` says. pub(crate) fn assemble_frame( value_type: DatapointValueType, rows: usize, directory: &[(i64, &str)], - raw: &[u8], + batch: &RecordBatch, + capacity: usize, + compression: FrameCompression, zstd_level: i32, ) -> Result { - if raw.len() > MAX_FRAME_RAW_BYTES { + let (compressed, raw_len) = match compression { + FrameCompression::Payload => { + let raw = ipc_stream(batch, capacity, None)?; + let compressed = + zstd::bulk::compress(&raw, zstd_level).map_err(|e| format!("zstd failed: {e}"))?; + (compressed, raw.len()) + } + FrameCompression::Buffers => { + let stream = ipc_stream(batch, capacity / 2, Some(zstd_level))?; + let raw_len = in_band_raw_len(&stream)?; + (stream, raw_len) + } + }; + if raw_len > MAX_FRAME_RAW_BYTES { return Err(format!( - "frame payload is {} bytes, the cap is {MAX_FRAME_RAW_BYTES}; split it", - raw.len() + "frame payload is {raw_len} bytes, the cap is {MAX_FRAME_RAW_BYTES}; split it" )); } - let compressed = - zstd::bulk::compress(raw, zstd_level).map_err(|e| format!("zstd failed: {e}"))?; let directory_len: usize = directory .iter() .map(|(_, external_id)| 8 + varint_size(external_id.len()) + external_id.len()) @@ -590,8 +684,8 @@ pub(crate) fn assemble_frame( frame.push(VERSION); frame.push(value_type.id()); frame.push(CODEC_ARROW_IPC); - frame.push(COMPRESSION_ZSTD); - for n in [rows, directory.len(), directory_len, compressed.len(), raw.len()] { + frame.push(compression.id()); + for n in [rows, directory.len(), directory_len, compressed.len(), raw_len] { frame.extend_from_slice(&(n as u32).to_le_bytes()); } for (id, external_id) in directory { @@ -604,7 +698,7 @@ pub(crate) fn assemble_frame( bytes: frame, rows, series: directory.len(), - raw_len: raw.len(), + raw_len, clamped: 0, }) } @@ -848,10 +942,10 @@ impl TimeSeriesService { .collect(); let writers = cut_into_writers(&items)?; - let level = options.zstd_level; + let (compression, level) = (options.compression, options.zstd_level); let builds = writers .into_iter() - .map(|writer| tokio::task::spawn_blocking(move || writer.build(level))); + .map(|writer| tokio::task::spawn_blocking(move || writer.build_with(compression, level))); let mut frames = Vec::new(); for built in join_all(builds).await { let frame = built @@ -1108,6 +1202,43 @@ mod tests { assert_eq!(values.values(), &[1.5, 2.0, 2.5]); } + /// Compression 2 stores what compression 1 does, and its raw length is the buffers' + /// uncompressed, padded to 8: compressible ones carry their length, tiny ones -1 and the bytes. + #[test] + fn buffers_compression_reads_back_as_payload_compression() { + use crate::timeseries::binary_read::{parse_read_response, END_OF_RESPONSE}; + let cases: [(DatapointValueType, &[&str]); 7] = [ + (DatapointValueType::Bigint, &["1", "-9000000000", "42"]), + (DatapointValueType::Float, &["1.5", "-2.25", "NaN"]), + (DatapointValueType::Float32, &["0.5", "3"]), + (DatapointValueType::Numeric, &["123456789012.345678", "-0.000001"]), + (DatapointValueType::Decimal32, &["12345.6789", "99999999"]), + (DatapointValueType::Text, &["open", "blåbær"]), + (DatapointValueType::Mixed, &["3.5", "fault", "-7"]), + ]; + for (value_type, values) in cases { + for rows in [values.len(), 5000] { + let mut writer = FrameWriter::new(value_type); + writer.series(1, "s").unwrap(); + for t in 0..rows { + writer.add(1, t as i64 * 1000, values[t % values.len()]).unwrap(); + } + let read = |frame: Frame| { + let mut body = frame.bytes; + body.extend_from_slice(END_OF_RESPONSE); + parse_read_response(&body).unwrap() + }; + let payload = writer.build_with(FrameCompression::Payload, 3).unwrap(); + let buffers = writer.build_with(FrameCompression::Buffers, 3).unwrap(); + assert_eq!(buffers.bytes[7], COMPRESSION_ARROW_ZSTD); + assert_eq!(u32_at(&buffers.bytes, 24) as usize, buffers.raw_len); + assert!(buffers.raw_len <= payload.raw_len, "{value_type:?}: buffers are part of the stream"); + let (buffers, payload) = (format!("{:?}", read(buffers)), format!("{:?}", read(payload))); + assert_eq!(buffers, payload, "{value_type:?}, {rows} rows"); + } + } + } + #[test] fn every_value_type_writes_its_canonical_schema() { for value_type in [ diff --git a/src/timeseries/binary_table.rs b/src/timeseries/binary_table.rs index 80280a8..3295cab 100644 --- a/src/timeseries/binary_table.rs +++ b/src/timeseries/binary_table.rs @@ -31,8 +31,8 @@ use oauth2::http::StatusCode; use crate::http::ResponseError; use crate::timeseries::binary::{ - assemble_frame, checked_text, ipc_stream, is_stale_series_rejection, parse_scaled, - unprocessable, BinaryIngestOptions, DatapointValueType, Frame, ResolvedSeries, + assemble_frame, checked_text, is_stale_series_rejection, parse_scaled, + unprocessable, BinaryIngestOptions, DatapointValueType, Frame, FrameCompression, ResolvedSeries, DECIMAL32_UNSCALED_MAX, MAX_SERIES_PER_FRAME, NUMERIC_UNSCALED_MAX, }; use crate::timeseries::TimeSeriesService; @@ -364,6 +364,7 @@ fn build_frame( spec: &FrameSpec, points: &[Points], series: &[ResolvedSeries], + compression: FrameCompression, zstd_level: i32, ) -> Result { let rows: usize = spec.pieces.iter().map(|(_, _, n)| n).sum(); @@ -389,8 +390,7 @@ fn build_frame( columns.push(column(&|p| &p.values[v])?); } let batch = RecordBatch::try_new(spec.value_type.schema(), columns).map_err(|e| e.to_string())?; - let raw = ipc_stream(&batch, rows * 32 + 2048)?; - assemble_frame(spec.value_type, rows, &directory, &raw, zstd_level) + assemble_frame(spec.value_type, rows, &directory, &batch, rows * 32 + 2048, compression, zstd_level) } /// A batch's frames, compressed on as many threads as there are cores and frames. @@ -398,6 +398,7 @@ fn table_frames( batch: &RecordBatch, timestamp: &str, series: &[ResolvedSeries], + compression: FrameCompression, zstd_level: i32, ) -> Result<(Vec, TableIngest), String> { let schema = batch.schema(); @@ -424,7 +425,7 @@ fn table_frames( loop { let i = next.fetch_add(1, Ordering::Relaxed); let Some(spec) = specs.get(i) else { return done }; - done.push((i, build_frame(spec, &all, series, zstd_level))); + done.push((i, build_frame(spec, &all, series, compression, zstd_level))); } }) }) @@ -513,9 +514,10 @@ impl TimeSeriesService { ) -> Result { let wanted: Vec<(Option, Option<&str>)> = names.iter().map(|n| (None, Some(n.as_str()))).collect(); let series = self.resolve_series(&wanted).await?; - let (batch, timestamp, level) = (batch.clone(), timestamp.to_string(), options.zstd_level); + let (batch, timestamp) = (batch.clone(), timestamp.to_string()); + let (compression, level) = (options.compression, options.zstd_level); let (frames, summary) = - tokio::task::spawn_blocking(move || table_frames(&batch, ×tamp, &series, level)) + tokio::task::spawn_blocking(move || table_frames(&batch, ×tamp, &series, compression, level)) .await .map_err(|e| ResponseError { status: StatusCode::INTERNAL_SERVER_ERROR, @@ -573,8 +575,12 @@ mod tests { decode(&[writer.build(1).unwrap()]) } + /// Built both ways, which must read back the same. fn through_table(batch: &RecordBatch, resolved: &[ResolvedSeries]) -> Vec { - decode(&table_frames(batch, "timestamp", resolved, 1).unwrap().0) + let payload = decode(&table_frames(batch, "timestamp", resolved, FrameCompression::Payload, 1).unwrap().0); + let buffers = decode(&table_frames(batch, "timestamp", resolved, FrameCompression::Buffers, 1).unwrap().0); + assert_eq!(format!("{payload:?}"), format!("{buffers:?}"), "compression 1 and 2 differ"); + payload } #[test] @@ -608,7 +614,7 @@ mod tests { ); let ints = Arc::new(arrow_array::Int32Array::from(vec![7, -3, 0])); let batch = table(ms(&[1, 2, 3]), vec![("numeric_from_float", floats), ("decimal32_from_decimal", decimals), ("numeric_from_int", ints)]); - let (frames, summary) = table_frames(&batch, "timestamp", &s, 1).unwrap(); + let (frames, summary) = table_frames(&batch, "timestamp", &s, FrameCompression::Payload, 1).unwrap(); let got = decode(&frames); let strings: [&[&str]; 3] = [ @@ -671,7 +677,7 @@ mod tests { Arc::new(Float64Array::from_iter((0..n).map(|i| (i % every == 0).then_some(1.0)))) }; let batch = table(ms(&ts), vec![("long", long), ("short_b", sparse(100_000)), ("short_a", sparse(50_000))]); - let (frames, summary) = table_frames(&batch, "timestamp", &s, 1).unwrap(); + let (frames, summary) = table_frames(&batch, "timestamp", &s, FrameCompression::Payload, 1).unwrap(); assert_eq!(summary.points, n + 3 + 5); // Ascending ids: the two short series (5 + 3 rows) open the first frame, and the long one @@ -687,22 +693,22 @@ mod tests { fn what_the_table_cannot_say_is_refused() { let float = [series(1, "v", DatapointValueType::Float)]; let naive = Arc::new(TimestampMillisecondArray::from(vec![1])); - assert!(table_frames(&table(naive, vec![("v", Arc::new(Float64Array::from(vec![1.0])))]), "timestamp", &float, 1) + assert!(table_frames(&table(naive, vec![("v", Arc::new(Float64Array::from(vec![1.0])))]), "timestamp", &float, FrameCompression::Payload, 1) .unwrap_err() .contains("no time zone")); let sub_ms = Arc::new(TimestampMicrosecondArray::from(vec![1_500]).with_timezone("UTC")); - assert!(table_frames(&table(sub_ms, vec![("v", Arc::new(Float64Array::from(vec![1.0])))]), "timestamp", &float, 1) + assert!(table_frames(&table(sub_ms, vec![("v", Arc::new(Float64Array::from(vec![1.0])))]), "timestamp", &float, FrameCompression::Payload, 1) .unwrap_err() .contains("finer than a millisecond")); let bigint = [series(1, "v", DatapointValueType::Bigint)]; - assert!(table_frames(&table(ms(&[1]), vec![("v", Arc::new(Float64Array::from(vec![1.0])))]), "timestamp", &bigint, 1) + assert!(table_frames(&table(ms(&[1]), vec![("v", Arc::new(Float64Array::from(vec![1.0])))]), "timestamp", &bigint, FrameCompression::Payload, 1) .unwrap_err() .contains("cannot hold")); let text = [series(1, "v", DatapointValueType::Text)]; - assert!(table_frames(&table(ms(&[1]), vec![("v", Arc::new(StringArray::from(vec!["x".repeat(65)])))]), "timestamp", &text, 1) + assert!(table_frames(&table(ms(&[1]), vec![("v", Arc::new(StringArray::from(vec!["x".repeat(65)])))]), "timestamp", &text, FrameCompression::Payload, 1) .unwrap_err() .contains("characters")); } diff --git a/src/timeseries/mod.rs b/src/timeseries/mod.rs index 1327c3e..73c905b 100644 --- a/src/timeseries/mod.rs +++ b/src/timeseries/mod.rs @@ -4,7 +4,9 @@ pub mod binary_table; pub mod datapoint_listen; mod test; -pub use binary::{BinaryIngestOptions, DatapointValueType, Frame, FrameWriter, ResolvedSeries}; +pub use binary::{ + BinaryIngestOptions, DatapointValueType, Frame, FrameCompression, FrameWriter, ResolvedSeries, +}; pub use binary_read::{BinaryDatapointStream, BinaryDatapoints, BinaryValues}; pub use binary_table::TableIngest; pub use datapoint_listen::{DatapointListener, LiveDatapoint}; From 5c7dfa51ffd845701838caa6e112fd5a71d0d909 Mon Sep 17 00:00:00 2001 From: jgjesdal Date: Wed, 14 Oct 2026 10:35:09 +0200 Subject: [PATCH 7/8] Default binary ingest to zstd level 3 Level 9 compressed noisy floats no smaller than level 1 and took several times as long, and the client is what pays for it. 1 and 9 stay available. Co-Authored-By: Claude Opus 5.5 Signed-off-by: jgjesdal --- AGENTS.md | 2 +- README.md | 2 +- .../python/intellistream_datahub_sdk/__init__.pyi | 2 +- datahub_python_bindings/src/timeseries/mod.rs | 4 ++-- datahub_python_bindings/src/timeseries/sync_service.rs | 2 +- src/timeseries/binary.rs | 4 ++-- 6 files changed, 8 insertions(+), 8 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index d211978..6d534c5 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -201,7 +201,7 @@ the generic default with `recognized: false`, not an error. ### Binary datapoints (`src/timeseries/binary.rs`, `src/timeseries/binary_read.rs`) -`TimeSeriesService::insert_datapoints_binary` is the second ingest path, to `POST /timeseries/data/binary`: the same `DatapointsCollection` input, resolved through `/timeseries/byids` (cached per service instance; needs read access on the dataset), checked against each series' value type locally, cut into Arrow IPC frames at the contract's caps, zstd-compressed per frame (mandatory: level 1, 3 or 9, default 9) and posted through `execute_post_bytes_request`. `BinaryIngestOptions::compression` picks where the zstd goes: `FrameCompression::Payload` (compression 1, the default: one stream around the Arrow payload) or `FrameCompression::Buffers` (compression 2: arrow-rs compresses each buffer in-band, the form a read answers with). At compression 2 the envelope's raw length is what the server recounts — every record batch buffer uncompressed, padded to 8 — so `in_band_raw_len` reads it back from the written stream rather than predicting which buffers arrow-rs writes; a mismatch is a 400. The api only takes compression 2 on ingest from platform `357794cce` on. Every refusal on this path is a problem document typed for its status — `invalid-frame` (400), `unknown-timeseries` (404), `request-too-large` (413), `unsupported-media-type` (415), `value-type-mismatch` and `external-id-mismatch` (422), `too-many-in-flight` (429) — with the kebab-case sub-case in a `reason` extension beside it. It was one type, `datapoint-block-rejected`, answering with all six statuses until the api split it (platform #120); the SDK matches the new slugs only, and nothing here recognises the old one — the binary path has never been in a release, so no caller can be on the other side of that change. `is_stale_series_rejection` reads the slug to decide the one re-resolve-and-retry, and reads it from the problem rather than the body text because the SDK's own pre-flight 404 names the series it could not resolve — an external id spelling `unknown-timeseries` used to trigger the retry. `FrameWriter` builds one frame and is public; `cut_into_writers` and `pack_requests` hold the caps. The byte layout is the platform's `binary_datapoints_format.md`. The arrow-rs crates (`arrow-array`, `arrow-schema`, `arrow-ipc`) serve both directions. In Python it is `insert_datapoints_binary` and `insert_from_lists_binary` on the **sync** client only, each taking an optional `zstd_level` and `compression` (`"payload"` or `"buffers"`); the async client has no binary methods. The ignored `timeseries::tests::test_datapoints_binary` is the live twin of `test_datapoints` and needs a backend that serves the endpoint; the writer's own tests in `binary.rs` run offline. `python_tests/test_timeseries_binary.py` round-trips the Python binding through the JSON read path. +`TimeSeriesService::insert_datapoints_binary` is the second ingest path, to `POST /timeseries/data/binary`: the same `DatapointsCollection` input, resolved through `/timeseries/byids` (cached per service instance; needs read access on the dataset), checked against each series' value type locally, cut into Arrow IPC frames at the contract's caps, zstd-compressed per frame (mandatory: level 1, 3 or 9, default 3; 9 compressed noisy floats no smaller than 1 and took several times as long) and posted through `execute_post_bytes_request`. `BinaryIngestOptions::compression` picks where the zstd goes: `FrameCompression::Payload` (compression 1, the default: one stream around the Arrow payload) or `FrameCompression::Buffers` (compression 2: arrow-rs compresses each buffer in-band, the form a read answers with). At compression 2 the envelope's raw length is what the server recounts — every record batch buffer uncompressed, padded to 8 — so `in_band_raw_len` reads it back from the written stream rather than predicting which buffers arrow-rs writes; a mismatch is a 400. The api only takes compression 2 on ingest from platform `357794cce` on. Every refusal on this path is a problem document typed for its status — `invalid-frame` (400), `unknown-timeseries` (404), `request-too-large` (413), `unsupported-media-type` (415), `value-type-mismatch` and `external-id-mismatch` (422), `too-many-in-flight` (429) — with the kebab-case sub-case in a `reason` extension beside it. It was one type, `datapoint-block-rejected`, answering with all six statuses until the api split it (platform #120); the SDK matches the new slugs only, and nothing here recognises the old one — the binary path has never been in a release, so no caller can be on the other side of that change. `is_stale_series_rejection` reads the slug to decide the one re-resolve-and-retry, and reads it from the problem rather than the body text because the SDK's own pre-flight 404 names the series it could not resolve — an external id spelling `unknown-timeseries` used to trigger the retry. `FrameWriter` builds one frame and is public; `cut_into_writers` and `pack_requests` hold the caps. The byte layout is the platform's `binary_datapoints_format.md`. The arrow-rs crates (`arrow-array`, `arrow-schema`, `arrow-ipc`) serve both directions. In Python it is `insert_datapoints_binary` and `insert_from_lists_binary` on the **sync** client only, each taking an optional `zstd_level` and `compression` (`"payload"` or `"buffers"`); the async client has no binary methods. The ignored `timeseries::tests::test_datapoints_binary` is the live twin of `test_datapoints` and needs a backend that serves the endpoint; the writer's own tests in `binary.rs` run offline. `python_tests/test_timeseries_binary.py` round-trips the Python binding through the JSON read path. The read is `TimeSeriesService::retrieve_datapoints_binary`, to `POST /timeseries/data/list/binary`: the JSON retrieve body in, the same frames out, one series per frame, then the 4-byte marker `DHDE`. Read frames are **compression 2**: the payload is an Arrow IPC stream whose record batches zstd-compress each buffer in-band (Arrow's body compression), because the api relays ClickHouse's compressed batches without decoding them; `arrow-ipc`'s `zstd` feature is what decodes that, and a frame holds whole batches up to the row cap, so most hold fewer rows than the cap. Compression 1 (one zstd stream around the payload, what ingest sends) is still read. At compression 2 the envelope's raw length is the batches' buffers uncompressed, which arrow-rs does not report, so only its cap is checked. A body without the marker was cut short (the server streams, so a failure after the first bytes can only end the body early) and is a **502** saying so, never fewer points; bytes after it are refused too. Codec 2 frames carry bucketed aggregates in their own schema (`avg`/`min`/`max`/`sum`, Float64, 50 000 buckets per frame) while the envelope's type byte stays the series' own. `parse_read_response` returns one columnar `BinaryDatapoints` per series, merging consecutive frames of one series; decimals stay exact as unscaled `i128` plus scale. `TimeSeriesService::stream_datapoints_binary` is the primitive: a `FrameAssembler` cuts the body into frames as chunks arrive (a frame may be split anywhere), each frame is an owned `Bytes` decoded on `spawn_blocking`, at most `min(cores, 8)` in flight through a `FuturesOrdered` so items stay in frame order, and a producer task feeds a bounded channel behind the `BinaryDatapointStream` (dropping it aborts the task and closes the connection). One item per frame, so a long series is several consecutive items. A refusal is the `Err` before there is a stream; a response that breaks off is an error item after the items that arrived, and that is the semantic difference stream consumers must handle. `retrieve_datapoints_binary` collects the stream and merges consecutive parts (each series allocated once at its final size); `parse_read_response` does the same for a body in memory, decoding on scoped threads. Measured on 100M float rows, 10 series: the server takes ~6.5 s; the buffered call adds ~0.24 s (the merge copy) and the stream into polars ~0.02 s. Single-threaded zstd used to add 1.3 s on its own. Differences from the JSON read worth knowing: no `cursor` (a 400; advance `start` past the last timestamp), a series with no points is absent rather than empty, and an unknown or unreadable series is a 400 for the whole request — `DataWrapper` serialises only `items`, so the request-level `ignoreUnknownIds` and `limit` are not reachable. `src/timeseries/fixtures/read_*.bin` are response bodies written by the platform's own `DatapointFrameWriter.relayStream` (`read_*_zstd.bin` are the compression 1 bodies its former `wrap` wrote), so the offline tests in `binary_read.rs` prove arrow-rs reads what the Java writer emits; regenerate them when the platform's writer changes. The ignored `timeseries::tests::test_datapoints_binary_read` is the live check against the JSON read. In Python it is `retrieve_datapoints_binary` on the sync client, plus `retrieve_datapoints_binary_stream`, whose single-use `BinaryDatapointReader` iterates per frame or is an Arrow stream of one long table (`timeseries_id`, `timestamp`, values; one schema, so a read mixing value types must be iterated). The long table carries no external id column on purpose: as a dictionary column, polars spent ~0.8 s per 100M rows making it categorical, more than everything else; `reader.external_ids` maps the ids instead. The buffered `retrieve_datapoints_binary` returns a `BinaryReadResult`, a sequence of per-series `BinaryDatapoints` with the same `external_ids`, long-table Arrow stream and `to_polars(wide=False)` as the reader: both call `python/intellistream_datahub_sdk/_tables.py`, so the two tables cannot drift. `wide=True` puts series side by side when every one has the same timestamps (free), pivots on a numeric key when they share one value column type (~1.7 s per 100M rows; joining names first doubled it, `concat(how="align")` tripled it), and aligns otherwise, which is how mixed value types sit side by side. Each per-series `BinaryDatapoints` implements the Arrow PyCapsule stream protocol (`__arrow_c_stream__`, via `arrow-array`'s `ffi` feature enabled in the bindings crate only), so polars, pyarrow and pandas 3 read them zero-copy; the package still declares no runtime dependency, and `to_polars`/`to_pandas` import theirs lazily. `python_tests/test_timeseries_binary_read.py` covers it offline against a stand-in server and live when the backend serves the endpoint. diff --git a/README.md b/README.md index 7982698..14478e1 100644 --- a/README.md +++ b/README.md @@ -124,7 +124,7 @@ the same collections as `insert_datapoints` and sends them through `POST /timeseries/data/binary`: each series is resolved once to its id and value type and cached, values are checked against that type locally, sorted and de-duplicated, cut into Arrow IPC frames at the contract's caps (100 000 points per numeric frame, 10 000 per text or mixed -frame, 32 frames per request), compressed with zstd (level 9 by default, 1 and 3 are the other +frame, 32 frames per request), compressed with zstd (level 3 by default, 1 and 9 are the other choices) and posted. A 204 means every frame was accepted. A series that does not exist is a 404 before anything is sent, a value that does not fit its diff --git a/datahub_python_bindings/python/intellistream_datahub_sdk/__init__.pyi b/datahub_python_bindings/python/intellistream_datahub_sdk/__init__.pyi index 15ec25f..9735c06 100644 --- a/datahub_python_bindings/python/intellistream_datahub_sdk/__init__.pyi +++ b/datahub_python_bindings/python/intellistream_datahub_sdk/__init__.pyi @@ -894,7 +894,7 @@ class TimeSeriesServiceSync: column named ``timestamp``, else the only timestamp-typed column. It has to carry a time zone and whole milliseconds, or be ``Int64`` epoch milliseconds. zstd_level - 1, 3 or 9, as for :meth:`insert_datapoints_binary`. + 1, 3 or 9; 3 by default. compression ``"payload"`` (the default) zstd-compresses each frame's Arrow stream whole; ``"buffers"`` compresses each Arrow buffer inside it, the form a binary read answers diff --git a/datahub_python_bindings/src/timeseries/mod.rs b/datahub_python_bindings/src/timeseries/mod.rs index 19c9685..570f728 100644 --- a/datahub_python_bindings/src/timeseries/mod.rs +++ b/datahub_python_bindings/src/timeseries/mod.rs @@ -481,8 +481,8 @@ pub fn register(m: &Bound<'_, PyModule>) -> PyResult<()> { Ok(()) } -/// Options for the binary ingest bindings. zstd level 1, 3 or 9; 9 is the default, as in the -/// Rust and Java SDKs, because the client is what pays for it. `compression` is `"payload"` +/// Options for the binary ingest bindings. zstd level 1, 3 or 9; 3 is the default, as in the +/// Rust SDK. `compression` is `"payload"` /// (the default) or `"buffers"`. pub(crate) fn binary_options( zstd_level: Option, diff --git a/datahub_python_bindings/src/timeseries/sync_service.rs b/datahub_python_bindings/src/timeseries/sync_service.rs index e35a077..a1f6b99 100644 --- a/datahub_python_bindings/src/timeseries/sync_service.rs +++ b/datahub_python_bindings/src/timeseries/sync_service.rs @@ -276,7 +276,7 @@ impl PyTimeSeriesServiceSync { }) } /// `POST /timeseries/data/binary`: the same collections as `insert_datapoints`, sent as - /// zstd-compressed Arrow frames. `zstd_level` is 1, 3 or 9 and defaults to 9; `compression` is + /// zstd-compressed Arrow frames. `zstd_level` is 1, 3 or 9 and defaults to 3; `compression` is /// `"payload"` (default) or `"buffers"`. #[pyo3(signature = (input, zstd_level=None, compression=None))] fn insert_datapoints_binary<'py>( diff --git a/src/timeseries/binary.rs b/src/timeseries/binary.rs index dac2460..1ed3548 100644 --- a/src/timeseries/binary.rs +++ b/src/timeseries/binary.rs @@ -193,7 +193,7 @@ pub struct BinaryIngestOptions { impl Default for BinaryIngestOptions { fn default() -> Self { BinaryIngestOptions { - zstd_level: 9, + zstd_level: 3, compression: FrameCompression::Payload, max_retries: 3, request_concurrency: 4, @@ -1391,7 +1391,7 @@ mod tests { #[test] fn options_accept_only_the_three_levels() { assert!(BinaryIngestOptions::default().validate().is_ok()); - assert_eq!(BinaryIngestOptions::default().zstd_level, 9); + assert_eq!(BinaryIngestOptions::default().zstd_level, 3); assert!(BinaryIngestOptions::new().zstd_level(3).validate().is_ok()); assert!(BinaryIngestOptions::new().zstd_level(0).validate().is_err()); assert!(BinaryIngestOptions::new().zstd_level(22).validate().is_err()); From cfd029aa08e4f2133af159d427d0413e403896c3 Mon Sep 17 00:00:00 2001 From: jgjesdal Date: Fri, 16 Oct 2026 10:25:40 +0200 Subject: [PATCH 8/8] Default binary ingest to compression 2, in-band buffers The consumer relays compression 2 batches into ClickHouse without decompressing them (platform c9e9351dd), while a compression 1 frame is decoded first. On 100M float points from polars, time until the last point was readable fell from 15.5 s to 13.2 s, at the same cost to the client. Payload stays available as compression="payload". bench_json_vs_binary takes DATAHUB_BENCH_ZSTD_LEVEL and DATAHUB_BENCH_COMPRESSION to compare the two. Co-Authored-By: Claude Opus 5.5 Signed-off-by: jgjesdal --- AGENTS.md | 2 +- .../intellistream_datahub_sdk/__init__.pyi | 7 +++--- datahub_python_bindings/src/timeseries/mod.rs | 10 ++++---- .../src/timeseries/sync_service.rs | 2 +- src/timeseries/binary.rs | 7 +++--- src/timeseries/test.rs | 23 +++++++++++++++---- 6 files changed, 33 insertions(+), 18 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index 6d534c5..36f9de6 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -201,7 +201,7 @@ the generic default with `recognized: false`, not an error. ### Binary datapoints (`src/timeseries/binary.rs`, `src/timeseries/binary_read.rs`) -`TimeSeriesService::insert_datapoints_binary` is the second ingest path, to `POST /timeseries/data/binary`: the same `DatapointsCollection` input, resolved through `/timeseries/byids` (cached per service instance; needs read access on the dataset), checked against each series' value type locally, cut into Arrow IPC frames at the contract's caps, zstd-compressed per frame (mandatory: level 1, 3 or 9, default 3; 9 compressed noisy floats no smaller than 1 and took several times as long) and posted through `execute_post_bytes_request`. `BinaryIngestOptions::compression` picks where the zstd goes: `FrameCompression::Payload` (compression 1, the default: one stream around the Arrow payload) or `FrameCompression::Buffers` (compression 2: arrow-rs compresses each buffer in-band, the form a read answers with). At compression 2 the envelope's raw length is what the server recounts — every record batch buffer uncompressed, padded to 8 — so `in_band_raw_len` reads it back from the written stream rather than predicting which buffers arrow-rs writes; a mismatch is a 400. The api only takes compression 2 on ingest from platform `357794cce` on. Every refusal on this path is a problem document typed for its status — `invalid-frame` (400), `unknown-timeseries` (404), `request-too-large` (413), `unsupported-media-type` (415), `value-type-mismatch` and `external-id-mismatch` (422), `too-many-in-flight` (429) — with the kebab-case sub-case in a `reason` extension beside it. It was one type, `datapoint-block-rejected`, answering with all six statuses until the api split it (platform #120); the SDK matches the new slugs only, and nothing here recognises the old one — the binary path has never been in a release, so no caller can be on the other side of that change. `is_stale_series_rejection` reads the slug to decide the one re-resolve-and-retry, and reads it from the problem rather than the body text because the SDK's own pre-flight 404 names the series it could not resolve — an external id spelling `unknown-timeseries` used to trigger the retry. `FrameWriter` builds one frame and is public; `cut_into_writers` and `pack_requests` hold the caps. The byte layout is the platform's `binary_datapoints_format.md`. The arrow-rs crates (`arrow-array`, `arrow-schema`, `arrow-ipc`) serve both directions. In Python it is `insert_datapoints_binary` and `insert_from_lists_binary` on the **sync** client only, each taking an optional `zstd_level` and `compression` (`"payload"` or `"buffers"`); the async client has no binary methods. The ignored `timeseries::tests::test_datapoints_binary` is the live twin of `test_datapoints` and needs a backend that serves the endpoint; the writer's own tests in `binary.rs` run offline. `python_tests/test_timeseries_binary.py` round-trips the Python binding through the JSON read path. +`TimeSeriesService::insert_datapoints_binary` is the second ingest path, to `POST /timeseries/data/binary`: the same `DatapointsCollection` input, resolved through `/timeseries/byids` (cached per service instance; needs read access on the dataset), checked against each series' value type locally, cut into Arrow IPC frames at the contract's caps, zstd-compressed per frame (mandatory: level 1, 3 or 9, default 3; 9 compressed noisy floats no smaller than 1 and took several times as long) and posted through `execute_post_bytes_request`. `BinaryIngestOptions::compression` picks where the zstd goes: `FrameCompression::Buffers` (compression 2, the default: arrow-rs compresses each buffer in-band, the form a read answers with) or `FrameCompression::Payload` (compression 1: one stream around the Arrow payload). Buffers is the default because the consumer relays those batches into ClickHouse without decompressing them (platform `c9e9351dd`), while a compression 1 frame is decoded first: on 100M float points from polars, time until the last point was readable fell from 15.5 s to 13.2 s, and the client's cost was the same. At compression 2 the envelope's raw length is what the server recounts — every record batch buffer uncompressed, padded to 8 — so `in_band_raw_len` reads it back from the written stream rather than predicting which buffers arrow-rs writes; a mismatch is a 400. The api only takes compression 2 on ingest from platform `357794cce` on. Every refusal on this path is a problem document typed for its status — `invalid-frame` (400), `unknown-timeseries` (404), `request-too-large` (413), `unsupported-media-type` (415), `value-type-mismatch` and `external-id-mismatch` (422), `too-many-in-flight` (429) — with the kebab-case sub-case in a `reason` extension beside it. It was one type, `datapoint-block-rejected`, answering with all six statuses until the api split it (platform #120); the SDK matches the new slugs only, and nothing here recognises the old one — the binary path has never been in a release, so no caller can be on the other side of that change. `is_stale_series_rejection` reads the slug to decide the one re-resolve-and-retry, and reads it from the problem rather than the body text because the SDK's own pre-flight 404 names the series it could not resolve — an external id spelling `unknown-timeseries` used to trigger the retry. `FrameWriter` builds one frame and is public; `cut_into_writers` and `pack_requests` hold the caps. The byte layout is the platform's `binary_datapoints_format.md`. The arrow-rs crates (`arrow-array`, `arrow-schema`, `arrow-ipc`) serve both directions. In Python it is `insert_datapoints_binary` and `insert_from_lists_binary` on the **sync** client only, each taking an optional `zstd_level` and `compression` (`"buffers"`, the default, or `"payload"`); the async client has no binary methods. The ignored `timeseries::tests::test_datapoints_binary` is the live twin of `test_datapoints` and needs a backend that serves the endpoint; the writer's own tests in `binary.rs` run offline. `python_tests/test_timeseries_binary.py` round-trips the Python binding through the JSON read path. The read is `TimeSeriesService::retrieve_datapoints_binary`, to `POST /timeseries/data/list/binary`: the JSON retrieve body in, the same frames out, one series per frame, then the 4-byte marker `DHDE`. Read frames are **compression 2**: the payload is an Arrow IPC stream whose record batches zstd-compress each buffer in-band (Arrow's body compression), because the api relays ClickHouse's compressed batches without decoding them; `arrow-ipc`'s `zstd` feature is what decodes that, and a frame holds whole batches up to the row cap, so most hold fewer rows than the cap. Compression 1 (one zstd stream around the payload, what ingest sends) is still read. At compression 2 the envelope's raw length is the batches' buffers uncompressed, which arrow-rs does not report, so only its cap is checked. A body without the marker was cut short (the server streams, so a failure after the first bytes can only end the body early) and is a **502** saying so, never fewer points; bytes after it are refused too. Codec 2 frames carry bucketed aggregates in their own schema (`avg`/`min`/`max`/`sum`, Float64, 50 000 buckets per frame) while the envelope's type byte stays the series' own. `parse_read_response` returns one columnar `BinaryDatapoints` per series, merging consecutive frames of one series; decimals stay exact as unscaled `i128` plus scale. `TimeSeriesService::stream_datapoints_binary` is the primitive: a `FrameAssembler` cuts the body into frames as chunks arrive (a frame may be split anywhere), each frame is an owned `Bytes` decoded on `spawn_blocking`, at most `min(cores, 8)` in flight through a `FuturesOrdered` so items stay in frame order, and a producer task feeds a bounded channel behind the `BinaryDatapointStream` (dropping it aborts the task and closes the connection). One item per frame, so a long series is several consecutive items. A refusal is the `Err` before there is a stream; a response that breaks off is an error item after the items that arrived, and that is the semantic difference stream consumers must handle. `retrieve_datapoints_binary` collects the stream and merges consecutive parts (each series allocated once at its final size); `parse_read_response` does the same for a body in memory, decoding on scoped threads. Measured on 100M float rows, 10 series: the server takes ~6.5 s; the buffered call adds ~0.24 s (the merge copy) and the stream into polars ~0.02 s. Single-threaded zstd used to add 1.3 s on its own. Differences from the JSON read worth knowing: no `cursor` (a 400; advance `start` past the last timestamp), a series with no points is absent rather than empty, and an unknown or unreadable series is a 400 for the whole request — `DataWrapper` serialises only `items`, so the request-level `ignoreUnknownIds` and `limit` are not reachable. `src/timeseries/fixtures/read_*.bin` are response bodies written by the platform's own `DatapointFrameWriter.relayStream` (`read_*_zstd.bin` are the compression 1 bodies its former `wrap` wrote), so the offline tests in `binary_read.rs` prove arrow-rs reads what the Java writer emits; regenerate them when the platform's writer changes. The ignored `timeseries::tests::test_datapoints_binary_read` is the live check against the JSON read. In Python it is `retrieve_datapoints_binary` on the sync client, plus `retrieve_datapoints_binary_stream`, whose single-use `BinaryDatapointReader` iterates per frame or is an Arrow stream of one long table (`timeseries_id`, `timestamp`, values; one schema, so a read mixing value types must be iterated). The long table carries no external id column on purpose: as a dictionary column, polars spent ~0.8 s per 100M rows making it categorical, more than everything else; `reader.external_ids` maps the ids instead. The buffered `retrieve_datapoints_binary` returns a `BinaryReadResult`, a sequence of per-series `BinaryDatapoints` with the same `external_ids`, long-table Arrow stream and `to_polars(wide=False)` as the reader: both call `python/intellistream_datahub_sdk/_tables.py`, so the two tables cannot drift. `wide=True` puts series side by side when every one has the same timestamps (free), pivots on a numeric key when they share one value column type (~1.7 s per 100M rows; joining names first doubled it, `concat(how="align")` tripled it), and aligns otherwise, which is how mixed value types sit side by side. Each per-series `BinaryDatapoints` implements the Arrow PyCapsule stream protocol (`__arrow_c_stream__`, via `arrow-array`'s `ffi` feature enabled in the bindings crate only), so polars, pyarrow and pandas 3 read them zero-copy; the package still declares no runtime dependency, and `to_polars`/`to_pandas` import theirs lazily. `python_tests/test_timeseries_binary_read.py` covers it offline against a stand-in server and live when the backend serves the endpoint. diff --git a/datahub_python_bindings/python/intellistream_datahub_sdk/__init__.pyi b/datahub_python_bindings/python/intellistream_datahub_sdk/__init__.pyi index 9735c06..bfa9780 100644 --- a/datahub_python_bindings/python/intellistream_datahub_sdk/__init__.pyi +++ b/datahub_python_bindings/python/intellistream_datahub_sdk/__init__.pyi @@ -896,9 +896,10 @@ class TimeSeriesServiceSync: zstd_level 1, 3 or 9; 3 by default. compression - ``"payload"`` (the default) zstd-compresses each frame's Arrow stream whole; - ``"buffers"`` compresses each Arrow buffer inside it, the form a binary read answers - with. The server stores the same points either way. + ``"buffers"`` (the default) compresses each Arrow buffer inside the frame, the form a + binary read answers with, which the server stores without decompressing; + ``"payload"`` zstd-compresses each frame's Arrow stream whole. The server stores the + same points either way. Returns ------- diff --git a/datahub_python_bindings/src/timeseries/mod.rs b/datahub_python_bindings/src/timeseries/mod.rs index 570f728..e6c14b9 100644 --- a/datahub_python_bindings/src/timeseries/mod.rs +++ b/datahub_python_bindings/src/timeseries/mod.rs @@ -482,8 +482,8 @@ pub fn register(m: &Bound<'_, PyModule>) -> PyResult<()> { } /// Options for the binary ingest bindings. zstd level 1, 3 or 9; 3 is the default, as in the -/// Rust SDK. `compression` is `"payload"` -/// (the default) or `"buffers"`. +/// Rust SDK. `compression` is `"buffers"` +/// (the default) or `"payload"`. pub(crate) fn binary_options( zstd_level: Option, compression: Option<&str>, @@ -494,11 +494,11 @@ pub(crate) fn binary_options( options = options.zstd_level(level); } options.compression = match compression { - None | Some("payload") => FrameCompression::Payload, - Some("buffers") => FrameCompression::Buffers, + None | Some("buffers") => FrameCompression::Buffers, + Some("payload") => FrameCompression::Payload, Some(other) => { return Err(pyo3::exceptions::PyValueError::new_err(format!( - "compression {other:?} is not \"payload\" or \"buffers\"" + "compression {other:?} is not \"buffers\" or \"payload\"" ))) } }; diff --git a/datahub_python_bindings/src/timeseries/sync_service.rs b/datahub_python_bindings/src/timeseries/sync_service.rs index a1f6b99..af8a83e 100644 --- a/datahub_python_bindings/src/timeseries/sync_service.rs +++ b/datahub_python_bindings/src/timeseries/sync_service.rs @@ -277,7 +277,7 @@ impl PyTimeSeriesServiceSync { } /// `POST /timeseries/data/binary`: the same collections as `insert_datapoints`, sent as /// zstd-compressed Arrow frames. `zstd_level` is 1, 3 or 9 and defaults to 3; `compression` is - /// `"payload"` (default) or `"buffers"`. + /// `"buffers"` (default) or `"payload"`. #[pyo3(signature = (input, zstd_level=None, compression=None))] fn insert_datapoints_binary<'py>( &self, diff --git a/src/timeseries/binary.rs b/src/timeseries/binary.rs index 1ed3548..aab6f5b 100644 --- a/src/timeseries/binary.rs +++ b/src/timeseries/binary.rs @@ -159,10 +159,11 @@ impl DatapointValueType { #[derive(Debug, Clone, Copy, PartialEq, Eq, Default)] pub enum FrameCompression { /// Compression 1: one zstd stream around the whole Arrow IPC payload. - #[default] Payload, /// Compression 2: every record batch compresses each buffer in-band (Arrow's body - /// compression), the form a binary read answers with. + /// compression), the form a binary read answers with. The consumer relays these batches to + /// ClickHouse without decompressing them, where compression 1 has to be decoded first. + #[default] Buffers, } @@ -194,7 +195,7 @@ impl Default for BinaryIngestOptions { fn default() -> Self { BinaryIngestOptions { zstd_level: 3, - compression: FrameCompression::Payload, + compression: FrameCompression::Buffers, max_retries: 3, request_concurrency: 4, } diff --git a/src/timeseries/test.rs b/src/timeseries/test.rs index bfcecff..8550af7 100644 --- a/src/timeseries/test.rs +++ b/src/timeseries/test.rs @@ -12,7 +12,7 @@ mod tests { use crate::generic::{DataWrapper, Datapoint, DatapointString, DatapointsCollection, DeleteFilter, IdAndExtId, RetrieveFilter}; use crate::http::ResponseError; use crate::timeseries::{TimeSeries, TimeSeriesFilter, TimeSeriesFilterForm, TimeSeriesUpdate, TimeSeriesUpdateCollection, TimeSeriesUpdateFields}; - use crate::timeseries::binary::{BinaryIngestOptions, DatapointValueType}; + use crate::timeseries::binary::{BinaryIngestOptions, DatapointValueType, FrameCompression}; use crate::timeseries::binary_read::{BinaryDatapoints, BinaryValues}; use crate::tests::cleanup::cleanup_timeseries; use crate::tests::ids::{unique_id, unique_token}; @@ -889,7 +889,9 @@ mod tests { /// clients can be compared. /// /// Sized by `DATAHUB_BENCH_POINTS` (default 10 million) across `DATAHUB_BENCH_SERIES` - /// series. The api must have its daily quota and rate limiter off for a run of any size: + /// series. `DATAHUB_BENCH_ZSTD_LEVEL` and `DATAHUB_BENCH_COMPRESSION` (`payload` or `buffers`) + /// override the binary path's defaults. The api must have its daily quota and rate limiter off + /// for a run of any size: /// `-Ddatahub.limits.quota.enabled=false -Ddatahub.limits.rate.enabled=false`. /// /// **Run it with `--release`.** `cargo test` builds with the `dev` profile at @@ -912,8 +914,16 @@ mod tests { .ok().and_then(|v| v.parse().ok()).unwrap_or(100); let chunk: usize = std::env::var("DATAHUB_BENCH_CHUNK") .ok().and_then(|v| v.parse().ok()).unwrap_or(1_000_000); + let zstd_level: i32 = std::env::var("DATAHUB_BENCH_ZSTD_LEVEL") + .ok().and_then(|v| v.parse().ok()).unwrap_or(BinaryIngestOptions::default().zstd_level); + let compression = match std::env::var("DATAHUB_BENCH_COMPRESSION").as_deref() { + Ok("buffers") => FrameCompression::Buffers, + Ok("payload") => FrameCompression::Payload, + Err(_) => BinaryIngestOptions::default().compression, + Ok(other) => panic!("DATAHUB_BENCH_COMPRESSION is payload or buffers, not {other}"), + }; let api_service = create_api_service(); - println!("\n=== {total} points across {series_count} series, float32 ==="); + println!("\n=== {total} points across {series_count} series, float32, binary zstd {zstd_level} {compression:?} ==="); // Which paths to run. A sweep over request sizes only needs the binary one, and the JSON // path is slow enough that running it every time makes the sweep the expensive part. @@ -967,7 +977,10 @@ mod tests { api_service.time_series .insert_datapoints_binary( &request, - &BinaryIngestOptions::default().request_concurrency(concurrency), + &BinaryIngestOptions::default() + .zstd_level(zstd_level) + .compression(compression) + .request_concurrency(concurrency), ) .await .unwrap_or_else(|e| panic!("{label} insert failed: {}", e.get_message())); @@ -1006,7 +1019,7 @@ mod tests { } println!("\n=== datapoint ingest from Rust ==="); - println!("{total} points across {series_count} series, {chunk} per call, float32\n"); + println!("{total} points across {series_count} series, {chunk} per call, float32, binary zstd {zstd_level} {compression:?}\n"); print!("{:<28}", "metric"); for r in &results { print!("{:>18}", r.0); } println!();