Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -24,7 +24,7 @@ test:
@# Note that extensive integration tests are also run in the bundle repo,
@# for both aw-server and aw-server-rust, but without code coverage.
python -c 'import aw_server'
python -m pytest tests/test_server.py tests/test_profile.py tests/test_profile_config.py tests/test_query_cache.py tests/test_import.py tests/test_host_header.py tests/test_database_concurrency.py tests/test_settings.py
python -m pytest tests/test_server.py tests/test_profile.py tests/test_profile_config.py tests/test_query_cache.py tests/test_import.py tests/test_host_header.py tests/test_database_concurrency.py tests/test_settings.py tests/test_csv_export.py

typecheck:
python -m mypy aw_server tests --ignore-missing-imports
Expand Down
29 changes: 23 additions & 6 deletions aw_server/csv_export.py
Original file line number Diff line number Diff line change
Expand Up @@ -6,10 +6,12 @@
Columns are ``id``, ``timestamp``, ``duration``, then the union of data keys
across all exported events (first-seen order); a data key that collides with
an earlier column name gets a ``data.`` prefix. Above ``MAX_CSV_DATA_COLUMNS``
keys, a single ``data`` column holds each event's data as JSON instead.
keys, or if any event has non-object data, a single ``data`` column holds
each event's data as JSON instead.
"""

import json
import math
from datetime import timedelta, timezone
from typing import Any, Dict, Iterable, Iterator, List

Expand All @@ -22,7 +24,13 @@
_FORMULA_STARTERS = ("=", "+", "-", "@")


class _Number(str):
"""A finite JSON number. It is not a formula, so it is exported unchanged."""


def _neutralize_formula(s: str) -> str:
if isinstance(s, _Number):
return s
if s.lstrip(_FORMULA_LEADING_WHITESPACE).startswith(_FORMULA_STARTERS):
return "'" + s
return s
Expand All @@ -41,7 +49,12 @@ def _csv_record(fields: Iterable[str]) -> str:


def _json(value: Any) -> str:
return json.dumps(value, separators=(",", ":"), ensure_ascii=False)
text = json.dumps(value, separators=(",", ":"), ensure_ascii=False)
# Keep -5 / -1.5 as valid JSON numbers; a leading "'" would break json.loads.
is_number = isinstance(value, (int, float)) and not isinstance(value, bool)
if is_number and math.isfinite(value):

Copy link
Copy Markdown

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

P1 Large integers stop CSV downloads

_json now calls math.isfinite on integers, which raises OverflowError for valid JSON integers such as 10**400. An event with data={"n": 10**400} therefore stops a CSV export after the header, even though json.dumps can preserve the value.

Treat integers as finite directly and call math.isfinite only for floats. Add a large-integer regression.

Suggested change
if is_number and math.isfinite(value):
if is_number and (isinstance(value, int) or math.isfinite(value)):

return _Number(text)
return text


def _duration(duration: timedelta) -> str:
Expand Down Expand Up @@ -83,12 +96,16 @@ def _header_names(data_keys: List[str]) -> List[str]:

def events_to_csv(events: List[Event]) -> Iterator[str]:
"""Yield CSV records for events, in the order given."""
# Previously accepted data may be a list or scalar. Choose the lossless
# JSON fallback before discovering object keys (scalars may not be iterable).
data_json_column = any(not isinstance(e.data, dict) for e in events)
Comment thread
TimeToBuildBob marked this conversation as resolved.
Resolved
keys: Dict[str, None] = {}
for e in events:
for k in e.data:
keys.setdefault(k)
if not data_json_column:
for e in events:
for k in e.data:
keys.setdefault(k)
data_keys = list(keys)
data_json_column = len(data_keys) > MAX_CSV_DATA_COLUMNS
data_json_column = data_json_column or len(data_keys) > MAX_CSV_DATA_COLUMNS
if data_json_column:
data_keys = ["data"]

Expand Down
153 changes: 153 additions & 0 deletions tests/test_csv_export.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,153 @@
"""CSV export must remain readable for previously accepted non-object data."""

import csv
import io
import json
from datetime import datetime, timedelta, timezone

import pytest
from aw_core.models import Event
from aw_server.csv_export import MAX_CSV_DATA_COLUMNS, events_to_csv
from aw_server.server import AWFlask


@pytest.mark.parametrize("legacy_data", [["x"], "legacy", 5, 1.5, True])
def test_legacy_nonobject_csv_stream_roundtrip(legacy_data):
app = AWFlask("127.0.0.1", testing=True)
client = app.test_client()
bucket = "test-legacy-csv"
response = client.post(
f"/api/0/buckets/{bucket}",
json={"client": "test", "type": "test", "hostname": "test"},
)
assert response.status_code == 200
payloads = [{"title": "=SUM(A1:A2)", "unicode": 'å\n"'}, legacy_data]
for index, data in enumerate(payloads):
response = client.post(
f"/api/0/buckets/{bucket}/events",
json={
"timestamp": f"2026-01-01T00:00:0{index}Z",
"duration": 1.25,
"data": data,
},
)
assert response.status_code == 200

# Headers alone cannot detect exceptions in the streamed body.
response = client.get(f"/api/0/buckets/{bucket}/export/csv")
assert response.status_code == 200
rows = list(csv.reader(io.StringIO(response.get_data(as_text=True))))
assert rows[0] == ["id", "timestamp", "duration", "data"]
assert len(rows) == 3
assert [json.loads(row[3]) for row in rows[1:]] == payloads[::-1]
assert [row[2] for row in rows[1:]] == ["1.250000000"] * 2
assert [row[1] for row in rows[1:]] == [
"2026-01-01T00:00:01+00:00",
"2026-01-01T00:00:00+00:00",
]
# Exporting must not rewrite the stored payloads.
stored = client.get(f"/api/0/buckets/{bucket}/events").get_json()
assert [event["data"] for event in stored] == payloads[::-1]


@pytest.mark.parametrize("data", [[], False, 0, ""])
def test_legacy_nonobject_csv_falsey_data_preserved(data):
# aw-core's constructor normalizes falsey data to {}. Seed historical
# values explicitly to test the exporter boundary without changing ingest.
event = Event(timestamp=datetime(2026, 1, 1, tzinfo=timezone.utc))
event.data = data
rows = list(csv.reader(events_to_csv([event])))
assert rows[0] == ["id", "timestamp", "duration", "data"]
assert json.loads(rows[1][3]) == data


def test_legacy_nonobject_csv_null_uses_model_semantics():
event = Event(timestamp=datetime(2026, 1, 1, tzinfo=timezone.utc), data=None)
assert event.data == {}
event.data = None
assert event.data == {} # The property also normalizes an explicitly stored null.
rows = list(csv.reader(events_to_csv([event])))
assert rows[0] == ["id", "timestamp", "duration"]
assert len(rows[1]) == 3


def test_legacy_nonobject_csv_object_only_compatibility():
event = Event(
id=7,
timestamp=datetime(2026, 1, 1, tzinfo=timezone(timedelta(hours=1))),
duration=timedelta(microseconds=1234567),
data={
"id": "=1+1",
"data.id": " @command",
"title": 'å,"line\nnext',
"null": None,
"nested": {"a": [1, True]},
},
)
second = Event(timestamp=event.timestamp, data={"other": "-formula"})
rows = list(csv.reader(events_to_csv([event, second])))
assert rows[0] == [
"id",
"timestamp",
"duration",
"data.id",
"data.data.id",
"title",
"null",
"nested",
"other",
]
assert rows[1] == [
"7",
"2025-12-31T23:00:00+00:00",
"1.234567000",
"'=1+1",
"' @command",
'å,"line\nnext',
"null",
'{"a":[1,true]}',
"",
]
assert rows[2][3:8] == [""] * 5
assert rows[2][8] == "'-formula"


def test_legacy_nonobject_csv_wide_object_fallback_unchanged():
data = {f"key{i}": i for i in range(MAX_CSV_DATA_COLUMNS + 1)}
rows = list(csv.reader(events_to_csv([Event(data=data)])))
assert rows[0] == ["id", "timestamp", "duration", "data"]
assert json.loads(rows[1][3]) == data


@pytest.mark.parametrize("value", [-5, -1.5, 0, 7])
def test_numeric_json_is_not_formula_neutralized(value):
event = Event(
timestamp=datetime(2026, 1, 1, tzinfo=timezone.utc), data={"n": value}
)
legacy = Event(timestamp=datetime(2026, 1, 2, tzinfo=timezone.utc))
legacy.data = value
wide = Event(
timestamp=datetime(2026, 1, 3, tzinfo=timezone.utc),
data={f"k{i}": value for i in range(MAX_CSV_DATA_COLUMNS + 1)},
)
# Per-key column, scalar data column and wide-object JSON column.
assert list(csv.reader(events_to_csv([event])))[1][3] == json.dumps(value)
assert json.loads(list(csv.reader(events_to_csv([legacy])))[1][3]) == value
assert json.loads(list(csv.reader(events_to_csv([wide])))[1][3])["k0"] == value


def test_formula_text_still_neutralized_next_to_numbers():
event = Event(
timestamp=datetime(2026, 1, 1, tzinfo=timezone.utc),
data={"title": "=SUM(A1:A2)", "text": "-5", "n": -5},
)
row = list(csv.reader(events_to_csv([event])))[1]
assert row[3:] == ["'=SUM(A1:A2)", "'-5", "-5"]


@pytest.mark.parametrize("value", [float("nan"), float("inf"), float("-inf")])
def test_nonfinite_numbers_are_not_trusted_as_numbers(value):
event = Event(timestamp=datetime(2026, 1, 1, tzinfo=timezone.utc))
event.data = value
cell = list(csv.reader(events_to_csv([event])))[1][3]
assert not cell.startswith(("-", "="))
Loading