diff --git a/Makefile b/Makefile index 8b26305c..92bf85bf 100644 --- a/Makefile +++ b/Makefile @@ -24,7 +24,7 @@ test: @# Note that extensive integration tests are also run in the bundle repo, @# for both aw-server and aw-server-rust, but without code coverage. python -c 'import aw_server' - python -m pytest tests/test_server.py tests/test_profile.py tests/test_profile_config.py tests/test_query_cache.py tests/test_import.py tests/test_host_header.py tests/test_database_concurrency.py tests/test_settings.py + python -m pytest tests/test_server.py tests/test_profile.py tests/test_profile_config.py tests/test_query_cache.py tests/test_import.py tests/test_host_header.py tests/test_database_concurrency.py tests/test_settings.py tests/test_csv_export.py typecheck: python -m mypy aw_server tests --ignore-missing-imports diff --git a/aw_server/csv_export.py b/aw_server/csv_export.py index ab5a46d5..43056139 100644 --- a/aw_server/csv_export.py +++ b/aw_server/csv_export.py @@ -6,10 +6,12 @@ Columns are ``id``, ``timestamp``, ``duration``, then the union of data keys across all exported events (first-seen order); a data key that collides with an earlier column name gets a ``data.`` prefix. Above ``MAX_CSV_DATA_COLUMNS`` -keys, a single ``data`` column holds each event's data as JSON instead. +keys, or if any event has non-object data, a single ``data`` column holds +each event's data as JSON instead. """ import json +import math from datetime import timedelta, timezone from typing import Any, Dict, Iterable, Iterator, List @@ -22,7 +24,13 @@ _FORMULA_STARTERS = ("=", "+", "-", "@") +class _Number(str): + """A finite JSON number. It is not a formula, so it is exported unchanged.""" + + def _neutralize_formula(s: str) -> str: + if isinstance(s, _Number): + return s if s.lstrip(_FORMULA_LEADING_WHITESPACE).startswith(_FORMULA_STARTERS): return "'" + s return s @@ -41,7 +49,12 @@ def _csv_record(fields: Iterable[str]) -> str: def _json(value: Any) -> str: - return json.dumps(value, separators=(",", ":"), ensure_ascii=False) + text = json.dumps(value, separators=(",", ":"), ensure_ascii=False) + # Keep -5 / -1.5 as valid JSON numbers; a leading "'" would break json.loads. + is_number = isinstance(value, (int, float)) and not isinstance(value, bool) + if is_number and math.isfinite(value): + return _Number(text) + return text def _duration(duration: timedelta) -> str: @@ -83,12 +96,16 @@ def _header_names(data_keys: List[str]) -> List[str]: def events_to_csv(events: List[Event]) -> Iterator[str]: """Yield CSV records for events, in the order given.""" + # Previously accepted data may be a list or scalar. Choose the lossless + # JSON fallback before discovering object keys (scalars may not be iterable). + data_json_column = any(not isinstance(e.data, dict) for e in events) keys: Dict[str, None] = {} - for e in events: - for k in e.data: - keys.setdefault(k) + if not data_json_column: + for e in events: + for k in e.data: + keys.setdefault(k) data_keys = list(keys) - data_json_column = len(data_keys) > MAX_CSV_DATA_COLUMNS + data_json_column = data_json_column or len(data_keys) > MAX_CSV_DATA_COLUMNS if data_json_column: data_keys = ["data"] diff --git a/tests/test_csv_export.py b/tests/test_csv_export.py new file mode 100644 index 00000000..5e1604a9 --- /dev/null +++ b/tests/test_csv_export.py @@ -0,0 +1,153 @@ +"""CSV export must remain readable for previously accepted non-object data.""" + +import csv +import io +import json +from datetime import datetime, timedelta, timezone + +import pytest +from aw_core.models import Event +from aw_server.csv_export import MAX_CSV_DATA_COLUMNS, events_to_csv +from aw_server.server import AWFlask + + +@pytest.mark.parametrize("legacy_data", [["x"], "legacy", 5, 1.5, True]) +def test_legacy_nonobject_csv_stream_roundtrip(legacy_data): + app = AWFlask("127.0.0.1", testing=True) + client = app.test_client() + bucket = "test-legacy-csv" + response = client.post( + f"/api/0/buckets/{bucket}", + json={"client": "test", "type": "test", "hostname": "test"}, + ) + assert response.status_code == 200 + payloads = [{"title": "=SUM(A1:A2)", "unicode": 'å\n"'}, legacy_data] + for index, data in enumerate(payloads): + response = client.post( + f"/api/0/buckets/{bucket}/events", + json={ + "timestamp": f"2026-01-01T00:00:0{index}Z", + "duration": 1.25, + "data": data, + }, + ) + assert response.status_code == 200 + + # Headers alone cannot detect exceptions in the streamed body. + response = client.get(f"/api/0/buckets/{bucket}/export/csv") + assert response.status_code == 200 + rows = list(csv.reader(io.StringIO(response.get_data(as_text=True)))) + assert rows[0] == ["id", "timestamp", "duration", "data"] + assert len(rows) == 3 + assert [json.loads(row[3]) for row in rows[1:]] == payloads[::-1] + assert [row[2] for row in rows[1:]] == ["1.250000000"] * 2 + assert [row[1] for row in rows[1:]] == [ + "2026-01-01T00:00:01+00:00", + "2026-01-01T00:00:00+00:00", + ] + # Exporting must not rewrite the stored payloads. + stored = client.get(f"/api/0/buckets/{bucket}/events").get_json() + assert [event["data"] for event in stored] == payloads[::-1] + + +@pytest.mark.parametrize("data", [[], False, 0, ""]) +def test_legacy_nonobject_csv_falsey_data_preserved(data): + # aw-core's constructor normalizes falsey data to {}. Seed historical + # values explicitly to test the exporter boundary without changing ingest. + event = Event(timestamp=datetime(2026, 1, 1, tzinfo=timezone.utc)) + event.data = data + rows = list(csv.reader(events_to_csv([event]))) + assert rows[0] == ["id", "timestamp", "duration", "data"] + assert json.loads(rows[1][3]) == data + + +def test_legacy_nonobject_csv_null_uses_model_semantics(): + event = Event(timestamp=datetime(2026, 1, 1, tzinfo=timezone.utc), data=None) + assert event.data == {} + event.data = None + assert event.data == {} # The property also normalizes an explicitly stored null. + rows = list(csv.reader(events_to_csv([event]))) + assert rows[0] == ["id", "timestamp", "duration"] + assert len(rows[1]) == 3 + + +def test_legacy_nonobject_csv_object_only_compatibility(): + event = Event( + id=7, + timestamp=datetime(2026, 1, 1, tzinfo=timezone(timedelta(hours=1))), + duration=timedelta(microseconds=1234567), + data={ + "id": "=1+1", + "data.id": " @command", + "title": 'å,"line\nnext', + "null": None, + "nested": {"a": [1, True]}, + }, + ) + second = Event(timestamp=event.timestamp, data={"other": "-formula"}) + rows = list(csv.reader(events_to_csv([event, second]))) + assert rows[0] == [ + "id", + "timestamp", + "duration", + "data.id", + "data.data.id", + "title", + "null", + "nested", + "other", + ] + assert rows[1] == [ + "7", + "2025-12-31T23:00:00+00:00", + "1.234567000", + "'=1+1", + "' @command", + 'å,"line\nnext', + "null", + '{"a":[1,true]}', + "", + ] + assert rows[2][3:8] == [""] * 5 + assert rows[2][8] == "'-formula" + + +def test_legacy_nonobject_csv_wide_object_fallback_unchanged(): + data = {f"key{i}": i for i in range(MAX_CSV_DATA_COLUMNS + 1)} + rows = list(csv.reader(events_to_csv([Event(data=data)]))) + assert rows[0] == ["id", "timestamp", "duration", "data"] + assert json.loads(rows[1][3]) == data + + +@pytest.mark.parametrize("value", [-5, -1.5, 0, 7]) +def test_numeric_json_is_not_formula_neutralized(value): + event = Event( + timestamp=datetime(2026, 1, 1, tzinfo=timezone.utc), data={"n": value} + ) + legacy = Event(timestamp=datetime(2026, 1, 2, tzinfo=timezone.utc)) + legacy.data = value + wide = Event( + timestamp=datetime(2026, 1, 3, tzinfo=timezone.utc), + data={f"k{i}": value for i in range(MAX_CSV_DATA_COLUMNS + 1)}, + ) + # Per-key column, scalar data column and wide-object JSON column. + assert list(csv.reader(events_to_csv([event])))[1][3] == json.dumps(value) + assert json.loads(list(csv.reader(events_to_csv([legacy])))[1][3]) == value + assert json.loads(list(csv.reader(events_to_csv([wide])))[1][3])["k0"] == value + + +def test_formula_text_still_neutralized_next_to_numbers(): + event = Event( + timestamp=datetime(2026, 1, 1, tzinfo=timezone.utc), + data={"title": "=SUM(A1:A2)", "text": "-5", "n": -5}, + ) + row = list(csv.reader(events_to_csv([event])))[1] + assert row[3:] == ["'=SUM(A1:A2)", "'-5", "-5"] + + +@pytest.mark.parametrize("value", [float("nan"), float("inf"), float("-inf")]) +def test_nonfinite_numbers_are_not_trusted_as_numbers(value): + event = Event(timestamp=datetime(2026, 1, 1, tzinfo=timezone.utc)) + event.data = value + cell = list(csv.reader(events_to_csv([event])))[1][3] + assert not cell.startswith(("-", "="))