Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
69 changes: 69 additions & 0 deletions tests/test_trace.py
Original file line number Diff line number Diff line change
@@ -1,4 +1,5 @@
"""Trace seam: load_trajectory / tool_usage / parse_catalog over synthetic transcripts."""
import json
import os

from tokeneconomizer import trace
Expand Down Expand Up @@ -312,6 +313,74 @@ def test_a_non_object_line_does_not_stop_a_rollout_from_being_converted(tmp_path
assert trace.codex_capability_trace(str(r), str(out)) == 2


def _assert_codex_runner_repair(tmp_path, wrap, missing_message, failure_code, expected):
from tokeneconomizer.microskills import EnvFactsDistiller
from tokeneconomizer.trace.codex import codex_exec_calls

rollout = tmp_path / "rollout.jsonl"
converted = tmp_path / "exec.jsonl"
commands = [
("pytest -q", missing_message, failure_code),
("python -m pytest -q", "1 passed in 0.1s", 0),
]
records = []
for index, (command, output, code) in enumerate(commands):
records.append({"type": "event_msg", "payload": {"type": "item_completed", "item": {
"type": "CommandExecution", "id": f"call-{index}", "command": wrap(command),
"aggregated_output": output, "exit_code": code,
"status": "failed" if code else "completed",
}}})
rollout.write_text("".join(json.dumps(record) + "\n" for record in records), encoding="utf-8")
assert trace.codex_rollout_to_exec(str(rollout), str(converted)) == len(commands)
calls = codex_exec_calls(converted.read_text(encoding="utf-8").splitlines())
assert [args["command"] for _, args, _, _ in calls] == expected
assert [ok for _, _, _, ok in calls] == [False, True]

distiller = EnvFactsDistiller("synthetic")
distiller.update(str(converted), iid="synthetic-session")
assert "form_repair:run_tests:pytest" in distiller.facts


def test_codex_powershell_rollout_keeps_the_script_visible_to_envfacts(tmp_path):
def wrap(command):
return [r"C:\Program Files\PowerShell\7\pwsh.exe", "-NoProfile", "-Command", command]

_assert_codex_runner_repair(
tmp_path, wrap, "The term 'pytest' is not recognized as a name of a cmdlet.", 1,
["pytest -q", "python -m pytest -q"],
)


def test_codex_bash_rollout_preserves_quoting_and_envfacts(tmp_path):
def wrap(command):
return ["/bin/bash", "-lc", command]

_assert_codex_runner_repair(
tmp_path, wrap, "/bin/bash: line 1: pytest: command not found", 127,
["/bin/bash -lc 'pytest -q'", "/bin/bash -lc 'python -m pytest -q'"],
)


def test_codex_powershell_inline_script_preserves_quotes_and_other_modes():
from tokeneconomizer.trace.codex import rollout_item_to_exec

def command_text(argv):
item = {"type": "CommandExecution", "command": argv}
return rollout_item_to_exec(item)["command"]

executable = r"C:\Windows\System32\WindowsPowerShell\v1.0\PowerShell.EXE"
script = 'Write-Output "a b"\nWrite-Output "c"'
assert command_text([executable, "-NoProfile", "-COMMAND", script]) == script
assert command_text(["powershell.exe", "-File", "path with spaces.ps1"]) == (
"powershell.exe -File 'path with spaces.ps1'"
)
assert command_text(["pwsh.exe", "-File", "fixture.ps1", "-Command", "value"]) == (
"pwsh.exe -File fixture.ps1 -Command value"
)
assert command_text([]) == ""
assert command_text("python -m pytest -q") == "python -m pytest -q"


def test_a_non_object_line_does_not_empty_a_tool_catalog():
system = {"type": "system", "tools": ["Bash"]}
checkpoint = {"type": "session.usage_checkpoint", "data": {"promptCacheBreakState": [
Expand Down
9 changes: 9 additions & 0 deletions tokeneconomizer/trace/codex.py
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,7 @@
unreadable rather than half-read.
"""
import json
import ntpath
import os
import re
import shlex
Expand Down Expand Up @@ -99,6 +100,14 @@ def codex_exec_calls(lines):

def _command_text(command):
if isinstance(command, list):
if len(command) >= 3:
executable = ntpath.basename(str(command[0])).lower()
startup_switches = {"-nologo", "-noprofile", "-noninteractive"}
if (executable in {"pwsh.exe", "powershell.exe"}
and all(str(arg).lower() in startup_switches for arg in command[1:-2])
and str(command[-2]).lower() == "-command"):
# Preserve the script verbatim; POSIX shell quoting hides it from envfacts.
return str(command[-1])
return shlex.join(str(c) for c in command)
return str(command or "")

Expand Down