Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
69 changes: 69 additions & 0 deletions tests/test_proposer_cost.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,69 @@
"""The micro-skills proposer is charged for its call even when its CLI reports no usage.

Every paid side call is charged to the ledger as overhead. The loop guard's judge and redirect
writer charge the usage their runner recovered, else an estimate (`pricing.
runner_effective_or_approx`: the prompt at 1x, the reply at 5x). The proposer charged only
recovered usage, so a call whose CLI reported none (Cursor's never does; others when the usage
block is missing) cost the ledger nothing."""
from fixtures import write_claude_transcript
from tokeneconomizer import pricing
from tokeneconomizer.loopguard import judge
from tokeneconomizer.microskills import GENERIC
from tokeneconomizer.microskills import distiller as dist

REPLY = '{"extractors": [], "facts": []}'


def _distiller(tmp_path, monkeypatch, runner):
monkeypatch.setattr(dist, "PROPOSE_EVERY", 1)
return dist.Distiller.load("acme/tool", str(tmp_path / "state"), runner=runner,
vocab=GENERIC)


def _session(tmp_path, n):
return write_claude_transcript(str(tmp_path / f"s{n}.jsonl"), [
("Bash", {"command": "make build"}, "built"),
("Bash", {"command": "make test"}, "ok ./... 0.01s")])


def _propose_once(tmp_path, monkeypatch, usage):
prompts = []

def runner(prompt, model):
prompts.append(prompt)
return REPLY, usage
d = _distiller(tmp_path, monkeypatch, runner)
d.update(_session(tmp_path, 0), iid="s0")
assert len(prompts) == 1
return d, prompts[0]


def test_a_call_with_no_usage_is_charged_the_estimate(tmp_path, monkeypatch):
d, prompt = _propose_once(tmp_path, monkeypatch, None)
want = pricing.approx_tokens(prompt) + 5 * pricing.approx_tokens(REPLY)
assert d.opt_tokens == want > 0


def test_it_is_the_estimate_the_judge_charges_for_the_same_call(tmp_path, monkeypatch):
d, prompt = _propose_once(tmp_path, monkeypatch, None)

def runner(p, m):
return REPLY, None
assert d.opt_tokens == pricing.runner_effective_or_approx(runner, prompt, REPLY, None)
v = judge.llm_stuck_judge("turns=30 repeated_identical_failures=4", runner)
assert v["_tokens"] == pricing.runner_effective_or_approx(
runner, judge.STUCK_JUDGE_SYSTEM + "\n\nRecent process digest:\n"
"turns=30 repeated_identical_failures=4\n\nReturn the JSON verdict:", REPLY, None)


def test_recovered_usage_is_charged_as_before(tmp_path, monkeypatch):
assert _propose_once(tmp_path, monkeypatch, 321.5)[0].opt_tokens == 321.5
assert _propose_once(tmp_path / "local", monkeypatch, 0)[0].opt_tokens == 0 # on-device


def test_a_failed_call_is_still_not_charged(tmp_path, monkeypatch):
def runner(prompt, model):
raise RuntimeError("no reply")
d = _distiller(tmp_path, monkeypatch, runner)
d.update(_session(tmp_path, 0), iid="s0")
assert d.opt_tokens == 0
6 changes: 4 additions & 2 deletions tokeneconomizer/microskills/distiller.py
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,7 @@
import re

from ..names import repo_filename
from ..pricing import runner_effective_or_approx
from ..trace import SHELL_TOOLS, call_cwds, tool_kind
from ..trace.claude import CONVERSATION
from .lifecycle import Forgetting, Lifecycle, edited_files, edited_pyfiles, write_json_atomic
Expand Down Expand Up @@ -667,8 +668,9 @@ def _maybe_propose(self):
except Exception:
self.last_propose = self.now
return
if eff is not None:
self.opt_tokens += eff
# Charged as every paid side call is: the usage the runner recovered, else an estimate on
# the same basis (prompt at 1x, reply at 5x), so a CLI that reports none is not free.
self.opt_tokens += runner_effective_or_approx(self.runner, prompt, raw or "", eff)
exts, facts = parse_payload(raw)
for e in exts:
self._admit_extractor(e) # append-only
Expand Down