Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
14 changes: 12 additions & 2 deletions lib/token-utils.ts
Original file line number Diff line number Diff line change
Expand Up @@ -2,8 +2,18 @@ import { SessionState, WithParts } from "./state"
import { AssistantMessage, UserMessage } from "@opencode-ai/sdk/v2"
import { Logger } from "./logger"
import * as _anthropicTokenizer from "@anthropic-ai/tokenizer"
const anthropicCountTokens = (_anthropicTokenizer.countTokens ??
(_anthropicTokenizer as any).default?.countTokens) as typeof _anthropicTokenizer.countTokens
const getAnthropicTokenizer = (_anthropicTokenizer.getTokenizer ??
(_anthropicTokenizer as any).default?.getTokenizer) as typeof _anthropicTokenizer.getTokenizer

// The package's countTokens() builds a new tokenizer on every call, which is slow.
// Build one once and reuse it, encoding the same way countTokens() does.
let anthropicTokenizer: ReturnType<typeof getAnthropicTokenizer> | undefined
function anthropicCountTokens(text: string): number {
if (!anthropicTokenizer) {
anthropicTokenizer = getAnthropicTokenizer()
}
return anthropicTokenizer.encode(text.normalize("NFKC"), "all").length
}
import { getLastUserMessage } from "./messages/query"

export function getCurrentTokenUsage(state: SessionState, messages: WithParts[]): number {
Expand Down
21 changes: 21 additions & 0 deletions tests/token-counting.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -4,11 +4,13 @@ import type { WithParts } from "../lib/state"
import {
COMPACTED_TOOL_OUTPUT_PLACEHOLDER,
countAllMessageTokens,
countTokens,
countToolTokens,
estimateTokensBatch,
extractCompletedToolOutput,
extractToolContent,
} from "../lib/token-utils"
import * as anthropicTokenizer from "@anthropic-ai/tokenizer"

function buildToolMessage(part: Record<string, any>): WithParts {
return {
Expand Down Expand Up @@ -172,3 +174,22 @@ test("counting uses the compacted tool placeholder for completed outputs", () =>
assert.equal(extractCompletedToolOutput(part), COMPACTED_TOOL_OUTPUT_PLACEHOLDER)
assertCounted(part, [JSON.stringify(input), COMPACTED_TOOL_OUTPUT_PLACEHOLDER])
})

test("countTokens matches the tokenizer package and stays fast across many calls", () => {
const samples = [
"hello world",
"Crème brûlée, fi ligature and emoji 🚀",
JSON.stringify({ path: "src/index.ts", edits: [{ line: 1, text: "const x = 1" }] }),
"<|endoftext|> special tokens are counted",
]
for (const sample of samples) {
assert.equal(countTokens(sample), anthropicTokenizer.countTokens(sample))
}

// Building a new tokenizer per call takes ~20 ms, so 200 calls took ~4.6 s.
const start = performance.now()
for (let i = 0; i < 200; i++) {
countTokens(`tool output ${i} `.repeat(50))
}
assert.ok(performance.now() - start < 2000, "countTokens should reuse one tokenizer")
})
Loading