From 10a38cdf47a41fb16252c874e549da19084ef31c Mon Sep 17 00:00:00 2001 From: Dan Smolsky Date: Mon, 16 Nov 2026 02:55:10 -0500 Subject: [PATCH] fix: reuse one tokenizer instead of building one per count --- lib/token-utils.ts | 14 ++++++++++++-- tests/token-counting.test.ts | 21 +++++++++++++++++++++ 2 files changed, 33 insertions(+), 2 deletions(-) diff --git a/lib/token-utils.ts b/lib/token-utils.ts index bba07fa1a..cc173b587 100644 --- a/lib/token-utils.ts +++ b/lib/token-utils.ts @@ -2,8 +2,18 @@ import { SessionState, WithParts } from "./state" import { AssistantMessage, UserMessage } from "@opencode-ai/sdk/v2" import { Logger } from "./logger" import * as _anthropicTokenizer from "@anthropic-ai/tokenizer" -const anthropicCountTokens = (_anthropicTokenizer.countTokens ?? - (_anthropicTokenizer as any).default?.countTokens) as typeof _anthropicTokenizer.countTokens +const getAnthropicTokenizer = (_anthropicTokenizer.getTokenizer ?? + (_anthropicTokenizer as any).default?.getTokenizer) as typeof _anthropicTokenizer.getTokenizer + +// The package's countTokens() builds a new tokenizer on every call, which is slow. +// Build one once and reuse it, encoding the same way countTokens() does. +let anthropicTokenizer: ReturnType | undefined +function anthropicCountTokens(text: string): number { + if (!anthropicTokenizer) { + anthropicTokenizer = getAnthropicTokenizer() + } + return anthropicTokenizer.encode(text.normalize("NFKC"), "all").length +} import { getLastUserMessage } from "./messages/query" export function getCurrentTokenUsage(state: SessionState, messages: WithParts[]): number { diff --git a/tests/token-counting.test.ts b/tests/token-counting.test.ts index 03e9439ce..362d9af7e 100644 --- a/tests/token-counting.test.ts +++ b/tests/token-counting.test.ts @@ -4,11 +4,13 @@ import type { WithParts } from "../lib/state" import { COMPACTED_TOOL_OUTPUT_PLACEHOLDER, countAllMessageTokens, + countTokens, countToolTokens, estimateTokensBatch, extractCompletedToolOutput, extractToolContent, } from "../lib/token-utils" +import * as anthropicTokenizer from "@anthropic-ai/tokenizer" function buildToolMessage(part: Record): WithParts { return { @@ -172,3 +174,22 @@ test("counting uses the compacted tool placeholder for completed outputs", () => assert.equal(extractCompletedToolOutput(part), COMPACTED_TOOL_OUTPUT_PLACEHOLDER) assertCounted(part, [JSON.stringify(input), COMPACTED_TOOL_OUTPUT_PLACEHOLDER]) }) + +test("countTokens matches the tokenizer package and stays fast across many calls", () => { + const samples = [ + "hello world", + "Crème brûlée, fi ligature and emoji 🚀", + JSON.stringify({ path: "src/index.ts", edits: [{ line: 1, text: "const x = 1" }] }), + "<|endoftext|> special tokens are counted", + ] + for (const sample of samples) { + assert.equal(countTokens(sample), anthropicTokenizer.countTokens(sample)) + } + + // Building a new tokenizer per call takes ~20 ms, so 200 calls took ~4.6 s. + const start = performance.now() + for (let i = 0; i < 200; i++) { + countTokens(`tool output ${i} `.repeat(50)) + } + assert.ok(performance.now() - start < 2000, "countTokens should reuse one tokenizer") +})