From ef60f6fdc8a796237388c62e88e9629eade3d87c Mon Sep 17 00:00:00 2001 From: Saleh Alghusson Date: Tue, 13 Oct 2026 13:17:16 +0000 Subject: [PATCH 1/5] Refresh Sonar migration models and request guidance --- skills/migrate-sonar-to-agent-api/SKILL.md | 4 +- .../references/integration-styles.md | 1 + .../references/models-and-presets.md | 29 ++++++------ .../references/request-mapping.md | 46 ++++++++++++++++--- .../references/response-and-streaming.md | 5 +- 5 files changed, 61 insertions(+), 24 deletions(-) diff --git a/skills/migrate-sonar-to-agent-api/SKILL.md b/skills/migrate-sonar-to-agent-api/SKILL.md index 2c865fc..9feb91e 100644 --- a/skills/migrate-sonar-to-agent-api/SKILL.md +++ b/skills/migrate-sonar-to-agent-api/SKILL.md @@ -86,7 +86,7 @@ Leave framework-internal LLMs pointed at other providers alone. A consumer that only exits on `response.completed` hangs on failed runs. 6. **No `perplexity/sonar-pro` slug exists.** - Map `sonar-pro` per the two-option rule (preset `"low"` vs `perplexity/sonar` + web_search) in [models-and-presets.md](references/models-and-presets.md). + Map `sonar-pro` per the two-option rule (preset `"fast"` vs `perplexity/sonar` + web_search) in [models-and-presets.md](references/models-and-presets.md). 7. **`max_output_tokens` is REQUIRED for `anthropic/*` models** (400 otherwise); values under 16 floor to 16. On presets and reasoning models, tight caps can be consumed by reasoning tokens before any text or search appears - size generously or omit. @@ -127,7 +127,7 @@ Leave framework-internal LLMs pointed at other providers alone. The live run is MANDATORY when an API key is available - compile-clean SDK code can still crash on call (see the typed-kwarg trap in integration-styles.md); if you truly cannot run it, say so explicitly in your summary. **Minimal-diff rule:** keep the author's structure, naming, and voice; do not refactor unrelated code. -When a Sonar feature has no Agent equivalent (images, related questions, academic search mode), apply the documented workaround from [request-mapping.md](references/request-mapping.md) if it is small. +When a Sonar feature has no direct Agent equivalent (related questions, academic search mode), apply the documented workaround from [request-mapping.md](references/request-mapping.md) if it is small. For `return_related_questions` specifically, use the lightweight prompt-delimiter recipe in request-mapping.md - it fits free-text flows too. If no workaround fits, remove the feature AND call the removal out in your migration notes; never drop user-facing output silently. diff --git a/skills/migrate-sonar-to-agent-api/references/integration-styles.md b/skills/migrate-sonar-to-agent-api/references/integration-styles.md index 8a510e4..d991255 100644 --- a/skills/migrate-sonar-to-agent-api/references/integration-styles.md +++ b/skills/migrate-sonar-to-agent-api/references/integration-styles.md @@ -176,6 +176,7 @@ Whichever you pick, state the choice explicitly in your migration notes. - `max_tool_calls: 0` disables tool calls entirely - the closest analog of `disable_search` when a preset forces tools you cannot remove (note: it disables ALL tools). - Built-in tools, one line each: - `web_search` - grounded search; filters per [request-mapping.md](request-mapping.md); `max_results` <= 50. + - `image_search` - image retrieval; `max_results` 1..30; domain/format filters per [request-mapping.md](request-mapping.md); returns `image_search_results` output items. - `fetch_url` - fetch pages; `max_urls` 1..10; `total_budget_tokens` <= 120000. - `people_search` - professional profiles; bills as `search_people`. - `finance_search` - quotes, fundamentals, filings; replaces `search_mode: "sec"`. diff --git a/skills/migrate-sonar-to-agent-api/references/models-and-presets.md b/skills/migrate-sonar-to-agent-api/references/models-and-presets.md index a6d75e9..fae5706 100644 --- a/skills/migrate-sonar-to-agent-api/references/models-and-presets.md +++ b/skills/migrate-sonar-to-agent-api/references/models-and-presets.md @@ -4,29 +4,31 @@ Read this when choosing a target model or preset for a migrated call site. ## Valid model slugs -`GET https://api.perplexity.ai/v1/models` (no auth required) is authoritative - always verify slugs there before writing them into code. -Agent API slugs are provider-prefixed, e.g. `perplexity/sonar`, `openai/gpt-5.1`, `anthropic/claude-sonnet-4-5`, `google/gemini-3-flash-preview`. +Verify slugs against the current [model reference](https://docs.perplexity.ai/docs/agent-api/models) or `GET https://api.perplexity.ai/v1/models` with your API key before writing them into code. +Agent API slugs are provider-prefixed, e.g. `perplexity/sonar` and `openai/gpt-6-luna`. +Check the [changelog](https://docs.perplexity.ai/docs/resources/changelog) for retirement dates when choosing a model. Do not invent slugs: there is NO `perplexity/sonar-pro` slug. ## Model mapping table | Sonar model | Migrate to | Rationale and caveats | |---|---|---| -| `sonar` | `model: "perplexity/sonar"` + `tools: [{"type": "web_search"}]`, or `preset: "fast"` | Closest by name. CAUTION: output is not byte-identical to chat `sonar` and pricing differs (see the public pricing page) - A/B compare quality and cost before cutover. | -| `sonar-pro` | Two options - pick per the heuristic below | Option 1, behavioral match: `preset: "low"` (pro-search, multi-step search). Option 2, lighter direct substitution: `model: "perplexity/sonar"` + web_search. | -| `sonar-reasoning` | `preset: "low"`, or a reasoning model (e.g. `openai/gpt-5.1` + `reasoning: {"effort": "medium"}` + web_search) | Retired; do not carry the name forward. Note the substitution in your migration notes. | -| `sonar-reasoning-pro` | `preset: "low"` (closest), or a reasoning model + `reasoning: {"effort": "medium"}` + web_search + a generously sized `max_output_tokens` | `medium` is a strictly heavier, costlier tier - offer it only as an explicit upgrade. | -| `sonar-deep-research` | `preset: "medium"` (deep-research) | The medium (deep-research) preset is the closest published equivalent of sonar-deep-research. `high` is a costlier upgrade, not an equivalent; offer it only as an explicit upsell note. | +| `sonar` | `preset: "fast"`, or `model: "perplexity/sonar"` + `tools: [{"type": "web_search"}]` | `fast` is the documented starting point. Output is not byte-identical and pricing differs - A/B compare quality and cost before cutover. | +| `sonar-pro` | Two options - pick per the heuristic below | Option 1, documented starting point: `preset: "fast"`. Option 2, model-slug substitution: `model: "perplexity/sonar"` + web_search. | +| `sonar-reasoning` | `preset: "low"`, or a current reasoning model + web_search | Retired; do not carry the name forward. Verify the model's supported reasoning efforts and note the substitution in your migration notes. | +| `sonar-reasoning-pro` | `preset: "low"`, or a current reasoning model + web_search + a generously sized `max_output_tokens` | `low` is the documented starting point; evaluate heavier presets separately. | +| `sonar-deep-research` | `preset: "high"` | The current migration guide recommends `high` for deep research. | + +These are starting points from the [migration guide](https://docs.perplexity.ai/docs/agent-api/migrate-from-sonar/how-to), not guarantees of equivalent behavior or cost. ### The sonar-pro two-option rule -- **Option 1 - behavioral match: `preset: "low"`.** - Caveats to state inline in migration notes: `preset` is a separate request field, not a model slug; it may resolve to a third-party model (`response.model` can echo e.g. a `google/*` model); higher latency and cost per call; via the OpenAI SDK pass `extra_body={"preset": "low"}` and omit `model`. -- **Option 2 - lighter substitution: `model: "perplexity/sonar"` + `tools: [{"type": "web_search"}]`.** +- **Option 1 - documented starting point: `preset: "fast"`.** + Caveats to state inline in migration notes: `preset` is a separate request field, not a model slug; it may resolve to a third-party model; compare quality, latency, and cost; via the OpenAI SDK pass `extra_body={"preset": "fast"}` and omit `model`. +- **Option 2 - model-slug substitution: `model: "perplexity/sonar"` + `tools: [{"type": "web_search"}]`.** Required when the code's contract is a model slug: a `--model` CLI flag, a model allowlist, or a framework model-class bridge that takes a model string (presets do not fit there). - Also preferred for lightweight or high-frequency call sites where pro-search depth is not the point. -Selection heuristic: model-slug contract -> option 2; hardcoded single model where search depth matters -> option 1; otherwise either, but document the substitution. +Selection heuristic: model-slug contract -> option 2; otherwise start with option 1, but document and evaluate the substitution. **Preset plus search filters:** per the presets documentation, a tool you list in the request overrides only the preset's tool of the same type; the preset's other tools stay enabled. When the flow needs guaranteed `search_domain_filter` / recency / date filters, the simplest setup to reason about is still option 2 - an explicit model plus a `web_search` tool carrying `filters` - because nothing else is bundled alongside it. @@ -35,7 +37,8 @@ When the flow needs guaranteed `search_domain_filter` / recency / date filters, Know these before using presets: -- Short names `fast | low | medium | high | xhigh` are the canonical preset names; `fast-search | pro-search | deep-research | advanced-deep-research` are accepted as previous names for the first four. +- Short names `fast | low | medium | high | xhigh` are the canonical preset names; `fast-search | pro-search | deep-research | advanced-deep-research | ultra` are their respective previous names. +- Preset models and tool configuration can change; consult the current [preset reference](https://docs.perplexity.ai/docs/agent-api/presets) rather than freezing a model from an old example. - Any request field set alongside `preset` overrides the preset's value for that field (request-over-preset precedence). In particular, set `max_output_tokens` in the request when you need a cap. - `instructions` alongside a preset REPLACES the preset's system prompt entirely; `""` clears it; omit the field to keep it. diff --git a/skills/migrate-sonar-to-agent-api/references/request-mapping.md b/skills/migrate-sonar-to-agent-api/references/request-mapping.md index 7cbc93f..19c2d82 100644 --- a/skills/migrate-sonar-to-agent-api/references/request-mapping.md +++ b/skills/migrate-sonar-to-agent-api/references/request-mapping.md @@ -27,7 +27,7 @@ Offering the tool does not guarantee the model calls it; for citation-critical f | Sonar param | Agent API | Notes | |---|---|---| -| `messages` | `input` | String for a single user turn; else array of `{role, content}` items (roles: user, assistant, system, developer). System prompt: keep as a `role:"system"` input item or move to top-level `instructions`. | +| `messages` | `input` | String for a single user turn; else array of `{type:"message", role, content}` items (roles: user, assistant, system, developer). Include the message `type` for SDK schema validation. System prompt: keep as a `role:"system"` input item or move to top-level `instructions`. | | (system message) | `instructions` or input item | With a preset, `instructions` REPLACES the preset prompt (never appends); `""` clears it. | | `model` | `model` | New slugs - see [models-and-presets.md](models-and-presets.md). Or `preset`, or `models` (fallback list, max 5). | | `max_tokens` / `max_completion_tokens` | `max_output_tokens` | REQUIRED when model is `anthropic/*` (else 400 "max_output_tokens is required when using Anthropic models"). Values under 16 are floored to 16. | @@ -39,7 +39,7 @@ Offering the tool does not guarantee the model calls it; for citation-critical f | `search_recency_filter` | `tools[web_search].filters.search_recency_filter` | `hour\|day\|week\|month\|year`; mutually exclusive with the after/before date filters. | | `search_after_date_filter`, `search_before_date_filter` | same names inside `tools[web_search].filters` | Format `MM/DD/YYYY`. | | `last_updated_after_filter`, `last_updated_before_filter` | same names inside `tools[web_search].filters` | Format `MM/DD/YYYY`. | -| `web_search_options.search_context_size` | `tools[web_search].search_context_size` | Sits on the tool object, NOT inside `filters`. | +| `web_search_options.search_context_size` | No named-size equivalent | Set explicit `max_tokens` and `max_tokens_per_page` budgets on the `web_search` tool, NOT inside `filters`. Do not forward `search_context_size`. | | `web_search_options.user_location` | `tools[web_search].user_location` | On the tool object, NOT inside `filters`. `{latitude, longitude, country, region, city}`. | | `num_search_results` | `tools[web_search].max_results` | 1..50. | | `tools` (custom functions) | `tools` with `{type:"function", name, description, parameters, strict}` | Name and description at the tool's top level (Responses style), not nested under `function`. Watch reserved names - see [integration-styles.md](integration-styles.md). | @@ -47,6 +47,15 @@ Offering the tool does not guarantee the model calls it; for citation-critical f | `parallel_tool_calls` | unchanged | Default true. | | `language_preference` | unchanged | | | `image_url` content part | `input_image` content part | `{"type":"input_image","image_url":"..."}` (data URI or HTTPS URL, <= 2048 chars). | +| `file_url` / `pdf_url` content part | `input_file` content part | Use exactly one of base64 `file_data` or public HTTPS `file_url`; see the document-input example below. Model/provider restrictions apply. | +| `return_images` | Add `{"type":"image_search"}` to `tools` | The model decides when to call it; force it with `tool_choice: {"type":"image_search"}` if needed. | +| `num_images` | `tools[image_search].max_results` | 1..30; defaults to 5. | +| `image_domain_filter` | `tools[image_search].filters.domain_filter` | Up to 10 domains; either allowlist or `-`-prefixed denylist, not both. | +| `image_format_filter` | `tools[image_search].filters.format_filter` | Up to 10 formats from `bmp`, `gif`, `jpeg`, `png`, `webp`, `svg`. | + +Image results are returned in `output[]` items with `type: "image_search_results"`, not Sonar's top-level `images`. +Read `results[].image_url` and `results[].origin_url`; handle empty results and tool errors. +See [Image Search](https://docs.perplexity.ai/docs/agent-api/tools/image-search). ## REMOVE list (any leftover = 400) @@ -55,8 +64,7 @@ Offering the tool does not guarantee the model calls it; for citation-critical f | `disable_search` | REMOVE | Omit the `web_search` tool. With a preset, preset tools stay enabled - there is no public field to disable them; `max_tool_calls: 0` disables ALL tool calls. | | `enable_search_classifier` | REMOVE | Provide `web_search` and let the model decide when to call it. | | `search_mode: "sec"` | REMOVE | Use the `finance_search` tool instead. | -| `search_mode: "academic"` | REMOVE | No equivalent. | -| `return_images`, `image_domain_filter`, `image_format_filter` | REMOVE | Prompt for image URLs plus a structured-output schema. | +| `search_mode: "academic"` | REMOVE | No direct equivalent; use `web_search` with domain filters and prompting. | | `return_related_questions` | REMOVE | Do not silently drop this user-facing feature - see the recipe below the table. | | `presence_penalty`, `frequency_penalty` | REMOVE | No equivalent. They may appear as echo-only zeros in responses; they are not settable. | | `stop` | REMOVE | No equivalent. | @@ -65,9 +73,33 @@ Offering the tool does not guarantee the model calls it; for citation-critical f | `logprobs`, `top_logprobs` | REMOVE | Some such fields are accepted but have no effect; do not carry them over. | | `web_search_options.search_type` / pro search | REMOVE | Use `max_steps` (1..100) or a preset for multi-step behavior. | | `search_language_filter` | REMOVE | No equivalent. | -| `return_videos`, `num_images`, `num_videos`, `media_response` | REMOVE | No equivalent. | +| `return_videos`, `num_videos`, `media_response` | REMOVE | No equivalent. | | `search_mode`, `web_search_options` (the container fields themselves) | REMOVE | Contents relocate per the table above; the containers must not remain. | -| `file_url` / `pdf_url` / `video_url` content parts | REMOVE | No public equivalent on the Agent API. | +| `video_url` content part | REMOVE | No public equivalent on the Agent API. | + +### Document input + +Put document parts inside a user message with an explicit `type: "message"`: + +```json +{ + "model": "openai/gpt-6-luna", + "input": [{ + "type": "message", + "role": "user", + "content": [ + {"type": "input_text", "text": "Summarize this document."}, + {"type": "input_file", "file_url": "https://example.com/report.pdf"} + ] + }] +} +``` + +For inline documents, replace `file_url` with base64 `file_data` and optionally `filename`. +Do not use `file_id`. Native input supports PDF, DOC, DOCX, TXT, and RTF, subject to model/provider restrictions. +The limit is 30 active documents and 50 MiB of combined decoded inline bytes, including inherited documents - not Sonar's per-file allowance. +Use `previous_response_id` to reuse inline documents without resending them; browser tools cannot be combined with document inputs or document continuations. +See [Working with Files](https://docs.perplexity.ai/docs/agent-api/working-with-files) and the [request schema](https://docs.perplexity.ai/api-reference/agent-post) before migrating attachments. ### Related-questions recipe (`return_related_questions` replacement) @@ -96,7 +128,7 @@ Only remove the feature if none of these fit, and call the removal out in the mi | Limit | Value / behavior | |---|---| -| Request body | <= 32 MiB, else 413 `payload_too_large` | +| Document request body | <= 72 MiB including base64, prompts, and other fields; decoded inline documents have a separate 50 MiB combined limit | | `search_domain_filter` | <= 20 entries, scheme-less, <= 253 chars each | | `models` fallback chain | <= 5 | | `metadata` | <= 16 keys; key <= 64; value <= 512 | diff --git a/skills/migrate-sonar-to-agent-api/references/response-and-streaming.md b/skills/migrate-sonar-to-agent-api/references/response-and-streaming.md index f09aa13..25c9087 100644 --- a/skills/migrate-sonar-to-agent-api/references/response-and-streaming.md +++ b/skills/migrate-sonar-to-agent-api/references/response-and-streaming.md @@ -15,7 +15,8 @@ Read this when rewriting response handling or a streaming consumer. | `usage.total_tokens` | `usage.total_tokens` | | `usage.cost.{input_tokens_cost, output_tokens_cost, request_cost, total_cost}` | `usage.cost.{input_cost, output_cost, tool_calls_cost, total_cost, currency:"USD", tool_calls_cost_details}` | | (search billing) | `usage.tool_calls_details` and `cost.tool_calls_cost_details` are keyed by billing tool names that differ from the request-side tool types: `search_web` (not web_search), `search_people` (not people_search), `fetch_url`, `finance_search`, `sandbox`. | -| `images`, `videos`, `related_questions` | GONE. Workaround: structured outputs plus prompting; for `related_questions` use the lightweight recipe in [request-mapping.md](request-mapping.md). | +| `images` | Enable `image_search` and read the `output[]` item with `type:"image_search_results"`. Its `results[]` entries contain `image_url` and `origin_url`; see [request-mapping.md](request-mapping.md). | +| `videos`, `related_questions` | No matching top-level fields. For `related_questions`, use the lightweight prompting/structured-output recipe in [request-mapping.md](request-mapping.md); video results have no public equivalent. | | `id` | `id` (`resp_...`), plus `object:"response"`, `created_at`, `completed_at`, `status`. | | `finish_reason` | No direct field. Check `response.status` (below) and `incomplete_details.reason` (`"max_output_tokens"` ~ finish_reason `"length"`). | @@ -112,7 +113,7 @@ Envelope: `{"error": {"message": str, "type": str, "code": int, "param"?: str}}` | 400 | `invalid_request`, `invalid_model`, `invalid_parameter`, `invalid_date_format`, `invalid_search_domain_filter`, ... | Unknown field (with `param`), bad model slug, bad filter, missing `max_output_tokens` on `anthropic/*`, reserved function name, expired reconnect cursor. | | 401/403 | `forbidden` | Bad or blocked key. | | 404 | `not_found` | Unknown response id (also returned for other accounts' ids and `store:false` responses). | -| 413 | `payload_too_large` | Body > 32 MiB. | +| 413 | `payload_too_large` | Request body exceeds the applicable limit; document requests allow up to 72 MiB with a separate 50 MiB decoded-inline-document limit. See [request limits](request-mapping.md#limits). | | 424 | `external_connector_error` | MCP server unreachable. | | 429 | `request_rate_limit_exceeded` | Rate limit; retry with backoff; `X-RateLimit-*` headers present. | | 5xx | `internal_error`, `bad_gateway`, `upstream_structured_output_failed`, `gateway_timeout` | Retry-worthy. | From 3027bc48065a1b94a6dffd1b8a0ae96871e46c87 Mon Sep 17 00:00:00 2001 From: Saleh Alghusson Date: Wed, 14 Oct 2026 18:50:29 +0000 Subject: [PATCH 2/5] Require API keys for model catalog checks and flag image params --- skills/migrate-sonar-to-agent-api/references/verification.md | 2 +- skills/migrate-sonar-to-agent-api/scripts/scan_sonar_usage.py | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/skills/migrate-sonar-to-agent-api/references/verification.md b/skills/migrate-sonar-to-agent-api/references/verification.md index 853d97a..e9d1400 100644 --- a/skills/migrate-sonar-to-agent-api/references/verification.md +++ b/skills/migrate-sonar-to-agent-api/references/verification.md @@ -48,4 +48,4 @@ For quality-sensitive flows, A/B compare a sample of real prompts before cutover Cross-check the live documentation - it is canonical for API facts and may be newer than this skill: the migration guide at https://docs.perplexity.ai/docs/agent-api/migrate-from-sonar, and the Agent API section around it (models, tools/web-search, output-control). Via the `perplexity-docs` MCP server, search with compact queries (2-6 terms, e.g. "web_search filters recency"); without MCP, fetch the docs pages directly. -Model and preset catalogs drift fastest - re-verify slugs against GET https://api.perplexity.ai/v1/models (no auth) rather than trusting any table, including this skill's. +Model and preset catalogs drift fastest - re-verify slugs against GET https://api.perplexity.ai/v1/models with your API key rather than trusting any table, including this skill's. diff --git a/skills/migrate-sonar-to-agent-api/scripts/scan_sonar_usage.py b/skills/migrate-sonar-to-agent-api/scripts/scan_sonar_usage.py index 79e3d84..94fd222 100644 --- a/skills/migrate-sonar-to-agent-api/scripts/scan_sonar_usage.py +++ b/skills/migrate-sonar-to-agent-api/scripts/scan_sonar_usage.py @@ -63,7 +63,8 @@ r"|search_after_date_filter|search_before_date_filter" r"|last_updated_(?:after|before)_filter|web_search_options" r"|search_mode|disable_search|enable_search_classifier" - r"|return_images|return_related_questions|num_search_results" + r"|return_images|image_domain_filter|image_format_filter|num_images" + r"|return_related_questions|num_search_results|pdf_url" r"|presence_penalty|frequency_penalty|top_k|max_tokens)\b"), "Sonar request param"), ("param", From 356ae0748d316ef58692a1cb096abcce91007049 Mon Sep 17 00:00:00 2001 From: Saleh Alghusson Date: Wed, 14 Oct 2026 18:55:43 +0000 Subject: [PATCH 3/5] Allow web_search token budgets in Sonar leftover checks --- evals/migrate-sonar-to-agent-api/check.sh | 30 +++++++++++++++++++ skills/migrate-sonar-to-agent-api/SKILL.md | 2 +- .../references/verification.md | 2 +- .../scripts/scan_sonar_usage.py | 18 +++++++++++ 4 files changed, 50 insertions(+), 2 deletions(-) diff --git a/evals/migrate-sonar-to-agent-api/check.sh b/evals/migrate-sonar-to-agent-api/check.sh index 7836ba1..be47e44 100755 --- a/evals/migrate-sonar-to-agent-api/check.sh +++ b/evals/migrate-sonar-to-agent-api/check.sh @@ -41,7 +41,37 @@ ABSENT=( "\\[[\"']citations[\"']\\]|\\.citations\\b|get\\([\"']citations[\"']" "[\"']sonar(-[a-z]+)*[\"']" ) +# web_search tool budgets (max_tokens next to web_search or max_tokens_per_page) +# are valid Agent API fields; only request-level max_tokens is a leftover. +request_max_tokens() { + hit "(^|[^_a-zA-Z])max_tokens\\b" | "$PY" -c ' +import re, sys +from collections import defaultdict +budget = re.compile(r"[\"\x27]web_search[\"\x27]|\bmax_tokens_per_page\b") +hits = defaultdict(list) +for row in sys.stdin: + path, num, _ = row.split(":", 2) + hits[path].append(int(num)) +for path, nums in hits.items(): + lines = open(path, encoding="utf-8", errors="ignore").read().splitlines() + for n in nums: + window = lines[max(0, n - 5):n] + if not any(budget.search(line) for line in window): + print(f"{path}:{n}:{lines[n - 1]}") +' +} + for pat in "${ABSENT[@]}"; do + if [ "$pat" = "(^|[^_a-zA-Z])max_tokens\\b" ]; then + if [ -n "$(request_max_tokens)" ]; then + echo "FAIL leftover: $pat" + request_max_tokens | head -3 + fail=1 + else + echo " ok: no $pat" + fi + continue + fi if hit "$pat" >/dev/null; then echo "FAIL leftover: $pat" hit "$pat" | head -3 diff --git a/skills/migrate-sonar-to-agent-api/SKILL.md b/skills/migrate-sonar-to-agent-api/SKILL.md index 9feb91e..b1fb200 100644 --- a/skills/migrate-sonar-to-agent-api/SKILL.md +++ b/skills/migrate-sonar-to-agent-api/SKILL.md @@ -136,7 +136,7 @@ If no workaround fits, remove the feature AND call the removal out in your migra - Compile or typecheck every changed file. - Run one real request per distinct call shape against production, with a small `max_output_tokens` and a cheap model (except on presets and reasoning models - see pitfall 7). - Capture real response JSON for any printed samples; never fabricate response JSON. -- Grep for leftovers: `chat.completions`, `choices[0]`, `max_tokens` (as a request field), top-level `search_recency_filter`, `citations`, `delta.content`, `sonar-` model slugs. +- Grep for leftovers: `chat.completions`, `choices[0]`, `max_tokens` (as a request field, not a `web_search` tool budget), top-level `search_recency_filter`, `citations`, `delta.content`, `sonar-` model slugs. Every hit must be intentional legacy prose. - Full checklist in [verification.md](references/verification.md). diff --git a/skills/migrate-sonar-to-agent-api/references/verification.md b/skills/migrate-sonar-to-agent-api/references/verification.md index e9d1400..1f7c58d 100644 --- a/skills/migrate-sonar-to-agent-api/references/verification.md +++ b/skills/migrate-sonar-to-agent-api/references/verification.md @@ -30,7 +30,7 @@ Grep the final diff; every hit must be intentional legacy prose (e.g. "migrated - `chat.completions` - `choices[0]` -- `max_tokens` (as a request field; `max_output_tokens` is fine) +- `max_tokens` (as a request field; `max_output_tokens` and `tools[web_search].max_tokens` are fine) - `search_recency_filter` at the top level of a request body (inside `tools[web_search].filters` is fine) - `citations` (top-level response access) - `delta.content` diff --git a/skills/migrate-sonar-to-agent-api/scripts/scan_sonar_usage.py b/skills/migrate-sonar-to-agent-api/scripts/scan_sonar_usage.py index 94fd222..e33d17c 100644 --- a/skills/migrate-sonar-to-agent-api/scripts/scan_sonar_usage.py +++ b/skills/migrate-sonar-to-agent-api/scripts/scan_sonar_usage.py @@ -116,6 +116,21 @@ def _inside_tool_filters(lines, idx): return any(NESTED_FILTERS_RX.search(lines[j]) for j in range(lo, idx + 1)) +WEB_SEARCH_BUDGET_RX = re.compile(r"[\"']web_search[\"']|\bmax_tokens_per_page\b") + + +def _web_search_budget(lines, idx): + """True when a max_tokens hit on line idx (0-based) is a web_search tool + budget, i.e. web_search or max_tokens_per_page appears on that line or in + the 4 lines before it.""" + lo = max(0, idx - 4) + return any(WEB_SEARCH_BUDGET_RX.search(lines[j]) for j in range(lo, idx + 1)) + + +def _only_max_tokens(rx, line): + return {m.group(0) for m in rx.finditer(line)} == {"max_tokens"} + + def scan_lines(lines, loc_fmt="L{n}"): """Scan an iterable of lines; yield (category, location, snippet, label).""" lines = list(lines) @@ -131,6 +146,9 @@ def scan_lines(lines, loc_fmt="L{n}"): and not re.search(r"\bmax_tokens\b|\bweb_search_options\b|\bsearch_mode\b", line) and _inside_tool_filters(lines, n - 1)): continue + if (label == "Sonar request param" and _only_max_tokens(rx, line) + and _web_search_budget(lines, n - 1)): + continue seen.add((category, label)) findings.append((category, loc_fmt.format(n=n), snippet(line), label)) for category, rx, label in STREAM_PATTERNS: From 1d3ec0d30b2555a5f9d488aebe7a6227d3cbaae6 Mon Sep 17 00:00:00 2001 From: saleh alghusson Date: Wed, 14 Oct 2026 21:09:55 -0400 Subject: [PATCH 4/5] Distinguish nested search budgets from request token limits --- evals/migrate-sonar-to-agent-api/check.sh | 15 +++--- .../test_search_budgets.py | 52 +++++++++++++++++++ .../scripts/scan_sonar_usage.py | 44 ++++++++++++---- 3 files changed, 94 insertions(+), 17 deletions(-) create mode 100644 evals/migrate-sonar-to-agent-api/test_search_budgets.py diff --git a/evals/migrate-sonar-to-agent-api/check.sh b/evals/migrate-sonar-to-agent-api/check.sh index be47e44..8041407 100755 --- a/evals/migrate-sonar-to-agent-api/check.sh +++ b/evals/migrate-sonar-to-agent-api/check.sh @@ -41,24 +41,25 @@ ABSENT=( "\\[[\"']citations[\"']\\]|\\.citations\\b|get\\([\"']citations[\"']" "[\"']sonar(-[a-z]+)*[\"']" ) -# web_search tool budgets (max_tokens next to web_search or max_tokens_per_page) -# are valid Agent API fields; only request-level max_tokens is a leftover. +# Only max_tokens belonging to the web_search object is a valid tool budget. request_max_tokens() { hit "(^|[^_a-zA-Z])max_tokens\\b" | "$PY" -c ' -import re, sys +import importlib.util, sys from collections import defaultdict -budget = re.compile(r"[\"\x27]web_search[\"\x27]|\bmax_tokens_per_page\b") +spec = importlib.util.spec_from_file_location("scanner", sys.argv[1]) +scanner = importlib.util.module_from_spec(spec) +spec.loader.exec_module(scanner) hits = defaultdict(list) for row in sys.stdin: path, num, _ = row.split(":", 2) hits[path].append(int(num)) for path, nums in hits.items(): lines = open(path, encoding="utf-8", errors="ignore").read().splitlines() + budget_lines = scanner.web_search_budget_lines(lines) for n in nums: - window = lines[max(0, n - 5):n] - if not any(budget.search(line) for line in window): + if n not in budget_lines: print(f"{path}:{n}:{lines[n - 1]}") -' +' "$(dirname "$0")/../../skills/migrate-sonar-to-agent-api/scripts/scan_sonar_usage.py" } for pat in "${ABSENT[@]}"; do diff --git a/evals/migrate-sonar-to-agent-api/test_search_budgets.py b/evals/migrate-sonar-to-agent-api/test_search_budgets.py new file mode 100644 index 0000000..6d67ac4 --- /dev/null +++ b/evals/migrate-sonar-to-agent-api/test_search_budgets.py @@ -0,0 +1,52 @@ +import importlib.util +from pathlib import Path +import subprocess +import tempfile +import unittest + + +ROOT = Path(__file__).resolve().parents[2] +spec = importlib.util.spec_from_file_location( + "scanner", ROOT / "skills/migrate-sonar-to-agent-api/scripts/scan_sonar_usage.py" +) +scanner = importlib.util.module_from_spec(spec) +spec.loader.exec_module(scanner) + + +class SearchBudgetTests(unittest.TestCase): + def test_only_nested_tool_budgets_are_exempt(self): + cases = [ + ('tools=[{"type": "web_search", "max_tokens": 2000}]', False), + ('tools=[{\n"max_tokens": 2000,\n"type": "web_search"\n}]', False), + ('responses.create(\n tools=[{"type": "web_search"}],\n max_tokens=100,\n)', True), + ('payload={"tools": [{"type": "web_search"}], "max_tokens": 100}', True), + ('payload={"max_tokens_per_page": 2000, "max_tokens": 100}', True), + ('tools=[{"type": "web_search", "max_tokens": 2000}]; max_tokens=100', True), + ('tools=[{"type": "web_search"}, {"type": "other", "max_tokens": 100}]', True), + ] + for source, expected in cases: + with self.subTest(source=source): + findings, _ = scanner.scan_lines(source.splitlines()) + self.assertEqual(any(row[3] == "Sonar request param" for row in findings), expected) + + def test_grader_rejects_request_budget_next_to_tool(self): + with tempfile.TemporaryDirectory() as directory: + path = Path(directory) / "app.py" + source = '''response = client.responses.create( + tools=[{"type": "web_search", "max_tokens": 2000}], + max_output_tokens=100, +) +search_results = response.status +''' + path.write_text(source) + grader = Path(__file__).with_name("check.sh") + accepted = subprocess.run([str(grader), directory, "--style", "sdk"], capture_output=True, text=True) + self.assertEqual(accepted.returncode, 0, accepted.stdout + accepted.stderr) + path.write_text(source.replace("max_output_tokens=100,", "max_output_tokens=100, max_tokens=100,")) + rejected = subprocess.run([str(grader), directory, "--style", "sdk"], capture_output=True, text=True) + self.assertNotEqual(rejected.returncode, 0, rejected.stdout + rejected.stderr) + self.assertIn("FAIL leftover:", rejected.stdout) + + +if __name__ == "__main__": + unittest.main() diff --git a/skills/migrate-sonar-to-agent-api/scripts/scan_sonar_usage.py b/skills/migrate-sonar-to-agent-api/scripts/scan_sonar_usage.py index e33d17c..4eb9479 100644 --- a/skills/migrate-sonar-to-agent-api/scripts/scan_sonar_usage.py +++ b/skills/migrate-sonar-to-agent-api/scripts/scan_sonar_usage.py @@ -12,9 +12,12 @@ """ import argparse +import ast +import io import json import re import sys +import tokenize from pathlib import Path SKIP_DIRS = { @@ -116,15 +119,35 @@ def _inside_tool_filters(lines, idx): return any(NESTED_FILTERS_RX.search(lines[j]) for j in range(lo, idx + 1)) -WEB_SEARCH_BUDGET_RX = re.compile(r"[\"']web_search[\"']|\bmax_tokens_per_page\b") - - -def _web_search_budget(lines, idx): - """True when a max_tokens hit on line idx (0-based) is a web_search tool - budget, i.e. web_search or max_tokens_per_page appears on that line or in - the 4 lines before it.""" - lo = max(0, idx - 4) - return any(WEB_SEARCH_BUDGET_RX.search(lines[j]) for j in range(lo, idx + 1)) +def web_search_budget_lines(lines): + """Only exempt budgets belonging to the same object as type: web_search.""" + try: + tokens = list(tokenize.generate_tokens(io.StringIO("\n".join(lines)).readline)) + except (tokenize.TokenError, IndentationError, SyntaxError): + return set() + stack = [] + tools = set() + budgets = {} + for i, token in enumerate(tokens): + value = token.string + if value in ("{", "[", "("): + stack.append((value, i)) + elif value in ("}", "]", ")"): + if stack: + stack.pop() + elif value.strip("\"'") == "max_tokens": + owner = stack[-1][1] if stack and stack[-1][0] == "{" else None + if i + 1 >= len(tokens) or tokens[i + 1].string != ":": + owner = None + budgets.setdefault(token.start[0], []).append(owner) + elif token.type == tokenize.STRING and stack and stack[-1][0] == "{": + try: + if (ast.literal_eval(value) == "type" and tokens[i + 1].string == ":" + and ast.literal_eval(tokens[i + 2].string) == "web_search"): + tools.add(stack[-1][1]) + except (ValueError, SyntaxError, IndexError): + pass + return {line for line, owners in budgets.items() if all(owner in tools for owner in owners)} def _only_max_tokens(rx, line): @@ -134,6 +157,7 @@ def _only_max_tokens(rx, line): def scan_lines(lines, loc_fmt="L{n}"): """Scan an iterable of lines; yield (category, location, snippet, label).""" lines = list(lines) + budget_lines = web_search_budget_lines(lines) findings = [] stream_hits = [] for n, line in enumerate(lines, 1): @@ -147,7 +171,7 @@ def scan_lines(lines, loc_fmt="L{n}"): and _inside_tool_filters(lines, n - 1)): continue if (label == "Sonar request param" and _only_max_tokens(rx, line) - and _web_search_budget(lines, n - 1)): + and n in budget_lines): continue seen.add((category, label)) findings.append((category, loc_fmt.format(n=n), snippet(line), label)) From 9ca21c042aa05a2a945a979161bfce32e1dfefe3 Mon Sep 17 00:00:00 2001 From: saleh alghusson Date: Wed, 14 Oct 2026 21:11:24 -0400 Subject: [PATCH 5/5] Recognize unquoted search tool fields --- evals/migrate-sonar-to-agent-api/test_search_budgets.py | 1 + skills/migrate-sonar-to-agent-api/scripts/scan_sonar_usage.py | 4 ++-- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/evals/migrate-sonar-to-agent-api/test_search_budgets.py b/evals/migrate-sonar-to-agent-api/test_search_budgets.py index 6d67ac4..3569bb0 100644 --- a/evals/migrate-sonar-to-agent-api/test_search_budgets.py +++ b/evals/migrate-sonar-to-agent-api/test_search_budgets.py @@ -17,6 +17,7 @@ class SearchBudgetTests(unittest.TestCase): def test_only_nested_tool_budgets_are_exempt(self): cases = [ ('tools=[{"type": "web_search", "max_tokens": 2000}]', False), + ('const tools = [{type: "web_search", max_tokens: 2000}];', False), ('tools=[{\n"max_tokens": 2000,\n"type": "web_search"\n}]', False), ('responses.create(\n tools=[{"type": "web_search"}],\n max_tokens=100,\n)', True), ('payload={"tools": [{"type": "web_search"}], "max_tokens": 100}', True), diff --git a/skills/migrate-sonar-to-agent-api/scripts/scan_sonar_usage.py b/skills/migrate-sonar-to-agent-api/scripts/scan_sonar_usage.py index 4eb9479..d92cc2a 100644 --- a/skills/migrate-sonar-to-agent-api/scripts/scan_sonar_usage.py +++ b/skills/migrate-sonar-to-agent-api/scripts/scan_sonar_usage.py @@ -140,9 +140,9 @@ def web_search_budget_lines(lines): if i + 1 >= len(tokens) or tokens[i + 1].string != ":": owner = None budgets.setdefault(token.start[0], []).append(owner) - elif token.type == tokenize.STRING and stack and stack[-1][0] == "{": + elif value.strip("\"'") == "type" and stack and stack[-1][0] == "{": try: - if (ast.literal_eval(value) == "type" and tokens[i + 1].string == ":" + if (tokens[i + 1].string == ":" and ast.literal_eval(tokens[i + 2].string) == "web_search"): tools.add(stack[-1][1]) except (ValueError, SyntaxError, IndexError):