From f077901e0e31c3f08415e4fc10be511f4993d82c Mon Sep 17 00:00:00 2001 From: Daniel Petty Date: Thu, 5 Nov 2026 15:44:06 +0000 Subject: [PATCH 01/10] docs(openspec): plan the tool-call metrics of the stage planner (#3240) --- .../.openspec.yaml | 2 + .../design.md | 31 +++++++ .../proposal.md | 69 +++++++++++++++ .../specs/metric-instruments/spec.md | 19 +++++ .../specs/project-metrics-dashboard/spec.md | 12 +++ .../specs/roadmap-service-alerts/spec.md | 35 ++++++++ .../specs/roadmap-stage/spec.md | 68 +++++++++++++++ .../tasks.md | 85 +++++++++++++++++++ 8 files changed, 321 insertions(+) create mode 100644 openspec/changes/3240-count-tool-calls-of-the-stage/.openspec.yaml create mode 100644 openspec/changes/3240-count-tool-calls-of-the-stage/design.md create mode 100644 openspec/changes/3240-count-tool-calls-of-the-stage/proposal.md create mode 100644 openspec/changes/3240-count-tool-calls-of-the-stage/specs/metric-instruments/spec.md create mode 100644 openspec/changes/3240-count-tool-calls-of-the-stage/specs/project-metrics-dashboard/spec.md create mode 100644 openspec/changes/3240-count-tool-calls-of-the-stage/specs/roadmap-service-alerts/spec.md create mode 100644 openspec/changes/3240-count-tool-calls-of-the-stage/specs/roadmap-stage/spec.md create mode 100644 openspec/changes/3240-count-tool-calls-of-the-stage/tasks.md diff --git a/openspec/changes/3240-count-tool-calls-of-the-stage/.openspec.yaml b/openspec/changes/3240-count-tool-calls-of-the-stage/.openspec.yaml new file mode 100644 index 000000000..1491b921e --- /dev/null +++ b/openspec/changes/3240-count-tool-calls-of-the-stage/.openspec.yaml @@ -0,0 +1,2 @@ +schema: spec-driven +created: 2026-11-05 diff --git a/openspec/changes/3240-count-tool-calls-of-the-stage/design.md b/openspec/changes/3240-count-tool-calls-of-the-stage/design.md new file mode 100644 index 000000000..7d34eef4b --- /dev/null +++ b/openspec/changes/3240-count-tool-calls-of-the-stage/design.md @@ -0,0 +1,31 @@ +# Design + +## Decisions + +- **Name the dimension `row`, not `job`.**: The collector sets the Prometheus label `job` from `service.name` after it copies the data point attributes, so an attribute `job` is lost with no log (Feasibility 3). `devops_cli_github_requests_total` already uses `row` for the job row that did the work. The value is `stage`. A check in `tests/telemetry/test_telemetry_instruments.py` fails on an `emit()` that sends the attribute `job` or `instance`, so the clash cannot come back. +- **Read the outcome and the size of each call from the messages of the run.**: `RunTrace.absorb` already reads the messages after the run, in the `finally` of `investigate`, also when the run stops. A `ToolReturnPart` holds the answer after the cut of its tool and its `outcome`. A `RetryPromptPart` holds a rejected call. A call with no answer is an error. This needs no wrapper of the tool functions and no hook, and it covers a call that failed in argument validation, which a hook on the execution does not see. +- **Keep the outcome and the size in `ToolCallRecord`.**: One list holds each call, in order. The comment line, the counter and the histograms all read it, so the three agree. `RunTrace.combined` joins the lists of two runs, as it does today. +- **An answer is `empty` when its text holds no non-whitespace character.**: It matches the words "return nothing" of the issue and a test can check it offline. A text such as "No match." is `ok`: the tool gave an answer that says why it found nothing. +- **Send the counts when a run ends, and the zero increments when it starts.**: A process sends its deltas when it flushes, so the order inside a run does not change the series. The start-of-run increments add 0 to `outcome="ok"` for every tool of the toolset, so each tool has a series before the run calls any. The end-of-run code adds 1 for each call, and observes the size of each answer, the calls of the run, and the limit. +- **Observe the calls of a run only when the run got a reply.**: A run whose first model request failed made no plan run. If it counted as a run, a model that is down for a week would make the alert fire for every tool. `trace.requests` is 0 for such a run. +- **Name the limit in `investigate`, once for each run.**: The hook that ends the tools knows which limit it hit. `_limit_reached` returns `calls`, `requests` or `window`, and `_offering` keeps the first one. A hard stop by `UsageLimitExceeded` has no hook call, so its limit comes from the count of replies against `request_limit`. The message of the error is not read, as `_stopped` in `stage_toolbox.py` does not read it. +- **Put the alert in a new group `roadmap-service-tools`.**: The plan of #2950 changes the requirement of the group `roadmap-service-activity` with `MODIFIED`. Two changes never modify one requirement, and a tool alert is no Service activity. A group of its own adds a requirement and modifies none, as `roadmap-service-items` did (#2919). +- **The alert groups by `row`, `tool` and `toolset`, and reads no `row` filter.**: The implement job can send the same series later with `row="implement"`. The `on (row, toolset)` join makes the rule count the runs of the same row and toolset. A tool of `extended` is not "unused" in a week when only `base` ran, because no run of `extended` happened. +- **The alert reads a series that starts in the window by its last value.**: A new series gets its first value in the window, and `increase()` needs two samples. The form `(last_over_time(x[7d]) unless last_over_time(x[7d] offset 7d)) or increase(x[7d])` of the other roadmap rules counts that first value (#2505). The same form reads the native histogram of the runs, and `histogram_count` gives the number of runs. +- **A tool that leaves the registry keeps its alert until its series leaves the window.**: Its series has no new sample, so for up to 7 days its sum stays 0 while runs happen. A person who removes a tool knows the alert follows. The rule has no list of tool names, so it needs no edit for a new tool. A list would drift from the registry (rejected below). +- **Put the panel at `x` 12 and `y` 34 in `project-metrics.json`, 12 wide and 8 high.**: It sits in the row "Release & Taxonomy Tracking", next to "Oldest Item Age by State". The panels below it keep their places. The panel has two queries: the calls by tool and outcome on the left axis, and the mean answer size by tool on the right axis. Both filter `job="roadmap-service"` and `row="stage"`. +- **Use the unit annotations `{char}` and `{call}` for the two histograms.**: The collector uses `UnderscoreEscapingWithoutSuffixes`, so the unit does not change a name. The UCUM annotation says what the number counts. + +## Rejected alternatives + +- **Wrap each tool function, or add a pydantic-ai `Hooks` capability, to tally each result.**: The refined design of the issue chose a wrapper. The messages already hold each answer with its `tool_call_id`, and `RunTrace` reads them. A wrapper of 12 tool methods must keep each signature and docstring, because pydantic-ai builds the tool schema from them. A hook on the execution sees no call that fails in validation. +- **Parse the text of the `RunTrace` comment after the run.**: It holds no outcome and no answer size, and the text is for a person. +- **Query Prometheus from the comment path.**: The comment must not depend on the metrics backend. +- **A label `job` on the instruments.**: See the first decision. The value would never reach Prometheus. +- **One generic counter with a free-form label.**: A free-form label is unbounded and cannot support the panel or the alert. Each label here has a closed set: the tools of the registry, `stage`, `base` and `extended`, three outcomes and three limits. +- **A static `absent_over_time` list of tool names in the rule.**: The list drifts from the registry and needs one expression for each tool. +- **A separate gauge for the tools that a toolset offers.**: It adds a fifth instrument that the issue does not ask for. +- **A new Roadmap dashboard file.**: No such file exists, and a new dashboard needs its own provisioning. +- **The raw return before the cut, or the tokens of the answer.**: The model reads the text after the cut, and no token counter exists for a tool return. +- **A threshold for `empty` such as an answer under 20 characters.**: A short answer can be valid, and the threshold is arbitrary. +- **Put the alert in the group `roadmap-service-activity`.**: It forces a `MODIFIED` of a requirement that #2950 modifies. diff --git a/openspec/changes/3240-count-tool-calls-of-the-stage/proposal.md b/openspec/changes/3240-count-tool-calls-of-the-stage/proposal.md new file mode 100644 index 000000000..982dfed37 --- /dev/null +++ b/openspec/changes/3240-count-tool-calls-of-the-stage/proposal.md @@ -0,0 +1,69 @@ +# Proposal + +## Why + +Issue #3240: a person who tunes the toolset of the stage planner cannot see which tools a plan run calls. The `RunTrace` comment on an item lists the calls of one run as text. No metric exists, so no trend shows across runs. The alternatives check ran 34 times and searched about 4 times, and nothing showed it (#2950). The same blind spot can hide a tool that the planner never calls. This change counts each tool call by tool and outcome, measures the size of each answer, and warns when a tool of the toolset has no call for 7 days. + +## What Changes + +- `src/devops_cli/telemetry/instruments.py` declares four instruments: the counters `devops_cli_roadmap_tool_calls_total` and `devops_cli_roadmap_tool_limit_hits_total`, and the histograms `devops_cli_roadmap_tool_answer_chars` and `devops_cli_roadmap_tool_calls_per_run`. The labels are `tool`, `row`, `toolset`, `outcome` and `limit`. The issue names the label `row` as `job`: see Feasibility 3. +- `RunTrace` in `src/devops_cli/roadmap/agent_run.py` records the outcome (`ok`, `empty` or `error`) and the answer size of each tool call, read from the `ToolReturnPart` of the run messages, and the limit that ended the tools of the run. +- `src/devops_cli/roadmap/tool_metrics.py` is new. It sends the zero increments at the start of a run and the counts of a run at its end, through `emit`. +- `ToolboxClient` in `src/devops_cli/roadmap/stage_toolbox.py` calls both functions around each plan run. +- `investigate` in `src/devops_cli/roadmap/refine_investigation.py` names the limit that ends the tools of a run: `calls`, `requests` or `window`. +- `_run_report` in `src/devops_cli/roadmap/stage.py` adds one line to the comment of a staged item: the calls and the empty answers of each tool. +- `k8s/monitoring/dashboards/project-metrics.json` gets one panel, "Planner Tool Calls and Answer Size", next to "Oldest Item Age by State". +- `k8s/monitoring/prometheus-values.yaml` gets the group `roadmap-service-tools` with the alert `RoadmapServiceToolUnused`. A `promtool` unit test covers it. The pins in `tests/prometheus/test_k8s_monitoring_alerting.py` and the section "Roadmap service" of `k8s/README.md` name it. +- `tests/telemetry/test_telemetry_instruments.py` gets a check that no `emit()` call sends the attribute `job` or `instance`. +- `docs/TELEMETRY.md` is generated from `INSTRUMENTS`, and says that the resource sets `job` and `instance`. +- The change has spec deltas: a metric, a comment line, a panel and an alert rule are observable (`openspec/config.yaml`). It fits one pull request. + +## Capabilities + +### New Capabilities + +None. + +### Modified Capabilities + +- `metric-instruments`: The tool-call instruments are declared and documented, No instrument attribute is named job or instance +- `roadmap-stage`: A plan run counts each tool call by outcome, A plan run measures the size of each answer, Each plan run gives every tool of its toolset a series, A plan run measures its calls and the limit that ended its tools, The comment of a staged item counts the calls of each tool +- `project-metrics-dashboard`: The items dashboard shows the tool calls of the planner +- `roadmap-service-alerts`: The planner tools have their own alert group + +## Impact + +- Code: `src/devops_cli/telemetry/instruments.py`, `src/devops_cli/roadmap/agent_run.py`, `src/devops_cli/roadmap/refine_investigation.py`, `src/devops_cli/roadmap/stage_toolbox.py`, `src/devops_cli/roadmap/tool_metrics.py`, `src/devops_cli/roadmap/stage.py`, `src/devops_cli/lang/en/messages.py` +- Manifests: `k8s/monitoring/dashboards/project-metrics.json`, `k8s/monitoring/prometheus-values.yaml` +- Tests: `tests/telemetry/test_telemetry_instruments.py`, `tests/roadmap/test_roadmap_stage_toolbox.py`, `tests/roadmap/test_roadmap_stage_apply.py`, `tests/roadmap/test_roadmap_refine_investigation.py`, `tests/prometheus/test_k8s_monitoring_alerting.py`, `tests/fixtures/prometheus_rules/roadmap_service_tools_tests.yaml` +- Documents: `docs/TELEMETRY.md` (generated), `k8s/README.md` +- Deleted code: `_within_budget`, `_within_window_of` and `_recording` in `refine_investigation.py`. `_limit_reached` and `_offering` replace them. + +## Dependencies + +- #3238 (merged) added the toolset `extended` and the label value `toolset`. Nothing to wait for. +- #2919 (merged) put the age panel in `project-metrics.json`. The free right half of its row holds the new panel. +- #2950 (open plan) adds three alternatives instruments, one alert in the group `roadmap-service-activity` and a panel row. Both plans edit `INSTRUMENTS` and `__all__`, the table in `docs/TELEMETRY.md`, `SERVED_SERIES`, the baseline rule list in `tests/prometheus/test_k8s_monitoring_alerting.py` and `k8s/README.md`. The change that merges second merges `main` and keeps both entries. This change adds its alert to a new group, so no requirement is in both plans. +- #3069 (open) may create a dashboard of its own. This change does not wait for it: the panel moves with a one-line edit of the file name. + +## Feasibility + +Checked on `origin/main` 2db71f29. Each premise has its output. + +1. The module declares and sends instruments as the issue says. `git grep -n "ROADMAP_STAGE_TOTAL" src` shows the instrument, its entry in `INSTRUMENTS` and its `emit` in `stage.py`. `git grep -n "devops_cli_roadmap_tool" -- . ':!openspec'` prints nothing: the four names are free. `docs/TELEMETRY.md` is generated by `generate_telemetry_docs` in `src/devops_cli/docs/generator.py` from `INSTRUMENTS`. +2. The issue asks for a panel in "the Roadmap dashboard". `ls k8s/monitoring/dashboards` lists 15 files, and none is a Roadmap dashboard. `project-metrics.json` holds the items panels. Its panel "Oldest Item Age by State" has `gridPos` `{"h": 8, "w": 12, "x": 0, "y": 34}`, so the right half at `x` 12 is free. The panel goes there. +3. The label `job` of the issue cannot work. The OTLP resource sets `service.name` as the Prometheus label `job`. `createAttributes` in `pkg/translator/prometheusremotewrite/helper.go` of opentelemetry-collector-contrib copies the data point attributes into a map first. It then runs `l[model.JobLabel] = val` for `service.name`, with no log. So a data point attribute named `job` is lost, and `job="stage"` would match nothing. The same holds for `instance`. `devops_cli_github_requests_total` names the same idea `row` (`src/devops_cli/github/request_scope.py`). The four instruments use `row` with the value `stage`, and the dashboard and the alert read `row`. +4. A zero increment reaches the collector. A test that called `increment_counter("devops_cli_roadmap_tool_calls_total", 0, attributes={"tool": "read_file"})` on the `otlp` fixture exported one sum data point with `asDouble` 0.0 and delta temporality. So the zero increment at the start of a run gives each tool a series. +5. pydantic-ai 2.59.0 already records each tool answer. `ToolReturnPart` holds `content` (the text that the model reads, after the cut of each tool), `tool_call_id` and `outcome`. A rejected call gives a `RetryPromptPart` with the same `tool_call_id`. `RunTrace.absorb` already reads the same messages. So the counts come from the messages, with no wrapper of the tool functions and no `Hooks` capability. A hook would miss a call that failed in argument validation. +6. The tools answer a failure as text, so an answer with text is `ok` even when it says "No match." The refined design of the issue defines `empty` as an answer with no non-whitespace character. An exception, a rejected call and a call that the run never answered are `error`. +7. The alert expression works in promtool 3.15.1, the version of `src/devops_cli/tools_lock/tools.lock` (`promtool --version`). A scratch rule over two counter series and one native histogram series, with the input `{{schema:0 sum:3 count:1 buckets:[1]}}`, passed `promtool check rules` and `promtool test rules`. The rule fired for the tool whose counter stayed at 0 and stayed quiet for the tool whose counter grew each hour. A first run of the same case, where the called tool had its last call 12 days before, correctly fired for both tools. +8. The limit hooks are in one module. `_within_budget`, `_within_window_of` and `_recording` in `refine_investigation.py` end the tools by `tool_calls_limit` (fewer than `CONST_ROADMAP_REFINE_INVESTIGATION_TURN_CALLS` calls left), by `request_limit` (one request left) or by the context window. Only `refine_investigation.py` reads them. `tests/roadmap/test_roadmap_stage_toolbox.py` already runs the three cases with `Script`. +9. A hard stop by `UsageLimitExceeded` can happen with no earlier cut: one reply with 201 calls does it (`test_one_reply_over_the_call_limit_fails_the_run_and_names_the_limit`). The message of that error is not read. The count of replies of the run against `request_limit` names the limit: `requests` when they are equal, else `calls`. + +Not verified: how a real Prometheus stores the zero sample of a new series from a Service pod, and the live values of the new series. No collector or cluster is reachable from this session. The promtool cases feed the label sets that `docs/TELEMETRY.md` describes. + +## Existing Tool Considered + +pydantic-ai, which the planner already uses, records every tool call and answer in the run messages. `RunTrace` reads them and no new library is needed. The OpenTelemetry SDK, through `emit()`, sends the values. Prometheus, `promtool` and Grafana store, test and show them. The alternatives check of 2026-11-05 found no open tool: agent instrumentation such as OpenLLMetry records tool calls as spans, and gives no `empty` outcome, no answer size after the cut, no calls per run and no limit hits as metrics. So this repository builds the four instruments, the tally and the rule. + +Cost to build: about 70,000 tokens and about 4 hours with the CI waits (seven groups, five of them with a failing test first). Cost to keep: about 3,000 tokens each quarter, when the toolset changes. Cost to fix one miss: about 8,000 tokens, with a failing test that names the label. The change deletes the three hook functions that `_limit_reached` and `_offering` replace. diff --git a/openspec/changes/3240-count-tool-calls-of-the-stage/specs/metric-instruments/spec.md b/openspec/changes/3240-count-tool-calls-of-the-stage/specs/metric-instruments/spec.md new file mode 100644 index 000000000..133e999ee --- /dev/null +++ b/openspec/changes/3240-count-tool-calls-of-the-stage/specs/metric-instruments/spec.md @@ -0,0 +1,19 @@ +## ADDED Requirements + +### Requirement: The tool-call instruments are declared and documented +`INSTRUMENTS` SHALL hold `devops_cli_roadmap_tool_calls_total` and `devops_cli_roadmap_tool_limit_hits_total` as counters with the unit `1`, `devops_cli_roadmap_tool_answer_chars` as a histogram with the unit `{char}`, and `devops_cli_roadmap_tool_calls_per_run` as a histogram with the unit `{call}`. `docs/TELEMETRY.md` SHALL list each one with its kind and its unit. + +#### Scenario: The registry and the document list the four instruments +- **WHEN** a test reads `INSTRUMENTS` and `docs/TELEMETRY.md` +- **THEN** both list the four names with the same kind and the same unit + +### Requirement: No instrument attribute is named job or instance +The resource of a process sets the Prometheus labels `job` and `instance`, and a data point attribute with either name is lost. So no `emit()` call SHALL send an attribute named `job` or `instance`. The instrument check SHALL name the file and line of each call that does. + +#### Scenario: The source tree keeps the rule +- **WHEN** the instrument check reads every module under `src/devops_cli/` +- **THEN** it finds no `emit()` call with the attribute `job` or `instance` + +#### Scenario: A module sends the attribute job +- **WHEN** a module calls `emit(COMMAND_TOTAL, 1.0, {"job": "stage"})` +- **THEN** the instrument check names that call, and it passes `emit(COMMAND_TOTAL, 1.0, {"row": "stage"})` diff --git a/openspec/changes/3240-count-tool-calls-of-the-stage/specs/project-metrics-dashboard/spec.md b/openspec/changes/3240-count-tool-calls-of-the-stage/specs/project-metrics-dashboard/spec.md new file mode 100644 index 000000000..f043e6177 --- /dev/null +++ b/openspec/changes/3240-count-tool-calls-of-the-stage/specs/project-metrics-dashboard/spec.md @@ -0,0 +1,12 @@ +## ADDED Requirements + +### Requirement: The items dashboard shows the tool calls of the planner +`k8s/monitoring/dashboards/project-metrics.json` SHALL hold one panel, "Planner Tool Calls and Answer Size". It SHALL chart the calls of each tool by outcome from `devops_cli_roadmap_tool_calls_total`, and the mean answer size of each tool from `devops_cli_roadmap_tool_answer_chars`. Each query SHALL select `job="roadmap-service"` and `row="stage"`. + +#### Scenario: The dashboard JSON holds the tool panel +- **WHEN** a test reads `k8s/monitoring/dashboards/project-metrics.json` +- **THEN** one panel has a query for `devops_cli_roadmap_tool_calls_total` and a query for `devops_cli_roadmap_tool_answer_chars`, each with `row="stage"` + +#### Scenario: The panel reads each series through the function of its kind +- **WHEN** the dashboard checks read the panel +- **THEN** the counter is the first argument of `increase(`, and the histogram is the first argument of `increase(` inside `histogram_avg(` diff --git a/openspec/changes/3240-count-tool-calls-of-the-stage/specs/roadmap-service-alerts/spec.md b/openspec/changes/3240-count-tool-calls-of-the-stage/specs/roadmap-service-alerts/spec.md new file mode 100644 index 000000000..3322b3860 --- /dev/null +++ b/openspec/changes/3240-count-tool-calls-of-the-stage/specs/roadmap-service-alerts/spec.md @@ -0,0 +1,35 @@ +## ADDED Requirements + +### Requirement: The planner tools have their own alert group +The rule group `roadmap-service-tools` in `k8s/monitoring/prometheus-values.yaml`, apart from the other groups of the Service, SHALL hold the warning alert `RoadmapServiceToolUnused`. Each `devops_cli_` selector of the group SHALL have `job="roadmap-service"`. Its description SHALL end with `(k8s/README.md, Roadmap service)`, and that section SHALL name the alert. promtool SHALL accept the rule. + +#### Scenario: The selectors of the group +- **WHEN** a test reads the selectors of the group +- **THEN** each `devops_cli_` selector has `job="roadmap-service"` + +#### Scenario: The README section +- **WHEN** a test reads the description of the alert and "Roadmap service" in `k8s/README.md` +- **THEN** the description points to the section, and the section names the alert + +### Requirement: A tool that no run called in 7 days warns +`RoadmapServiceToolUnused` SHALL fire, by `row`, `tool` and `toolset`, when the count of calls of the tool in 7 days is 0 and the Service made plan runs of the same `row` and `toolset` in the same 7 days. The count of calls SHALL add the first value of a series that starts in the 7 days. The summary SHALL name the tool and the toolset. + +#### Scenario: A tool is not called while runs happen +- **WHEN** `promtool test rules` evaluates a counter that stays at 0 for `repo_map`, a counter that grows for `read_file`, and a histogram of calls per run that grows, all of the same `toolset` +- **THEN** the alert fires for `repo_map` and not for `read_file` + +#### Scenario: No run happened +- **WHEN** `promtool test rules` evaluates a counter that stays at 0 for one tool and a histogram of calls per run that does not grow +- **THEN** the alert stays quiet + +#### Scenario: A series that starts in the window +- **WHEN** `promtool test rules` evaluates a counter whose first value is 3, 10 hours before the evaluation, and a histogram that grows +- **THEN** the alert stays quiet for that tool + +#### Scenario: A tool of another toolset +- **WHEN** `promtool test rules` evaluates a counter at 0 for a tool of the toolset `extended`, and runs of the toolset `base` only +- **THEN** the alert stays quiet for that tool + +#### Scenario: A series of another job +- **WHEN** `promtool test rules` evaluates the same series with `job="devops-cli"` +- **THEN** the alert stays quiet diff --git a/openspec/changes/3240-count-tool-calls-of-the-stage/specs/roadmap-stage/spec.md b/openspec/changes/3240-count-tool-calls-of-the-stage/specs/roadmap-stage/spec.md new file mode 100644 index 000000000..f7ef454fd --- /dev/null +++ b/openspec/changes/3240-count-tool-calls-of-the-stage/specs/roadmap-stage/spec.md @@ -0,0 +1,68 @@ +## ADDED Requirements + +### Requirement: A plan run counts each tool call by outcome +A plan run of the toolbox SHALL add 1 to `devops_cli_roadmap_tool_calls_total` for each tool call of the toolset. The labels SHALL be `tool`, `row` (`stage`), `toolset` (`base` or `extended`) and `outcome`. The `outcome` SHALL be `ok` for an answer with text, `empty` for an answer with no non-whitespace character, and `error` for a call that got no answer. + +#### Scenario: One call of each outcome +- **WHEN** a plan run calls `read_file` for a file with text, calls a tool whose answer is blank, and calls `read_file` with no `path` +- **THEN** the counter holds one point for each of the outcomes `ok`, `empty` and `error`, and each point has the labels `tool`, `row` `stage` and `toolset` + +#### Scenario: A call that no tool answers +- **WHEN** a tool call is rejected, a tool fails, the run no longer offers the tool, or a limit stops the call +- **THEN** the call counts with `outcome` `error` + +#### Scenario: A tool that the toolset does not offer +- **WHEN** the model calls a name that is no tool of the toolset +- **THEN** the counter holds no point with that name as `tool` + +### Requirement: A plan run measures the size of each answer +A plan run SHALL observe `devops_cli_roadmap_tool_answer_chars` once for each tool call that got an answer, with the labels `tool`, `row` and `toolset`. The value SHALL be the count of characters of the answer after the cut of the tool. A call with no answer SHALL add no observation. + +#### Scenario: A long answer is cut +- **WHEN** a plan run reads a file of 300 long lines and the tool cuts the answer +- **THEN** the histogram observes the length of the cut answer that the model read, and not the length of the file + +### Requirement: Each plan run gives every tool of its toolset a series +A plan run SHALL add 0 to `devops_cli_roadmap_tool_calls_total` with `outcome` `ok` for each tool of its toolset, before its first model request. + +#### Scenario: A run that calls no tool +- **WHEN** a plan run with the toolset `base` makes no tool call +- **THEN** the counter holds nine points with the value 0 and `outcome` `ok`, one for each tool + +### Requirement: A plan run measures its calls and the limit that ended its tools +A plan run that got at least one model reply SHALL observe `devops_cli_roadmap_tool_calls_per_run` once, with its count of tool calls and the labels `row` and `toolset`. A run that ends its tools on a limit SHALL add 1, once, to `devops_cli_roadmap_tool_limit_hits_total` with the labels `row`, `toolset` and `limit`. The `limit` SHALL be `calls`, `requests` or `window`. + +#### Scenario: The three limits +- **WHEN** a plan run ends its tools because fewer calls are left than a turn needs, because one request is left, or because the messages fill the context window +- **THEN** the limit counter holds one point with `limit` `calls`, `requests` or `window` in that order + +#### Scenario: A run stops at the call limit +- **WHEN** a plan run makes calls until the request that offers no tool because the call budget is spent +- **THEN** the histogram observes one run, and the limit counter holds one point with `limit` `calls` + +#### Scenario: A run fills the context window +- **WHEN** a plan run reads a large file in each turn, with a context window of 12000 tokens +- **THEN** the limit counter holds one point with `limit` `window` + +#### Scenario: One reply over the call limit +- **WHEN** one reply of the model holds one more parallel tool call than `CONST_ROADMAP_STAGE_TOOL_MAX_CALLS` +- **THEN** the run fails, and the limit counter holds one point with `limit` `calls` + +#### Scenario: A run with no model reply +- **WHEN** the first model request of a plan run fails +- **THEN** the histogram of calls per run holds no observation + +### Requirement: The comment of a staged item counts the calls of each tool +The comment of an item that the toolbox planned SHALL add one line after the list of tool calls. The line SHALL name each tool that the plan runs called, in alphabetical order, with its count of calls and its count of empty answers, as `Tool calls: read_file=3 (empty=0), search_code=2 (empty=1)`. The comment SHALL hold no such line when the plan runs made no tool call. + +#### Scenario: Two tools and one empty answer +- **WHEN** a plan run calls `read_file` twice and `list_files` once, and one `list_files` answer is blank +- **THEN** the comment holds the line `Tool calls: list_files=1 (empty=1), read_file=2 (empty=0)` + +#### Scenario: An item with two plan runs +- **WHEN** the first plan of an item is refused and each of the two plan runs calls `read_file` once +- **THEN** the line of the comment holds `read_file=2 (empty=0)` + +#### Scenario: A run that made no tool call +- **WHEN** a plan run makes no tool call +- **THEN** the comment holds no line that starts with `Tool calls:` diff --git a/openspec/changes/3240-count-tool-calls-of-the-stage/tasks.md b/openspec/changes/3240-count-tool-calls-of-the-stage/tasks.md new file mode 100644 index 000000000..dd2969389 --- /dev/null +++ b/openspec/changes/3240-count-tool-calls-of-the-stage/tasks.md @@ -0,0 +1,85 @@ +# Tasks + +## 1. Declare the four tool-call instruments + +Reads: `src/devops_cli/telemetry/instruments.py`, `tests/telemetry/test_telemetry_instruments.py`. Changes: `src/devops_cli/telemetry/instruments.py`, `tests/telemetry/test_telemetry_instruments.py`. + +- [ ] 1.1 Add a test to `tests/telemetry/test_telemetry_instruments.py` that finds `devops_cli_roadmap_tool_calls_total` and `devops_cli_roadmap_tool_limit_hits_total` as counters with the unit `1`, `devops_cli_roadmap_tool_answer_chars` as a histogram with the unit `{char}`, and `devops_cli_roadmap_tool_calls_per_run` as a histogram with the unit `{call}`, in `INSTRUMENTS` + - Verification: `uv run devops ci test tests/telemetry/test_telemetry_instruments.py` fails on the base +- [ ] 1.2 Add `ROADMAP_TOOL_CALLS_TOTAL`, `ROADMAP_TOOL_ANSWER_CHARS`, `ROADMAP_TOOL_CALLS_PER_RUN` and `ROADMAP_TOOL_LIMIT_HITS_TOTAL` to `src/devops_cli/telemetry/instruments.py` after `ROADMAP_STAGE_TOTAL`, each with a description in the style of its neighbors. Add the four names to `INSTRUMENTS` and `__all__` + - Verification: `uv run devops ci test tests/telemetry/test_telemetry_instruments.py` passes + +## 2. Keep the labels job and instance out of emit + +Reads: `tests/telemetry/test_telemetry_instruments.py`. Changes: `tests/telemetry/test_telemetry_instruments.py`. + +- [ ] 2.1 Add a check to `tests/telemetry/test_telemetry_instruments.py`, beside `test_every_metric_is_a_declared_instrument_sent_through_emit`, that names each `emit()` call under `src/devops_cli/` whose attribute dict has the key `job` or `instance`. Add a test of the check on a module text: `emit(COMMAND_TOTAL, 1.0, {"job": "stage"})` is named, and `emit(COMMAND_TOTAL, 1.0, {"row": "stage"})` is not + - Verification: `uv run devops ci test tests/telemetry/test_telemetry_instruments.py` + +## 3. Record the outcome, the size and the limit of each tool call in the trace + +Reads: `src/devops_cli/roadmap/agent_run.py`, `src/devops_cli/roadmap/refine_investigation.py`, `tests/roadmap/test_roadmap_stage_toolbox.py`, `tests/support/scripted_model.py`. Changes: `src/devops_cli/roadmap/agent_run.py`, `src/devops_cli/roadmap/refine_investigation.py`, `tests/roadmap/test_roadmap_stage_toolbox.py`. + +- [ ] 3.1 Add tests to `tests/roadmap/test_roadmap_stage_toolbox.py` for the records of `client.traces`: a call with text is `ok` with the length of its answer, a blank answer is `empty`, a call with no `path` is `error`, and a call after the tools were removed is `error`. A tool answer of 300 long lines gives the length of the cut answer that the model read. The run that spends its call budget has the limit `calls`, the run that fills a window of 12000 tokens has `window`, the run that spends its requests has `requests`, one reply over the call limit has `calls`, and a run that stops on no limit has none. `RunTrace.combined` keeps the last limit + - Verification: `uv run devops ci test tests/roadmap/test_roadmap_stage_toolbox.py` fails on the base +- [ ] 3.2 In `src/devops_cli/roadmap/agent_run.py`, add `ToolOutcome` and `RunLimit` as string enums. Add `outcome` and `chars` to `ToolCallRecord` and `limit` to `RunTrace`. Make `RunTrace.absorb` read the answer of each call from the `ToolReturnPart` or the `RetryPromptPart` with the same `tool_call_id`, and take a `limit`. Add `RunTrace.tool_counts`. In `src/devops_cli/roadmap/refine_investigation.py`, replace `_within_budget`, `_within_window_of` and `_recording` with `_limit_reached` and `_offering`, keep the first limit of the run, name the limit of a `UsageLimitExceeded` from the count of replies, and pass it to `trace.absorb` + - Verification: `uv run devops ci test tests/roadmap/test_roadmap_stage_toolbox.py` passes +- [ ] 3.3 Run the other tests of the investigation, which share the hooks + - Verification: `uv run devops ci test tests/roadmap/test_roadmap_refine_investigation.py` + +## 4. Send the counts of each plan run + +Reads: `src/devops_cli/roadmap/stage_toolbox.py`, `src/devops_cli/telemetry/instruments.py`, `tests/support/fakes/otlp.py`, `tests/roadmap/test_roadmap_stage_toolbox.py`. Changes: `src/devops_cli/roadmap/tool_metrics.py`, `src/devops_cli/roadmap/stage_toolbox.py`, `tests/roadmap/test_roadmap_stage_toolbox.py`. + +- [ ] 4.1 Add tests with the `otlp` fixture to `tests/roadmap/test_roadmap_stage_toolbox.py`: a run that calls `read_file` with text, a blank tool and `read_file` with no `path` gives one point of the counter for each outcome with the labels `tool`, `row`, `toolset` and `outcome`; a run with the toolset `base` that makes no call gives nine zero points; the histogram of answers holds the cut length; a run that spends its call budget gives one observation of calls per run and one limit point `calls`; a run whose first request fails observes no calls per run; every label value is in its closed set + - Verification: `uv run devops ci test tests/roadmap/test_roadmap_stage_toolbox.py` fails on the base +- [ ] 4.2 Add `src/devops_cli/roadmap/tool_metrics.py` with `start_run`, which adds 0 to the counter for each tool of the toolset, and `end_run`, which sends the counter, the answer histogram, the calls per run and the limit of a `RunTrace`. Call them in `ToolboxClient.chat_structured` before and after the run, with `row` `stage` and the toolset of the client + - Verification: `uv run devops ci test tests/roadmap/test_roadmap_stage_toolbox.py` passes + +## 5. Count the calls of each tool in the comment + +Reads: `src/devops_cli/roadmap/stage.py`, `src/devops_cli/lang/en/messages.py`, `tests/roadmap/test_roadmap_stage_apply.py`. Changes: `src/devops_cli/roadmap/stage.py`, `src/devops_cli/lang/en/messages.py`, `tests/roadmap/test_roadmap_stage_apply.py`. + +- [ ] 5.1 Add tests to `tests/roadmap/test_roadmap_stage_apply.py`: the comment of a run with two tools holds `Tool calls: list_files=1 (empty=0), read_file=2 (empty=0)`; the comment of an item with two plan runs sums the counts; the comment of a run with no tool call holds no line `Tool calls:` + - Verification: `uv run devops ci test tests/roadmap/test_roadmap_stage_apply.py` fails on the base +- [ ] 5.2 Add `stage_comment_tool_counts` and `stage_comment_tool_count` to `src/devops_cli/lang/en/messages.py`. In `_run_report` of `src/devops_cli/roadmap/stage.py`, append the line from `RunTrace.tool_counts` after the list of calls + - Verification: `uv run devops ci test tests/roadmap/test_roadmap_stage_apply.py` passes + +## 6. Add the unused-tool alert, its promtool cases and the pins + +Reads: `k8s/monitoring/prometheus-values.yaml`, `tests/prometheus/test_k8s_monitoring_alerting.py`, `tests/fixtures/prometheus_rules/roadmap_service_items_tests.yaml`, `k8s/README.md`. Changes: `k8s/monitoring/prometheus-values.yaml`, `tests/prometheus/test_k8s_monitoring_alerting.py`, `tests/fixtures/prometheus_rules/roadmap_service_tools_tests.yaml`, `k8s/README.md`. + +- [ ] 6.1 Add `tests/fixtures/prometheus_rules/roadmap_service_tools_tests.yaml` with the cases of the spec, with the real label sets (`job="roadmap-service"`, `instance="roadmap-service"`, `row`, `toolset`, `tool` and `outcome`) and a native histogram for the runs. In `tests/prometheus/test_k8s_monitoring_alerting.py`, add the group `roadmap-service-tools` to `ROADMAP_SERVICE_GROUP_TESTS`, `RoadmapServiceToolUnused` to `test_the_baseline_rule_set`, and the two series names to `SERVED_SERIES` + - Verification: `uv run devops ci test tests/prometheus/test_k8s_monitoring_alerting.py` fails on the base +- [ ] 6.2 Add the group `roadmap-service-tools` with the alert `RoadmapServiceToolUnused` to `k8s/monitoring/prometheus-values.yaml`: `for: 0m`, severity `warning`, the expression of design.md, and a summary and a description that name the tool and the toolset and end with `(k8s/README.md, Roadmap service)`. Add its row to the alert tables of the section "Roadmap service" in `k8s/README.md`, and add the fixture to the list of the unit-test files in that section + - Verification: `uv run devops ci test tests/prometheus/test_k8s_monitoring_alerting.py` passes with promtool 3.15.1 on the PATH + +## 7. Add the panel to the items dashboard + +Reads: `k8s/monitoring/dashboards/project-metrics.json`, `tests/telemetry/test_telemetry_instruments.py`. Changes: `k8s/monitoring/dashboards/project-metrics.json`, `tests/telemetry/test_telemetry_instruments.py`. + +- [ ] 7.1 Add a test to `tests/telemetry/test_telemetry_instruments.py`, beside the test of "Oldest Item Age by State", that pins the panel "Planner Tool Calls and Answer Size" and its two queries, each with `row="stage"` + - Verification: `uv run devops ci test tests/telemetry/test_telemetry_instruments.py` fails on the base +- [ ] 7.2 Add the panel (id 25, `gridPos` `{"h": 8, "w": 12, "x": 12, "y": 34}`) to `k8s/monitoring/dashboards/project-metrics.json`, with a description that says the values are approximate, the datasource of the other panels, and the second axis for the answer size + - Verification: `uv run devops ci test tests/telemetry/test_telemetry_instruments.py` passes + +## 8. Document the instruments + +Reads: `docs/TELEMETRY.md`, `src/devops_cli/docs/generator.py`. Changes: `docs/TELEMETRY.md`, `src/devops_cli/docs/generator.py`. + +- [ ] 8.1 In `generate_telemetry_docs` of `src/devops_cli/docs/generator.py`, say that an attribute named `job` or `instance` is lost because the resource sets both labels. Run `uv run devops docs generate --sync-readme` to write the four instruments and the sentence to `docs/TELEMETRY.md` + - Verification: `uv run devops ci docs` passes +- [ ] 8.2 Run the whole gate + - Verification: `uv run devops ci` passes + +## Criteria + +- The four instruments are declared, no label has an unbounded value, and `docs/TELEMETRY.md` lists them.: groups 1 and 8 +- A test shows one call of each outcome and checks the labels.: group 4 +- The trace comment shows the counts for each tool.: group 5 +- The alert rule and its unit test pass.: group 6 +- `uv run devops ci` passes.: group 8 + +## Workflow follow-up + +- Pending a person: review the diff in tuicr (tuicr -r ..HEAD). From e07ea0be4854581e128c774eda154b84ce918e54 Mon Sep 17 00:00:00 2001 From: Daniel Petty Date: Thu, 5 Nov 2026 15:49:57 +0000 Subject: [PATCH 02/10] feat(telemetry): declare the tool-call instruments and the planner panel (#3240) --- docs/TELEMETRY.md | 8 +- k8s/monitoring/dashboards/devops-cli.json | 80 ++++++++++++ .../design.md | 3 +- .../proposal.md | 17 ++- .../specs/metric-instruments/spec.md | 11 ++ .../specs/project-metrics-dashboard/spec.md | 12 -- .../tasks.md | 16 +-- src/devops_cli/docs/generator.py | 4 +- src/devops_cli/telemetry/instruments.py | 39 ++++++ tests/telemetry/test_telemetry_instruments.py | 119 ++++++++++++++++++ 10 files changed, 277 insertions(+), 32 deletions(-) delete mode 100644 openspec/changes/3240-count-tool-calls-of-the-stage/specs/project-metrics-dashboard/spec.md diff --git a/docs/TELEMETRY.md b/docs/TELEMETRY.md index 611136de5..245c7b917 100644 --- a/docs/TELEMETRY.md +++ b/docs/TELEMETRY.md @@ -17,7 +17,9 @@ were collected. The collector's `deltatocumulative` processor adds the deltas of each stream (its resource, scope, metric and attributes) into a running total before Prometheus stores it. Prometheus names a series by `service.name` (`job`), `service.instance.id` -(`instance`) and the metric's attributes. The `service.instance.id` is the host name, +(`instance`) and the metric's attributes. The collector sets `job` and `instance` from +the resource after it copies the attributes, so an attribute with either name is lost: a +job row is the `row` attribute. The `service.instance.id` is the host name, which every command on the host shares. The standard `OTEL_RESOURCE_ATTRIBUTES` sets another one: roadmap-service sets `service.instance.id=roadmap-service`, so on the OTLP path a new pod continues the series of the pod before it. The `/metrics` scrape adds the @@ -92,6 +94,10 @@ writes by name. | `devops_cli_roadmap_refine_comments_capped_total` | Counter | `1` | Refine comments that the cap on refine's comments on one item in 24 hours stopped: one count for each newest comment on an item that the cap left with no answer. The item waits on a person and takes no place under the round's limit. A later round that finds no newer comment on the item does not count it again. | | `devops_cli_roadmap_alternatives_total` | Counter | `1` | Checks of open items for an open tool that covers them, by verdict: covers, partly, none or not-applicable. One count for each check that the alternatives job writes to an item. A check that fails, or that the job skips because a person changed the item, counts nothing. | | `devops_cli_roadmap_stage_total` | Counter | `1` | Items that a stage round handled, one count for each item, by result: staged for an item written as a branch and a draft pull request, needs_split for an item stopped on a task group over the budget, failed for an item that failed in the plan or in the writes, and skipped for an item the job left out with a reason. A plan with no write counts nothing. | +| `devops_cli_roadmap_tool_calls_total` | Counter | `1` | Tool calls of the roadmap planner, by tool, row, toolset and outcome: ok for an answer with text, empty for an answer with no text, or error for a call that got no answer. The row is the job row that ran the planner, such as stage. Each plan run adds 0 for every tool of its toolset before its first model request, so a tool that no run calls keeps a series. | +| `devops_cli_roadmap_tool_answer_chars` | Histogram | `{char}` | Characters in the answer of each tool call of the roadmap planner, after the cut of the tool, by tool, row and toolset. A call that got no answer adds no observation. | +| `devops_cli_roadmap_tool_calls_per_run` | Histogram | `{call}` | Tool calls of each plan run of the roadmap planner, by row and toolset. A run that got no model reply adds no observation. | +| `devops_cli_roadmap_tool_limit_hits_total` | Counter | `1` | Plan runs of the roadmap planner whose tools ended on a limit, by row, toolset and limit: calls when fewer tool calls were left than a turn needs, requests when one model request was left, or window when the messages filled the context window. One count for each run. | | `devops_cli_roadmap_item_age_seconds` | Gauge | `s` | Age of the oldest open roadmap item in each status, by repo, state and plan. The state is the status. Plan is true when the item holds a stage pull request. That pull request waits for a person. The age is the time since the status label was set. Zero when no item is in the pair. | | `devops_cli_roadmap_oldest_item_number` | Gauge | `1` | Issue number of the oldest open roadmap item in each pair of repo, state and plan. The pair is the one that devops_cli_roadmap_item_age_seconds reports. Zero when no item is in the pair. | | `devops_cli_roadmap_claim_age_seconds` | Gauge | `s` | Age of the oldest claim on an open roadmap item, by repo. The age is the time since the owner label was set. Zero when no open item holds a claim. | diff --git a/k8s/monitoring/dashboards/devops-cli.json b/k8s/monitoring/dashboards/devops-cli.json index 95274ef6c..0c4448b42 100644 --- a/k8s/monitoring/dashboards/devops-cli.json +++ b/k8s/monitoring/dashboards/devops-cli.json @@ -1211,6 +1211,86 @@ "refId": "A" } ] + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 94 + }, + "id": 33, + "title": "Planner Tools", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "Tool calls of the stage planner in the last hour, by tool and outcome (ok, empty or error), on the left axis, and the mean size of the answers of each tool in characters, after the cut of the tool, on the right axis. Approximate, and low for sparse series: increase() needs two samples of a series inside its window, so a series' first sample, and any series seen only once in the window, add nothing. Delta points from one process can arrive out of order, and totals then come out low (#792). A tool that no run calls shows a line at 0.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "drawStyle": "line" + }, + "unit": "short" + }, + "overrides": [ + { + "matcher": { + "id": "byFrameRefID", + "options": "B" + }, + "properties": [ + { + "id": "custom.axisPlacement", + "value": "right" + } + ] + } + ] + }, + "gridPos": { + "h": 8, + "w": 24, + "x": 0, + "y": 95 + }, + "id": 34, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom" + } + }, + "title": "Planner Tool Calls and Answer Size", + "type": "timeseries", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum by (tool, outcome) (increase(devops_cli_roadmap_tool_calls_total{job=\"roadmap-service\", row=\"stage\"}[1h]))", + "legendFormat": "{{tool}} {{outcome}}", + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "histogram_avg(sum by (tool) (increase(devops_cli_roadmap_tool_answer_chars{job=\"roadmap-service\", row=\"stage\"}[1h])))", + "legendFormat": "{{tool}} mean chars", + "refId": "B" + } + ] } ], "refresh": "10s", diff --git a/openspec/changes/3240-count-tool-calls-of-the-stage/design.md b/openspec/changes/3240-count-tool-calls-of-the-stage/design.md index 7d34eef4b..266e7c3be 100644 --- a/openspec/changes/3240-count-tool-calls-of-the-stage/design.md +++ b/openspec/changes/3240-count-tool-calls-of-the-stage/design.md @@ -13,7 +13,7 @@ - **The alert groups by `row`, `tool` and `toolset`, and reads no `row` filter.**: The implement job can send the same series later with `row="implement"`. The `on (row, toolset)` join makes the rule count the runs of the same row and toolset. A tool of `extended` is not "unused" in a week when only `base` ran, because no run of `extended` happened. - **The alert reads a series that starts in the window by its last value.**: A new series gets its first value in the window, and `increase()` needs two samples. The form `(last_over_time(x[7d]) unless last_over_time(x[7d] offset 7d)) or increase(x[7d])` of the other roadmap rules counts that first value (#2505). The same form reads the native histogram of the runs, and `histogram_count` gives the number of runs. - **A tool that leaves the registry keeps its alert until its series leaves the window.**: Its series has no new sample, so for up to 7 days its sum stays 0 while runs happen. A person who removes a tool knows the alert follows. The rule has no list of tool names, so it needs no edit for a new tool. A list would drift from the registry (rejected below). -- **Put the panel at `x` 12 and `y` 34 in `project-metrics.json`, 12 wide and 8 high.**: It sits in the row "Release & Taxonomy Tracking", next to "Oldest Item Age by State". The panels below it keep their places. The panel has two queries: the calls by tool and outcome on the left axis, and the mean answer size by tool on the right axis. Both filter `job="roadmap-service"` and `row="stage"`. +- **Put the panel in a new row "Planner Tools" of `devops-cli.json`, at `y` 95, 24 wide and 8 high.**: `project-metrics.json` holds snapshot gauges, and a test of `tests/grafana/test_grafana_dashboards.py` asserts that it has no `increase(` query. `devops-cli.json` holds the counters of the Service. The row takes the ids 33 and 34, which the plan of #2950 does not use. The panel has two queries: the calls by tool and outcome on the left axis, and the mean answer size by tool on the right axis. Both filter `job="roadmap-service"` and `row="stage"`. - **Use the unit annotations `{char}` and `{call}` for the two histograms.**: The collector uses `UnderscoreEscapingWithoutSuffixes`, so the unit does not change a name. The UCUM annotation says what the number counts. ## Rejected alternatives @@ -26,6 +26,7 @@ - **A static `absent_over_time` list of tool names in the rule.**: The list drifts from the registry and needs one expression for each tool. - **A separate gauge for the tools that a toolset offers.**: It adds a fifth instrument that the issue does not ask for. - **A new Roadmap dashboard file.**: No such file exists, and a new dashboard needs its own provisioning. +- **The panel in `project-metrics.json`, with the assertion of `tests/grafana/test_grafana_dashboards.py` edited.**: The assertion says that file has no `increase(` query. An edit that lets one query pass is a lowered check, which only the owner can allow. - **The raw return before the cut, or the tokens of the answer.**: The model reads the text after the cut, and no token counter exists for a tool return. - **A threshold for `empty` such as an answer under 20 characters.**: A short answer can be valid, and the threshold is arbitrary. - **Put the alert in the group `roadmap-service-activity`.**: It forces a `MODIFIED` of a requirement that #2950 modifies. diff --git a/openspec/changes/3240-count-tool-calls-of-the-stage/proposal.md b/openspec/changes/3240-count-tool-calls-of-the-stage/proposal.md index 982dfed37..283cd7354 100644 --- a/openspec/changes/3240-count-tool-calls-of-the-stage/proposal.md +++ b/openspec/changes/3240-count-tool-calls-of-the-stage/proposal.md @@ -12,7 +12,7 @@ Issue #3240: a person who tunes the toolset of the stage planner cannot see whic - `ToolboxClient` in `src/devops_cli/roadmap/stage_toolbox.py` calls both functions around each plan run. - `investigate` in `src/devops_cli/roadmap/refine_investigation.py` names the limit that ends the tools of a run: `calls`, `requests` or `window`. - `_run_report` in `src/devops_cli/roadmap/stage.py` adds one line to the comment of a staged item: the calls and the empty answers of each tool. -- `k8s/monitoring/dashboards/project-metrics.json` gets one panel, "Planner Tool Calls and Answer Size", next to "Oldest Item Age by State". +- `k8s/monitoring/dashboards/devops-cli.json` gets the row "Planner Tools" with one panel, "Planner Tool Calls and Answer Size". The refined design names `project-metrics.json`: see Feasibility 2. - `k8s/monitoring/prometheus-values.yaml` gets the group `roadmap-service-tools` with the alert `RoadmapServiceToolUnused`. A `promtool` unit test covers it. The pins in `tests/prometheus/test_k8s_monitoring_alerting.py` and the section "Roadmap service" of `k8s/README.md` name it. - `tests/telemetry/test_telemetry_instruments.py` gets a check that no `emit()` call sends the attribute `job` or `instance`. - `docs/TELEMETRY.md` is generated from `INSTRUMENTS`, and says that the resource sets `job` and `instance`. @@ -26,15 +26,14 @@ None. ### Modified Capabilities -- `metric-instruments`: The tool-call instruments are declared and documented, No instrument attribute is named job or instance +- `metric-instruments`: The tool-call instruments are declared and documented, No instrument attribute is named job or instance, The devops-cli dashboard shows the tool calls of the planner - `roadmap-stage`: A plan run counts each tool call by outcome, A plan run measures the size of each answer, Each plan run gives every tool of its toolset a series, A plan run measures its calls and the limit that ended its tools, The comment of a staged item counts the calls of each tool -- `project-metrics-dashboard`: The items dashboard shows the tool calls of the planner -- `roadmap-service-alerts`: The planner tools have their own alert group +- `roadmap-service-alerts`: The planner tools have their own alert group, A tool that no run called in 7 days warns ## Impact - Code: `src/devops_cli/telemetry/instruments.py`, `src/devops_cli/roadmap/agent_run.py`, `src/devops_cli/roadmap/refine_investigation.py`, `src/devops_cli/roadmap/stage_toolbox.py`, `src/devops_cli/roadmap/tool_metrics.py`, `src/devops_cli/roadmap/stage.py`, `src/devops_cli/lang/en/messages.py` -- Manifests: `k8s/monitoring/dashboards/project-metrics.json`, `k8s/monitoring/prometheus-values.yaml` +- Manifests: `k8s/monitoring/dashboards/devops-cli.json`, `k8s/monitoring/prometheus-values.yaml` - Tests: `tests/telemetry/test_telemetry_instruments.py`, `tests/roadmap/test_roadmap_stage_toolbox.py`, `tests/roadmap/test_roadmap_stage_apply.py`, `tests/roadmap/test_roadmap_refine_investigation.py`, `tests/prometheus/test_k8s_monitoring_alerting.py`, `tests/fixtures/prometheus_rules/roadmap_service_tools_tests.yaml` - Documents: `docs/TELEMETRY.md` (generated), `k8s/README.md` - Deleted code: `_within_budget`, `_within_window_of` and `_recording` in `refine_investigation.py`. `_limit_reached` and `_offering` replace them. @@ -42,16 +41,16 @@ None. ## Dependencies - #3238 (merged) added the toolset `extended` and the label value `toolset`. Nothing to wait for. -- #2919 (merged) put the age panel in `project-metrics.json`. The free right half of its row holds the new panel. -- #2950 (open plan) adds three alternatives instruments, one alert in the group `roadmap-service-activity` and a panel row. Both plans edit `INSTRUMENTS` and `__all__`, the table in `docs/TELEMETRY.md`, `SERVED_SERIES`, the baseline rule list in `tests/prometheus/test_k8s_monitoring_alerting.py` and `k8s/README.md`. The change that merges second merges `main` and keeps both entries. This change adds its alert to a new group, so no requirement is in both plans. -- #3069 (open) may create a dashboard of its own. This change does not wait for it: the panel moves with a one-line edit of the file name. +- #2919 (merged) put the age panels in `project-metrics.json`. That dashboard holds gauges only (see Feasibility 2), so this change does not use it. +- #2950 (open plan) adds three alternatives instruments, one alert in the group `roadmap-service-activity` and a panel row in `devops-cli.json`. Both plans edit `INSTRUMENTS` and `__all__`, the table in `docs/TELEMETRY.md`, `SERVED_SERIES`, the baseline rule list in `tests/prometheus/test_k8s_monitoring_alerting.py`, `k8s/README.md` and the end of `devops-cli.json`. The plan of #2950 puts its row at `y` 94, as this change does with the ids 33 and 34. The change that merges second merges `main`, keeps both entries and moves its row below the other. This change adds its alert to a new group, so no requirement is in both plans. +- #3069 (open) may create a dashboard of its own. This change does not wait for it: the row moves with a one-line edit of the file name. ## Feasibility Checked on `origin/main` 2db71f29. Each premise has its output. 1. The module declares and sends instruments as the issue says. `git grep -n "ROADMAP_STAGE_TOTAL" src` shows the instrument, its entry in `INSTRUMENTS` and its `emit` in `stage.py`. `git grep -n "devops_cli_roadmap_tool" -- . ':!openspec'` prints nothing: the four names are free. `docs/TELEMETRY.md` is generated by `generate_telemetry_docs` in `src/devops_cli/docs/generator.py` from `INSTRUMENTS`. -2. The issue asks for a panel in "the Roadmap dashboard". `ls k8s/monitoring/dashboards` lists 15 files, and none is a Roadmap dashboard. `project-metrics.json` holds the items panels. Its panel "Oldest Item Age by State" has `gridPos` `{"h": 8, "w": 12, "x": 0, "y": 34}`, so the right half at `x` 12 is free. The panel goes there. +2. The issue asks for a panel in "the Roadmap dashboard". `ls k8s/monitoring/dashboards` lists 15 files, and none is a Roadmap dashboard. The refined design puts the panel in `project-metrics.json`, which holds the items panels. A panel with `increase()` cannot go there: `test_grafana_dashboard_projections_and_dcgm_grid_layout` in `tests/grafana/test_grafana_dashboards.py` asserts that the file has no `increase(` query, because it shows snapshots as gauges. A change that edits that assertion would lower a check. `devops-cli.json` (`devops-cli Telemetry`) holds the counters of the Service, such as "GitHub Requests by Resource", which reads the same `row` label. Its last panel has `gridPos` `{"h": 8, "w": 6, "x": 18, "y": 86}`, so the new row goes at `y` 94. 3. The label `job` of the issue cannot work. The OTLP resource sets `service.name` as the Prometheus label `job`. `createAttributes` in `pkg/translator/prometheusremotewrite/helper.go` of opentelemetry-collector-contrib copies the data point attributes into a map first. It then runs `l[model.JobLabel] = val` for `service.name`, with no log. So a data point attribute named `job` is lost, and `job="stage"` would match nothing. The same holds for `instance`. `devops_cli_github_requests_total` names the same idea `row` (`src/devops_cli/github/request_scope.py`). The four instruments use `row` with the value `stage`, and the dashboard and the alert read `row`. 4. A zero increment reaches the collector. A test that called `increment_counter("devops_cli_roadmap_tool_calls_total", 0, attributes={"tool": "read_file"})` on the `otlp` fixture exported one sum data point with `asDouble` 0.0 and delta temporality. So the zero increment at the start of a run gives each tool a series. 5. pydantic-ai 2.59.0 already records each tool answer. `ToolReturnPart` holds `content` (the text that the model reads, after the cut of each tool), `tool_call_id` and `outcome`. A rejected call gives a `RetryPromptPart` with the same `tool_call_id`. `RunTrace.absorb` already reads the same messages. So the counts come from the messages, with no wrapper of the tool functions and no `Hooks` capability. A hook would miss a call that failed in argument validation. diff --git a/openspec/changes/3240-count-tool-calls-of-the-stage/specs/metric-instruments/spec.md b/openspec/changes/3240-count-tool-calls-of-the-stage/specs/metric-instruments/spec.md index 133e999ee..68c003f77 100644 --- a/openspec/changes/3240-count-tool-calls-of-the-stage/specs/metric-instruments/spec.md +++ b/openspec/changes/3240-count-tool-calls-of-the-stage/specs/metric-instruments/spec.md @@ -17,3 +17,14 @@ The resource of a process sets the Prometheus labels `job` and `instance`, and a #### Scenario: A module sends the attribute job - **WHEN** a module calls `emit(COMMAND_TOTAL, 1.0, {"job": "stage"})` - **THEN** the instrument check names that call, and it passes `emit(COMMAND_TOTAL, 1.0, {"row": "stage"})` + +### Requirement: The devops-cli dashboard shows the tool calls of the planner +`k8s/monitoring/dashboards/devops-cli.json` SHALL have the row "Planner Tools" with the panel "Planner Tool Calls and Answer Size". The panel SHALL chart the calls of each tool by outcome from `devops_cli_roadmap_tool_calls_total`, and the mean answer size of each tool from `devops_cli_roadmap_tool_answer_chars`. Each query SHALL select `job="roadmap-service"` and `row="stage"`. + +#### Scenario: The dashboard JSON holds the tool panel +- **WHEN** a test reads `k8s/monitoring/dashboards/devops-cli.json` +- **THEN** one panel has a query for `devops_cli_roadmap_tool_calls_total` and a query for `devops_cli_roadmap_tool_answer_chars`, each with `row="stage"` + +#### Scenario: The panel reads each series through the function of its kind +- **WHEN** the dashboard checks read the panel +- **THEN** the counter is the first argument of `increase(`, and the histogram is the first argument of `increase(` inside `histogram_avg(` diff --git a/openspec/changes/3240-count-tool-calls-of-the-stage/specs/project-metrics-dashboard/spec.md b/openspec/changes/3240-count-tool-calls-of-the-stage/specs/project-metrics-dashboard/spec.md deleted file mode 100644 index f043e6177..000000000 --- a/openspec/changes/3240-count-tool-calls-of-the-stage/specs/project-metrics-dashboard/spec.md +++ /dev/null @@ -1,12 +0,0 @@ -## ADDED Requirements - -### Requirement: The items dashboard shows the tool calls of the planner -`k8s/monitoring/dashboards/project-metrics.json` SHALL hold one panel, "Planner Tool Calls and Answer Size". It SHALL chart the calls of each tool by outcome from `devops_cli_roadmap_tool_calls_total`, and the mean answer size of each tool from `devops_cli_roadmap_tool_answer_chars`. Each query SHALL select `job="roadmap-service"` and `row="stage"`. - -#### Scenario: The dashboard JSON holds the tool panel -- **WHEN** a test reads `k8s/monitoring/dashboards/project-metrics.json` -- **THEN** one panel has a query for `devops_cli_roadmap_tool_calls_total` and a query for `devops_cli_roadmap_tool_answer_chars`, each with `row="stage"` - -#### Scenario: The panel reads each series through the function of its kind -- **WHEN** the dashboard checks read the panel -- **THEN** the counter is the first argument of `increase(`, and the histogram is the first argument of `increase(` inside `histogram_avg(` diff --git a/openspec/changes/3240-count-tool-calls-of-the-stage/tasks.md b/openspec/changes/3240-count-tool-calls-of-the-stage/tasks.md index dd2969389..8ce42054d 100644 --- a/openspec/changes/3240-count-tool-calls-of-the-stage/tasks.md +++ b/openspec/changes/3240-count-tool-calls-of-the-stage/tasks.md @@ -4,16 +4,16 @@ Reads: `src/devops_cli/telemetry/instruments.py`, `tests/telemetry/test_telemetry_instruments.py`. Changes: `src/devops_cli/telemetry/instruments.py`, `tests/telemetry/test_telemetry_instruments.py`. -- [ ] 1.1 Add a test to `tests/telemetry/test_telemetry_instruments.py` that finds `devops_cli_roadmap_tool_calls_total` and `devops_cli_roadmap_tool_limit_hits_total` as counters with the unit `1`, `devops_cli_roadmap_tool_answer_chars` as a histogram with the unit `{char}`, and `devops_cli_roadmap_tool_calls_per_run` as a histogram with the unit `{call}`, in `INSTRUMENTS` +- [x] 1.1 Add a test to `tests/telemetry/test_telemetry_instruments.py` that finds `devops_cli_roadmap_tool_calls_total` and `devops_cli_roadmap_tool_limit_hits_total` as counters with the unit `1`, `devops_cli_roadmap_tool_answer_chars` as a histogram with the unit `{char}`, and `devops_cli_roadmap_tool_calls_per_run` as a histogram with the unit `{call}`, in `INSTRUMENTS` - Verification: `uv run devops ci test tests/telemetry/test_telemetry_instruments.py` fails on the base -- [ ] 1.2 Add `ROADMAP_TOOL_CALLS_TOTAL`, `ROADMAP_TOOL_ANSWER_CHARS`, `ROADMAP_TOOL_CALLS_PER_RUN` and `ROADMAP_TOOL_LIMIT_HITS_TOTAL` to `src/devops_cli/telemetry/instruments.py` after `ROADMAP_STAGE_TOTAL`, each with a description in the style of its neighbors. Add the four names to `INSTRUMENTS` and `__all__` +- [x] 1.2 Add `ROADMAP_TOOL_CALLS_TOTAL`, `ROADMAP_TOOL_ANSWER_CHARS`, `ROADMAP_TOOL_CALLS_PER_RUN` and `ROADMAP_TOOL_LIMIT_HITS_TOTAL` to `src/devops_cli/telemetry/instruments.py` after `ROADMAP_STAGE_TOTAL`, each with a description in the style of its neighbors. Add the four names to `INSTRUMENTS` and `__all__` - Verification: `uv run devops ci test tests/telemetry/test_telemetry_instruments.py` passes ## 2. Keep the labels job and instance out of emit Reads: `tests/telemetry/test_telemetry_instruments.py`. Changes: `tests/telemetry/test_telemetry_instruments.py`. -- [ ] 2.1 Add a check to `tests/telemetry/test_telemetry_instruments.py`, beside `test_every_metric_is_a_declared_instrument_sent_through_emit`, that names each `emit()` call under `src/devops_cli/` whose attribute dict has the key `job` or `instance`. Add a test of the check on a module text: `emit(COMMAND_TOTAL, 1.0, {"job": "stage"})` is named, and `emit(COMMAND_TOTAL, 1.0, {"row": "stage"})` is not +- [x] 2.1 Add a check to `tests/telemetry/test_telemetry_instruments.py`, beside `test_every_metric_is_a_declared_instrument_sent_through_emit`, that names each `emit()` call under `src/devops_cli/` whose attribute dict has the key `job` or `instance`. Add a test of the check on a module text: `emit(COMMAND_TOTAL, 1.0, {"job": "stage"})` is named, and `emit(COMMAND_TOTAL, 1.0, {"row": "stage"})` is not - Verification: `uv run devops ci test tests/telemetry/test_telemetry_instruments.py` ## 3. Record the outcome, the size and the limit of each tool call in the trace @@ -54,20 +54,20 @@ Reads: `k8s/monitoring/prometheus-values.yaml`, `tests/prometheus/test_k8s_monit - [ ] 6.2 Add the group `roadmap-service-tools` with the alert `RoadmapServiceToolUnused` to `k8s/monitoring/prometheus-values.yaml`: `for: 0m`, severity `warning`, the expression of design.md, and a summary and a description that name the tool and the toolset and end with `(k8s/README.md, Roadmap service)`. Add its row to the alert tables of the section "Roadmap service" in `k8s/README.md`, and add the fixture to the list of the unit-test files in that section - Verification: `uv run devops ci test tests/prometheus/test_k8s_monitoring_alerting.py` passes with promtool 3.15.1 on the PATH -## 7. Add the panel to the items dashboard +## 7. Add the panel to the devops-cli dashboard -Reads: `k8s/monitoring/dashboards/project-metrics.json`, `tests/telemetry/test_telemetry_instruments.py`. Changes: `k8s/monitoring/dashboards/project-metrics.json`, `tests/telemetry/test_telemetry_instruments.py`. +Reads: `k8s/monitoring/dashboards/devops-cli.json`, `tests/telemetry/test_telemetry_instruments.py`. Changes: `k8s/monitoring/dashboards/devops-cli.json`, `tests/telemetry/test_telemetry_instruments.py`. -- [ ] 7.1 Add a test to `tests/telemetry/test_telemetry_instruments.py`, beside the test of "Oldest Item Age by State", that pins the panel "Planner Tool Calls and Answer Size" and its two queries, each with `row="stage"` +- [x] 7.1 Add a test to `tests/telemetry/test_telemetry_instruments.py`, beside the test of "Oldest Item Age by State", that pins the panel "Planner Tool Calls and Answer Size", its two queries, each with `row="stage"`, and its place - Verification: `uv run devops ci test tests/telemetry/test_telemetry_instruments.py` fails on the base -- [ ] 7.2 Add the panel (id 25, `gridPos` `{"h": 8, "w": 12, "x": 12, "y": 34}`) to `k8s/monitoring/dashboards/project-metrics.json`, with a description that says the values are approximate, the datasource of the other panels, and the second axis for the answer size +- [x] 7.2 Add the row "Planner Tools" (id 33, `y` 94) and the panel (id 34, `gridPos` `{"h": 8, "w": 24, "x": 0, "y": 95}`) to `k8s/monitoring/dashboards/devops-cli.json`, with a description that says the values are approximate, the datasource of the other panels, and the second axis for the answer size - Verification: `uv run devops ci test tests/telemetry/test_telemetry_instruments.py` passes ## 8. Document the instruments Reads: `docs/TELEMETRY.md`, `src/devops_cli/docs/generator.py`. Changes: `docs/TELEMETRY.md`, `src/devops_cli/docs/generator.py`. -- [ ] 8.1 In `generate_telemetry_docs` of `src/devops_cli/docs/generator.py`, say that an attribute named `job` or `instance` is lost because the resource sets both labels. Run `uv run devops docs generate --sync-readme` to write the four instruments and the sentence to `docs/TELEMETRY.md` +- [x] 8.1 In `generate_telemetry_docs` of `src/devops_cli/docs/generator.py`, say that an attribute named `job` or `instance` is lost because the resource sets both labels. Run `uv run devops docs generate --sync-readme` to write the four instruments and the sentence to `docs/TELEMETRY.md` - Verification: `uv run devops ci docs` passes - [ ] 8.2 Run the whole gate - Verification: `uv run devops ci` passes diff --git a/src/devops_cli/docs/generator.py b/src/devops_cli/docs/generator.py index 3828ac1d4..bb96e9a15 100644 --- a/src/devops_cli/docs/generator.py +++ b/src/devops_cli/docs/generator.py @@ -1094,7 +1094,9 @@ def generate_telemetry_docs(self) -> str: "The collector's `deltatocumulative` processor adds the deltas of each stream (its", "resource, scope, metric and attributes) into a running total before Prometheus stores", "it. Prometheus names a series by `service.name` (`job`), `service.instance.id`", - "(`instance`) and the metric's attributes. The `service.instance.id` is the host name,", + "(`instance`) and the metric's attributes. The collector sets `job` and `instance` from", + "the resource after it copies the attributes, so an attribute with either name is lost: a", + "job row is the `row` attribute. The `service.instance.id` is the host name,", "which every command on the host shares. The standard `OTEL_RESOURCE_ATTRIBUTES` sets", "another one: roadmap-service sets `service.instance.id=roadmap-service`, so on the OTLP", "path a new pod continues the series of the pod before it. The `/metrics` scrape adds the", diff --git a/src/devops_cli/telemetry/instruments.py b/src/devops_cli/telemetry/instruments.py index 2db70bc63..0b3c222b2 100644 --- a/src/devops_cli/telemetry/instruments.py +++ b/src/devops_cli/telemetry/instruments.py @@ -257,6 +257,37 @@ class Instrument: "group over the budget, failed for an item that failed in the plan or in the writes, and " "skipped for an item the job left out with a reason. A plan with no write counts nothing", ) +ROADMAP_TOOL_CALLS_TOTAL = Instrument( + "devops_cli_roadmap_tool_calls_total", + InstrumentKind.COUNTER, + "1", + "Tool calls of the roadmap planner, by tool, row, toolset and outcome: ok for an answer with " + "text, empty for an answer with no text, or error for a call that got no answer. The row is " + "the job row that ran the planner, such as stage. Each plan run adds 0 for every tool of its " + "toolset before its first model request, so a tool that no run calls keeps a series", +) +ROADMAP_TOOL_ANSWER_CHARS = Instrument( + "devops_cli_roadmap_tool_answer_chars", + InstrumentKind.HISTOGRAM, + "{char}", + "Characters in the answer of each tool call of the roadmap planner, after the cut of the " + "tool, by tool, row and toolset. A call that got no answer adds no observation", +) +ROADMAP_TOOL_CALLS_PER_RUN = Instrument( + "devops_cli_roadmap_tool_calls_per_run", + InstrumentKind.HISTOGRAM, + "{call}", + "Tool calls of each plan run of the roadmap planner, by row and toolset. A run that got no " + "model reply adds no observation", +) +ROADMAP_TOOL_LIMIT_HITS_TOTAL = Instrument( + "devops_cli_roadmap_tool_limit_hits_total", + InstrumentKind.COUNTER, + "1", + "Plan runs of the roadmap planner whose tools ended on a limit, by row, toolset and limit: " + "calls when fewer tool calls were left than a turn needs, requests when one model request " + "was left, or window when the messages filled the context window. One count for each run", +) ROADMAP_AGE_LABELS = ("repo", "state", "plan") """The labels of `ROADMAP_ITEM_AGE` and `ROADMAP_OLDEST_ITEM`, in the order the dashboard names them.""" ROADMAP_ITEM_AGE = Instrument( @@ -485,6 +516,10 @@ class Instrument: ROADMAP_REFINE_COMMENTS_CAPPED_TOTAL, ROADMAP_ALTERNATIVES_TOTAL, ROADMAP_STAGE_TOTAL, + ROADMAP_TOOL_CALLS_TOTAL, + ROADMAP_TOOL_ANSWER_CHARS, + ROADMAP_TOOL_CALLS_PER_RUN, + ROADMAP_TOOL_LIMIT_HITS_TOTAL, ROADMAP_ITEM_AGE, ROADMAP_OLDEST_ITEM, ROADMAP_CLAIM_AGE, @@ -614,6 +649,10 @@ def backend_name(served_by: str | None) -> str: "ROADMAP_OLDEST_ITEM", "ROADMAP_REFINE_COMMENTS_CAPPED_TOTAL", "ROADMAP_STAGE_TOTAL", + "ROADMAP_TOOL_ANSWER_CHARS", + "ROADMAP_TOOL_CALLS_PER_RUN", + "ROADMAP_TOOL_CALLS_TOTAL", + "ROADMAP_TOOL_LIMIT_HITS_TOTAL", "SUBCOMMAND_DURATION", "SUBPROCESS_DURATION", "TELEMETRY_TESTS_TOTAL", diff --git a/tests/telemetry/test_telemetry_instruments.py b/tests/telemetry/test_telemetry_instruments.py index 49312de00..b8dc76891 100644 --- a/tests/telemetry/test_telemetry_instruments.py +++ b/tests/telemetry/test_telemetry_instruments.py @@ -367,6 +367,71 @@ def test_the_instrument_check_names_a_metric_sent_by_name_and_an_undeclared_inst ] +_RESOURCE_LABELS = frozenset({"job", "instance"}) +"""The labels that the resource of a process sets: the collector overwrites a data point +attribute with either name, with no log (#3240).""" + + +def _emit_attributes(call: ast.Call) -> ast.expr | None: + """The attributes argument of a call, by position or by keyword, or None.""" + if len(call.args) > 2: + return call.args[2] + return next((keyword.value for keyword in call.keywords if keyword.arg == "attributes"), None) + + +def _emits_with_a_resource_label(tree: ast.AST, label: str) -> list[str]: + """Each `emit()` call of a module, as `