From 48be3114a21d441b4ef44b75ca7023dfb49fad22 Mon Sep 17 00:00:00 2001 From: "warp-agent-staging[bot]" <240773466+warp-agent-staging[bot]@users.noreply.github.com> Date: Wed, 9 Sep 2026 19:33:25 +0000 Subject: [PATCH 1/6] spec: placeholder for GROW-6131 spec draft PR --- .factory-draft-pr-placeholder | 1 + 1 file changed, 1 insertion(+) create mode 100644 .factory-draft-pr-placeholder diff --git a/.factory-draft-pr-placeholder b/.factory-draft-pr-placeholder new file mode 100644 index 00000000..18728075 --- /dev/null +++ b/.factory-draft-pr-placeholder @@ -0,0 +1 @@ +Temporary factory draft PR placeholder. Removed before ready for review. From 8c689ae8decc50426a8ec9059d2b3e0dae18aa5a Mon Sep 17 00:00:00 2001 From: "warp-agent-staging[bot]" <240773466+warp-agent-staging[bot]@users.noreply.github.com> Date: Wed, 9 Sep 2026 20:57:06 +0000 Subject: [PATCH 2/6] docs: nest Scorers, Self-improvement, and Benchmarks under Measure and improve (GROW-6131) Split Scorers and Self-improvement out of measure-and-improve.mdx into dedicated pages, nest all three (plus an overview) under one expanded 'Measure and improve' sidebar group, and repoint internal cross-references and the terminology.md Scorer glossary entry at the new pages. Benchmarks keeps its existing URL and content; only its sidebar parent changes, so no vercel.json redirects are needed. The old #configure-scorers and #configure-and-review-self-improvement anchors remain valid as short pointer stubs on the overview page. --- .agents/references/terminology.md | 2 +- .factory-draft-pr-placeholder | 1 - .../docs/factories/factory-as-code.mdx | 2 +- .../docs/factories/factory-dashboard.mdx | 4 +-- src/content/docs/factories/factory-skills.mdx | 2 +- .../docs/factories/how-factories-work.mdx | 2 +- src/content/docs/factories/index.mdx | 2 +- .../docs/factories/measure-and-improve.mdx | 36 +++++-------------- .../factories/measure-and-improve/scorers.mdx | 33 +++++++++++++++++ .../measure-and-improve/self-improvement.mdx | 20 +++++++++++ .../build-a-self-improving-agent.mdx | 2 +- src/sidebar.ts | 18 ++++++++-- 12 files changed, 86 insertions(+), 38 deletions(-) delete mode 100644 .factory-draft-pr-placeholder create mode 100644 src/content/docs/factories/measure-and-improve/scorers.mdx create mode 100644 src/content/docs/factories/measure-and-improve/self-improvement.mdx diff --git a/.agents/references/terminology.md b/.agents/references/terminology.md index 0344575a..c786e3ee 100644 --- a/.agents/references/terminology.md +++ b/.agents/references/terminology.md @@ -362,7 +362,7 @@ Docs match the screen; the fix belongs in the app. *Usage note:* Lowercase common noun. Scoped to a factory's definition; distinct from the general [cloud agent runner](/platform/runners/) reference, which covers the same concept for standalone cloud agents outside a factory. - **Scorer** — A configured LLM judge, scoped per factory to chosen agents and sampled at a set rate, that classifies completed runs against criteria you write, such as "did the agent run the tests before opening a PR?" A Scorer assigns a label (a classification with a score), not a freeform numeric grade. Feeds the **Dashboard** page's Scorer cards, benchmarks, and Self-improvement. - *Usage note:* Capitalize "Scorer"/"Scorers" when referring to the feature or a configured instance ("create a Scorer," "Scorer cards"); lowercase only for a generic instance count or file listing ("two scorers," alongside "skills" in an example tree). Say "classify," never "grade" — the docs draw this distinction deliberately. The unit a Scorer evaluates is a **run** (a single agent execution), not a "conversation" or "completed work." `measure-and-improve.mdx` is the canonical page for what a Scorer is and how to configure one; other pages link there rather than repeating the definition. + *Usage note:* Capitalize "Scorer"/"Scorers" when referring to the feature or a configured instance ("create a Scorer," "Scorer cards"); lowercase only for a generic instance count or file listing ("two scorers," alongside "skills" in an example tree). Say "classify," never "grade" — the docs draw this distinction deliberately. The unit a Scorer evaluates is a **run** (a single agent execution), not a "conversation" or "completed work." `measure-and-improve/scorers.mdx` is the canonical page for what a Scorer is and how to configure one; other pages link there rather than repeating the definition. - **AI sovereignty** — Warp Factories' positioning around customer ownership and control of inference, hosting, and data exhaust (agent conversations, evals, memories) for their factory. diff --git a/.factory-draft-pr-placeholder b/.factory-draft-pr-placeholder deleted file mode 100644 index 18728075..00000000 --- a/.factory-draft-pr-placeholder +++ /dev/null @@ -1 +0,0 @@ -Temporary factory draft PR placeholder. Removed before ready for review. diff --git a/src/content/docs/factories/factory-as-code.mdx b/src/content/docs/factories/factory-as-code.mdx index 9340a0ca..53bcd7cf 100644 --- a/src/content/docs/factories/factory-as-code.mdx +++ b/src/content/docs/factories/factory-as-code.mdx @@ -404,7 +404,7 @@ Optional. The GitHub or GitLab repositories and exact commits the task starts fr ## `scorers//scorer.md` -Optional. Each file defines a scorer: an LLM judge that classifies a sample of an agent's finished runs against a rubric. The directory segment is only a stable filesystem slug — the required `name` field is the scorer's identity. The YAML frontmatter declares the classification contract, and the Markdown body after the closing `---` fence is the rubric. See [Configure Scorers](/factories/measure-and-improve/#configure-scorers) for how scores are used. +Optional. Each file defines a scorer: an LLM judge that classifies a sample of an agent's finished runs against a rubric. The directory segment is only a stable filesystem slug — the required `name` field is the scorer's identity. The YAML frontmatter declares the classification contract, and the Markdown body after the closing `---` fence is the rubric. See [Configuring Scorers](/factories/measure-and-improve/scorers/) for how scores are used. ```markdown title="scorers/tests-run/scorer.md" --- diff --git a/src/content/docs/factories/factory-dashboard.mdx b/src/content/docs/factories/factory-dashboard.mdx index d7cde667..b25f2d5f 100644 --- a/src/content/docs/factories/factory-dashboard.mdx +++ b/src/content/docs/factories/factory-dashboard.mdx @@ -70,7 +70,7 @@ When an agent proposes a change to a Warp-managed definition, a spec review for ## Score and benchmark -**Scorers** is where you create Scorers and read their results. **Self-improvement** lists the pull requests the self-improvement flow opens after analyzing runs your Scorers mark as failing, and **Benchmarks** compares model and runner configurations against a fixed set of tasks. See [Configure Scorers](/factories/measure-and-improve/#configure-scorers), [Configure and review Self-improvement](/factories/measure-and-improve/#configure-and-review-self-improvement), and [benchmarking factory agent configurations](/factories/benchmarks/) for what each one is and how to use it. +**Scorers** is where you create Scorers and read their results. **Self-improvement** lists the pull requests the self-improvement flow opens after analyzing runs your Scorers mark as failing, and **Benchmarks** compares model and runner configurations against a fixed set of tasks. See [Configuring Scorers](/factories/measure-and-improve/scorers/), [Configuring and reviewing Self-improvement](/factories/measure-and-improve/self-improvement/), and [benchmarking factory agent configurations](/factories/benchmarks/) for what each one is and how to use it. ## Change factory settings @@ -79,7 +79,7 @@ When an agent proposes a change to a Warp-managed definition, a spec review for * **Identity** - The factory's name, avatar, and [**Foreman name**](/factories/factory-as-code/#alias), the handle your team @-mentions. * **Repositories** - The repos the factory works in. * **Pull request authorship** - Whether pull requests are authored by the agent or the run creator (the definition's [`credentialStrategy`](/factories/factory-as-code/#credentialstrategy)). -* **Analysis model** - The model [Self-improvement](/factories/measure-and-improve/#configure-and-review-self-improvement) uses to analyze failed runs. +* **Analysis model** - The model [Self-improvement](/factories/measure-and-improve/self-improvement/) uses to analyze failed runs. * **Runners** - The compute the factory's runs execute on. * **Integrations** - The integrations this factory can access. * **Deletion** - Deletes the factory. This cannot be undone. diff --git a/src/content/docs/factories/factory-skills.mdx b/src/content/docs/factories/factory-skills.mdx index a5c5ad33..3024a0cf 100644 --- a/src/content/docs/factories/factory-skills.mdx +++ b/src/content/docs/factories/factory-skills.mdx @@ -65,7 +65,7 @@ For worked examples, including a factory-wide skill and a per-agent skill togeth ## Skills and self-improvement -A factory can propose changes to a skill. When [Self-improvement](/factories/measure-and-improve/#configure-and-review-self-improvement) is on for a Scorer and it flags a recurring failure, it can edit the responsible skill in a follow-up run, the same way it can edit application code. The change still arrives as a pull request for your team to review, whether that's through the factory dashboard or your Git host. +A factory can propose changes to a skill. When [Self-improvement](/factories/measure-and-improve/self-improvement/) is on for a Scorer and it flags a recurring failure, it can edit the responsible skill in a follow-up run, the same way it can edit application code. The change still arrives as a pull request for your team to review, whether that's through the factory dashboard or your Git host. ## Related pages diff --git a/src/content/docs/factories/how-factories-work.mdx b/src/content/docs/factories/how-factories-work.mdx index a051a018..bc3af48a 100644 --- a/src/content/docs/factories/how-factories-work.mdx +++ b/src/content/docs/factories/how-factories-work.mdx @@ -65,7 +65,7 @@ The first two are workflow policy, written into the foreman's instructions; edit ## How the factory improves itself -Your factory is self-improving, and you define what "better" means. [Scorers](/factories/measure-and-improve/) classify completed runs against criteria you write, and [Self-improvement](/factories/measure-and-improve/#configure-and-review-self-improvement) groups the failures they flag into follow-up runs that propose fixes — to the application code or to the factory's own definition. Every proposal arrives as a change for your review; nothing is adopted on its own. +Your factory is self-improving, and you define what "better" means. [Scorers](/factories/measure-and-improve/) classify completed runs against criteria you write, and [Self-improvement](/factories/measure-and-improve/self-improvement/) groups the failures they flag into follow-up runs that propose fixes — to the application code or to the factory's own definition. Every proposal arrives as a change for your review; nothing is adopted on its own. The factory's definition is open to the same loop. Anyone on the team, or an agent, can propose changes to its instructions, skills, models, or other [definition files](/factories/factory-as-code/), and definitions stored in GitHub go through pull request review and [configuration checks](/factories/factory-as-code/#pull-request-checks-for-github-backed-factories) before a change reaches the production branch. diff --git a/src/content/docs/factories/index.mdx b/src/content/docs/factories/index.mdx index 67f74eaa..23649ec6 100644 --- a/src/content/docs/factories/index.mdx +++ b/src/content/docs/factories/index.mdx @@ -52,7 +52,7 @@ Warp Factories is designed for engineering teams with repeatable work that exten * **Definitions as code** - [Version-controlled definition files](/factories/factory-as-code/) describe your repositories, agents, automations, runners, [skills](/factories/factory-skills/), and MCP servers, so factory changes get the same review, history, and rollback as code changes. * **Integrations and the Factory MCP** - Work flows in from [Slack](/factories/integrations/slack/), [GitHub](/factories/integrations/github/), [GitLab](/factories/integrations/gitlab/), [Linear](/factories/integrations/linear/), and [Jira](/factories/integrations/jira/), plus [custom webhooks](/factories/webhooks/), direct runs, and schedules. The [Factory MCP](/factories/factory-mcp/) connects coding agents and other MCP clients. * **Model and harness choice** - Each agent can use a different model and [supported harness](/platform/harnesses/), including the Warp Agent, Claude Code, and Codex. -* **Measurement and self-improvement** - The [factory dashboard](/factories/factory-dashboard/) shows work-item status, runs, automations, costs, and benchmarks. [Scorers](/factories/measure-and-improve/) classify completed runs, [Benchmarks](/factories/benchmarks/) compare fixed tasks across configurations, and [Self-improvement](/factories/measure-and-improve/#configure-and-review-self-improvement) turns repeated failures into follow-up work the factory proposes for review. +* **Measurement and self-improvement** - The [factory dashboard](/factories/factory-dashboard/) shows work-item status, runs, automations, costs, and benchmarks. [Scorers](/factories/measure-and-improve/) classify completed runs, [Benchmarks](/factories/benchmarks/) compare fixed tasks across configurations, and [Self-improvement](/factories/measure-and-improve/self-improvement/) turns repeated failures into follow-up work the factory proposes for review. * **Infrastructure control** - Run on Warp-hosted infrastructure, or self-host execution on an eligible Enterprise plan. Teams can also connect supported inference providers, scope secrets, and (if eligible) store transcripts, artifacts, and run attachments in their own S3 or GCS buckets. See [infrastructure and security](/factories/infrastructure-and-security/) for the available controls. ## How Warp Factories relates to other Warp products diff --git a/src/content/docs/factories/measure-and-improve.mdx b/src/content/docs/factories/measure-and-improve.mdx index ea32b2f1..bfd473b7 100644 --- a/src/content/docs/factories/measure-and-improve.mdx +++ b/src/content/docs/factories/measure-and-improve.mdx @@ -1,8 +1,8 @@ --- title: Measure and improve a factory description: >- - Measure factory activity and costs, evaluate completed runs, compare - agent configurations, and turn failures into follow-up work. + An overview of how Warp Factories measures and improves a factory, with + links to Scorers, Self-improvement, and Benchmarks. sidebar: label: "Measure and improve" --- @@ -42,24 +42,7 @@ Use the **Dashboard** page to pick which runs to investigate, not to conclude wh ## Configure Scorers -A **Scorer** uses an LLM judge to classify completed runs against criteria you write — for example, "did the agent run the tests before opening a PR?" It assigns a label, not a numeric grade, so keep each Scorer focused on one question its failures can point back to. Create Scorers on the factory dashboard's **Scorers** page, which also holds each Scorer's results. For Scorers defined as files in a factory definition, see the [`scorers//scorer.md` syntax](/factories/factory-as-code/#scorersnamescorermd) and the two scorers in [`02-sdlc-issue-to-pr`](https://github.com/warpdotdev/warp-factory-examples/tree/main/examples/02-sdlc-issue-to-pr) in the [warp-factory-examples](https://github.com/warpdotdev/warp-factory-examples) repository. - -Configure these fields: - -* **Agent(s) to evaluate** - The agents this Scorer applies to. Select at least one. -* **Judge instructions** - The criteria the judge checks for. -* **Judge model** - The model that acts as the judge. -* **Classifications** - The labels the judge can assign, each with a score. -* **Pass threshold** - The score a run needs to pass. -* **Sample rate** - The share of the selected agents' completed runs to evaluate. - -{/* VISUAL: The Scorer create/edit form (judge instructions, judge model, classifications, pass threshold, sample rate). */} - -While the sample rate is above 0, scoring happens automatically: shortly after a sampled run completes, the judge evaluates it and records a classification, a score, and its reasoning. To stop automatic scoring, set the sample rate to 0. - -You can also score any single run on demand, which is useful for testing new judge instructions before raising the sample rate. Scoring a run again replaces its previous result from that Scorer. - -Changing **Pass threshold** updates how past scores display as pass or fail; the recorded results don't change. +A **Scorer** uses an LLM judge to classify completed runs against criteria you write — for example, "did the agent run the tests before opening a PR?" See [Configuring Scorers](/factories/measure-and-improve/scorers/) for its fields and how automatic and on-demand scoring work. ## Compare configurations with benchmarks @@ -67,11 +50,7 @@ A benchmark compares model and runner configurations for a single agent on the s ## Configure and review Self-improvement -{/* VISUAL: The Self-improvement pull request list, or a Benchmarks suite run -- this section is text-only today. */} - -Turn on **Self-improvement** for each Scorer whose failures you want investigated automatically. Self-improvement groups related failures and files follow-up tasks as ordinary agent runs. A follow-up run can propose changes to application code. It can also improve the factory itself: when you manage your factory as [definitions as code](/factories/factory-as-code/), its prompts, skills, and configuration are version-controlled files, so a follow-up run can open a pull request against the factory definition the same way it would against application code. Nothing is adopted without your review. - -The factory dashboard's **Self-improvement** page lists the pull requests these follow-up runs open. Each pull request includes a **Regressions addressed** section that links the failing runs and Scorer results behind it, so you can trace the change back to its evidence. +**Self-improvement** turns a Scorer's repeated failures into follow-up pull requests, against application code or the factory's own definition. See [Configuring and reviewing Self-improvement](/factories/measure-and-improve/self-improvement/) for how to turn it on and review its pull requests. ## Run a practical improvement loop @@ -96,6 +75,9 @@ flowchart LR 5. **Review and adopt.** If the evidence supports the change, make it. Review Self-improvement pull requests with the same standards as human-authored ones. 6. **Keep monitoring.** Leave the Scorer active and compare new results against your baseline. Revise the Scorer, or set its sample rate to 0, when its criteria no longer match what your team needs. -## Next step +## Related pages -Record an adopted change in [factory definitions as code](/factories/factory-as-code/) so your team can review the factory configuration. +* [Configuring Scorers](/factories/measure-and-improve/scorers/) - What a Scorer is, its fields, and how it runs. +* [Configuring and reviewing Self-improvement](/factories/measure-and-improve/self-improvement/) - Turn on Self-improvement and review its pull requests. +* [Benchmarking factory agent configurations](/factories/benchmarks/) - Compare model and runner configurations on the same tasks. +* [Definitions as code](/factories/factory-as-code/) - Record an adopted change so your team can review the factory configuration. diff --git a/src/content/docs/factories/measure-and-improve/scorers.mdx b/src/content/docs/factories/measure-and-improve/scorers.mdx new file mode 100644 index 00000000..4d9a177f --- /dev/null +++ b/src/content/docs/factories/measure-and-improve/scorers.mdx @@ -0,0 +1,33 @@ +--- +title: Configuring Scorers +description: >- + Configure a Scorer to have an LLM judge classify completed factory runs + against criteria you write. +sidebar: + label: "Scorers" +--- + +A **Scorer** uses an LLM judge to classify completed runs against criteria you write — for example, "did the agent run the tests before opening a PR?" It assigns a label, not a numeric grade, so keep each Scorer focused on one question its failures can point back to. Create Scorers on the factory dashboard's **Scorers** page, which also holds each Scorer's results. For Scorers defined as files in a factory definition, see the [`scorers//scorer.md` syntax](/factories/factory-as-code/#scorersnamescorermd) and the two scorers in [`02-sdlc-issue-to-pr`](https://github.com/warpdotdev/warp-factory-examples/tree/main/examples/02-sdlc-issue-to-pr) in the [warp-factory-examples](https://github.com/warpdotdev/warp-factory-examples) repository. + +Configure these fields: + +* **Agent(s) to evaluate** - The agents this Scorer applies to. Select at least one. +* **Judge instructions** - The criteria the judge checks for. +* **Judge model** - The model that acts as the judge. +* **Classifications** - The labels the judge can assign, each with a score. +* **Pass threshold** - The score a run needs to pass. +* **Sample rate** - The share of the selected agents' completed runs to evaluate. + +{/* VISUAL: The Scorer create/edit form (judge instructions, judge model, classifications, pass threshold, sample rate). */} + +While the sample rate is above 0, scoring happens automatically: shortly after a sampled run completes, the judge evaluates it and records a classification, a score, and its reasoning. To stop automatic scoring, set the sample rate to 0. + +You can also score any single run on demand, which is useful for testing new judge instructions before raising the sample rate. Scoring a run again replaces its previous result from that Scorer. + +Changing **Pass threshold** updates how past scores display as pass or fail; the recorded results don't change. + +## Related pages + +* [Measure and improve a factory](/factories/measure-and-improve/) - Read dashboard metrics and run a practical improvement loop. +* [Configuring and reviewing Self-improvement](/factories/measure-and-improve/self-improvement/) - Turn repeated Scorer failures into follow-up pull requests. +* [Benchmarking factory agent configurations](/factories/benchmarks/) - Compare model and runner configurations on the same tasks. diff --git a/src/content/docs/factories/measure-and-improve/self-improvement.mdx b/src/content/docs/factories/measure-and-improve/self-improvement.mdx new file mode 100644 index 00000000..3dc792b3 --- /dev/null +++ b/src/content/docs/factories/measure-and-improve/self-improvement.mdx @@ -0,0 +1,20 @@ +--- +title: Configuring and reviewing Self-improvement +description: >- + Turn on Self-improvement for a Scorer to turn its repeated failures into + follow-up pull requests you review. +sidebar: + label: "Self-improvement" +--- + +{/* VISUAL: The Self-improvement pull request list, or a Benchmarks suite run -- this section is text-only today. */} + +Turn on **Self-improvement** for each Scorer whose failures you want investigated automatically. Self-improvement groups related failures and files follow-up tasks as ordinary agent runs. A follow-up run can propose changes to application code. It can also improve the factory itself: when you manage your factory as [definitions as code](/factories/factory-as-code/), its prompts, skills, and configuration are version-controlled files, so a follow-up run can open a pull request against the factory definition the same way it would against application code. Nothing is adopted without your review. + +The factory dashboard's **Self-improvement** page lists the pull requests these follow-up runs open. Each pull request includes a **Regressions addressed** section that links the failing runs and Scorer results behind it, so you can trace the change back to its evidence. + +## Related pages + +* [Measure and improve a factory](/factories/measure-and-improve/) - Read dashboard metrics and run a practical improvement loop. +* [Configuring Scorers](/factories/measure-and-improve/scorers/) - Define the criteria whose failures Self-improvement investigates. +* [Benchmarking factory agent configurations](/factories/benchmarks/) - Compare model and runner configurations on the same tasks. diff --git a/src/content/docs/guides/agent-workflows/build-a-self-improving-agent.mdx b/src/content/docs/guides/agent-workflows/build-a-self-improving-agent.mdx index b8cfe828..3b6b9128 100644 --- a/src/content/docs/guides/agent-workflows/build-a-self-improving-agent.mdx +++ b/src/content/docs/guides/agent-workflows/build-a-self-improving-agent.mdx @@ -120,7 +120,7 @@ Over time, the companion skill accumulates a clear description of how your team ## Next steps -* [Self-improvement in Warp Factories](/factories/measure-and-improve/#configure-and-review-self-improvement) — The managed version of this outer loop, run inside a factory. +* [Self-improvement in Warp Factories](/factories/measure-and-improve/self-improvement/) — The managed version of this outer loop, run inside a factory. * [Set up your software factory](/guides/agent-workflows/set-up-a-software-factory) — The inner loop the outer loop improves. * [Run a software factory in the cloud](/guides/agent-workflows/run-a-software-factory-in-the-cloud) — Move the loop to the {VARS.WARP_AUTOMATION_PLATFORM} for team-wide visibility. * [Scheduled agents](/platform/triggers/scheduled-agents) — Full reference for running cloud agents on a cadence. diff --git a/src/sidebar.ts b/src/sidebar.ts index 9433b3ed..51141801 100644 --- a/src/sidebar.ts +++ b/src/sidebar.ts @@ -457,8 +457,22 @@ export const sidebarTopics: StarlightSidebarTopicsUserConfig = [ items: [ { slug: 'factories/factory-inbox', label: 'Factory inbox' }, { slug: 'factories/factory-dashboard', label: 'Factory dashboard' }, - { slug: 'factories/measure-and-improve', label: 'Measure and improve' }, - { slug: 'factories/benchmarks', label: 'Benchmarks' }, + { + // Nests Scorers, Self-improvement, and Benchmarks as coequal + // children, matching the factory dashboard's own "Score and + // benchmark" tab grouping instead of splitting them across a + // bundled page and a standalone one. Expanded by default (unlike + // Harnesses/Environments above): the point of nesting is to make + // the three siblings immediately visible, not tucked behind a click. + label: 'Measure and improve', + collapsed: false, + items: [ + { slug: 'factories/measure-and-improve', label: 'Overview' }, + { slug: 'factories/measure-and-improve/scorers', label: 'Scorers' }, + { slug: 'factories/measure-and-improve/self-improvement', label: 'Self-improvement' }, + { slug: 'factories/benchmarks', label: 'Benchmarks' }, + ], + }, ], }, // Troubleshooting sits outside the groups, last in the tab. It was in From 59e5b4fbaaeb3dc881a23f377425719f12f3cbc5 Mon Sep 17 00:00:00 2001 From: Rachael Rose Renk <91027132+rachaelrenk@users.noreply.github.com> Date: Wed, 9 Sep 2026 15:58:13 -0600 Subject: [PATCH 3/6] Update src/content/docs/factories/measure-and-improve.mdx Co-authored-by: warp-for-oss[bot] <277970191+warp-for-oss[bot]@users.noreply.github.com> --- src/content/docs/factories/measure-and-improve.mdx | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/content/docs/factories/measure-and-improve.mdx b/src/content/docs/factories/measure-and-improve.mdx index bfd473b7..8658d3cb 100644 --- a/src/content/docs/factories/measure-and-improve.mdx +++ b/src/content/docs/factories/measure-and-improve.mdx @@ -1,8 +1,8 @@ --- title: Measure and improve a factory description: >- - An overview of how Warp Factories measures and improves a factory, with - links to Scorers, Self-improvement, and Benchmarks. + Measure factory activity, compare benchmarks, and use Scorers and + Self-improvement to turn repeated failures into follow-up work. sidebar: label: "Measure and improve" --- From b6f0ba1a036639119b43f3172a4f3184d8941b16 Mon Sep 17 00:00:00 2001 From: Rachael Rose Renk <91027132+rachaelrenk@users.noreply.github.com> Date: Wed, 9 Sep 2026 15:58:23 -0600 Subject: [PATCH 4/6] Update src/content/docs/factories/how-factories-work.mdx Co-authored-by: warp-for-oss[bot] <277970191+warp-for-oss[bot]@users.noreply.github.com> --- src/content/docs/factories/how-factories-work.mdx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/content/docs/factories/how-factories-work.mdx b/src/content/docs/factories/how-factories-work.mdx index bc3af48a..99e9e433 100644 --- a/src/content/docs/factories/how-factories-work.mdx +++ b/src/content/docs/factories/how-factories-work.mdx @@ -65,7 +65,7 @@ The first two are workflow policy, written into the foreman's instructions; edit ## How the factory improves itself -Your factory is self-improving, and you define what "better" means. [Scorers](/factories/measure-and-improve/) classify completed runs against criteria you write, and [Self-improvement](/factories/measure-and-improve/self-improvement/) groups the failures they flag into follow-up runs that propose fixes — to the application code or to the factory's own definition. Every proposal arrives as a change for your review; nothing is adopted on its own. +Your factory is self-improving, and you define what "better" means. [Scorers](/factories/measure-and-improve/scorers/) classify completed runs against criteria you write, and [Self-improvement](/factories/measure-and-improve/self-improvement/) groups the failures they flag into follow-up runs that propose fixes — to the application code or to the factory's own definition. Every proposal arrives as a change for your review; nothing is adopted on its own. The factory's definition is open to the same loop. Anyone on the team, or an agent, can propose changes to its instructions, skills, models, or other [definition files](/factories/factory-as-code/), and definitions stored in GitHub go through pull request review and [configuration checks](/factories/factory-as-code/#pull-request-checks-for-github-backed-factories) before a change reaches the production branch. From 5634fff1f3f4fd3ee6fa246c7eb4e6d176a9cb72 Mon Sep 17 00:00:00 2001 From: Rachael Rose Renk <91027132+rachaelrenk@users.noreply.github.com> Date: Wed, 9 Sep 2026 15:58:56 -0600 Subject: [PATCH 5/6] Update src/content/docs/factories/index.mdx Co-authored-by: warp-for-oss[bot] <277970191+warp-for-oss[bot]@users.noreply.github.com> --- src/content/docs/factories/index.mdx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/content/docs/factories/index.mdx b/src/content/docs/factories/index.mdx index 23649ec6..dd620acd 100644 --- a/src/content/docs/factories/index.mdx +++ b/src/content/docs/factories/index.mdx @@ -52,7 +52,7 @@ Warp Factories is designed for engineering teams with repeatable work that exten * **Definitions as code** - [Version-controlled definition files](/factories/factory-as-code/) describe your repositories, agents, automations, runners, [skills](/factories/factory-skills/), and MCP servers, so factory changes get the same review, history, and rollback as code changes. * **Integrations and the Factory MCP** - Work flows in from [Slack](/factories/integrations/slack/), [GitHub](/factories/integrations/github/), [GitLab](/factories/integrations/gitlab/), [Linear](/factories/integrations/linear/), and [Jira](/factories/integrations/jira/), plus [custom webhooks](/factories/webhooks/), direct runs, and schedules. The [Factory MCP](/factories/factory-mcp/) connects coding agents and other MCP clients. * **Model and harness choice** - Each agent can use a different model and [supported harness](/platform/harnesses/), including the Warp Agent, Claude Code, and Codex. -* **Measurement and self-improvement** - The [factory dashboard](/factories/factory-dashboard/) shows work-item status, runs, automations, costs, and benchmarks. [Scorers](/factories/measure-and-improve/) classify completed runs, [Benchmarks](/factories/benchmarks/) compare fixed tasks across configurations, and [Self-improvement](/factories/measure-and-improve/self-improvement/) turns repeated failures into follow-up work the factory proposes for review. +* **Measurement and self-improvement** - The [factory dashboard](/factories/factory-dashboard/) shows work-item status, runs, automations, costs, and benchmarks. [Scorers](/factories/measure-and-improve/scorers/) classify completed runs, [Benchmarks](/factories/benchmarks/) compare fixed tasks across configurations, and [Self-improvement](/factories/measure-and-improve/self-improvement/) turns repeated failures into follow-up work the factory proposes for review. * **Infrastructure control** - Run on Warp-hosted infrastructure, or self-host execution on an eligible Enterprise plan. Teams can also connect supported inference providers, scope secrets, and (if eligible) store transcripts, artifacts, and run attachments in their own S3 or GCS buckets. See [infrastructure and security](/factories/infrastructure-and-security/) for the available controls. ## How Warp Factories relates to other Warp products From 16b32c2e84b1a1c439a40b81afabd8130625d671 Mon Sep 17 00:00:00 2001 From: Rachael Rose Renk <91027132+rachaelrenk@users.noreply.github.com> Date: Fri, 11 Sep 2026 10:36:10 -0600 Subject: [PATCH 6/6] docs: address factory measurement review feedback Co-Authored-By: Warp --- .agents/skills/draft_docs/SKILL.md | 1 + src/content/docs/factories/measure-and-improve.mdx | 6 +++--- .../docs/factories/measure-and-improve/scorers.mdx | 11 +++++------ .../measure-and-improve/self-improvement.mdx | 6 +++++- src/sidebar.ts | 9 +-------- 5 files changed, 15 insertions(+), 18 deletions(-) diff --git a/.agents/skills/draft_docs/SKILL.md b/.agents/skills/draft_docs/SKILL.md index 50f891a0..c30f05de 100644 --- a/.agents/skills/draft_docs/SKILL.md +++ b/.agents/skills/draft_docs/SKILL.md @@ -156,6 +156,7 @@ These rules are frequently violated by agents. Apply them carefully during draft - **Keep error messages out of the main flow** — Do not weave full error strings through conceptual or procedural sections. Put them in a dedicated `## Troubleshooting` section near the end, formatted symptom → cause → fix. ✅ one Troubleshooting section with the exact error as a bold lead-in ❌ repeating the same error callout after every step - **Use callouts sparingly** — Prefer body prose. At most one or two callouts per page unless the content type template requires more, never two callouts back to back, and at most one per section. ✅ a single `:::note` for a non-obvious prerequisite ❌ a `:::note` / `:::tip` after every subsection - **No AI-ism buzzwords or meta-openers** — Never open with "This page covers/explains/walks through..."; state the thing itself. Avoid marketing adjectives (seamless, powerful, robust, comprehensive), inflated verbs (leverage, streamline, empower, unlock), restated cause-and-effect ("This process ensures..."), and recap lines. See AGENTS.md → Voice & tone for the full lists. ✅ "Run agents directly in your GitHub Actions workflows using `oz-agent-action`." ❌ "This page covers how the integration works, how to set it up, and common automation patterns." +- **Keep source comments rare** — Do not add comments that restate a sidebar item's hierarchy, label, or obvious grouping. Add a short comment only for a non-obvious invariant, compatibility constraint, or behavior that the code cannot make clear. Never compare unrelated sidebar sections merely to justify a local choice. - **Cut length before shipping** — Brevity is a drafting rule, not only a review checklist item. After the first complete draft, run a deletion-only "Cut again" pass (AGENTS.md → Voice & tone → Cut again): remove framing lines, restated cause-and-effect, internal implementation detail the reader cannot act on, and boilerplate a parent page already owns. Prefer one concrete sentence over a paragraph that re-explains the same step. A page past ~1500 words usually still has padding — cut first; split only if the remaining content still does not fit. ✅ a setup section that states only the required clicks and links out for credential types ❌ a multi-paragraph walkthrough of harness internals on a how-to page - **Document the user-visible model, not internal architecture** — Internal components (orchestrators, control planes, lifecycle state machines) get at most one sentence, and only when the reader can act on them. ✅ "Warp tracks every run. Check its status from the CLI, the API, or the dashboard." ❌ "The orchestration layer runs on Warp's servers (cloud control plane) and tracks lifecycle state (created → running → completed/failed)." - **Descriptive link text, and no dead-end pages** — Never use "here", "this page", or a bare URL as link text. End every new page with `## Next steps` (quickstarts and tutorials) or `## Related pages` (every other type), containing at least one internal link whose anchor names the destination topic. Do not use "Further reading" or "See also". ✅ `Learn more about [Codebase Context](/code/codebase-context/)` ❌ `Click [here](/code/codebase-context/)` ❌ ending a new feature page with no cross-links diff --git a/src/content/docs/factories/measure-and-improve.mdx b/src/content/docs/factories/measure-and-improve.mdx index 8658d3cb..191b1479 100644 --- a/src/content/docs/factories/measure-and-improve.mdx +++ b/src/content/docs/factories/measure-and-improve.mdx @@ -1,10 +1,10 @@ --- title: Measure and improve a factory description: >- - Measure factory activity, compare benchmarks, and use Scorers and - Self-improvement to turn repeated failures into follow-up work. + Read factory activity and cost metrics, then find where to configure + Scorers, Self-improvement, and Benchmarks. sidebar: - label: "Measure and improve" + label: "Overview" --- Warp Factories tracks what your factory produces and how well it performs, so you can spot a problem, test a fix, and decide whether to keep it. diff --git a/src/content/docs/factories/measure-and-improve/scorers.mdx b/src/content/docs/factories/measure-and-improve/scorers.mdx index 4d9a177f..56f0a883 100644 --- a/src/content/docs/factories/measure-and-improve/scorers.mdx +++ b/src/content/docs/factories/measure-and-improve/scorers.mdx @@ -1,13 +1,13 @@ --- title: Configuring Scorers description: >- - Configure a Scorer to have an LLM judge classify completed factory runs - against criteria you write. + Configure an LLM-based evaluation that checks completed factory runs from + selected agents against criteria you define. sidebar: label: "Scorers" --- -A **Scorer** uses an LLM judge to classify completed runs against criteria you write — for example, "did the agent run the tests before opening a PR?" It assigns a label, not a numeric grade, so keep each Scorer focused on one question its failures can point back to. Create Scorers on the factory dashboard's **Scorers** page, which also holds each Scorer's results. For Scorers defined as files in a factory definition, see the [`scorers//scorer.md` syntax](/factories/factory-as-code/#scorersnamescorermd) and the two scorers in [`02-sdlc-issue-to-pr`](https://github.com/warpdotdev/warp-factory-examples/tree/main/examples/02-sdlc-issue-to-pr) in the [warp-factory-examples](https://github.com/warpdotdev/warp-factory-examples) repository. +A **Scorer** is an LLM-based evaluation, or eval, that checks whether completed runs from selected agents meet criteria you define. For example, it can check, "Did the agent run the tests before opening a pull request?" A Scorer assigns a classification, not a numeric grade, so keep each one focused on a question its failures can point back to. Create Scorers on the factory dashboard's **Scorers** page, which also holds each Scorer's results. For Scorers defined as files in a factory definition, see the [`scorers//scorer.md` syntax](/factories/factory-as-code/#scorersnamescorermd) and the two scorers in [`02-sdlc-issue-to-pr`](https://github.com/warpdotdev/warp-factory-examples/tree/main/examples/02-sdlc-issue-to-pr) in the [warp-factory-examples](https://github.com/warpdotdev/warp-factory-examples) repository. Configure these fields: @@ -16,15 +16,14 @@ Configure these fields: * **Judge model** - The model that acts as the judge. * **Classifications** - The labels the judge can assign, each with a score. * **Pass threshold** - The score a run needs to pass. -* **Sample rate** - The share of the selected agents' completed runs to evaluate. +* **Sample rate** - The percentage of completed runs from selected agents to evaluate. For example, a 10% sample rate evaluates about one in ten runs. {/* VISUAL: The Scorer create/edit form (judge instructions, judge model, classifications, pass threshold, sample rate). */} -While the sample rate is above 0, scoring happens automatically: shortly after a sampled run completes, the judge evaluates it and records a classification, a score, and its reasoning. To stop automatic scoring, set the sample rate to 0. +With a sample rate above 0%, the Scorer automatically evaluates sampled runs after they complete and records a classification, score, and reasoning. You can also score any single run on demand, which is useful for testing new judge instructions before raising the sample rate. Scoring a run again replaces its previous result from that Scorer. -Changing **Pass threshold** updates how past scores display as pass or fail; the recorded results don't change. ## Related pages diff --git a/src/content/docs/factories/measure-and-improve/self-improvement.mdx b/src/content/docs/factories/measure-and-improve/self-improvement.mdx index 3dc792b3..8cf6b2f1 100644 --- a/src/content/docs/factories/measure-and-improve/self-improvement.mdx +++ b/src/content/docs/factories/measure-and-improve/self-improvement.mdx @@ -9,7 +9,11 @@ sidebar: {/* VISUAL: The Self-improvement pull request list, or a Benchmarks suite run -- this section is text-only today. */} -Turn on **Self-improvement** for each Scorer whose failures you want investigated automatically. Self-improvement groups related failures and files follow-up tasks as ordinary agent runs. A follow-up run can propose changes to application code. It can also improve the factory itself: when you manage your factory as [definitions as code](/factories/factory-as-code/), its prompts, skills, and configuration are version-controlled files, so a follow-up run can open a pull request against the factory definition the same way it would against application code. Nothing is adopted without your review. +Turn on **Self-improvement** for each Scorer whose failures you want investigated automatically. A failure is a score below the Scorer's pass threshold. The scheduled check starts a Self-improvement run after an agent has 25 unreviewed failures, or when its oldest unreviewed failure is seven days old. It groups the failures for each agent into a follow-up run that proposes a fix. + +To run the check without waiting for the scheduled threshold, click **Run now** on the factory dashboard's **Self-improvement** page. An ad hoc run can include an agent with one unreviewed failure. You cannot choose which agents or failures it processes. + +A follow-up run can propose changes to application code. It can also improve the factory itself: when you manage your factory as [definitions as code](/factories/factory-as-code/), its prompts, skills, and configuration are version-controlled files, so a follow-up run can open a pull request against the factory definition the same way it would against application code. Nothing is adopted without your review. The factory dashboard's **Self-improvement** page lists the pull requests these follow-up runs open. Each pull request includes a **Regressions addressed** section that links the failing runs and Scorer results behind it, so you can trace the change back to its evidence. diff --git a/src/sidebar.ts b/src/sidebar.ts index 51141801..179bac26 100644 --- a/src/sidebar.ts +++ b/src/sidebar.ts @@ -423,8 +423,7 @@ export const sidebarTopics: StarlightSidebarTopicsUserConfig = [ // // The per-service pages are listed directly rather than in a nested // Integrations subgroup, which would have rendered as - // Integrations > Integrations > Slack. Flattening also drops the tab - // to two levels, matching every other group in it. + // Integrations > Integrations > Slack. // // 'Connect your factory' leads because it is the overview for this // group; Factory MCP trails because it is a connection mechanism @@ -458,12 +457,6 @@ export const sidebarTopics: StarlightSidebarTopicsUserConfig = [ { slug: 'factories/factory-inbox', label: 'Factory inbox' }, { slug: 'factories/factory-dashboard', label: 'Factory dashboard' }, { - // Nests Scorers, Self-improvement, and Benchmarks as coequal - // children, matching the factory dashboard's own "Score and - // benchmark" tab grouping instead of splitting them across a - // bundled page and a standalone one. Expanded by default (unlike - // Harnesses/Environments above): the point of nesting is to make - // the three siblings immediately visible, not tucked behind a click. label: 'Measure and improve', collapsed: false, items: [