Repository navigation
Expand file tree
/
Copy pathtesting-layers.json
More file actions
279 lines (279 loc) · 12.9 KB
/
Copy pathtesting-layers.json
File metadata and controls
279 lines (279 loc) · 12.9 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
{
"version": 1,
"note": "One manifest, checked by .github/scripts/testing-layers.mjs: every test file in the repository belongs to a suite row, and every workflow in .github/workflows/ has a row. The tables in docs/testing-layers.md are generated from this file, so the two cannot drift (#749).",
"layers": {
"L1": {
"name": "commit",
"trigger": "Push to a branch, a pull request, and every queue entry, through `.github/workflows/test.yml`.",
"guarantee": "The cheapest layer that can catch a defect: it must be deterministic and hermetic — no network, no model, no wall-clock or calendar dependence, no ordering between files.",
"mustNot": "Reach the network, call a model, read the clock, or depend on a machine's home directory.",
"budget": "The whole layer under 10 minutes on a cold runner. A suite that cannot fit is either split or moved up, not added.",
"command": "node .github/scripts/testing-layers.mjs --check && npm --prefix core test && npm --prefix plugins/social test && npm --prefix plugins/tree test && npm --prefix plugins/timeline test && npm --prefix plugins/workflow test && node --test \".github/scripts/*.test.mjs\" \"policy/*.test.mjs\" && npm run test:interop"
},
"L2": {
"name": "merge",
"trigger": "Every merge into `main`, through the merge queue: `test.yml` on the queue entry plus `merge-gate.yml` on that entry.",
"guarantee": "Everything a merge must not break: every L1 suite, the ds-bot pin and patch set, the packaged plugin entry points, the strict typecheck of the plugin API, well-formed changelog fragments, the workflow lint, and the verdict/`hold` judgement at the exact commit being merged.",
"mustNot": "Be bypassed. `main` requires these checks; a ref push that never built a queue entry cannot land.",
"budget": "The checkout-to-green time of `test.yml` plus `merge-gate` on one queue entry; the queue's own limit applies.",
"command": "gh pr merge <n> --auto --squash --match-head-commit <sha>"
},
"L3": {
"name": "self-test",
"trigger": "The product self-test routine (09:00 and 21:00 Asia/Shanghai) and `live-e2e.yml` on `main` daily; a person drives the former.",
"guarantee": "What a user meets: two real homes, a real Bot Social, a real model, and a person watching. It is not deterministic, so it produces issues with reproduction steps instead of failing a merge.",
"mustNot": "Gate a merge, or be the only copy of a case that can be reproduced offline: an L3 case that turns out deterministic moves down to L1 and leaves a pointer.",
"budget": "A full self-test pass is minutes of wall-clock time with a real model; a live-e2e run is the same on `main`.",
"command": "node dev/live-e2e/run.mjs <bench> <port> --model <id> --harness <dsh checkout> --tap <bench>/tap.jsonl --tz Asia/Shanghai # needs DEEPSEEK_API_KEY; live-e2e.yml is the automated form"
}
},
"suites": [
{
"id": "core",
"layer": "L1",
"paths": [
"core/test/**/*.test.js"
],
"command": "npm --prefix core test",
"note": "The host, the client (through the generated `client.js`), the plugin API and the feed; this is the suite the `ubuntu-latest` job runs with coverage."
},
{
"id": "plugins/social",
"layer": "L1",
"paths": [
"plugins/social/test/**/*.test.js"
],
"command": "npm --prefix plugins/social test",
"note": "Peer plugin; its CI job also runs the plugin lint and fails a skipped test."
},
{
"id": "plugins/tree",
"layer": "L1",
"paths": [
"plugins/tree/test/**/*.test.js"
],
"command": "npm --prefix plugins/tree test",
"note": "Peer plugin; its CI job also runs the plugin lint and fails a skipped test."
},
{
"id": "plugins/project",
"layer": "L1",
"paths": [
"plugins/project/test/**/*.test.js"
],
"command": "npm --prefix plugins/project test",
"note": "Peer plugin; its CI job also runs the plugin lint and fails a skipped test."
},
{
"id": "plugins/timeline",
"layer": "L1",
"paths": [
"plugins/timeline/test/**/*.test.js"
],
"command": "npm --prefix plugins/timeline test",
"note": "Peer plugin; its CI job also runs the plugin lint and fails a skipped test."
},
{
"id": "plugins/workflow",
"layer": "L1",
"paths": [
"plugins/workflow/test/**/*.test.js"
],
"command": "npm --prefix plugins/workflow test",
"note": "Peer plugin; its CI job also runs the plugin lint and fails a skipped test."
},
{
"id": "plane",
"layer": "L1",
"paths": [
"plane/test/**/*.test.js"
],
"command": "npm --prefix plane test",
"note": "Management-plane skeleton: credential separation, ownership, create/start/stop, compose materialization; no network and no Docker."
},
{
"id": "scripts",
"layer": "L1",
"paths": [
".github/scripts/*.test.mjs",
"policy/*.test.mjs"
],
"command": "node --test \".github/scripts/*.test.mjs\" \"policy/*.test.mjs\"",
"note": "The repository's own gates: risk tier, review gate, merge gate, enqueue guard, changelog, labels, and this manifest's check."
},
{
"id": "interop",
"layer": "L1",
"paths": [
"dev/interop/**/*.test.mjs"
],
"command": "npm run test:interop",
"note": "The offline cross-runtime harness; `dev/interop/no-network.mjs` fails the suite if it reaches the network."
},
{
"id": "patch-payload",
"layer": "none",
"paths": [
"patches/ds-bot/files/**"
],
"reason": "Generated copies of the files the patches carry. They are compared against `core/` by `node scripts/ds-bot-patches.mjs check` (L2) and never run on their own; the code under test is the copy in `core/`."
},
{
"id": "upstream",
"layer": "none",
"paths": [
"ds-bot/test/**"
],
"reason": "The pristine upstream checkout at the pinned version. FlyBot never edits or runs it directly — `core/` is `ds-bot/` plus the patches, and the `core` suite runs the patched copies."
},
{
"id": "live-e2e-unit",
"layer": "none",
"paths": [
"dev/live-e2e/test/**"
],
"reason": "Offline unit tests for the live end-to-end harness (a fake adapter, no model). They run in the root `npm run verify` only; no CI gate runs them today. Moving them into L1 is a separate change, and the layer that does drive a real model is `live-e2e.yml`, whose row is below."
}
],
"workflows": [
{
"id": "test",
"workflow": ".github/workflows/test.yml",
"kind": "gate",
"layer": "L2",
"gate": true,
"runs": "push to main, pull_request, merge_group",
"role": "Runs every L1 suite, the packaging and patch-pin checks, the strict typecheck, the changelog check and the workflow lint on `push`, `pull_request` and `merge_group`, and aggregates them as `tests-ok`, which `main` requires."
},
{
"id": "merge-gate",
"workflow": ".github/workflows/merge-gate.yml",
"kind": "gate",
"layer": "L2",
"gate": true,
"runs": "merge_group (reports on pull_request)",
"role": "Re-reads `hold` and the newest conclusion at the entry's head, and fails the queue entry when either refuses it. On `pull_request` it also reports work `main` merged that the branch deletes (silent-revert, #914) — report only."
},
{
"id": "enqueue-guard",
"workflow": ".github/workflows/enqueue-guard.yml",
"kind": "gate",
"layer": "L2",
"gate": true,
"runs": "pull_request_target, pull_request_review, issue_comment, workflow_dispatch",
"role": "Writes the `enqueue-guard` check run a merge on the fork requires: the decision of `enqueue-guard.mjs`, on the live head, from code taken from `main`. The ruleset refuses a merge while it is red (#812)."
},
{
"id": "docker",
"workflow": ".github/workflows/docker.yml",
"kind": "workflow-only",
"layer": "L2",
"gate": true,
"runs": "pull_request that touches the image",
"role": "Builds the container image and starts it, so a packaging change cannot merge without the image proving it starts."
},
{
"id": "live-e2e",
"workflow": ".github/workflows/live-e2e.yml",
"kind": "workflow-only",
"layer": "L3",
"gate": false,
"runs": "daily on main",
"role": "Drives the real end-to-end suite with a real model against `main`; it is the automation behind the self-test layer, and it never gates a merge."
},
{
"id": "review-gate",
"workflow": ".github/workflows/review-gate.yml",
"kind": "not-a-layer",
"layer": "none",
"gate": false,
"reason": "Validates and posts the conclusion on a pull request. It gates nothing by itself: `merge-gate` re-judges the entry at merge time."
},
{
"id": "risk-tier",
"workflow": ".github/workflows/risk-tier.yml",
"kind": "not-a-layer",
"layer": "none",
"gate": false,
"reason": "Reports the computed risk tier on every pull request. Nothing reads it to block or merge; the tier that decides is recomputed by `merge-gate`."
},
{
"id": "controller",
"workflow": ".github/workflows/controller.yml",
"kind": "not-a-layer",
"layer": "none",
"gate": false,
"reason": "The label poll: sets `agent-pr` and `risk:T*`, demotes a stale claim. It runs no tests and gates nothing."
},
{
"id": "audit-merges",
"workflow": ".github/workflows/audit-merges.yml",
"kind": "not-a-layer",
"layer": "none",
"gate": false,
"reason": "A daily read-only audit that fails on a merge newer than its last green run. Advisory: it is not a required check of `main`."
},
{
"id": "label-hygiene",
"workflow": ".github/workflows/label-hygiene.yml",
"kind": "workflow-only",
"layer": "none",
"gate": false,
"runs": "daily 03:30 Asia/Shanghai, and workflow_dispatch",
"reason": "Scheduled cleanup of status labels on closed issues and pull requests (#738). A workflow-only layer with no test file of its own, and it gates nothing."
},
{
"id": "nightly",
"workflow": ".github/workflows/nightly.yml",
"kind": "workflow-only",
"layer": "none",
"gate": false,
"runs": "daily 00:00 Asia/Shanghai, Sunday 01:00 compat matrix, and workflow_dispatch",
"reason": "The compatibility matrix (other OS, Node and DSH versions) on a schedule. It widens coverage but is not a gate, and its cases are not part of L1 or L2."
},
{
"id": "release",
"workflow": ".github/workflows/release.yml",
"kind": "not-a-layer",
"layer": "none",
"gate": false,
"reason": "The mechanical release: preflight, release PR, tag. It runs no tests of its own."
},
{
"id": "pass-label",
"workflow": ".github/workflows/pass-label.yml",
"kind": "workflow-only",
"layer": "none",
"gate": false,
"runs": "issue_comment (created) and workflow_dispatch",
"reason": "Applies the `pass` label to a pull request whose conclusion was a PASS, and backfills merged ones (#739). A workflow-only layer with no test file of its own, and it gates nothing."
},
{
"id": "review-verdict",
"workflow": ".github/workflows/review-verdict.yml",
"kind": "workflow-only",
"layer": "none",
"gate": false,
"runs": "pull_request_review (submitted)",
"reason": "Says, in one warning and the run summary, that a PASS posted as a review body whose Written-by names someone outside the policy's writers does not count (#801). Read-only by construction (permissions: contents: read), so a pull request that edits it can only change what its own review job prints; a workflow-only layer with no gate."
},
{
"id": "agent-audience",
"workflow": ".github/workflows/agent-audience.yml",
"kind": "workflow-only",
"layer": "none",
"gate": false,
"runs": "pull_request (opened, edited, synchronize, reopened, ready_for_review), issue_comment (created) and workflow_dispatch",
"reason": "Reads the body's Owner:/Reviewer: lines and the calls written after the current tip, and prints where they disagree (#911). Report only: continue-on-error, read-only permissions, no comment and no label, so it gates nothing."
},
{
"id": "release-beat",
"workflow": ".github/workflows/release-beat.yml",
"kind": "workflow-only",
"layer": "none",
"gate": false,
"runs": "daily 02:30 Asia/Shanghai, and workflow_dispatch",
"reason": "The release-beat watchdog (#1209, second half): reads the newest `v*` tag, and files at most one `Release beat missing: <date>` issue per day when no tag carries today. It runs no tests of its own, does not touch release.yml, and gates nothing."
}
]
}