diff --git a/AGENTS.md b/AGENTS.md index 040bd2da4..a4ad74db9 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -27,6 +27,16 @@ Names are snake_case with flat primitive params. gtag only exists on Foundation builds (`VITE_FOUNDATION_BUILD`, see `index.html`), so OSS and local dev log events to the console instead. +## Accessibility scans + +E2e scenarios call `app.expectNoNewAccessibilityViolations()` (axe, in +`src/e2e/axe.ts`) on the states they reach. It fails only on violations missing +from `knownViolations` in that file: delete an entry when you fix it, and scan +new UI states from the scenario that reaches them. +The known list is recorded against the Foundation's private theme. Without the +theme package installed the colour contrast rule is skipped, as the open source +default colours give different results. + ## Commands - Unit tests: `npm test` (vitest). E2e: run headlessly via diff --git a/package-lock.json b/package-lock.json index ad05a3df3..aa3f8e35f 100644 --- a/package-lock.json +++ b/package-lock.json @@ -47,6 +47,7 @@ "xterm-addon-fit": "^0.5.0" }, "devDependencies": { + "@axe-core/playwright": "^4.14.0", "@csstools/postcss-cascade-layers": "^6.0.0", "@microbit/eslint-config": "^0.2.1", "@microbit/i18n-tools": "^0.2.5", @@ -487,6 +488,22 @@ "node": ">=18.0.0" } }, + "node_modules/@axe-core/playwright": { + "version": "4.14.0", + "resolved": "https://registry.npmjs.org/@axe-core/playwright/-/playwright-4.14.0.tgz", + "integrity": "sha512-SGUhG5PIHd2eMdpgBF2PyQLOhX+qyDR2PT4cmoS4Qk2ak+jX/rrSvst9l0PGv5WTHolKpUMvVmoHpr4cZgf1RQ==", + "dev": true, + "license": "MPL-2.0", + "dependencies": { + "axe-core": "~4.14.0" + }, + "engines": { + "node": ">=22.12.0" + }, + "peerDependencies": { + "playwright-core": ">= 1.0.0" + } + }, "node_modules/@babel/code-frame": { "version": "7.29.7", "resolved": "https://registry.npmjs.org/@babel/code-frame/-/code-frame-7.29.7.tgz", @@ -6723,6 +6740,16 @@ "url": "https://github.com/sponsors/ljharb" } }, + "node_modules/axe-core": { + "version": "4.14.0", + "resolved": "https://registry.npmjs.org/axe-core/-/axe-core-4.14.0.tgz", + "integrity": "sha512-9WTZxEjsZ7b13TH8JPmbV2z8CHbl80/2hm3XPEG4JgNdQLK81IBRXmSxHfMAOkSqQeRxT/0dwNDz2GOm3zzpcQ==", + "dev": true, + "license": "MPL-2.0", + "engines": { + "node": ">=4" + } + }, "node_modules/axios": { "version": "1.20.0", "resolved": "https://registry.npmjs.org/axios/-/axios-1.20.0.tgz", diff --git a/package.json b/package.json index b2df235cb..21c528932 100644 --- a/package.json +++ b/package.json @@ -52,6 +52,7 @@ "xterm-addon-fit": "^0.5.0" }, "devDependencies": { + "@axe-core/playwright": "^4.14.0", "@csstools/postcss-cascade-layers": "^6.0.0", "@microbit/eslint-config": "^0.2.1", "@microbit/i18n-tools": "^0.2.5", diff --git a/src/e2e/accessibility.test.ts b/src/e2e/accessibility.test.ts index 77247b7eb..d39bed3aa 100644 --- a/src/e2e/accessibility.test.ts +++ b/src/e2e/accessibility.test.ts @@ -39,3 +39,69 @@ test.describe("accessibility", () => { await app.assertFocusBeforeEditor(); }); }); + +// Most states are scanned by the scenario that reaches them. These are the +// ones the page object's helpers pass straight through. +test.describe("accessibility scans", () => { + test("initial load", async ({ app }) => { + await app.saveButton.waitFor(); + await app.expectNoNewAccessibilityViolations(); + }); + + test("settings menu and dialog", async ({ app }) => { + await app.page.getByTestId("settings").click(); + const settings = app.page.getByRole("menuitem", { name: "Settings" }); + await settings.waitFor(); + await app.expectNoNewAccessibilityViolations(); + + await settings.click(); + await app.page.getByText("Font size").waitFor(); + await app.expectNoNewAccessibilityViolations(); + }); + + test("language dialog", async ({ app }) => { + await app.page.getByTestId("settings").click(); + await app.page.getByTestId("language").click(); + await app.page.getByTestId("fr").waitFor(); + await app.expectNoNewAccessibilityViolations(); + }); + + test("connect dialog", async ({ app }) => { + await app.page.getByTestId("more-connect-options").click(); + const connect = app.page.getByRole("menuitem", { name: "Connect" }); + await connect.waitFor(); + await app.expectNoNewAccessibilityViolations(); + + await connect.click(); + const next = app.page.getByRole("button", { name: "Next" }); + await next.waitFor(); + await app.expectNoNewAccessibilityViolations(); + + await next.click(); + await app.page.getByRole("button", { name: "Back" }).waitFor(); + await app.expectNoNewAccessibilityViolations(); + }); + + test("create file dialog", async ({ app }) => { + await app.switchTab("Project"); + await app.page.getByRole("button", { name: "Create file" }).click(); + const name = app.page.getByLabel("Name*"); + await name.waitFor(); + await app.expectNoNewAccessibilityViolations(); + + await name.fill("not a valid name!"); + await app.expectNoNewAccessibilityViolations(); + }); + + test("file actions menu", async ({ app }) => { + await app.switchTab("Project"); + await app.projectTab.openFileActionsMenu("main.py"); + await app.expectNoNewAccessibilityViolations(); + }); + + test("search results", async ({ app }) => { + await app.search("loop"); + await app.page.getByRole("dialog").getByRole("link").first().waitFor(); + await app.expectNoNewAccessibilityViolations(); + }); +}); diff --git a/src/e2e/app.ts b/src/e2e/app.ts index 363bd056b..2af7cb65d 100644 --- a/src/e2e/app.ts +++ b/src/e2e/app.ts @@ -16,6 +16,7 @@ import path from "path"; import { fileURLToPath } from "url"; import { readFileSync } from "fs"; import { DeviceErrorCode } from "@microbit/microbit-connection"; +import { expectNoNewViolations } from "./axe.js"; export enum LoadDialogType { CONFIRM, @@ -285,6 +286,10 @@ export class App { await this.editor.waitFor(); } + async expectNoNewAccessibilityViolations(): Promise { + await expectNoNewViolations(this.page); + } + async setProjectName(projectName: string): Promise { await this.page.getByRole("button", { name: "Edit project name" }).click(); await this.page.getByLabel("Name*").fill(projectName); diff --git a/src/e2e/autocomplete.test.ts b/src/e2e/autocomplete.test.ts index e6385a649..c4b7b4fb8 100644 --- a/src/e2e/autocomplete.test.ts +++ b/src/e2e/autocomplete.test.ts @@ -16,6 +16,7 @@ test.describe("autocomplete", () => { // Initial completions await app.expectCompletionOptions(["scroll", "set_pixel", "show"]); await app.expectCompletionActiveOption("scroll(text)"); + await app.expectNoNewAccessibilityViolations(); // Further refinement await app.page.keyboard.press("h"); diff --git a/src/e2e/axe.ts b/src/e2e/axe.ts new file mode 100644 index 000000000..54f37317c --- /dev/null +++ b/src/e2e/axe.ts @@ -0,0 +1,221 @@ +/** + * (c) 2026, Micro:bit Educational Foundation and contributors + * + * SPDX-License-Identifier: MIT + */ +import AxeBuilder from "@axe-core/playwright"; +import { Page, test } from "@playwright/test"; +import { createRequire } from "module"; +import path from "path"; + +interface KnownViolation { + /** axe rule id. */ + rule: string; + /** + * What the rule tests, from axe's tags for it: a WCAG conformance level, or + * an axe best practice that no WCAG success criterion requires. + */ + level: "A" | "AA" | "best-practice"; + /** The WCAG success criterion, for A and AA. */ + criterion?: string; + /** Elements the rule is known to fail on, as a CSS selector. */ + selector: string; + reason: string; +} + +/** + * Violations we know about and have not fixed yet. A scan fails only on + * violations that aren't listed here, so fixing one means deleting its entry. + * + * The list is recorded against the Foundation's private theme + * (@microbit-foundation/python-editor-v3-microbit), which is what CI and + * the deployed editor use. Colours come from the theme, so without it (a fork, + * or a checkout that hasn't installed it) the colour-contrast results are + * different and that rule is skipped. See themeDependentRules. + */ +const knownViolations: KnownViolation[] = [ + { + rule: "aria-input-field-name", + level: "A", + criterion: "4.1.2 Name, Role, Value", + selector: ".cm-content", + reason: + "CodeMirror's content element has no accessible name, in the main editor and in documentation code examples.", + }, + { + rule: "aria-prohibited-attr", + level: "A", + criterion: "4.1.2 Name, Role, Value", + selector: 'div[aria-label]:not([role])[style*="left"]', + reason: + "Simulator range sensor threshold marks put aria-label on a div with no role.", + }, + { + rule: "role-img-alt", + level: "A", + criterion: "1.1.1 Non-text Content", + selector: 'div[role="img"]', + reason: + "The sidebar header logos and simulator sensor icons are role=img wrappers with no label.", + }, + { + rule: "color-contrast", + level: "AA", + criterion: "1.4.3 Contrast (Minimum)", + selector: ".cm-gutterElement", + reason: "Editor line numbers are 2.02:1 against the gutter.", + }, + { + rule: "color-contrast", + level: "AA", + criterion: "1.4.3 Contrast (Minimum)", + selector: ".sidebar-tab p", + reason: + "Unselected sidebar tab labels are 4.3:1 against the tab bar at 13px.", + }, + { + rule: "heading-order", + level: "best-practice", + selector: "h4", + reason: "API documentation entries use h4 without an h3 above them.", + }, + { + rule: "page-has-heading-one", + level: "best-practice", + selector: "html", + reason: "The app has no level-one heading.", + }, + { + rule: "region", + level: "best-practice", + selector: ".splitView__splitter", + reason: "The sidebar and simulator resize handles sit between landmarks.", + }, + { + rule: "region", + level: "best-practice", + selector: '[data-testid="simulator"] *', + reason: "The simulator's sensor controls are outside any landmark.", + }, +]; + +/** + * Rules whose results depend on the theme's colours. The known violations + * above only describe the Foundation theme, so these rules are skipped when + * the app is running with the open source default theme. + */ +const themeDependentRules = ["color-contrast"]; + +const hasFoundationTheme = (() => { + try { + createRequire(path.join(process.cwd(), "package.json")).resolve( + "@microbit-foundation/python-editor-v3-microbit/panda-preset" + ); + return true; + } catch { + return false; + } +})(); + +interface NodeSummary { + target: string; + html: string; + failureSummary?: string; +} + +interface ViolationSummary { + id: string; + impact?: string | null; + help: string; + helpUrl: string; + nodes: NodeSummary[]; +} + +const format = (violations: ViolationSummary[]): string => { + const lines = [ + `${violations.length} accessibility violation(s) not in knownViolations (src/e2e/axe.ts):`, + ]; + for (const v of violations) { + lines.push("", `${v.id} (${v.impact ?? "n/a"}): ${v.help}`, v.helpUrl); + for (const n of v.nodes) { + lines.push(` ${n.target}`, ` ${n.html}`); + if (n.failureSummary) { + lines.push( + ...n.failureSummary.split("\n").map((line) => ` ${line}`) + ); + } + } + } + return lines.join("\n"); +}; + +/** + * Scans the page in its current state with axe and fails on any violation + * that isn't in knownViolations. + */ +export const expectNoNewViolations = async (page: Page): Promise => { + // A scan part-way through a fade reports the blended colours as contrast + // failures. Looping animations (the editor cursor) never finish. + await page.evaluate(() => + Promise.allSettled( + document + .getAnimations() + .filter((a) => a.effect?.getComputedTiming().endTime !== Infinity) + .map((a) => a.finished) + ) + ); + + let builder = new AxeBuilder({ page }) + // The simulator is a separate, cross-origin deployment with its own + // tests. Its content also hides page-level results for this document: + // its h1 satisfies page-has-heading-one. + .exclude("iframe"); + if (!hasFoundationTheme) { + builder = builder.disableRules(themeDependentRules); + } + const { violations } = await builder.analyze(); + + const candidates = violations.flatMap((v) => + v.nodes.map((n) => ({ + target: n.target.join(" "), + known: knownViolations + .filter((k) => k.rule === v.id) + .map((k) => k.selector), + })) + ); + const isKnown = await page.evaluate( + (candidates) => + candidates.map(({ target, known }) => { + const element = document.querySelector(target); + return !!element && known.some((selector) => element.matches(selector)); + }), + candidates + ); + + let index = 0; + const unexpected: ViolationSummary[] = []; + for (const v of violations) { + const nodes = v.nodes.filter(() => !isKnown[index++]); + if (nodes.length > 0) { + unexpected.push({ + id: v.id, + impact: v.impact, + help: v.help, + helpUrl: v.helpUrl, + nodes: nodes.map((n) => ({ + target: n.target.join(" "), + html: n.html, + failureSummary: n.failureSummary, + })), + }); + } + } + + if (unexpected.length > 0) { + await test.info().attach("axe-violations.json", { + body: JSON.stringify(unexpected, null, 2), + contentType: "application/json", + }); + throw new Error(format(unexpected)); + } +}; diff --git a/src/e2e/connect.test.ts b/src/e2e/connect.test.ts index 45bede5ae..e18672bfc 100644 --- a/src/e2e/connect.test.ts +++ b/src/e2e/connect.test.ts @@ -16,6 +16,7 @@ test.describe("connect", () => { // Connect and disconnect wait for serial to be shown/hidden await app.connect(); await app.expectConnected(); + await app.expectNoNewAccessibilityViolations(); await app.disconnect(); await app.expectDisconnected(); }); @@ -54,6 +55,7 @@ test.describe("connect", () => { await app.mockDeviceConnectFailure("no-device-selected"); await app.connect(); await app.expectDialog("No micro:bit found"); + await app.expectNoNewAccessibilityViolations(); await app.connectViaTryAgain(); await app.connectViaConnectHelp(); await app.expectConnected(); @@ -76,6 +78,7 @@ test.describe("connect", () => { await app.mockDeviceConnectFailure("firmware-update-required"); await app.connect(); await app.expectDialog("Firmware update required"); + await app.expectNoNewAccessibilityViolations(); await app.connectViaTryAgain(); await app.connectViaConnectHelp(); await app.expectConnected(); @@ -90,5 +93,6 @@ test.describe("connect", () => { await app.expectDialog("This browser does not support WebUSB"); await app.closeDialog(); await app.expectDialog("Transfer saved hex file to micro:bit"); + await app.expectNoNewAccessibilityViolations(); }); }); diff --git a/src/e2e/documentation.test.ts b/src/e2e/documentation.test.ts index 7a63976a6..a3da31044 100644 --- a/src/e2e/documentation.test.ts +++ b/src/e2e/documentation.test.ts @@ -12,6 +12,7 @@ test.describe("documentation", () => { "API", "For usage and examples, see" ); + await app.expectNoNewAccessibilityViolations(); }); test("Copy code and paste in editor", async ({ app }) => { @@ -67,6 +68,7 @@ test.describe("documentation", () => { "Loops", "Count and repeat sets of instructions" ); + await app.expectNoNewAccessibilityViolations(); }); test("Ideas tab navigation", async ({ app }) => { @@ -82,5 +84,6 @@ test.describe("documentation", () => { await app.switchTab("Ideas"); await app.selectDocumentationIdea(ideaName); await app.expectDocumentationTopLevelHeading(ideaName); + await app.expectNoNewAccessibilityViolations(); }); }); diff --git a/src/e2e/multiple-files.test.ts b/src/e2e/multiple-files.test.ts index a33f94bfa..c00a80a20 100644 --- a/src/e2e/multiple-files.test.ts +++ b/src/e2e/multiple-files.test.ts @@ -23,6 +23,7 @@ test.describe("multiple-files", () => { await app.expectEditorContainText(/Your new file/); await app.expectProjectFiles(["main.py", "test.py"]); + await app.expectNoNewAccessibilityViolations(); }); test("Prevents deleting main.py", async ({ app }) => { diff --git a/src/e2e/settings.test.ts b/src/e2e/settings.test.ts index 147665e01..95d9054df 100644 --- a/src/e2e/settings.test.ts +++ b/src/e2e/settings.test.ts @@ -20,6 +20,7 @@ test.describe("settings", () => { test("switches language", async ({ app }) => { await app.switchLanguage("fr"); await app.expectProjectName("Projet sans titre"); + await app.expectNoNewAccessibilityViolations(); await app.switchLanguage("en"); await app.expectProjectName("Untitled project"); }); diff --git a/src/e2e/simulator.test.ts b/src/e2e/simulator.test.ts index 32645a0d1..257bc4f4f 100644 --- a/src/e2e/simulator.test.ts +++ b/src/e2e/simulator.test.ts @@ -32,6 +32,7 @@ test.describe("simulator", () => { await app.simulator.simulatorSelectGesture("freefall"); await app.simulator.sendGestureButton.click(); await app.simulator.expectResponse(); + await app.expectNoNewAccessibilityViolations(); }); test("responds to a range sensor change", async ({ app }) => { diff --git a/src/simulator/Simulator.tsx b/src/simulator/Simulator.tsx index 0f197d7eb..6eead899d 100644 --- a/src/simulator/Simulator.tsx +++ b/src/simulator/Simulator.tsx @@ -100,6 +100,7 @@ const Simulator = ({ flexDirection="column" height="100%" position="relative" + data-testid="simulator" >