From 11bbd0116e5f975b759d31f90fe7918160721554 Mon Sep 17 00:00:00 2001 From: Matt Hillsdon Date: Thu, 12 Nov 2026 16:27:54 +0000 Subject: [PATCH 1/2] Run axe accessibility scans in e2e tests Scenarios scan the states they reach and fail on any violation not in the known list in src/e2e/axe.ts. Existing violations are recorded there rather than fixed here. The known list is recorded against the Foundation theme. Without the theme package the colour contrast rule is skipped, as the open source default colours fail in different places. --- AGENTS.md | 10 ++ package-lock.json | 27 +++++ package.json | 1 + src/e2e/accessibility.test.ts | 66 +++++++++++ src/e2e/app.ts | 5 + src/e2e/autocomplete.test.ts | 1 + src/e2e/axe.ts | 200 +++++++++++++++++++++++++++++++++ src/e2e/connect.test.ts | 4 + src/e2e/documentation.test.ts | 3 + src/e2e/multiple-files.test.ts | 1 + src/e2e/settings.test.ts | 1 + src/e2e/simulator.test.ts | 1 + src/simulator/Simulator.tsx | 1 + 13 files changed, 321 insertions(+) create mode 100644 src/e2e/axe.ts diff --git a/AGENTS.md b/AGENTS.md index 040bd2da4..a4ad74db9 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -27,6 +27,16 @@ Names are snake_case with flat primitive params. gtag only exists on Foundation builds (`VITE_FOUNDATION_BUILD`, see `index.html`), so OSS and local dev log events to the console instead. +## Accessibility scans + +E2e scenarios call `app.expectNoNewAccessibilityViolations()` (axe, in +`src/e2e/axe.ts`) on the states they reach. It fails only on violations missing +from `knownViolations` in that file: delete an entry when you fix it, and scan +new UI states from the scenario that reaches them. +The known list is recorded against the Foundation's private theme. Without the +theme package installed the colour contrast rule is skipped, as the open source +default colours give different results. + ## Commands - Unit tests: `npm test` (vitest). E2e: run headlessly via diff --git a/package-lock.json b/package-lock.json index ad05a3df3..aa3f8e35f 100644 --- a/package-lock.json +++ b/package-lock.json @@ -47,6 +47,7 @@ "xterm-addon-fit": "^0.5.0" }, "devDependencies": { + "@axe-core/playwright": "^4.14.0", "@csstools/postcss-cascade-layers": "^6.0.0", "@microbit/eslint-config": "^0.2.1", "@microbit/i18n-tools": "^0.2.5", @@ -487,6 +488,22 @@ "node": ">=18.0.0" } }, + "node_modules/@axe-core/playwright": { + "version": "4.14.0", + "resolved": "https://registry.npmjs.org/@axe-core/playwright/-/playwright-4.14.0.tgz", + "integrity": "sha512-SGUhG5PIHd2eMdpgBF2PyQLOhX+qyDR2PT4cmoS4Qk2ak+jX/rrSvst9l0PGv5WTHolKpUMvVmoHpr4cZgf1RQ==", + "dev": true, + "license": "MPL-2.0", + "dependencies": { + "axe-core": "~4.14.0" + }, + "engines": { + "node": ">=22.12.0" + }, + "peerDependencies": { + "playwright-core": ">= 1.0.0" + } + }, "node_modules/@babel/code-frame": { "version": "7.29.7", "resolved": "https://registry.npmjs.org/@babel/code-frame/-/code-frame-7.29.7.tgz", @@ -6723,6 +6740,16 @@ "url": "https://github.com/sponsors/ljharb" } }, + "node_modules/axe-core": { + "version": "4.14.0", + "resolved": "https://registry.npmjs.org/axe-core/-/axe-core-4.14.0.tgz", + "integrity": "sha512-9WTZxEjsZ7b13TH8JPmbV2z8CHbl80/2hm3XPEG4JgNdQLK81IBRXmSxHfMAOkSqQeRxT/0dwNDz2GOm3zzpcQ==", + "dev": true, + "license": "MPL-2.0", + "engines": { + "node": ">=4" + } + }, "node_modules/axios": { "version": "1.20.0", "resolved": "https://registry.npmjs.org/axios/-/axios-1.20.0.tgz", diff --git a/package.json b/package.json index b2df235cb..21c528932 100644 --- a/package.json +++ b/package.json @@ -52,6 +52,7 @@ "xterm-addon-fit": "^0.5.0" }, "devDependencies": { + "@axe-core/playwright": "^4.14.0", "@csstools/postcss-cascade-layers": "^6.0.0", "@microbit/eslint-config": "^0.2.1", "@microbit/i18n-tools": "^0.2.5", diff --git a/src/e2e/accessibility.test.ts b/src/e2e/accessibility.test.ts index 77247b7eb..d39bed3aa 100644 --- a/src/e2e/accessibility.test.ts +++ b/src/e2e/accessibility.test.ts @@ -39,3 +39,69 @@ test.describe("accessibility", () => { await app.assertFocusBeforeEditor(); }); }); + +// Most states are scanned by the scenario that reaches them. These are the +// ones the page object's helpers pass straight through. +test.describe("accessibility scans", () => { + test("initial load", async ({ app }) => { + await app.saveButton.waitFor(); + await app.expectNoNewAccessibilityViolations(); + }); + + test("settings menu and dialog", async ({ app }) => { + await app.page.getByTestId("settings").click(); + const settings = app.page.getByRole("menuitem", { name: "Settings" }); + await settings.waitFor(); + await app.expectNoNewAccessibilityViolations(); + + await settings.click(); + await app.page.getByText("Font size").waitFor(); + await app.expectNoNewAccessibilityViolations(); + }); + + test("language dialog", async ({ app }) => { + await app.page.getByTestId("settings").click(); + await app.page.getByTestId("language").click(); + await app.page.getByTestId("fr").waitFor(); + await app.expectNoNewAccessibilityViolations(); + }); + + test("connect dialog", async ({ app }) => { + await app.page.getByTestId("more-connect-options").click(); + const connect = app.page.getByRole("menuitem", { name: "Connect" }); + await connect.waitFor(); + await app.expectNoNewAccessibilityViolations(); + + await connect.click(); + const next = app.page.getByRole("button", { name: "Next" }); + await next.waitFor(); + await app.expectNoNewAccessibilityViolations(); + + await next.click(); + await app.page.getByRole("button", { name: "Back" }).waitFor(); + await app.expectNoNewAccessibilityViolations(); + }); + + test("create file dialog", async ({ app }) => { + await app.switchTab("Project"); + await app.page.getByRole("button", { name: "Create file" }).click(); + const name = app.page.getByLabel("Name*"); + await name.waitFor(); + await app.expectNoNewAccessibilityViolations(); + + await name.fill("not a valid name!"); + await app.expectNoNewAccessibilityViolations(); + }); + + test("file actions menu", async ({ app }) => { + await app.switchTab("Project"); + await app.projectTab.openFileActionsMenu("main.py"); + await app.expectNoNewAccessibilityViolations(); + }); + + test("search results", async ({ app }) => { + await app.search("loop"); + await app.page.getByRole("dialog").getByRole("link").first().waitFor(); + await app.expectNoNewAccessibilityViolations(); + }); +}); diff --git a/src/e2e/app.ts b/src/e2e/app.ts index 363bd056b..2af7cb65d 100644 --- a/src/e2e/app.ts +++ b/src/e2e/app.ts @@ -16,6 +16,7 @@ import path from "path"; import { fileURLToPath } from "url"; import { readFileSync } from "fs"; import { DeviceErrorCode } from "@microbit/microbit-connection"; +import { expectNoNewViolations } from "./axe.js"; export enum LoadDialogType { CONFIRM, @@ -285,6 +286,10 @@ export class App { await this.editor.waitFor(); } + async expectNoNewAccessibilityViolations(): Promise { + await expectNoNewViolations(this.page); + } + async setProjectName(projectName: string): Promise { await this.page.getByRole("button", { name: "Edit project name" }).click(); await this.page.getByLabel("Name*").fill(projectName); diff --git a/src/e2e/autocomplete.test.ts b/src/e2e/autocomplete.test.ts index e6385a649..c4b7b4fb8 100644 --- a/src/e2e/autocomplete.test.ts +++ b/src/e2e/autocomplete.test.ts @@ -16,6 +16,7 @@ test.describe("autocomplete", () => { // Initial completions await app.expectCompletionOptions(["scroll", "set_pixel", "show"]); await app.expectCompletionActiveOption("scroll(text)"); + await app.expectNoNewAccessibilityViolations(); // Further refinement await app.page.keyboard.press("h"); diff --git a/src/e2e/axe.ts b/src/e2e/axe.ts new file mode 100644 index 000000000..eb316828d --- /dev/null +++ b/src/e2e/axe.ts @@ -0,0 +1,200 @@ +/** + * (c) 2026, Micro:bit Educational Foundation and contributors + * + * SPDX-License-Identifier: MIT + */ +import AxeBuilder from "@axe-core/playwright"; +import { Page, test } from "@playwright/test"; +import { createRequire } from "module"; +import path from "path"; + +interface KnownViolation { + /** axe rule id. */ + rule: string; + /** Elements the rule is known to fail on, as a CSS selector. */ + selector: string; + reason: string; +} + +/** + * Violations we know about and have not fixed yet. A scan fails only on + * violations that aren't listed here, so fixing one means deleting its entry. + * + * The list is recorded against the Foundation's private theme + * (@microbit-foundation/python-editor-v3-microbit), which is what CI and + * the deployed editor use. Colours come from the theme, so without it (a fork, + * or a checkout that hasn't installed it) the colour-contrast results are + * different and that rule is skipped. See themeDependentRules. + */ +const knownViolations: KnownViolation[] = [ + { + rule: "aria-input-field-name", + selector: ".cm-content", + reason: + "CodeMirror's content element has no accessible name, in the main editor and in documentation code examples.", + }, + { + rule: "aria-prohibited-attr", + selector: 'div[aria-label]:not([role])[style*="left"]', + reason: + "Simulator range sensor threshold marks put aria-label on a div with no role.", + }, + { + rule: "color-contrast", + selector: ".cm-gutterElement", + reason: "Editor line numbers are 2.02:1 against the gutter.", + }, + { + rule: "color-contrast", + selector: ".sidebar-tab p", + reason: + "Unselected sidebar tab labels are 4.3:1 against the tab bar at 13px.", + }, + { + rule: "heading-order", + selector: "h4", + reason: "API documentation entries use h4 without an h3 above them.", + }, + { + rule: "page-has-heading-one", + selector: "html", + reason: "The app has no level-one heading.", + }, + { + rule: "region", + selector: ".splitView__splitter", + reason: "The sidebar and simulator resize handles sit between landmarks.", + }, + { + rule: "region", + selector: '[data-testid="simulator"] *', + reason: "The simulator's sensor controls are outside any landmark.", + }, + { + rule: "role-img-alt", + selector: 'div[role="img"]', + reason: + "The sidebar header logos and simulator sensor icons are role=img wrappers with no label.", + }, +]; + +/** + * Rules whose results depend on the theme's colours. The known violations + * above only describe the Foundation theme, so these rules are skipped when + * the app is running with the open source default theme. + */ +const themeDependentRules = ["color-contrast"]; + +const hasFoundationTheme = (() => { + try { + createRequire(path.join(process.cwd(), "package.json")).resolve( + "@microbit-foundation/python-editor-v3-microbit/panda-preset" + ); + return true; + } catch { + return false; + } +})(); + +interface NodeSummary { + target: string; + html: string; + failureSummary?: string; +} + +interface ViolationSummary { + id: string; + impact?: string | null; + help: string; + helpUrl: string; + nodes: NodeSummary[]; +} + +const format = (violations: ViolationSummary[]): string => { + const lines = [ + `${violations.length} accessibility violation(s) not in knownViolations (src/e2e/axe.ts):`, + ]; + for (const v of violations) { + lines.push("", `${v.id} (${v.impact ?? "n/a"}): ${v.help}`, v.helpUrl); + for (const n of v.nodes) { + lines.push(` ${n.target}`, ` ${n.html}`); + if (n.failureSummary) { + lines.push( + ...n.failureSummary.split("\n").map((line) => ` ${line}`) + ); + } + } + } + return lines.join("\n"); +}; + +/** + * Scans the page in its current state with axe and fails on any violation + * that isn't in knownViolations. + */ +export const expectNoNewViolations = async (page: Page): Promise => { + // A scan part-way through a fade reports the blended colours as contrast + // failures. Looping animations (the editor cursor) never finish. + await page.evaluate(() => + Promise.allSettled( + document + .getAnimations() + .filter((a) => a.effect?.getComputedTiming().endTime !== Infinity) + .map((a) => a.finished) + ) + ); + + let builder = new AxeBuilder({ page }) + // The simulator is a separate, cross-origin deployment with its own + // tests. Its content also hides page-level results for this document: + // its h1 satisfies page-has-heading-one. + .exclude("iframe"); + if (!hasFoundationTheme) { + builder = builder.disableRules(themeDependentRules); + } + const { violations } = await builder.analyze(); + + const candidates = violations.flatMap((v) => + v.nodes.map((n) => ({ + target: n.target.join(" "), + known: knownViolations + .filter((k) => k.rule === v.id) + .map((k) => k.selector), + })) + ); + const isKnown = await page.evaluate( + (candidates) => + candidates.map(({ target, known }) => { + const element = document.querySelector(target); + return !!element && known.some((selector) => element.matches(selector)); + }), + candidates + ); + + let index = 0; + const unexpected: ViolationSummary[] = []; + for (const v of violations) { + const nodes = v.nodes.filter(() => !isKnown[index++]); + if (nodes.length > 0) { + unexpected.push({ + id: v.id, + impact: v.impact, + help: v.help, + helpUrl: v.helpUrl, + nodes: nodes.map((n) => ({ + target: n.target.join(" "), + html: n.html, + failureSummary: n.failureSummary, + })), + }); + } + } + + if (unexpected.length > 0) { + await test.info().attach("axe-violations.json", { + body: JSON.stringify(unexpected, null, 2), + contentType: "application/json", + }); + throw new Error(format(unexpected)); + } +}; diff --git a/src/e2e/connect.test.ts b/src/e2e/connect.test.ts index 45bede5ae..e18672bfc 100644 --- a/src/e2e/connect.test.ts +++ b/src/e2e/connect.test.ts @@ -16,6 +16,7 @@ test.describe("connect", () => { // Connect and disconnect wait for serial to be shown/hidden await app.connect(); await app.expectConnected(); + await app.expectNoNewAccessibilityViolations(); await app.disconnect(); await app.expectDisconnected(); }); @@ -54,6 +55,7 @@ test.describe("connect", () => { await app.mockDeviceConnectFailure("no-device-selected"); await app.connect(); await app.expectDialog("No micro:bit found"); + await app.expectNoNewAccessibilityViolations(); await app.connectViaTryAgain(); await app.connectViaConnectHelp(); await app.expectConnected(); @@ -76,6 +78,7 @@ test.describe("connect", () => { await app.mockDeviceConnectFailure("firmware-update-required"); await app.connect(); await app.expectDialog("Firmware update required"); + await app.expectNoNewAccessibilityViolations(); await app.connectViaTryAgain(); await app.connectViaConnectHelp(); await app.expectConnected(); @@ -90,5 +93,6 @@ test.describe("connect", () => { await app.expectDialog("This browser does not support WebUSB"); await app.closeDialog(); await app.expectDialog("Transfer saved hex file to micro:bit"); + await app.expectNoNewAccessibilityViolations(); }); }); diff --git a/src/e2e/documentation.test.ts b/src/e2e/documentation.test.ts index 7a63976a6..a3da31044 100644 --- a/src/e2e/documentation.test.ts +++ b/src/e2e/documentation.test.ts @@ -12,6 +12,7 @@ test.describe("documentation", () => { "API", "For usage and examples, see" ); + await app.expectNoNewAccessibilityViolations(); }); test("Copy code and paste in editor", async ({ app }) => { @@ -67,6 +68,7 @@ test.describe("documentation", () => { "Loops", "Count and repeat sets of instructions" ); + await app.expectNoNewAccessibilityViolations(); }); test("Ideas tab navigation", async ({ app }) => { @@ -82,5 +84,6 @@ test.describe("documentation", () => { await app.switchTab("Ideas"); await app.selectDocumentationIdea(ideaName); await app.expectDocumentationTopLevelHeading(ideaName); + await app.expectNoNewAccessibilityViolations(); }); }); diff --git a/src/e2e/multiple-files.test.ts b/src/e2e/multiple-files.test.ts index a33f94bfa..c00a80a20 100644 --- a/src/e2e/multiple-files.test.ts +++ b/src/e2e/multiple-files.test.ts @@ -23,6 +23,7 @@ test.describe("multiple-files", () => { await app.expectEditorContainText(/Your new file/); await app.expectProjectFiles(["main.py", "test.py"]); + await app.expectNoNewAccessibilityViolations(); }); test("Prevents deleting main.py", async ({ app }) => { diff --git a/src/e2e/settings.test.ts b/src/e2e/settings.test.ts index 147665e01..95d9054df 100644 --- a/src/e2e/settings.test.ts +++ b/src/e2e/settings.test.ts @@ -20,6 +20,7 @@ test.describe("settings", () => { test("switches language", async ({ app }) => { await app.switchLanguage("fr"); await app.expectProjectName("Projet sans titre"); + await app.expectNoNewAccessibilityViolations(); await app.switchLanguage("en"); await app.expectProjectName("Untitled project"); }); diff --git a/src/e2e/simulator.test.ts b/src/e2e/simulator.test.ts index 32645a0d1..257bc4f4f 100644 --- a/src/e2e/simulator.test.ts +++ b/src/e2e/simulator.test.ts @@ -32,6 +32,7 @@ test.describe("simulator", () => { await app.simulator.simulatorSelectGesture("freefall"); await app.simulator.sendGestureButton.click(); await app.simulator.expectResponse(); + await app.expectNoNewAccessibilityViolations(); }); test("responds to a range sensor change", async ({ app }) => { diff --git a/src/simulator/Simulator.tsx b/src/simulator/Simulator.tsx index 0f197d7eb..6eead899d 100644 --- a/src/simulator/Simulator.tsx +++ b/src/simulator/Simulator.tsx @@ -100,6 +100,7 @@ const Simulator = ({ flexDirection="column" height="100%" position="relative" + data-testid="simulator" > Date: Thu, 12 Nov 2026 16:51:53 +0000 Subject: [PATCH 2/2] Record the WCAG level of each known axe violation Separates conformance failures from axe's best-practice rules, to help prioritise fixes. --- src/e2e/axe.ts | 33 +++++++++++++++++++++++++++------ 1 file changed, 27 insertions(+), 6 deletions(-) diff --git a/src/e2e/axe.ts b/src/e2e/axe.ts index eb316828d..54f37317c 100644 --- a/src/e2e/axe.ts +++ b/src/e2e/axe.ts @@ -11,6 +11,13 @@ import path from "path"; interface KnownViolation { /** axe rule id. */ rule: string; + /** + * What the rule tests, from axe's tags for it: a WCAG conformance level, or + * an axe best practice that no WCAG success criterion requires. + */ + level: "A" | "AA" | "best-practice"; + /** The WCAG success criterion, for A and AA. */ + criterion?: string; /** Elements the rule is known to fail on, as a CSS selector. */ selector: string; reason: string; @@ -29,53 +36,67 @@ interface KnownViolation { const knownViolations: KnownViolation[] = [ { rule: "aria-input-field-name", + level: "A", + criterion: "4.1.2 Name, Role, Value", selector: ".cm-content", reason: "CodeMirror's content element has no accessible name, in the main editor and in documentation code examples.", }, { rule: "aria-prohibited-attr", + level: "A", + criterion: "4.1.2 Name, Role, Value", selector: 'div[aria-label]:not([role])[style*="left"]', reason: "Simulator range sensor threshold marks put aria-label on a div with no role.", }, + { + rule: "role-img-alt", + level: "A", + criterion: "1.1.1 Non-text Content", + selector: 'div[role="img"]', + reason: + "The sidebar header logos and simulator sensor icons are role=img wrappers with no label.", + }, { rule: "color-contrast", + level: "AA", + criterion: "1.4.3 Contrast (Minimum)", selector: ".cm-gutterElement", reason: "Editor line numbers are 2.02:1 against the gutter.", }, { rule: "color-contrast", + level: "AA", + criterion: "1.4.3 Contrast (Minimum)", selector: ".sidebar-tab p", reason: "Unselected sidebar tab labels are 4.3:1 against the tab bar at 13px.", }, { rule: "heading-order", + level: "best-practice", selector: "h4", reason: "API documentation entries use h4 without an h3 above them.", }, { rule: "page-has-heading-one", + level: "best-practice", selector: "html", reason: "The app has no level-one heading.", }, { rule: "region", + level: "best-practice", selector: ".splitView__splitter", reason: "The sidebar and simulator resize handles sit between landmarks.", }, { rule: "region", + level: "best-practice", selector: '[data-testid="simulator"] *', reason: "The simulator's sensor controls are outside any landmark.", }, - { - rule: "role-img-alt", - selector: 'div[role="img"]', - reason: - "The sidebar header logos and simulator sensor icons are role=img wrappers with no label.", - }, ]; /**