test_runner: add option to forbid exclusive tests

doc/api/cli.md

@@ -3222,6 +3222,22 @@ added: v22.8.0
 Require a minimum percent of covered lines. If code coverage does not reach
 the threshold specified, the process will exit with code `1`.
 
+### `--test-forbid-only`
+
+<!-- YAML
+added: REPLACEME
+-->
+
+Fails the test run if a test or suite is declared with `only: true`, including
+the `test.only()`, `it.only()`, `suite.only()`, and `describe.only()` shorthands,
+or if `testContext.runOnly(true)` is called. This is useful in CI to prevent
+accidentally committing exclusive tests that exclude the rest of a test suite.
+
+This option applies with either test isolation mode and takes precedence over
+`--test-only`. Name filtering, TODO markers, and expected failures do not suppress
+detected violations. Only declarations evaluated during the run are checked;
+code inside a skipped test callback is not evaluated.
+
 ### `--test-force-exit`
 
 <!-- YAML
@@ -4339,6 +4355,7 @@ one is included in the list below.
 * `--test-coverage-include-all`
 * `--test-coverage-include`
 * `--test-coverage-lines`
+* `--test-forbid-only`
 * `--test-global-setup`
 * `--test-isolation`
 * `--test-name-pattern`

doc/api/test.md

@@ -353,6 +353,14 @@ it.todo('should do the thing', { expectFailure: true }, () => {
 
 ## `only` tests
 
+To prevent exclusive tests from accidentally excluding tests in CI, use
+[`--test-forbid-only`][] or the `forbidOnly` option of [`run()`][]. When enabled,
+declaring a test or suite with `only: true` or calling `testContext.runOnly(true)`
+fails the run. This includes the `.only()` shorthands and applies with either
+isolation mode, even when `--test-only` is also enabled. Name filtering, TODO
+markers, and expected failures cannot suppress detected violations. Declarations
+inside callbacks that are never executed are not checked.
+
 If Node.js is started with the [`--test-only`][] command-line option, or test
 isolation is disabled, it is possible to skip all tests except for a selected
 subset by passing the `only` option to the tests that should run. When a test
@@ -1704,6 +1712,9 @@ added:
   - v18.9.0
   - v16.19.0
 changes:
+  - version: REPLACEME
+    pr-url: https://github.com/nodejs/node/pull/66578
+    description: Added the `forbidOnly` option.
   - version:
      - v26.2.0
      - v24.19.0
@@ -1760,6 +1771,9 @@ changes:
   * `forceExit` {boolean} Configures the test runner to exit the process once
     all known tests have finished executing even if the event loop would
     otherwise remain active. **Default:** `false`.
+  * `forbidOnly` {boolean} Fail the run when a test or suite is declared with
+    `only: true` or `testContext.runOnly(true)` is called. See [`only` tests](#only-tests).
+    **Default:** the value of [`--test-forbid-only`][].
   * `globPatterns` {Array} An array containing the list of glob patterns to
     match test files. This option cannot be used together with `files`.
     **Default:** Same as [running tests from the command line][].
@@ -5105,6 +5119,7 @@ test.describe('my suite', (suite) => {
 [`--test-concurrency`]: cli.md#--test-concurrency
 [`--test-coverage-exclude`]: cli.md#--test-coverage-exclude
 [`--test-coverage-include`]: cli.md#--test-coverage-include
+[`--test-forbid-only`]: cli.md#--test-forbid-only
 [`--test-name-pattern`]: cli.md#--test-name-pattern
 [`--test-only`]: cli.md#--test-only
 [`--test-reporter-destination`]: cli.md#--test-reporter-destination

doc/node.1

@@ -1609,6 +1609,16 @@ filtering as the rest of the report.
 Require a minimum percent of covered lines. If code coverage does not reach
 the threshold specified, the process will exit with code \fB1\fR.
 .
+.It Fl -test-forbid-only
+Fails the test run if a test or suite is declared with \fBonly: true\fR, including
+the \fBtest.only()\fR, \fBit.only()\fR, \fBsuite.only()\fR, and \fBdescribe.only()\fR shorthands,
+or if \fBtestContext.runOnly(true)\fR is called. This is useful in CI to prevent
+accidentally committing exclusive tests that exclude the rest of a test suite.
+This option applies with either test isolation mode and takes precedence over
+\fB--test-only\fR. Name filtering, TODO markers, and expected failures do not suppress
+detected violations. Only declarations evaluated during the run are checked;
+code inside a skipped test callback is not evaluated.
+.
 .It Fl -test-force-exit
 Configures the test runner to exit the process once all known tests have
 finished executing even if the event loop would otherwise remain active.
@@ -2413,6 +2423,8 @@ one is included in the list below.
 .It
 \fB--test-coverage-lines\fR
 .It
+\fB--test-forbid-only\fR
+.It
 \fB--test-global-setup\fR
 .It
 \fB--test-isolation\fR

lib/internal/test_runner/runner.js

@@ -206,6 +206,7 @@ function getRunArgs(path, { forceExit,
                             testSkipPatterns,
                             testTagFilterExpressions,
                             only,
+                            forbidOnly,
                             hasFiles,
                             testFiles,
                             globPatterns,
@@ -252,6 +253,9 @@ function getRunArgs(path, { forceExit,
   if (only === true) {
     ArrayPrototypePush(runArgs, '--test-only');
   }
+  if (forbidOnly != null && forbidOnly !== getOptionValue('--test-forbid-only')) {
+    ArrayPrototypePush(runArgs, forbidOnly ? '--test-forbid-only' : '--no-test-forbid-only');
+  }
   if (timeout != null) {
     ArrayPrototypePush(runArgs, `--test-timeout=${timeout}`);
   }
@@ -788,6 +792,7 @@ function run(options = kEmptyObject) {
     globalSetupPath,
     only,
     globPatterns,
+    forbidOnly,
     coverage = false,
     coverageIncludeAll = false,
     lineCoverage = 0,
@@ -820,6 +825,9 @@ function run(options = kEmptyObject) {
   if (only != null) {
     validateBoolean(only, 'options.only');
   }
+  if (forbidOnly != null) {
+    validateBoolean(forbidOnly, 'options.forbidOnly');
+  }
   if (globPatterns != null) {
     validateArray(globPatterns, 'options.globPatterns');
   }
@@ -1020,6 +1028,10 @@ function run(options = kEmptyObject) {
     testTagFilters,
   };
 
+  if (forbidOnly != null) {
+    globalOptions.forbidOnly = forbidOnly;
+  }
+
   if (isolation === 'none') {
     if (testNamePatterns != null) {
       globalOptions.testNamePatterns = testNamePatterns;
@@ -1066,6 +1078,7 @@ function run(options = kEmptyObject) {
     globPatterns,
     only,
     forceExit,
+    forbidOnly,
     cwd,
     isolation,
     argv,

lib/internal/test_runner/test.js

@@ -370,6 +370,9 @@ class TestContext {
   }
 
   runOnly(value) {
+    if (value && this.#test.config.forbidOnly) {
+      failForbiddenOnly(this.#test.root, 'runOnly() is not allowed when forbidOnly is enabled');
+    }
     this.#test.runOnlySubtests = !!value;
   }
 
@@ -568,6 +571,14 @@ function parseExpectFailure(expectFailure) {
   return { __proto__: null, label: undefined, match: expectFailure };
 }
 
+function failForbiddenOnly(root, message) {
+  // Filtering, TODO, and expected failures must not hide forbidden exclusivity.
+  root.harness.success = false;
+  root.diagnostic(message);
+  process.exitCode = kGenericUserError;
+  throw new ERR_TEST_FAILURE(message, kTestCodeFailure);
+}
+
 class Test extends AsyncResource {
   reportedType = 'test';
   abortController;
@@ -634,6 +645,10 @@ class Test extends AsyncResource {
         parent.nesting + 1;
       const { config, isFilteringByName, isFilteringByOnly, isFilteringByTags } = parent.root.harness;
 
+      if (only && config.forbidOnly) {
+        failForbiddenOnly(parent.root, 'The only option is not allowed when forbidOnly is enabled');
+      }
+
       this.root = parent.root;
       this.harness = null;
       this.config = config;

lib/internal/test_runner/utils.js

@@ -268,6 +268,7 @@ function parseCommandLine() {
   let destinations;
   let isolation;
   let only = getOptionValue('--test-only');
+  const forbidOnly = getOptionValue('--test-forbid-only');
   let reporters;
   let shard;
   let testNamePatterns = mapPatternFlagToRegExArray('--test-name-pattern');
@@ -422,6 +423,7 @@ function parseCommandLine() {
     functionCoverage,
     lineCoverage,
     only,
+    forbidOnly,
     reporters,
     setup,
     globalSetupPath,

src/node_options.cc

@@ -1266,6 +1266,12 @@ EnvironmentOptionsParser::EnvironmentOptionsParser() {
             kAllowedInEnvvar,
             false,
             OptionNamespaces::kTestRunnerNamespace);
+  AddOption("--test-forbid-only",
+            "fail tests that use the 'only' option",
+            BOOL_FIELD(test_forbid_only),
+            kAllowedInEnvvar,
+            false,
+            OptionNamespaces::kTestRunnerNamespace);
   AddOption("--test-shard",
             "run test at specific shard",
             &EnvironmentOptions::test_shard,

src/node_options.h

@@ -265,6 +265,7 @@ class EnvironmentOptions : public Options {
   DEFINE_BOOL_FIELD(test_randomize) = false;
   DEFINE_BOOL_FIELD(has_test_random_seed) = false;
   DEFINE_BOOL_FIELD(test_only) = false;
+  DEFINE_BOOL_FIELD(test_forbid_only) = false;
   DEFINE_BOOL_FIELD(test_udp_no_try_send) = false;
   DEFINE_BOOL_FIELD(coverage_include_all) = false;
   DEFINE_BOOL_FIELD(throw_deprecation) = false;

test/parallel/test-runner-forbid-only.js

@@ -0,0 +1,153 @@
+'use strict';
+const common = require('../common');
+const assert = require('node:assert');
+const { writeFileSync } = require('node:fs');
+const { test, run } = require('node:test');
+const tmpdir = require('../common/tmpdir');
+
+tmpdir.refresh();
+const fixture = tmpdir.resolve('only.test.js');
+writeFileSync(fixture, `
+  const { test } = require('node:test');
+  test.only('focused test', () => {});
+  test('ordinary test', () => {});
+`);
+const env = { ...process.env, NODE_OPTIONS: '', FORCE_COLOR: '0' };
+delete env.NODE_TEST_CONTEXT;
+const onlyIfNodeOptionsSupport = { skip: process.config.variables.node_without_node_options };
+
+function createFixture(name, source) {
+  const file = tmpdir.resolve(`${name}.js`);
+  writeFileSync(file, `
+    const { test, it, suite, describe } = require('node:test');
+    ${source}
+  `);
+  return file;
+}
+
+function assertForbidden({ code, signal, stdout, stderr }) {
+  assert.strictEqual(signal, null);
+  assert.strictEqual(code, 1, stdout + stderr);
+  assert.match(stdout + stderr, /(?:only|runOnly).*not allowed/);
+  assert.doesNotMatch(stdout, /BODY_EXECUTED/);
+}
+
+const cases = {
+  'test-only': "test.only('focused test', () => { console.log('BODY_EXECUTED'); });",
+  'it-only': "it.only('focused test', () => { console.log('BODY_EXECUTED'); });",
+  'suite-only': "suite.only('focused suite', () => { console.log('BODY_EXECUTED'); });",
+  'describe-only': "describe.only('focused suite', () => { console.log('BODY_EXECUTED'); });",
+  'only-option': "test('focused test', { only: true }, () => { console.log('BODY_EXECUTED'); });",
+  'suite-option': "suite('focused suite', { only: true }, () => { console.log('BODY_EXECUTED'); });",
+  'subtest-option': "test('parent', async (t) => { await t.test('child', { only: true }, () => {}); });",
+  'run-only': "test('parent', (t) => { t.runOnly(true); console.log('BODY_EXECUTED'); });",
+  'skipped-only': "test('focused test', { only: true, skip: true }, () => {});",
+  'todo-only': "test('focused test', { only: true, todo: true }, () => {});",
+  'expected-failure-only': "test('focused test', { only: true, expectFailure: true }, () => {});",
+  'todo-parent': "test.todo('parent', () => { test.only('child', () => {}); });",
+  'expected-failure-parent': "test('parent', { expectFailure: true }, (t) => { t.runOnly(true); });",
+};
+const caseFiles = Object.entries(cases).map(([name, source]) => [name, createFixture(name, source)]);
+
+for (const isolation of ['process', 'none']) {
+  test(`run() rejects only with isolation=${isolation}`, async () => {
+    const runner = tmpdir.resolve(`runner-${isolation}.js`);
+    writeFileSync(runner, `
+        const { run } = require('node:test');
+        const { tap } = require('node:test/reporters');
+        run({
+          files: [${JSON.stringify(fixture)}],
+          isolation: ${JSON.stringify(isolation)},
+          forbidOnly: true,
+        }).on('test:fail', () => { process.exitCode = 1; }).compose(tap).pipe(process.stdout);
+      `);
+    const { code, signal, stdout, stderr } = await common.spawnPromisified(process.execPath, [runner], { env });
+    assertForbidden({ code, signal, stdout, stderr });
+  });
+
+  for (const [name, file] of caseFiles) {
+    test(`CLI rejects ${name} with isolation=${isolation}`, async () => {
+      assertForbidden(await common.spawnPromisified(process.execPath, [
+        '--test', `--test-isolation=${isolation}`, '--test-reporter=tap', '--test-forbid-only', file,
+      ], { env }));
+    });
+  }
+
+  for (const args of [['--test-only'], ['--test-name-pattern=ordinary'], ['--test-skip-pattern=focused']]) {
+    test(`filter ${args[0]} cannot hide only with isolation=${isolation}`, async () => {
+      assertForbidden(await common.spawnPromisified(process.execPath, [
+        '--test', `--test-isolation=${isolation}`, '--test-reporter=tap', '--test-forbid-only', ...args, fixture,
+      ], { env }));
+    });
+  }
+
+  test(`NODE_OPTIONS enables forbidOnly with isolation=${isolation}`, onlyIfNodeOptionsSupport, async () => {
+    assertForbidden(await common.spawnPromisified(process.execPath, [
+      '--test', `--test-isolation=${isolation}`, '--test-reporter=tap', fixture,
+    ], { env: { ...env, NODE_OPTIONS: '--test-forbid-only' } }));
+  });
+
+  test(`filtered suites cannot hide only with isolation=${isolation}`, async () => {
+    const file = createFixture(`filtered-${isolation}`, `
+      suite('filtered', () => { it.only('child', () => {}); });
+      test('ordinary', () => {});
+    `);
+    assertForbidden(await common.spawnPromisified(process.execPath, [
+      '--test', `--test-isolation=${isolation}`, '--test-reporter=tap',
+      '--test-forbid-only', '--test-name-pattern=ordinary', file,
+    ], { env }));
+  });
+
+  for (const source of ['CLI', 'NODE_OPTIONS']) {
+    const options = source === 'NODE_OPTIONS' ? onlyIfNodeOptionsSupport : {};
+    test(`run() can override ${source} with forbidOnly=false and isolation=${isolation}`, options, async () => {
+      const runner = createFixture(`override-${source}-${isolation}`, `
+        const { run } = require('node:test');
+        const { tap } = require('node:test/reporters');
+        run({ files: [${JSON.stringify(fixture)}], isolation: ${JSON.stringify(isolation)}, forbidOnly: false })
+          .on('test:fail', () => { process.exitCode = 1; }).compose(tap).pipe(process.stdout);
+      `);
+      const args = source === 'CLI' ? ['--test-forbid-only', runner] : [runner];
+      const childEnv = source === 'NODE_OPTIONS' ? { ...env, NODE_OPTIONS: '--test-forbid-only' } : env;
+      const { code, signal, stdout, stderr } = await common.spawnPromisified(process.execPath, args, { env: childEnv });
+      assert.strictEqual(signal, null);
+      assert.strictEqual(code, 0, stdout + stderr);
+      assert.match(stdout, /ok 1 - focused test/);
+    });
+  }
+
+  test(`ordinary tests still run with isolation=${isolation}`, async () => {
+    const file = createFixture(`ordinary-${isolation}`, `
+      test('ordinary', () => {});
+      test('explicit false', { only: false }, () => {});
+      test('runOnly false', (t) => { t.runOnly(false); });
+    `);
+    const { code, signal, stdout, stderr } = await common.spawnPromisified(process.execPath, [
+      '--test', `--test-isolation=${isolation}`, '--test-reporter=tap', '--test-forbid-only', file,
+    ], { env });
+    assert.strictEqual(signal, null);
+    assert.strictEqual(code, 0, stdout + stderr);
+    // Explicit only:false is filtered out in the existing none-isolation mode.
+    assert.match(stdout, isolation === 'none' ? /# pass 2/ : /# pass 3/);
+  });
+
+  for (const flags of [[], ['--test-forbid-only', '--no-test-forbid-only']]) {
+    test(`only still filters when disabled (${flags}) with isolation=${isolation}`, async () => {
+      const { code, signal, stdout, stderr } = await common.spawnPromisified(process.execPath, [
+        '--test', `--test-isolation=${isolation}`, '--test-reporter=tap', '--test-only', ...flags, fixture,
+      ], { env });
+      assert.strictEqual(signal, null);
+      assert.strictEqual(code, 0, stdout + stderr);
+      assert.match(stdout, /# pass 1/);
+      assert.doesNotMatch(stdout, /ordinary test/);
+    });
+  }
+}
+
+test('run() validates forbidOnly', () => {
+  for (const forbidOnly of [0, 1, '', 'true', {}, []]) {
+    assert.throws(() => run({ forbidOnly }), {
+      code: 'ERR_INVALID_ARG_TYPE',
+    });
+  }
+});