From 8199e0bcc3f9c69bfeb85020516e358763995d89 Mon Sep 17 00:00:00 2001 From: AdityaHirapara Date: Thu, 10 Sep 2026 14:55:23 +0530 Subject: [PATCH 01/10] feat(v8): route framework: 'cucumber' to the CLI/binary flow Add 'cucumber' to CLIUtils.CLISupportedFrameworks, give setupTestFramework an explicit 'webdriverio-cucumber' branch (previously no else arm, so testFramework stayed null and every event was dropped without an error), and widen the field to the TestFramework base type every consumer already uses. Membership alone is not safe to ship. The legacy InsightsHandler -> Listener -> api/v1/batch transport is gated only on TESTOPS_BUILD_COMPLETED and BROWSERSTACK_TESTHUB_JWT, both of which the CLI flow sets itself, and none of the cucumber lifecycle hooks were CLI-guarded. Opening the gate alone would keep the legacy path POSTing TestRunStarted/TestRunFinished/CBTSessionCreated under the binary-issued JWT alongside the tracker, reporting every scenario twice under two uuids with no error. The guards for the emitting hooks therefore land here, not in a follow-up. beforeStep/afterStep are deliberately left unguarded: they emit nothing and only build the step list afterScenario reads back via hasTestStepFailures(), a read that is not CLI-gated and feeds the process exit code. --- .../browserstack-service/src/cli/cliUtils.ts | 2 +- .../frameworks/wdioCucumberTestFramework.ts | 11 ++++++++++ .../browserstack-service/src/cli/index.ts | 13 ++++++++++-- packages/browserstack-service/src/service.ts | 20 ++++++++++++++++--- 4 files changed, 40 insertions(+), 6 deletions(-) create mode 100644 packages/browserstack-service/src/cli/frameworks/wdioCucumberTestFramework.ts diff --git a/packages/browserstack-service/src/cli/cliUtils.ts b/packages/browserstack-service/src/cli/cliUtils.ts index 423ea915..9944df42 100644 --- a/packages/browserstack-service/src/cli/cliUtils.ts +++ b/packages/browserstack-service/src/cli/cliUtils.ts @@ -39,7 +39,7 @@ import APIUtils from './apiUtils.js' export class CLIUtils { static automationFrameworkDetail = {} static testFrameworkDetail = {} - static CLISupportedFrameworks = ['mocha'] + static CLISupportedFrameworks = ['mocha', 'cucumber'] static isDevelopmentEnv() { return process.env.BROWSERSTACK_CLI_ENV === 'development' diff --git a/packages/browserstack-service/src/cli/frameworks/wdioCucumberTestFramework.ts b/packages/browserstack-service/src/cli/frameworks/wdioCucumberTestFramework.ts new file mode 100644 index 00000000..b2a2b2f0 --- /dev/null +++ b/packages/browserstack-service/src/cli/frameworks/wdioCucumberTestFramework.ts @@ -0,0 +1,11 @@ +import TestFramework from './testFramework.js' + +/** + * Routing target for `framework: 'cucumber'` on the CLI flow. + * + * Deliberately inert: the cucumber event model (scenario/step/hook state machine) is not + * implemented, so this inherits `TestFramework`'s logging-only `trackEvent` and emits nothing. + * It exists so the factory has an explicit cucumber branch instead of leaving `testFramework` + * null, which silently drops every event with no error. + */ +export default class WdioCucumberTestFramework extends TestFramework {} diff --git a/packages/browserstack-service/src/cli/index.ts b/packages/browserstack-service/src/cli/index.ts index 6b8b447a..31416141 100644 --- a/packages/browserstack-service/src/cli/index.ts +++ b/packages/browserstack-service/src/cli/index.ts @@ -16,6 +16,8 @@ import { BROWSERSTACK_ACCESSIBILITY, BROWSERSTACK_OBSERVABILITY, BROWSERSTACK_TE import type { Options } from '@wdio/types' import TestOpsConfig from '../testOps/testOpsConfig.js' import WdioMochaTestFramework from './frameworks/wdioMochaTestFramework.js' +import WdioCucumberTestFramework from './frameworks/wdioCucumberTestFramework.js' +import type TestFramework from './frameworks/testFramework.js' import WdioAutomationFramework from './frameworks/wdioAutomationFramework.js' import WebdriverIOModule from './modules/webdriverIOModule.js' import AccessibilityModule from './modules/accessibilityModule.js' @@ -44,7 +46,7 @@ export class BrowserstackCLI { isChildConnected = false binSessionId: string | null = null modules: Record = {} - testFramework: WdioMochaTestFramework|null = null + testFramework: TestFramework|null = null cliParams: Record | null = null automationFramework: WdioAutomationFramework|null = null SDK_CLI_BIN_PATH: string | null = null @@ -488,8 +490,15 @@ export class BrowserstackCLI { */ setupTestFramework() { const testFrameworkDetail = CLIUtils.getTestFrameworkDetail() - if (testFrameworkDetail.name.toLowerCase() === 'webdriverio-mocha') { + switch (testFrameworkDetail.name.toLowerCase()) { + case 'webdriverio-mocha': this.testFramework = new WdioMochaTestFramework([testFrameworkDetail.name], testFrameworkDetail.version, this.binSessionId as string) + break + case 'webdriverio-cucumber': + this.testFramework = new WdioCucumberTestFramework([testFrameworkDetail.name], testFrameworkDetail.version, this.binSessionId as string) + break + default: + this.logger.debug(`setupTestFramework: no CLI test framework for name=${testFrameworkDetail.name}`) } } diff --git a/packages/browserstack-service/src/service.ts b/packages/browserstack-service/src/service.ts index c14c85cf..3d6a7a86 100644 --- a/packages/browserstack-service/src/service.ts +++ b/packages/browserstack-service/src/service.ts @@ -624,7 +624,13 @@ export default class BrowserstackService implements Services.ServiceInstance { this._suiteTitle = feature.name await this._setSessionName(feature.name) await this._setAnnotation(`Feature: ${feature.name}`) - await this._insightsHandler?.beforeFeature(uri, feature) + // The legacy InsightsHandler -> Listener -> api/v1/batch transport is gated only on + // TESTOPS_BUILD_COMPLETED and BROWSERSTACK_TESTHUB_JWT, both of which the CLI flow itself + // sets. Left unguarded it keeps POSTing scenario events under the binary-issued JWT on top + // of whatever the tracker emits, reporting every scenario twice under two different uuids. + if (!BrowserstackCLI.getInstance().isRunning()) { + await this._insightsHandler?.beforeFeature(uri, feature) + } } /** @@ -635,7 +641,10 @@ export default class BrowserstackService implements Services.ServiceInstance { async beforeScenario (world: ITestCaseHookParameter) { this._currentTest = world await this._accessibilityHandler?.beforeScenario(world) - await this._insightsHandler?.beforeScenario(world) + // legacy transport — see beforeFeature + if (!BrowserstackCLI.getInstance().isRunning()) { + await this._insightsHandler?.beforeScenario(world) + } const scenarioName = world.pickle.name || 'unknown scenario' await this._setAnnotation(`Scenario: ${scenarioName}`) } @@ -676,14 +685,19 @@ export default class BrowserstackService implements Services.ServiceInstance { } await this._accessibilityHandler?.afterScenario(world) - await this._insightsHandler?.afterScenario(world) + // legacy transport — see beforeFeature if (!BrowserstackCLI.getInstance().isRunning()) { + await this._insightsHandler?.afterScenario(world) await this._percyHandler?.afterScenario() } } @PerformanceTester.Measure(PERFORMANCE_SDK_EVENTS.EVENTS.SDK_HOOK, { hookType: 'beforeStep' }) async beforeStep (step: Frameworks.PickleStep, scenario: Pickle) { + // NOT CLI-guarded, unlike the scenario hooks: beforeStep/afterStep emit nothing, they only + // build the step list that afterScenario reads back via hasTestStepFailures() to separate + // step failures from hook failures. That read is not CLI-gated and feeds the process exit + // code, so guarding these would make every step failure invisible under ignoreHooksStatus. await this._insightsHandler?.beforeStep(step, scenario) await this._setAnnotation(`Step: ${step.keyword}${step.text}`) } From f738561a80ff170728d33f0e13ca8afb8465a3aa Mon Sep 17 00:00:00 2001 From: AdityaHirapara Date: Thu, 10 Sep 2026 16:36:45 +0530 Subject: [PATCH 02/10] feat(v8): report cucumber scenarios and hooks through the CLI framework class MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit WdioCucumberTestFramework goes from an inert stub to the cucumber event tracker. It extends the base TestFramework, not the mocha one: WDIO never calls beforeTest/afterTest for cucumber and its hook invocations carry no title, so mocha's INIT_TEST/TEST/hook boundaries have no source here. A scenario raises TEST/PRE at beforeScenario and TEST/POST at afterScenario. That is forced rather than chosen — the binary's WDIO language index dispatches only on TEST and ^(BEFORE_|AFTER_), and silently ignores anything else, so a scenario-specific state would have produced a green build with no tests. Raising TEST is also what fills automateModule's sessionMap, which is why the session is now named and marked. Hooks are classified from a state machine over the class's own cucumber bookkeeping, never from a title. util.ts getHookType() is left alone: it matches Mocha's quoted titles and widening it would change mocha and jasmine too, so service.beforeHook/afterHook discriminate on the framework instead. Before this, every cucumber hook boundary threw inside an awaited WDIO hook and the hook was lost with a logged stack trace in a green build. The step-depth counter is deliberately not reset per scenario, matching the legacy handler: one missed afterStep classifies every later AFTER_EACH as unreported for the rest of the run, and reproducing that is parity. bdd_meta_info.feature.path is sent ABSOLUTE. The binary re-bases it against the project path for file_name/location and the git root for vc_filepath; pre-relativising on this side makes both fields depend on the binary's own cwd (SDK-7233). A failed BeforeAll abandons the whole feature, so every scenario it never reached (Rule-nested included) is reported skipped. Those rows are built as detached instances and sent straight to TestHub — routing them through the observers would rename the session, stop accessibility and run a Percy teardown per row. A hook finish with no recorded start drops both the finish and the cascade, as the legacy path does. accessibilityModule now owns the per-scenario Web A11y scan on this flow. It observes the TEST states raised above, and leaving the legacy handler live alongside it ran sendTestStopEvent twice per scenario against the same test run uuid — while that handler is only half-initialised here, since service.before() never calls its before(). preferScenarioName was a silent no-op on this flow: the only _updateJob carrying the name is gated off while the binary is up, and the session had already been named after the feature. It is written explicitly now, after the EXECUTE/POST tracker call rather than racing it. Unreachable for mocha and jasmine, whose scenario array is never populated. Co-Authored-By: Claude Opus 5 (1M context) --- .../frameworks/wdioCucumberTestFramework.ts | 648 +++++++++++++++++- packages/browserstack-service/src/service.ts | 190 ++++- .../wdioCucumberTestFramework.test.ts | 266 +++++++ 3 files changed, 1087 insertions(+), 17 deletions(-) create mode 100644 packages/browserstack-service/tests/cli/frameworks/wdioCucumberTestFramework.test.ts diff --git a/packages/browserstack-service/src/cli/frameworks/wdioCucumberTestFramework.ts b/packages/browserstack-service/src/cli/frameworks/wdioCucumberTestFramework.ts index b2a2b2f0..bfa3fbe0 100644 --- a/packages/browserstack-service/src/cli/frameworks/wdioCucumberTestFramework.ts +++ b/packages/browserstack-service/src/cli/frameworks/wdioCucumberTestFramework.ts @@ -1,11 +1,647 @@ +import { v4 as uuidv4 } from 'uuid' +import path from 'node:path' + import TestFramework from './testFramework.js' +import { TestFrameworkState } from '../states/testFrameworkState.js' +import { HookState } from '../states/hookState.js' +import TestFrameworkInstance from '../instances/testFrameworkInstance.js' +import TrackedInstance from '../instances/trackedInstance.js' +import { CLIUtils } from '../cliUtils.js' +import { TestFrameworkConstants } from './constants/testFrameworkConstants.js' +import { BStackLogger as logger } from '../cliLogger.js' +import { TEST_ANALYTICS_ID } from '../../constants.js' +import { getScenarioExamples, removeAnsiColors } from '../../util.js' + +import type { Frameworks } from '@wdio/types' +import type { CucumberHook, Feature, FeatureChild, ITestCaseHookParameter, Pickle, Scenario, Step } from '../../cucumber-types.js' /** - * Routing target for `framework: 'cucumber'` on the CLI flow. + * Wire keys the binary's WebdriverIO-cucumber module consumes that have no entry in + * TestFrameworkConstants. Kept local rather than appended to that file, which mocha shares. + */ +const KEY_TEST_DURATION = 'test_duration' +const KEY_BDD_META_INFO = 'bdd_meta_info' +const KEY_TEST_SKIPPED_CASCADE = 'test_skipped_cascade' +const KEY_HOOK_SCOPE = 'hook_scope' +const KEY_HOOK_RETRIES = 'hook_retries' +const KEY_HOOK_DURATION = 'hook_duration' + +type CucumberHookType = 'BEFORE_ALL' | 'AFTER_ALL' | 'BEFORE_EACH' | 'AFTER_EACH' + +const HOOK_STATES: Record = { + BEFORE_ALL: TestFrameworkState.BEFORE_ALL, + AFTER_ALL: TestFrameworkState.AFTER_ALL, + BEFORE_EACH: TestFrameworkState.BEFORE_EACH, + AFTER_EACH: TestFrameworkState.AFTER_EACH, +} + +interface StepMeta { + id: string + text: string + keyword: string + started_at?: string + finished_at?: string + result?: string + duration?: unknown + failure?: string +} + +/** + * CLI test framework for `framework: 'cucumber'` under WebdriverIO. + * + * Extends the BASE TestFramework, never WdioMochaTestFramework: WDIO does not call + * `beforeTest`/`afterTest` or deliver titled hooks for cucumber, so mocha's INIT_TEST / TEST / + * hook-boundary semantics have no source here. * - * Deliberately inert: the cucumber event model (scenario/step/hook state machine) is not - * implemented, so this inherits `TestFramework`'s logging-only `trackEvent` and emits nothing. - * It exists so the factory has an explicit cucumber branch instead of leaving `testFramework` - * null, which silently drops every event with no error. + * A scenario raises TEST/PRE at `beforeScenario` and TEST/POST at `afterScenario`. That is not a + * preference: the binary's WDIO language index dispatches only on `TEST` and `^(BEFORE_|AFTER_)`, + * so an additional state would be silently discarded on the far side. */ -export default class WdioCucumberTestFramework extends TestFramework {} +export default class WdioCucumberTestFramework extends TestFramework { + static KEY_HOOK_LAST_STARTED = 'test_hook_last_started' + static KEY_HOOK_LAST_FINISHED = 'test_hook_last_finished' + + /** + * The bookkeeping hook classification is derived from. A cucumber hook invocation carries no + * title and `BeforeAll`/`AfterAll` pass no hook object at all, so `util.ts → getHookType()` + * (Mocha quoted-title matching) can only return 'unknown' or throw on the property access. + * + * `stepDepth` is deliberately NOT reset per scenario. Legacy pushes to and pops from + * `_cucumberData.steps` with no per-scenario reset, so a step that never reaches `afterStep` + * leaves the depth above zero for the rest of the run and every later AFTER_EACH is + * classified null and dropped. That is v8 behaviour and therefore the parity target + * (pre-existing-bugs PB-V8-1); the per-scenario reset exists only on the v9 line. + */ + private cucumberData: { + feature?: Feature + uri?: string + scenario?: Pickle + scenariosStarted: boolean + stepsStarted: boolean + stepDepth: number + } = { scenariosStarted: false, stepsStarted: false, stepDepth: 0 } + + /** + * Steps of the scenario in flight, mirroring legacy's per-scenario `_tests[uniqueId].steps`. + * Re-allocated (never cleared in place) at each scenario start, so a payload already built + * from the previous scenario cannot observe this one's steps through a retained reference. + */ + private scenarioSteps: StepMeta[] = [] + + /** The hook started and not yet finished on this worker — legacy's `_currentHook`. */ + private openHook: { key: string, hookId: string } | null = null + + /** + * Set when a hook finish arrived with no recorded start. Legacy throws at that point and the + * wrapper swallows it, which drops the HookRunFinished *and* the BEFORE_ALL cascade in the + * same call; this flag reproduces the second half. + */ + private lastHookFinishOrphaned = false + + constructor(testFrameworks: string[], testFrameworkVersions: Record, binSessionId: string) { + super(testFrameworks, testFrameworkVersions, binSessionId) + logger.debug('WdioCucumberTestFramework: constructed') + } + + /** + * Feature bookkeeping. Raises no state — cucumber has no feature-level wire event, and + * `beforeSuite`/`afterSuite` are not part of its WDIO surface. + */ + onFeatureStart(uri: string, feature: Feature) { + logger.debug(`WdioCucumberTestFramework.onFeatureStart: uri=${uri} feature=${feature?.name}`) + this.cucumberData.scenariosStarted = false + this.cucumberData.feature = feature + this.cucumberData.uri = uri + } + + /** Step bookkeeping. Steps travel inside the scenario payload's BDD meta, never as events. */ + onStepStart(step: Frameworks.PickleStep) { + this.cucumberData.stepsStarted = true + this.cucumberData.stepDepth++ + this.scenarioSteps.push({ + id: step.id, + text: step.text, + keyword: step.keyword, + started_at: (new Date()).toISOString(), + }) + } + + onStepEnd(step: Frameworks.PickleStep, result: Frameworks.PickleResult) { + // Math.max mirrors Array.pop() on an empty array, which legacy relies on. + this.cucumberData.stepDepth = Math.max(0, this.cucumberData.stepDepth - 1) + const stepMeta = this.scenarioSteps.find(item => item.id === step.id) + if (!stepMeta) { + return + } + stepMeta.finished_at = (new Date()).toISOString() + stepMeta.result = result.passed ? 'PASSED' : 'FAILED' + stepMeta.duration = result.duration + if (result.error) { + stepMeta.failure = removeAnsiColors(result.error) + } + } + + /** + * Classify a cucumber hook invocation from the bookkeeping state — legacy's + * `getCucumberHookType()` algorithm. + * + * Returns null for a step-scoped hook (`BeforeStep`/`AfterStep`), which is never reported; + * reporting one would change the dashboard hook count. + */ + classifyHookType(test: CucumberHook | undefined): CucumberHookType | null { + if (!test) { + return this.cucumberData.scenariosStarted ? 'AFTER_ALL' : 'BEFORE_ALL' + } + if (!this.cucumberData.stepsStarted) { + return 'BEFORE_EACH' + } + if (this.cucumberData.stepDepth > 0) { + return null + } + return 'AFTER_EACH' + } + + /** The TestFrameworkState a cucumber hook maps to, or null when it is never reported. */ + classifyHookState(test: CucumberHook | undefined): State | null { + const hookType = this.classifyHookType(test) + return hookType ? HOOK_STATES[hookType] : null + } + + /** + * Whether a failed BEFORE_ALL should run the skip cascade. False when the finish that just + * arrived had no recorded start — see `lastHookFinishOrphaned`. + */ + shouldCascadeSkippedScenarios(): boolean { + return !this.lastHookFinishOrphaned + } + + /** ` for `; the separator is a literal ' for '. */ + private hookName(hookType: CucumberHookType): string { + switch (hookType) { + case 'BEFORE_EACH': + case 'AFTER_EACH': + return `${hookType} for ${this.cucumberData.scenario?.name}` + case 'BEFORE_ALL': + case 'AFTER_ALL': + return `${hookType} for ${this.cucumberData.feature?.name}` + } + } + + /** + * The ABSOLUTE feature path. `beforeFeature`'s uri is already absolute; path.resolve only + * normalises it. The binary re-bases this itself — `path.relative(session.pathProject, …)` + * for file_name/location and against the git root for vc_filepath — so sending a + * pre-relativised value makes both fields depend on the binary's own cwd (SDK-7233). + */ + private featurePath(): string | undefined { + const uri = this.cucumberData.uri + return uri ? path.resolve(process.cwd(), uri) : undefined + } + + private featureFilePathEntries() { + const featurePath = this.featurePath() + return { + [TestFrameworkConstants.KEY_TEST_FILE_PATH]: featurePath, + [TestFrameworkConstants.KEY_TEST_LOCATION]: featurePath, + } + } + + async trackEvent(testFrameworkState: State, hookState: State, args: Record = {}) { + logger.debug(`WdioCucumberTestFramework.trackEvent: testFrameworkState=${testFrameworkState} hookState=${hookState}`) + await super.trackEvent(testFrameworkState, hookState, args) + + const instance = this.resolveInstance(testFrameworkState, hookState) + if (!instance) { + // Console output before the first scenario has nothing to attach to; legacy drops it + // just as silently (appendTestItemLog needs a test or hook uuid and has neither). + const detail = `trackEvent: no instance for testFrameworkState=${testFrameworkState} hookState=${hookState}` + if (testFrameworkState === TestFrameworkState.LOG) { + logger.debug(detail) + } else { + logger.error(detail) + } + return + } + + const shortState = testFrameworkState.toString().split('.')[1] + const isHook = CLIUtils.matchHookRegex(shortState) + + try { + if (isHook && hookState === HookState.PRE) { + instance.updateMultipleEntries({ + [TestFrameworkConstants.KEY_HOOK_ID]: uuidv4(), + }) + } + + if (testFrameworkState === TestFrameworkState.TEST) { + if (hookState === HookState.PRE) { + this.loadScenarioData(instance, args.world as ITestCaseHookParameter) + instance.updateMultipleEntries({ + [TestFrameworkConstants.KEY_TEST_STARTED_AT]: new Date().toISOString(), + }) + } else if (hookState === HookState.POST) { + instance.updateMultipleEntries({ + [TestFrameworkConstants.KEY_TEST_ENDED_AT]: new Date().toISOString(), + }) + this.loadScenarioResult(instance, args) + } + } else if (testFrameworkState === TestFrameworkState.LOG) { + this.loadLogEntry(instance, args.logEntry as Record) + } + + if (isHook && !this.trackHookEvents(instance, shortState, hookState, args)) { + // Orphaned hook finish: legacy emits nothing for it at all. + return + } + } catch (error) { + logger.error(`trackEvent: Error in tracking events: ${error} hookState=${hookState} testFrameworkState=${testFrameworkState}`) + } + + args.instance = instance + await this.runHooks(instance, testFrameworkState, hookState, args) + } + + /** + * One instance per scenario. `TestFramework.instances` is keyed on + * `sha256(CLIUtils.getCurrentInstanceName())` = `sha256(':')`, and every + * `cli/modules/*` reads it through the single `TestFramework.getTrackedInstance()` lookup, so + * that key is the only one to register under. + * + * WDIO forks a worker process per spec and registers its own `beforeScenario`/`afterScenario` + * as cucumber Before/After hooks that cucumber awaits sequentially, so the previous scenario + * has always finished before the next one mints its instance — the slot cannot be clobbered + * mid-scenario the way mocha's two instance-creation triggers can. + */ + private resolveInstance(testFrameworkState: State, hookState: State): TestFrameworkInstance | null { + let instance = TestFramework.getTrackedInstance() + const isHook = CLIUtils.matchHookRegex(testFrameworkState.toString().split('.')[1]) + + if (testFrameworkState === TestFrameworkState.TEST && hookState === HookState.PRE) { + this.trackWdioCucumberInstance(testFrameworkState) + } else if (isHook && hookState === HookState.PRE && !instance) { + // BEFORE_ALL runs before any scenario exists. Every other hook reuses the live + // instance: WDIO's beforeScenario is registered before the user's Before hooks and + // its afterScenario runs after their After hooks, so an EACH hook always lands on + // its own scenario's instance and an ALL hook on the last scenario's — which is the + // uuid legacy stamps as test_run_id. + this.trackWdioCucumberInstance(testFrameworkState) + } + + instance = TestFramework.getTrackedInstance() + if (!instance) { + logger.debug(`resolveInstance: no instance for testFrameworkState=${testFrameworkState} hookState=${hookState}`) + return null + } + this.updateInstanceState(instance, testFrameworkState, hookState) + return instance + } + + private trackWdioCucumberInstance(testFrameworkState: State) { + const target = CLIUtils.getCurrentInstanceName() + const trackedContext = TrackedInstance.createContext(target) + + const instance = new TestFrameworkInstance( + trackedContext, + this.getTestFrameworks(), + this.getTestFrameworksVersions(), + testFrameworkState, + HookState.NONE + ) + + const frameworkName = this.getTestFrameworks()[0] + const testUuid = uuidv4() + + instance.updateMultipleEntries({ + [TestFrameworkConstants.KEY_TEST_FRAMEWORK_NAME]: frameworkName, + [TestFrameworkConstants.KEY_TEST_FRAMEWORK_VERSION]: this.getTestFrameworksVersions()[frameworkName], + [TestFrameworkConstants.KEY_TEST_LOGS]: [], + [TestFrameworkConstants.KEY_HOOKS_FINISHED]: new Map(), + [TestFrameworkConstants.KEY_HOOKS_STARTED]: new Map(), + [TestFrameworkConstants.KEY_TEST_UUID]: testUuid, + [TestFrameworkConstants.KEY_TEST_RESULT]: TestFrameworkConstants.DEFAULT_TEST_RESULT, + }) + + // Read by the Web-A11y and App-A11y scan paths to stamp the scan with this test run. + process.env[TEST_ANALYTICS_ID] = testUuid + + TestFramework.setTrackedInstance(trackedContext, instance) + logger.debug(`trackWdioCucumberInstance: contextId=${trackedContext.getId()} target=${target} testUuid=${testUuid}`) + } + + /** Scenario identity. The asymmetries between the fields are deliberate. */ + private loadScenarioData(instance: TestFrameworkInstance, world: ITestCaseHookParameter) { + if (!world?.pickle) { + logger.error('loadScenarioData: no pickle on the world object; scenario identity will be empty') + return + } + const pickle = world.pickle + const feature = world.gherkinDocument?.feature + this.cucumberData.scenario = pickle + this.cucumberData.scenariosStarted = true + this.cucumberData.stepsStarted = false + // stepDepth is NOT reset here — see the cucumberData doc comment (PB-V8-1). + this.scenarioSteps = [] + + const examples = getScenarioExamples(world) + // One space before '(' and ', ' between cells, no trailing space. + const qualifiedName = examples + ? pickle.name + ' (' + examples.join(', ') + ')' + : pickle.name + + instance.updateMultipleEntries({ + // The RAW pickle name, deliberately without the examples qualifier that name/scope + // carry: the binary maps this to `identifier`, and collapsing the two would fold + // every row of a Scenario Outline into one dashboard test. + [TestFrameworkConstants.KEY_TEST_ID]: pickle.name, + [TestFrameworkConstants.KEY_TEST_NAME]: qualifiedName, + [TestFrameworkConstants.KEY_TEST_SCOPE]: qualifiedName, + [TestFrameworkConstants.KEY_TEST_SCOPES]: [feature?.name || ''], + // Step source is never reported for cucumber, unlike the mocha path's test.body. + [TestFrameworkConstants.KEY_TEST_CODE]: null, + // Gherkin tag text INCLUDING the leading '@', source order, no dedupe, no casing + // change. `.map` allocates a new array; the pickle's own collection is never mutated. + [TestFrameworkConstants.KEY_TEST_TAGS]: pickle.tags.map(({ name }: { name: string }) => name), + ...this.featureFilePathEntries(), + [KEY_BDD_META_INFO]: this.buildBddMetaInfo(pickle, feature, examples), + }) + } + + private buildBddMetaInfo(pickle: Pickle, feature: Feature | undefined, examples: string[] | undefined) { + return { + feature: { + name: feature?.name, + path: this.featurePath(), + description: feature?.description, + }, + scenario: { name: pickle.name }, + steps: this.scenarioSteps.map(step => ({ ...step })), + examples: examples ?? [], + } + } + + /** + * Scenario result. + * + * KEY_TEST_RESULT_AT is set even though `testHubModule.onAllTestEvents()` cannot actually + * defer a completion on this line — its `sendTestFrameworkEvent()` call sits outside the + * else-if chain and runs regardless. Setting it keeps `test_deferred` off the wire and the + * "dropping due to lack of results" line out of the log. + */ + private loadScenarioResult(instance: TestFrameworkInstance, args: Record) { + const world = args.world as ITestCaseHookParameter | undefined + const pickle = world?.pickle ?? this.cucumberData.scenario + const feature = world?.gherkinDocument?.feature ?? this.cucumberData.feature + + const updates: Record = { + [TestFrameworkConstants.KEY_TEST_RESULT_AT]: new Date().toISOString(), + } + + if (pickle) { + updates[KEY_BDD_META_INFO] = this.buildBddMetaInfo(pickle, feature, getScenarioExamples(world as ITestCaseHookParameter)) + } + + const result = world?.result + if (result) { + let testResult = result.status.toLowerCase() + if (testResult !== 'passed' && testResult !== 'failed') { + // UNKNOWN / UNDEFINED / AMBIGUOUS / PENDING / SKIPPED all collapse to skipped. + testResult = 'skipped' + } + + // A scenario that failed only in a hook reports passed to Observability when the user + // declared ignoreHooksStatus. `hasStepFailures` is read from the same step store + // service.afterScenario() consults, so the o11y result and the session status cannot + // disagree; absent that store, legacy treats the failure as a step failure. + const hasStepFailures = args.hasStepFailures === undefined ? true : args.hasStepFailures === true + if (args.ignoreHooksStatus === true && testResult === 'failed' && !hasStepFailures) { + testResult = 'passed' + } + + updates[TestFrameworkConstants.KEY_TEST_RESULT] = testResult + // Cucumber's own protobuf Duration, never an ended_at - started_at delta. + updates[KEY_TEST_DURATION] = result.duration + ? result.duration.seconds * 1000 + result.duration.nanos / 1000000 + : undefined + + if (testResult === 'failed') { + const message = result.message + // A ONE-element backtrace; the mocha path sends message + stack. + updates[TestFrameworkConstants.KEY_TEST_FAILURE] = [ + { backtrace: [message ? removeAnsiColors(message) : 'unknown'] } + ] + updates[TestFrameworkConstants.KEY_TEST_FAILURE_REASON] = message ? removeAnsiColors(message) : message + if (message) { + updates[TestFrameworkConstants.KEY_TEST_FAILURE_TYPE] = message.match(/AssertionError/) + ? 'AssertionError' + : 'UnhandledError' + } + } + } + + instance.updateMultipleEntries(updates) + this.cucumberData.scenario = undefined + } + + /** + * One detached instance per scenario the feature never reached, for the BEFORE_ALL cascade + * (Rule-nested scenarios included). + * + * Detached is load-bearing: these are not registered via `setTrackedInstance`, so neither the + * live instance nor `process.env[TEST_ANALYTICS_ID]` is disturbed, and the caller sends them + * straight to TestHub. Routing them through the observers would rename the Automate session, + * stop the accessibility scan and run a Percy teardown per row, none of which legacy does. + */ + buildSkippedScenarioInstances(): TestFrameworkInstance[] { + const feature = this.cucumberData.feature + if (!feature) { + logger.debug('buildSkippedScenarioInstances: no feature recorded; nothing to cascade') + return [] + } + + const scenarios: Scenario[] = [] + for (const child of (feature.children || []) as FeatureChild[]) { + if (child.rule) { + for (const ruleChild of (child.rule.children || [])) { + if (ruleChild.scenario) { + scenarios.push(ruleChild.scenario) + } + } + } else if (child.scenario) { + scenarios.push(child.scenario) + } + } + + return scenarios.map(scenario => this.buildSkippedScenarioInstance(scenario, feature)) + } + + private buildSkippedScenarioInstance(scenario: Scenario, feature: Feature): TestFrameworkInstance { + const now = new Date().toISOString() + const instance = new TestFrameworkInstance( + TrackedInstance.createContext(CLIUtils.getCurrentInstanceName()), + this.getTestFrameworks(), + this.getTestFrameworksVersions(), + TestFrameworkState.TEST, + HookState.POST + ) + + const frameworkName = this.getTestFrameworks()[0] + instance.updateMultipleEntries({ + [TestFrameworkConstants.KEY_TEST_FRAMEWORK_NAME]: frameworkName, + [TestFrameworkConstants.KEY_TEST_FRAMEWORK_VERSION]: this.getTestFrameworksVersions()[frameworkName], + [TestFrameworkConstants.KEY_TEST_LOGS]: [], + [TestFrameworkConstants.KEY_HOOKS_STARTED]: new Map(), + [TestFrameworkConstants.KEY_HOOKS_FINISHED]: new Map(), + [TestFrameworkConstants.KEY_TEST_UUID]: uuidv4(), + // The feature never ran, so no Examples row was selected and there is nothing to + // qualify: the raw scenario name is name, scope and identifier alike. No tags either + // — legacy's cascade payload has no world to read them from. + [TestFrameworkConstants.KEY_TEST_ID]: scenario.name, + [TestFrameworkConstants.KEY_TEST_NAME]: scenario.name, + [TestFrameworkConstants.KEY_TEST_SCOPE]: scenario.name, + [TestFrameworkConstants.KEY_TEST_SCOPES]: [feature.name || ''], + [TestFrameworkConstants.KEY_TEST_CODE]: null, + [TestFrameworkConstants.KEY_TEST_RESULT]: 'skipped', + [TestFrameworkConstants.KEY_TEST_STARTED_AT]: now, + [TestFrameworkConstants.KEY_TEST_ENDED_AT]: now, + [TestFrameworkConstants.KEY_TEST_RESULT_AT]: now, + ...this.featureFilePathEntries(), + [KEY_TEST_SKIPPED_CASCADE]: true, + [KEY_BDD_META_INFO]: { + feature: { name: feature.name, path: this.featurePath(), description: feature.description }, + scenario: { name: scenario.name }, + steps: (scenario.steps || []).map((step: Step) => ({ + id: step.id, + text: step.text, + keyword: step.keyword, + result: 'skipped', + })), + examples: [], + }, + }) + + return instance + } + + /** + * Whether the scenario in flight failed in a STEP rather than only in a hook. Kept for + * callers that have no access to the legacy step store. + */ + hasStepFailures(): boolean { + return this.scenarioSteps.some(step => step.result === 'FAILED') + } + + /** + * Route a console log to the row it belongs on: the open hook's uuid while a hook is in + * flight and unfinished, otherwise the scenario's. + * + * The record always goes into KEY_TEST_LOGS, because the send path + * (`testHubModule.onAllTestEvents`) collects the test logs plus the last FINISHED hook's — a + * record parked on a still-open hook's own array would never be picked up. Routing rides on + * KEY_HOOK_ID, which `sendLogCreatedEvent` reads in preference to the test uuid. + * + * Not delegated to `WdioMochaTestFramework.loadLogEntries()`: its hook check is + * `matchHookRegex(instance.getCurrentTestState().toString())`, which tests the fully-qualified + * `TestFrameworkState.BEFORE_ALL` against an anchored `^(BEFORE_|AFTER_)` and never matches. + */ + private loadLogEntry(instance: TestFrameworkInstance, logEntry: Record) { + if (!logEntry) { + return + } + const { level, message, timestamp, kind } = logEntry + const logRecord: Record = { + // SDK-6277 forwards a saveScreenshot() result through this same state with + // kind: 'TEST_SCREENSHOT'; hardcoding KIND_LOG would reclassify it as a log line. + kind: (kind as string) ?? TestFrameworkConstants.KIND_LOG, + message: Buffer.from(message as string), + level, + timestamp, + } + + if (this.openHook) { + logRecord[TestFrameworkConstants.KEY_HOOK_ID] = this.openHook.hookId + } + + const entries = TestFramework.getState(instance, TestFrameworkConstants.KEY_TEST_LOGS) as unknown[] + entries.push(logRecord) + instance.updateMultipleEntries({ + [TestFrameworkConstants.KEY_TEST_LOGS]: entries, + }) + } + + /** + * Hook lifecycle. Entries are keyed by the short state name, matching how the binary looks + * them up via `event.test_hooks_started[request.testFrameworkState]`. + * + * @returns false when a finish arrived with no recorded start, so the caller emits nothing. + */ + private trackHookEvents(instance: TestFrameworkInstance, key: string, hookState: State, args: Record): boolean { + const hooksStarted = TestFramework.getState(instance, TestFrameworkConstants.KEY_HOOKS_STARTED) as Map + if (!hooksStarted.has(key)) { + hooksStarted.set(key, []) + } + const hooksFinished = TestFramework.getState(instance, TestFrameworkConstants.KEY_HOOKS_FINISHED) as Map + if (!hooksFinished.has(key)) { + hooksFinished.set(key, []) + } + + const updates: Record = { + [TestFrameworkConstants.KEY_HOOKS_STARTED]: hooksStarted, + [TestFrameworkConstants.KEY_HOOKS_FINISHED]: hooksFinished, + } + + if (hookState === HookState.PRE) { + const hookId = (TestFramework.getState(instance, TestFrameworkConstants.KEY_HOOK_ID) || '') as string + const hook: Record = { + key, + [TestFrameworkConstants.KEY_HOOK_ID]: hookId, + [TestFrameworkConstants.KEY_HOOK_RESULT]: TestFrameworkConstants.DEFAULT_HOOK_RESULT, + [TestFrameworkConstants.KEY_EVENT_STARTED_AT]: new Date().toISOString(), + [TestFrameworkConstants.KEY_HOOK_LOGS]: [], + [TestFrameworkConstants.KEY_HOOK_NAME]: this.hookName(key as CucumberHookType), + // A hook's scope is the FEATURE name for all four types. `event.test_scope` is the + // examples-qualified scenario name, and an ALL hook has no scenario data at all, + // so the binary reads this key off the hook record first. + [KEY_HOOK_SCOPE]: this.cucumberData.feature?.name, + ...this.featureFilePathEntries(), + } + hooksStarted.get(key)?.push(hook) + updates[WdioCucumberTestFramework.KEY_HOOK_LAST_STARTED] = key + this.openHook = { key, hookId } + this.lastHookFinishOrphaned = false + instance.updateMultipleEntries(updates) + logger.debug(`trackHookEvents: hook started hookState=${key} name=${hook[TestFrameworkConstants.KEY_HOOK_NAME]}`) + return true + } + + const hooksList = hooksStarted.get(key) || [] + if (hooksList.length === 0) { + logger.warn(`trackHookEvents: hook finish for hookState=${key} has no recorded start — dropping the finish and any cascade`) + this.openHook = null + this.lastHookFinishOrphaned = true + instance.updateMultipleEntries(updates) + return false + } + + const hook = hooksList.pop() as Record + const hookResult = args.result as Frameworks.TestResult | undefined + if (hookResult) { + // passed / failed only — the hook path has no 'skipped' arm, unlike the scenario path. + // Leaving hook_result at 'pending' makes the binary coerce a finished hook to + // 'passed', so a failing before-hook would show green while the runner exits 1. + hook[TestFrameworkConstants.KEY_HOOK_RESULT] = hookResult.passed ? 'passed' : 'failed' + // WDIO reports hook duration as plain ms on the result, not as cucumber's Duration. + hook[KEY_HOOK_RETRIES] = hookResult.retries + hook[KEY_HOOK_DURATION] = hookResult.duration + } else { + logger.warn(`trackHookEvents: no result on the hook finish for '${key}'; result stays pending`) + } + hook[TestFrameworkConstants.KEY_EVENT_ENDED_AT] = new Date().toISOString() + hooksFinished.get(key)?.push(hook) + updates[WdioCucumberTestFramework.KEY_HOOK_LAST_FINISHED] = key + this.openHook = null + this.lastHookFinishOrphaned = false + instance.updateMultipleEntries(updates) + logger.debug(`trackHookEvents: hook finished hookState=${key} result=${hook[TestFrameworkConstants.KEY_HOOK_RESULT]}`) + return true + } +} diff --git a/packages/browserstack-service/src/service.ts b/packages/browserstack-service/src/service.ts index 3d6a7a86..01068288 100644 --- a/packages/browserstack-service/src/service.ts +++ b/packages/browserstack-service/src/service.ts @@ -46,6 +46,8 @@ import type AutomationFrameworkInstance from './cli/instances/automationFramewor import util from 'node:util' import APIUtils from './cli/apiUtils.js' import { CLIUtils } from './cli/cliUtils.js' +import WdioCucumberTestFramework from './cli/frameworks/wdioCucumberTestFramework.js' +import type TestHubModule from './cli/modules/testHubModule.js' export default class BrowserstackService implements Services.ServiceInstance { private _sessionBaseUrl = `${APIUtils.BROWSERSTACK_AUTOMATE_API_URL}/automate/sessions` @@ -385,7 +387,16 @@ export default class BrowserstackService implements Services.ServiceInstance { // HookRunStarted/HookRunFinished never reach the dashboard. if (BrowserstackCLI.getInstance().isRunning()) { const framework = BrowserstackCLI.getInstance().getTestFramework() - if (framework) { + if (framework instanceof WdioCucumberTestFramework) { + // A cucumber hook invocation carries no title and BeforeAll/AfterAll pass no hook + // object at all, so getHookType — Mocha quoted-title matching — throws on the + // property access here. The framework classifies from its own bookkeeping and + // returns null for the step-scoped hooks that are deliberately never reported. + const hookFrameworkState = framework.classifyHookState(test as CucumberHook|undefined) + if (hookFrameworkState) { + await framework.trackEvent(hookFrameworkState, HookState.PRE, { test }) + } + } else if (framework) { const hookFrameworkState = TestFrameworkState[getHookType((test as Frameworks.Test).title) as keyof typeof TestFrameworkState] if (hookFrameworkState) { await framework.trackEvent(hookFrameworkState, HookState.PRE, { test }) @@ -413,6 +424,23 @@ export default class BrowserstackService implements Services.ServiceInstance { // CLI flow: mirror beforeHook — close the hook via the TestFramework tracker (gRPC). if (BrowserstackCLI.getInstance().isRunning()) { const framework = BrowserstackCLI.getInstance().getTestFramework() + if (framework instanceof WdioCucumberTestFramework) { + // Cucumber's taxonomy, not Mocha's titles — see beforeHook. + const hookFrameworkState = framework.classifyHookState(test as CucumberHook|undefined) + if (hookFrameworkState) { + await framework.trackEvent(hookFrameworkState, HookState.POST, { test, result }) + } + // The cascade fires on BEFORE_ALL only — never BEFORE_EACH or AFTER_EACH — and + // not at all when the finish had no recorded start: legacy throws at that point + // and loses the cascade along with the hook event. reportSuiteSkipped() is not + // reachable from here; it walks a Mocha suite tree and drives a four-event + // sequence whose INIT_TEST and LOG_REPORT states cucumber never emits. + if (hookFrameworkState === TestFrameworkState.BEFORE_ALL && result && !result.passed + && framework.shouldCascadeSkippedScenarios()) { + await this._reportCucumberScenariosSkipped(framework) + } + return + } if (framework) { const hookFrameworkState = TestFrameworkState[getHookType((test as Frameworks.Test).title) as keyof typeof TestFrameworkState] if (hookFrameworkState) { @@ -501,6 +529,14 @@ export default class BrowserstackService implements Services.ServiceInstance { // Only 1 Scenario ran and option enabled => Redefine session name to Scenario's name if (preferScenarioName && this._scenariosThatRan.length === 1){ this._fullTitle = this._scenariosThatRan.pop() + // On the CLI flow the _updateJob below that carries this name is gated off, and + // automateModule has already named the session from the feature name it saw at + // TEST/PRE — so without this write the option is a silent no-op. Sequenced after + // the EXECUTE/POST tracker call above, not racing it. Unreachable for mocha and + // jasmine: _scenariosThatRan is only ever pushed from afterScenario. + if (setSessionName && BrowserstackCLI.getInstance().isRunning()) { + await this._updateJob({ name: this._fullTitle }) + } } await PerformanceTester.measureWrapper(PERFORMANCE_SDK_EVENTS.AUTOMATE_EVENTS.SESSION_STATUS, async () => { @@ -619,11 +655,110 @@ export default class BrowserstackService implements Services.ServiceInstance { * For CucumberJS */ + private _cliCucumberFramework(): WdioCucumberTestFramework|null { + if (!BrowserstackCLI.getInstance().isRunning()) { + return null + } + const framework = BrowserstackCLI.getInstance().getTestFramework() + return framework instanceof WdioCucumberTestFramework ? framework : null + } + + /** + * A `Frameworks.Test`-shaped view of a scenario, for the cli/modules that read `args.test`. + * automateModule.onBeforeTest is the first observer on TEST/PRE and dereferences `test.title` + * unguarded, so an absent view takes down every later observer for that key — including the + * one that emits the scenario event. + * + * `title` is left undefined and `fullName` populated deliberately. Legacy calls + * `_setSessionName(feature.name)` with NO test argument, so `sessionNameFormat` receives + * `undefined` as its fourth argument and the mocha arm (`test && !test.fullName`, which + * activates sessionNamePrependTopLevelSuiteTitle / sessionNameOmitTestTitle) is never taken. + * Both properties reproduce that. + */ + private _cucumberTestView(world: ITestCaseHookParameter): Frameworks.Test { + return { + title: undefined, + fullName: world.pickle?.name || this._suiteTitle || 'unknown scenario', + parent: this._suiteTitle ?? '', + file: world.gherkinDocument?.uri, + } as unknown as Frameworks.Test + } + + /** + * A `Frameworks.TestResult`-shaped view of a scenario result, for automateModule's session + * marking — which on the CLI flow is the only session-status authority, since `after()`'s + * accumulation is gated off while the binary is up. + * + * Which statuses fail the SESSION is `_failureStatuses`, not the passed/failed collapse the + * o11y result applies: UNDEFINED / AMBIGUOUS / UNKNOWN fail the session while reporting to + * Observability as `skipped`, and PENDING joins them only under `cucumberOpts.strict`. + */ + private _cucumberTestResult(world: ITestCaseHookParameter): Frameworks.TestResult { + const status = world.result?.status?.toLowerCase() + const ignoreHooksStatus = this._options.testObservabilityOptions?.ignoreHooksStatus === true + const hasStepFailures = this._insightsHandler ? this._insightsHandler.hasTestStepFailures(world) : true + const hookOnlyFailure = ignoreHooksStatus && status === 'failed' && !hasStepFailures + + const passed = status === 'passed' || hookOnlyFailure + const failed = !passed && status !== undefined && this._failureStatuses.includes(status) + + // A status that fails only via `_failureStatuses` carries no `world.result.message`, so + // the reason is synthesised exactly as afterScenario() does for `_failReasons`. + let error: Error | undefined + if (failed) { + error = new Error(world.result?.message || (status === 'pending' + ? `Some steps/hooks are pending for scenario "${world.pickle.name}"` + : 'Unknown Error')) + } else if (world.result?.message) { + error = new Error(world.result.message) + } + + return { + passed, + skipped: !passed && !failed, + error, + duration: 0, + retries: { attempts: 0, limit: 0 }, + } as unknown as Frameworks.TestResult + } + + /** + * Cucumber abandons the whole feature when a `BeforeAll` fails, so every scenario it never + * reached (Rule-nested included) is reported skipped rather than vanishing. + * + * Sent straight to TestHub rather than through `trackEvent()`: legacy called + * `listener.testFinished()` directly, so driving the observers per row would rename the + * Automate session, stop the accessibility scan and run a Percy teardown for each one. Both + * halves are sent because TestHub's v2 pipeline creates the row from the START event. + */ + private async _reportCucumberScenariosSkipped(framework: WdioCucumberTestFramework) { + try { + const testHubModule = BrowserstackCLI.getInstance().modules.TestHubModule as TestHubModule | undefined + if (!testHubModule) { + BStackLogger.debug('BEFORE_ALL cascade: TestHub module not loaded; skipped scenarios will not be reported') + return + } + + const instances = framework.buildSkippedScenarioInstances() + for (const instance of instances) { + instance.setCurrentTestState(TestFrameworkState.TEST) + instance.setCurrentHookState(HookState.PRE) + await testHubModule.sendTestFrameworkEvent({ instance }) + instance.setCurrentHookState(HookState.POST) + await testHubModule.sendTestFrameworkEvent({ instance }) + } + BStackLogger.debug(`BEFORE_ALL cascade: reported ${instances.length} scenario(s) as skipped`) + } catch (error) { + BStackLogger.debug(`Exception reporting the BEFORE_ALL skip cascade: ${util.format(error)}`) + } + } + @PerformanceTester.Measure(PERFORMANCE_SDK_EVENTS.EVENTS.SDK_HOOK, { hookType: 'beforeFeature' }) async beforeFeature(uri: string, feature: Feature) { this._suiteTitle = feature.name await this._setSessionName(feature.name) await this._setAnnotation(`Feature: ${feature.name}`) + this._cliCucumberFramework()?.onFeatureStart(uri, feature) // The legacy InsightsHandler -> Listener -> api/v1/batch transport is gated only on // TESTOPS_BUILD_COMPLETED and BROWSERSTACK_TESTHUB_JWT, both of which the CLI flow itself // sets. Left unguarded it keeps POSTing scenario events under the binary-issued JWT on top @@ -640,12 +775,29 @@ export default class BrowserstackService implements Services.ServiceInstance { @PerformanceTester.Measure(PERFORMANCE_SDK_EVENTS.EVENTS.SDK_HOOK, { hookType: 'beforeScenario' }) async beforeScenario (world: ITestCaseHookParameter) { this._currentTest = world - await this._accessibilityHandler?.beforeScenario(world) - // legacy transport — see beforeFeature - if (!BrowserstackCLI.getInstance().isRunning()) { - await this._insightsHandler?.beforeScenario(world) - } const scenarioName = world.pickle.name || 'unknown scenario' + + // The scenario IS the unit of work, so it raises TEST/PRE — the state every cli/modules + // observer subscribes to. WDIO never calls beforeTest for cucumber, so there is no other + // moment at which the modules could be driven. + const cliFramework = this._cliCucumberFramework() + if (cliFramework) { + // accessibilityModule now owns the per-scenario scan on this flow (it observes the + // TEST/PRE raised just below). Leaving the legacy handler live too would run + // sendTestStopEvent twice per scenario against the same TEST_ANALYTICS_ID, and the + // handler is half-initialised here anyway — service.before() skips its before(), so + // its _sessionId is unset and its browser patching never happened. + await cliFramework.trackEvent(TestFrameworkState.TEST, HookState.PRE, { + world, + test: this._cucumberTestView(world), + suiteTitle: this._suiteTitle, + }) + await this._setAnnotation(`Scenario: ${scenarioName}`) + return + } + + await this._accessibilityHandler?.beforeScenario(world) + await this._insightsHandler?.beforeScenario(world) await this._setAnnotation(`Scenario: ${scenarioName}`) } @@ -684,12 +836,26 @@ export default class BrowserstackService implements Services.ServiceInstance { } } - await this._accessibilityHandler?.afterScenario(world) - // legacy transport — see beforeFeature - if (!BrowserstackCLI.getInstance().isRunning()) { - await this._insightsHandler?.afterScenario(world) - await this._percyHandler?.afterScenario() + const cliFramework = this._cliCucumberFramework() + if (cliFramework) { + // hasStepFailures is read from the same step store this method's own + // ignoreHooksStatus branch consults, so the o11y result and the session status cannot + // disagree. percyModule.onAfterTest and accessibilityModule.onAfterTest observe the + // TEST/POST raised here, which is why the legacy handlers are not also called. + await cliFramework.trackEvent(TestFrameworkState.TEST, HookState.POST, { + world, + test: this._cucumberTestView(world), + suiteTitle: this._suiteTitle, + result: this._cucumberTestResult(world), + ignoreHooksStatus: this._options.testObservabilityOptions?.ignoreHooksStatus === true, + hasStepFailures: this._insightsHandler ? this._insightsHandler.hasTestStepFailures(world) : true, + }) + return } + + await this._accessibilityHandler?.afterScenario(world) + await this._insightsHandler?.afterScenario(world) + await this._percyHandler?.afterScenario() } @PerformanceTester.Measure(PERFORMANCE_SDK_EVENTS.EVENTS.SDK_HOOK, { hookType: 'beforeStep' }) @@ -699,12 +865,14 @@ export default class BrowserstackService implements Services.ServiceInstance { // step failures from hook failures. That read is not CLI-gated and feeds the process exit // code, so guarding these would make every step failure invisible under ignoreHooksStatus. await this._insightsHandler?.beforeStep(step, scenario) + this._cliCucumberFramework()?.onStepStart(step) await this._setAnnotation(`Step: ${step.keyword}${step.text}`) } @PerformanceTester.Measure(PERFORMANCE_SDK_EVENTS.EVENTS.SDK_HOOK, { hookType: 'afterStep' }) async afterStep (step: Frameworks.PickleStep, scenario: Pickle, result: Frameworks.PickleResult) { await this._insightsHandler?.afterStep(step, scenario, result) + this._cliCucumberFramework()?.onStepEnd(step, result) } @PerformanceTester.Measure(PERFORMANCE_SDK_EVENTS.EVENTS.SDK_HOOK, { hookType: 'onReload' }) diff --git a/packages/browserstack-service/tests/cli/frameworks/wdioCucumberTestFramework.test.ts b/packages/browserstack-service/tests/cli/frameworks/wdioCucumberTestFramework.test.ts new file mode 100644 index 00000000..4cd79278 --- /dev/null +++ b/packages/browserstack-service/tests/cli/frameworks/wdioCucumberTestFramework.test.ts @@ -0,0 +1,266 @@ +import { describe, expect, it, beforeEach, vi } from 'vitest' + +import WdioCucumberTestFramework from '../../../src/cli/frameworks/wdioCucumberTestFramework.js' +import TestFramework from '../../../src/cli/frameworks/testFramework.js' +import { TestFrameworkState } from '../../../src/cli/states/testFrameworkState.js' +import { HookState } from '../../../src/cli/states/hookState.js' +import { TestFrameworkConstants } from '../../../src/cli/frameworks/constants/testFrameworkConstants.js' +import type TestFrameworkInstance from '../../../src/cli/instances/testFrameworkInstance.js' + +vi.mock('../../../src/bstackLogger.js') + +const FEATURE_URI = '/abs/project/features/login.feature' + +const feature = (children: unknown[] = []) => ({ + name: 'Login feature', + description: 'Login feature description', + children, +} as any) + +const pickle = (name: string, tags: string[] = [], astNodeIds: string[] = ['scenario-1']) => ({ + name, + uri: 'features/login.feature', + astNodeIds, + tags: tags.map(t => ({ name: t })), +} as any) + +const world = (p: any, f: any, result?: any) => ({ + pickle: p, + gherkinDocument: { uri: 'features/login.feature', feature: f }, + result, +} as any) + +const step = (id: string, text = 'I do a thing', keyword = 'Given ') => ({ id, text, keyword } as any) + +const newFramework = () => new WdioCucumberTestFramework(['WebdriverIO-cucumber'], { 'WebdriverIO-cucumber': '8.50.0' }, 'bin-session') + +const liveInstance = () => TestFramework.getTrackedInstance() as TestFrameworkInstance +const dataOf = (instance: TestFrameworkInstance) => Object.fromEntries(instance.getAllData()) + +describe('WdioCucumberTestFramework', () => { + let framework: WdioCucumberTestFramework + + beforeEach(() => { + TestFramework.instances.clear() + framework = newFramework() + }) + + describe('hook classification (state machine, never a title)', () => { + it('classifies an absent hook object as BEFORE_ALL before any scenario and AFTER_ALL after one', async () => { + framework.onFeatureStart(FEATURE_URI, feature()) + expect(framework.classifyHookType(undefined)).toBe('BEFORE_ALL') + + await framework.trackEvent(TestFrameworkState.TEST, HookState.PRE, { world: world(pickle('S'), feature()) }) + expect(framework.classifyHookType(undefined)).toBe('AFTER_ALL') + }) + + it('classifies a hook object as BEFORE_EACH before the first step and AFTER_EACH after the last', async () => { + framework.onFeatureStart(FEATURE_URI, feature()) + await framework.trackEvent(TestFrameworkState.TEST, HookState.PRE, { world: world(pickle('S'), feature()) }) + + expect(framework.classifyHookType({ id: 'h', hookId: 'h1' } as any)).toBe('BEFORE_EACH') + + framework.onStepStart(step('s1')) + framework.onStepEnd(step('s1'), { passed: true } as any) + expect(framework.classifyHookType({ id: 'h', hookId: 'h1' } as any)).toBe('AFTER_EACH') + }) + + it('classifies a step-scoped hook as null so it is never reported', async () => { + framework.onFeatureStart(FEATURE_URI, feature()) + await framework.trackEvent(TestFrameworkState.TEST, HookState.PRE, { world: world(pickle('S'), feature()) }) + + framework.onStepStart(step('s1')) + expect(framework.classifyHookType({ id: 'h', hookId: 'h1' } as any)).toBeNull() + expect(framework.classifyHookState({ id: 'h', hookId: 'h1' } as any)).toBeNull() + }) + + // PB-V8-1: legacy never resets _cucumberData.steps per scenario, so one missed afterStep + // silently drops every later AFTER_EACH. Reproducing it is parity; the v9 line resets. + it('does NOT reset the step depth per scenario, so a missed step end poisons later AFTER_EACH', async () => { + framework.onFeatureStart(FEATURE_URI, feature()) + await framework.trackEvent(TestFrameworkState.TEST, HookState.PRE, { world: world(pickle('S1'), feature()) }) + framework.onStepStart(step('s1')) + // no onStepEnd — the step never completed + + await framework.trackEvent(TestFrameworkState.TEST, HookState.PRE, { world: world(pickle('S2'), feature()) }) + framework.onStepStart(step('s2')) + framework.onStepEnd(step('s2'), { passed: true } as any) + + expect(framework.classifyHookType({ id: 'h', hookId: 'h1' } as any)).toBeNull() + }) + }) + + describe('scenario identity', () => { + it('keeps test_id raw while name and scope carry the examples qualifier', async () => { + const outlineFeature = feature([{ + scenario: { + id: 'scenario-1', + examples: [{ tableBody: [{ id: 'row-1', cells: [{ value: 'alpha' }, { value: 'one' }] }] }], + }, + }]) + framework.onFeatureStart(FEATURE_URI, outlineFeature) + await framework.trackEvent(TestFrameworkState.TEST, HookState.PRE, { + world: world(pickle('Add to cart', [], ['scenario-1', 'row-1']), outlineFeature), + }) + + const data = dataOf(liveInstance()) + expect(data[TestFrameworkConstants.KEY_TEST_NAME]).toBe('Add to cart (alpha, one)') + expect(data[TestFrameworkConstants.KEY_TEST_SCOPE]).toBe('Add to cart (alpha, one)') + expect(data[TestFrameworkConstants.KEY_TEST_ID]).toBe('Add to cart') + }) + + it('reports a one-element scopes array, a null test_code and the feature path absolute', async () => { + framework.onFeatureStart(FEATURE_URI, feature()) + await framework.trackEvent(TestFrameworkState.TEST, HookState.PRE, { world: world(pickle('Add to cart'), feature()) }) + + const data = dataOf(liveInstance()) as Record + expect(data[TestFrameworkConstants.KEY_TEST_SCOPES]).toEqual(['Login feature']) + expect(data[TestFrameworkConstants.KEY_TEST_CODE]).toBeNull() + expect(data.bdd_meta_info.feature.path).toBe(FEATURE_URI) + }) + + it('sends tags with the leading @, in source order, duplicates kept, without mutating the pickle', async () => { + const p = pickle('Add to cart', ['@smoke', '@smoke', '@regression']) + framework.onFeatureStart(FEATURE_URI, feature()) + await framework.trackEvent(TestFrameworkState.TEST, HookState.PRE, { world: world(p, feature()) }) + + expect(dataOf(liveInstance())[TestFrameworkConstants.KEY_TEST_TAGS]).toEqual(['@smoke', '@smoke', '@regression']) + expect(p.tags).toEqual([{ name: '@smoke' }, { name: '@smoke' }, { name: '@regression' }]) + }) + }) + + describe('scenario result', () => { + const finish = async (fw: WdioCucumberTestFramework, result: any, args: Record = {}) => { + fw.onFeatureStart(FEATURE_URI, feature()) + await fw.trackEvent(TestFrameworkState.TEST, HookState.PRE, { world: world(pickle('S'), feature()) }) + const instance = liveInstance() + await fw.trackEvent(TestFrameworkState.TEST, HookState.POST, { world: world(pickle('S'), feature(), result), ...args }) + return dataOf(instance) as Record + } + + it('collapses every non passed/failed status to skipped and stamps the result timestamp', async () => { + const data = await finish(framework, { status: 'UNDEFINED', duration: { seconds: 1, nanos: 500000000 } }) + expect(data[TestFrameworkConstants.KEY_TEST_RESULT]).toBe('skipped') + expect(data[TestFrameworkConstants.KEY_TEST_RESULT_AT]).toBeTruthy() + }) + + it("derives test_duration from cucumber's Duration, not from the timestamp delta", async () => { + const data = await finish(framework, { status: 'PASSED', duration: { seconds: 2, nanos: 250000000 } }) + expect(data.test_duration).toBe(2250) + }) + + it('reports a one-element backtrace and the AssertionError failure type', async () => { + const data = await finish(framework, { status: 'FAILED', message: 'AssertionError: nope', duration: { seconds: 0, nanos: 0 } }) + expect(data[TestFrameworkConstants.KEY_TEST_FAILURE]).toEqual([{ backtrace: ['AssertionError: nope'] }]) + expect(data[TestFrameworkConstants.KEY_TEST_FAILURE_TYPE]).toBe('AssertionError') + }) + + it('overrides a hook-only failure to passed only when ignoreHooksStatus is declared', async () => { + const failed = { status: 'FAILED', message: 'boom', duration: { seconds: 0, nanos: 0 } } + + const overridden = await finish(newFramework(), failed, { ignoreHooksStatus: true, hasStepFailures: false }) + expect(overridden[TestFrameworkConstants.KEY_TEST_RESULT]).toBe('passed') + + TestFramework.instances.clear() + const notOverridden = await finish(newFramework(), failed, { ignoreHooksStatus: true, hasStepFailures: true }) + expect(notOverridden[TestFrameworkConstants.KEY_TEST_RESULT]).toBe('failed') + }) + }) + + describe('hook lifecycle', () => { + const openScenario = async () => { + framework.onFeatureStart(FEATURE_URI, feature()) + await framework.trackEvent(TestFrameworkState.TEST, HookState.PRE, { world: world(pickle('Add to cart'), feature()) }) + return liveInstance() + } + + it('names the hook after the scenario for EACH and the feature for ALL, and scopes both to the feature', async () => { + const instance = await openScenario() + await framework.trackEvent(TestFrameworkState.BEFORE_EACH, HookState.PRE, { test: { id: 'h', hookId: 'h1' } }) + + const started = dataOf(instance)[TestFrameworkConstants.KEY_HOOKS_STARTED] as Map + const hook = started.get('BEFORE_EACH')![0] + expect(hook[TestFrameworkConstants.KEY_HOOK_NAME]).toBe('BEFORE_EACH for Add to cart') + expect(hook.hook_scope).toBe('Login feature') + expect(hook.test_file_path).toBe(FEATURE_URI) + expect(hook[TestFrameworkConstants.KEY_TEST_TAGS]).toBeUndefined() + }) + + it('records passed/failed on the finish rather than leaving hook_result pending', async () => { + const instance = await openScenario() + await framework.trackEvent(TestFrameworkState.BEFORE_EACH, HookState.PRE, { test: { id: 'h', hookId: 'h1' } }) + await framework.trackEvent(TestFrameworkState.BEFORE_EACH, HookState.POST, { + test: { id: 'h', hookId: 'h1' }, + result: { passed: false, duration: 12, retries: { attempts: 0, limit: 0 } }, + }) + + const finished = dataOf(instance)[TestFrameworkConstants.KEY_HOOKS_FINISHED] as Map + const hook = finished.get('BEFORE_EACH')![0] + expect(hook[TestFrameworkConstants.KEY_HOOK_RESULT]).toBe('failed') + expect(hook.hook_duration).toBe(12) + expect(hook.hook_retries).toEqual({ attempts: 0, limit: 0 }) + }) + + it('drops an orphaned hook finish and suppresses the cascade that would have followed it', async () => { + const instance = await openScenario() + await framework.trackEvent(TestFrameworkState.BEFORE_ALL, HookState.POST, { + test: undefined, + result: { passed: false }, + }) + + const finished = dataOf(instance)[TestFrameworkConstants.KEY_HOOKS_FINISHED] as Map + expect(finished.get('BEFORE_ALL')).toEqual([]) + expect(framework.shouldCascadeSkippedScenarios()).toBe(false) + }) + }) + + describe('BEFORE_ALL skip cascade', () => { + it('builds one detached instance per scenario, Rule-nested rows included', () => { + const cascadeFeature = feature([ + { scenario: { name: 'Plain one', steps: [step('s1')] } }, + { rule: { children: [{ scenario: { name: 'Rule nested', steps: [step('s2')] } }] } }, + ]) + framework.onFeatureStart(FEATURE_URI, cascadeFeature) + + const instances = framework.buildSkippedScenarioInstances() + expect(instances.map(i => dataOf(i)[TestFrameworkConstants.KEY_TEST_NAME])).toEqual(['Plain one', 'Rule nested']) + + const first = dataOf(instances[0]) as Record + expect(first[TestFrameworkConstants.KEY_TEST_ID]).toBe('Plain one') + expect(first[TestFrameworkConstants.KEY_TEST_SCOPE]).toBe('Plain one') + expect(first[TestFrameworkConstants.KEY_TEST_RESULT]).toBe('skipped') + expect(first.test_skipped_cascade).toBe(true) + expect(first[TestFrameworkConstants.KEY_TEST_TAGS]).toBeUndefined() + expect(first.bdd_meta_info.steps).toEqual([{ id: 's1', text: 'I do a thing', keyword: 'Given ', result: 'skipped' }]) + + // Detached: the live tracked instance is untouched. + expect(TestFramework.getTrackedInstance()).toBeUndefined() + }) + }) + + describe('log routing', () => { + it('stamps the open hook uuid while a hook is unfinished and the test uuid otherwise', async () => { + framework.onFeatureStart(FEATURE_URI, feature()) + await framework.trackEvent(TestFrameworkState.TEST, HookState.PRE, { world: world(pickle('S'), feature()) }) + const instance = liveInstance() + + await framework.trackEvent(TestFrameworkState.BEFORE_EACH, HookState.PRE, { test: { id: 'h', hookId: 'h1' } }) + await framework.trackEvent(TestFrameworkState.LOG, HookState.POST, { + logEntry: { level: 'INFO', message: 'inside the hook', timestamp: 'now' }, + }) + await framework.trackEvent(TestFrameworkState.BEFORE_EACH, HookState.POST, { + test: { id: 'h', hookId: 'h1' }, result: { passed: true }, + }) + await framework.trackEvent(TestFrameworkState.LOG, HookState.POST, { + logEntry: { level: 'INFO', message: 'inside the step', timestamp: 'now', kind: TestFrameworkConstants.KIND_SCREENSHOT }, + }) + + const logs = dataOf(instance)[TestFrameworkConstants.KEY_TEST_LOGS] as Record[] + expect(logs[0][TestFrameworkConstants.KEY_HOOK_ID]).toBeTruthy() + expect(logs[0].kind).toBe(TestFrameworkConstants.KIND_LOG) + expect(logs[1][TestFrameworkConstants.KEY_HOOK_ID]).toBeUndefined() + // SDK-6277: an incoming screenshot kind must survive rather than be flattened to a log. + expect(logs[1].kind).toBe(TestFrameworkConstants.KIND_SCREENSHOT) + }) + }) +}) From 9633a1e2d4051844d75b6227141d170408e9fe3f Mon Sep 17 00:00:00 2001 From: Aditya H Date: Thu, 10 Sep 2026 18:59:02 +0530 Subject: [PATCH 03/10] feat(cli): widen cucumber's shared dispatch gates, hook-log attribution and session verdict MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Hook-scoped logs now carry their hook's state on the log record, so the binary keys them to hook_run_uuid instead of stamping a hook uuid into test_run_uuid. automateModule gains a build-level hook observer, so a failed BeforeAll/AfterAll fails the session as legacy's after() did; keys per-scenario results on the scenario rather than the feature name; and treats a skipped scenario as legacy's _failureStatuses does. All cucumber-gated — mocha's path is unchanged. accessibilityModule's scan decision passes the world through, restoring the tag-aware filter cucumber has on legacy. --- .../constants/testFrameworkConstants.ts | 1 + .../frameworks/wdioCucumberTestFramework.ts | 4 + .../src/cli/modules/accessibilityModule.ts | 6 +- .../src/cli/modules/automateModule.ts | 112 +++++++++++++++++- .../src/cli/modules/testHubModule.ts | 7 +- packages/browserstack-service/src/service.ts | 9 +- 6 files changed, 131 insertions(+), 8 deletions(-) diff --git a/packages/browserstack-service/src/cli/frameworks/constants/testFrameworkConstants.ts b/packages/browserstack-service/src/cli/frameworks/constants/testFrameworkConstants.ts index e3213521..1dd45e24 100644 --- a/packages/browserstack-service/src/cli/frameworks/constants/testFrameworkConstants.ts +++ b/packages/browserstack-service/src/cli/frameworks/constants/testFrameworkConstants.ts @@ -29,6 +29,7 @@ export const TestFrameworkConstants = { KEY_EVENT_STARTED_AT : 'event_started_at', KEY_EVENT_ENDED_AT : 'event_ended_at', KEY_HOOK_ID : 'hook_id', + KEY_HOOK_STATE : 'hook_state', KEY_HOOK_RESULT : 'hook_result', KEY_HOOK_LOGS : 'hook_logs', KEY_HOOK_NAME : 'hook_name', diff --git a/packages/browserstack-service/src/cli/frameworks/wdioCucumberTestFramework.ts b/packages/browserstack-service/src/cli/frameworks/wdioCucumberTestFramework.ts index bfa3fbe0..2b225160 100644 --- a/packages/browserstack-service/src/cli/frameworks/wdioCucumberTestFramework.ts +++ b/packages/browserstack-service/src/cli/frameworks/wdioCucumberTestFramework.ts @@ -559,6 +559,10 @@ export default class WdioCucumberTestFramework extends TestFramework { if (this.openHook) { logRecord[TestFrameworkConstants.KEY_HOOK_ID] = this.openHook.hookId + // The uuid alone is not enough: the binary reads the log's own testFrameworkState to + // decide hook_run_uuid vs test_run_uuid, and at flush time that is 'LOG'. Carrying + // the hook's state on the record is what makes the pairing survive (row 24). + logRecord[TestFrameworkConstants.KEY_HOOK_STATE] = this.openHook.key } const entries = TestFramework.getState(instance, TestFrameworkConstants.KEY_TEST_LOGS) as unknown[] diff --git a/packages/browserstack-service/src/cli/modules/accessibilityModule.ts b/packages/browserstack-service/src/cli/modules/accessibilityModule.ts index e568f14d..833909dd 100644 --- a/packages/browserstack-service/src/cli/modules/accessibilityModule.ts +++ b/packages/browserstack-service/src/cli/modules/accessibilityModule.ts @@ -203,7 +203,11 @@ export default class AccessibilityModule extends BaseModule { const sessionId = AutomationFramework.getState(autoInstance, AutomationFrameworkConstants.KEY_FRAMEWORK_SESSION_ID) const accessibilityOptions = this.config.accessibilityOptions - const shouldScanTest = this.autoScanning && shouldScanTestForAccessibility(suiteTitle, test.title, accessibilityOptions as { [key: string]: any } | undefined) && this.accessibility + // `world` is set only by service.beforeScenario, so this stays the 3-arg substring + // form for mocha. Without it a cucumber scenario has no test.title and the tag-aware + // branch is never entered, so include/excludeTagsInTestingScope are ignored (row 48). + const world = args.world + const shouldScanTest = this.autoScanning && shouldScanTestForAccessibility(suiteTitle, test.title, accessibilityOptions as { [key: string]: any } | undefined, world, Boolean(world)) && this.accessibility this.accessibilityMap.set(sessionId, shouldScanTest) // Create test metadata similar to accessibility-handler diff --git a/packages/browserstack-service/src/cli/modules/automateModule.ts b/packages/browserstack-service/src/cli/modules/automateModule.ts index b5d7b48f..cfc8917b 100644 --- a/packages/browserstack-service/src/cli/modules/automateModule.ts +++ b/packages/browserstack-service/src/cli/modules/automateModule.ts @@ -43,6 +43,10 @@ export default class AutomateModule extends BaseModule { TestFramework.registerObserver(TestFrameworkState.TEST, HookState.PRE, this.onBeforeTest.bind(this)) TestFramework.registerObserver(TestFrameworkState.TEST, HookState.POST, this.onAfterTest.bind(this)) TestFramework.registerObserver(AutomationFrameworkState.EXECUTE, HookState.POST, this.onAfterExecute.bind(this)) + // Build-level hooks carry no scenario result, so they reach the session verdict only + // through their own state. See onBuildLevelHookEnd — cucumber-gated inside the handler. + TestFramework.registerObserver(TestFrameworkState.BEFORE_ALL, HookState.POST, this.onBuildLevelHookEnd.bind(this, 'BEFORE_ALL')) + TestFramework.registerObserver(TestFrameworkState.AFTER_ALL, HookState.POST, this.onBuildLevelHookEnd.bind(this, 'AFTER_ALL')) } getModuleName(): string { @@ -98,14 +102,20 @@ export default class AutomateModule extends BaseModule { async onAfterTest(args: Record) { this.logger.debug('onAfterTest: inside automate module after test hook!') const instace = args.instance as TestFrameworkInstance - const { error, passed } = args.result as { error: Error | null, passed: boolean } + const { error, passed, skipped } = args.result as { error: Error | null, passed: boolean, skipped?: boolean } const _failReasons: string[] = [] - if (!passed) { + // A skipped cucumber scenario must not fail the session: legacy accumulates _failReasons + // only for _failureStatuses (failed/ambiguous/undefined/unknown), which excludes skipped. + // Cucumber-scoped on purpose — mocha's collapse is its own long-standing behaviour on this + // flow and changing it here would alter a framework already shipping on the CLI. + const treatAsPassed = passed || Boolean(skipped && this.isCucumberInstance(instace)) + + if (!treatAsPassed) { _failReasons.push((error && error.message) || 'Unknown Error') } - const status = passed ? 'passed' : 'failed' + const status = treatAsPassed ? 'passed' : 'failed' const reason = _failReasons.length > 0 ? _failReasons.join('\n') : undefined const autoInstance = AutomationFramework.getTrackedInstance() @@ -145,7 +155,13 @@ export default class AutomateModule extends BaseModule { reason: reason } - sessionData.testResults.set(name, testResult) + // `name` is the session NAME, which for cucumber is the Feature title and therefore + // shared by every scenario in the file — keying on it collapses N scenarios into one + // last-write-wins entry, so a feature whose last scenario passes reports a passed + // session however many earlier ones failed. Mocha leaves `fullName` undefined, so its + // key is unchanged. + const resultKey = (test && test.fullName) ? String(test.fullName) : name + sessionData.testResults.set(resultKey, testResult) this.sessionMap.set(sessionId, sessionData) } @@ -153,6 +169,89 @@ export default class AutomateModule extends BaseModule { TestFramework.setState(instace, TestFrameworkConstants.KEY_AUTOMATE_SESSION_REASON, reason) } + /** + * A `BeforeAll` / `AfterAll` failure produces no scenario result, so it can never enter the + * per-test `testResults` map that onAfterExecute aggregates — a run whose BeforeAll blew up + * reports its session as PASSED. Legacy pushed the hook error into `_failReasons` and + * `after()` marked the session failed; that whole accumulation is gated + * `setSessionStatus && !BrowserstackCLI.isRunning()`, so it is dead while the binary is up. + * + * Cucumber-gated deliberately. `wdio_mocha` has the identical latent shape on this flow, but + * legacy mocha behaved the same way, so repairing it here would be an unrequested behaviour + * change to the one framework already working on the CLI flow. + */ + async onBuildLevelHookEnd(hookKey: string, args: Record) { + try { + const instance = (args?.instance as TestFrameworkInstance) || TestFramework.getTrackedInstance() + if (!instance || !this.isCucumberInstance(instance)) { + return + } + + const result = args?.result as { passed?: boolean, error?: Error } | undefined + if (!result || result.passed) { + return + } + + const testContextOptions = this.config.testContextOptions as TestContextOptions + if (testContextOptions?.skipSessionStatus) { + return + } + + const autoInstance = AutomationFramework.getTrackedInstance() + const sessionId = AutomationFramework.getState(autoInstance, AutomationFrameworkConstants.KEY_FRAMEWORK_SESSION_ID) + if (!sessionId) { + this.logger.debug(`onBuildLevelHookEnd: no session id resolved for ${hookKey}; nothing to mark`) + return + } + + const sessionData = this.sessionMap.get(sessionId) + // Keyed on the absence of scenario results, never on the flag alone: legacy's + // `ignoreHooksStatus && this._specsRan` arm needs BOTH, and with no scenario recorded + // it falls through to marking `failed` regardless of the flag. The count is final + // here — a BeforeAll failure aborts the run, and by AfterAll every scenario is in. + const specsRan = (sessionData?.testResults.size ?? 0) > 0 + if (specsRan && isTrue(args?.ignoreHooksStatus)) { + this.logger.debug(`onBuildLevelHookEnd: ${hookKey} failed but ignoreHooksStatus is set; not failing the session`) + return + } + + if (!sessionData) { + // A BeforeAll can fail before any scenario ran, so the session may not be + // registered yet. `lastTestName` stays empty on purpose: onAfterExecute's naming + // call is what consumes it, and an empty name is what beforeFeature's own + // (un-gated) _setSessionName has already applied. + this.sessionMap.set(sessionId, { lastTestName: '', testResults: new Map() }) + } + + const name = this.resolveHookName(instance, hookKey) + this.sessionMap.get(sessionId)!.testResults.set(name, { + testName: name, + status: 'failed', + reason: (result.error && result.error.message) || 'Hook failed' + }) + this.logger.info(`onBuildLevelHookEnd: recorded ${hookKey} failure against session ${sessionId}`) + } catch (error) { + this.logger.error(`Exception in automate onBuildLevelHookEnd: ${error}`) + } + } + + private isCucumberInstance(instance: TestFrameworkInstance): boolean { + const frameworkName = String(TestFramework.getState(instance, TestFrameworkConstants.KEY_TEST_FRAMEWORK_NAME) || '') + return frameworkName.toLowerCase().includes('cucumber') + } + + /** The hook's reported name (`BEFORE_ALL for `), so the session reason names the hook. */ + private resolveHookName(instance: TestFrameworkInstance, hookKey: string): string { + try { + const finished = TestFramework.getState(instance, TestFrameworkConstants.KEY_HOOKS_FINISHED) as Map[]> | undefined + const hooks = finished?.get(hookKey) + const hookName = hooks?.length ? hooks[hooks.length - 1][TestFrameworkConstants.KEY_HOOK_NAME] : undefined + return (hookName as string) || hookKey + } catch { + return hookKey + } + } + async onAfterExecute() { this.logger.debug('onAfterExecute: inside automate module after execute hook!') @@ -178,7 +277,10 @@ export default class AutomateModule extends BaseModule { } } - if (!testContextOptions.skipSessionName) { + // An empty name means nothing ever named this session — a BeforeAll that failed + // before any feature loaded, so beforeFeature never ran. Legacy makes no naming + // call at all in that state; PUTting '' would be an API call it never made. + if (!testContextOptions.skipSessionName && sessionData.lastTestName) { await this.markSessionName(sessionId, sessionData.lastTestName, { user: userName, key: accessKey }) } diff --git a/packages/browserstack-service/src/cli/modules/testHubModule.ts b/packages/browserstack-service/src/cli/modules/testHubModule.ts index d3e34a71..f621b00e 100644 --- a/packages/browserstack-service/src/cli/modules/testHubModule.ts +++ b/packages/browserstack-service/src/cli/modules/testHubModule.ts @@ -239,11 +239,16 @@ export default class TestHubModule extends BaseModule { executionContext } for (const logEntry of logEntries) { + // The uuid below may be a HOOK's, but the state is read at flush time and is + // always LOG — and the binary picks hook_run_uuid vs test_run_uuid off exactly + // this field (`^(BEFORE_|AFTER_)`), so a hook log would arrive labelled as a + // test's. A framework that knows the hook state stamps it on the record. + const entryHookState = logEntry[TestFrameworkConstants.KEY_HOOK_STATE] as string | undefined // eslint-disable-next-line camelcase const logData: LogCreatedEventRequest_LogEntry = { testFrameworkName, testFrameworkVersion, - testFrameworkState, + testFrameworkState: entryHookState || testFrameworkState, uuid: logEntry[TestFrameworkConstants.KEY_HOOK_ID] || TestFramework.getState(instance, TestFrameworkConstants.KEY_TEST_UUID), kind: logEntry.kind as string, message: logEntry.message as Uint8Array, diff --git a/packages/browserstack-service/src/service.ts b/packages/browserstack-service/src/service.ts index 01068288..accb55ad 100644 --- a/packages/browserstack-service/src/service.ts +++ b/packages/browserstack-service/src/service.ts @@ -428,7 +428,14 @@ export default class BrowserstackService implements Services.ServiceInstance { // Cucumber's taxonomy, not Mocha's titles — see beforeHook. const hookFrameworkState = framework.classifyHookState(test as CucumberHook|undefined) if (hookFrameworkState) { - await framework.trackEvent(hookFrameworkState, HookState.POST, { test, result }) + // ignoreHooksStatus travels with the event because automateModule's build-level + // hook observer is the only surviving authority on a hook-only session verdict + // — after()'s own three-way logic is gated off while the binary is up. + await framework.trackEvent(hookFrameworkState, HookState.POST, { + test, + result, + ignoreHooksStatus: this._options.testObservabilityOptions?.ignoreHooksStatus === true, + }) } // The cascade fires on BEFORE_ALL only — never BEFORE_EACH or AFTER_EACH — and // not at all when the finish had no recorded start: legacy throws at that point From 73d491619df27560a76047ad8a0d6ed37abf5627 Mon Sep 17 00:00:00 2001 From: Aditya H Date: Thu, 10 Sep 2026 19:53:15 +0530 Subject: [PATCH 04/10] fix(cli): register the Automate session even when setSessionName is false MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit setSessionName suppresses the session NAME, not its registration. Skipping registration left onAfterExecute with nothing to status-mark, so a failing run under setSessionName: false reported no status at all where legacy marked it — its after() status block gates on setSessionStatus alone. Applies to every framework on the CLI flow, wdio_mocha included: a mocha session that is unmarked today becomes status-marked. The name stays suppressed, since onAfterExecute's naming call is guarded on both the flag and a non-empty name. --- .../src/cli/modules/automateModule.ts | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/packages/browserstack-service/src/cli/modules/automateModule.ts b/packages/browserstack-service/src/cli/modules/automateModule.ts index cfc8917b..1cc38716 100644 --- a/packages/browserstack-service/src/cli/modules/automateModule.ts +++ b/packages/browserstack-service/src/cli/modules/automateModule.ts @@ -64,8 +64,20 @@ export default class AutomateModule extends BaseModule { const suiteTitle = args.suiteTitle as string const testContextOptions = this.config.testContextOptions as TestContextOptions - if (testContextOptions.skipSessionName || !isBrowserstackSession(browser)) { + if (!isBrowserstackSession(browser)) { + return + } + + // `setSessionName: false` suppresses the NAME, not the registration. The session still has + // to enter sessionMap or onAfterExecute has nothing to status-mark, and legacy marks it + // either way — its after() status block gates on setSessionStatus alone. Registering with + // an empty lastTestName is safe because onAfterExecute's naming call is guarded on both + // the flag and a non-empty name, so no name can be sent from here. + if (testContextOptions.skipSessionName) { this.logger.info('Skipping session name update as per configuration') + if (sessionId && !this.sessionMap.has(sessionId)) { + this.sessionMap.set(sessionId, { lastTestName: '', testResults: new Map() }) + } return } From 09276cc344794d657e8abe52706cf5398ef34c63 Mon Sep 17 00:00:00 2001 From: "aditya.h" Date: Thu, 10 Sep 2026 21:49:19 +0530 Subject: [PATCH 05/10] fix(cli): register the driver without a TestHub product, and contain observer failures MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit An Automate-only CLI run (every TestHub product off) never registered its driver: service.before() raised AutomationFrameworkState.CREATE/POST only inside the shouldProcessEventForTesthub block, so webdriverIOModule never recorded the session id or capabilities. automateModule's sessionMap stayed empty and the Automate session was left unmarked, where the legacy flow marked it. Session tracking does not depend on TestHub, so the raise is now gated on the CLI being up and nothing else. On cucumber the same gap also failed every scenario: testHubModule's session event dereferenced the missing session id, re-threw, and — because the event was neither awaited nor caught — surfaced as an unhandled rejection inside the user's own cucumber Before hook, skipping every step. The event is now awaited, and eventDispatcher gives each observer its own boundary so one module's failure neither aborts the observers registered after it nor escapes into the framework hook that raised the state. Co-Authored-By: Claude Opus 5 (1M context) --- .../src/cli/eventDispatcher.ts | 13 ++++++++++++- .../src/cli/modules/testHubModule.ts | 10 ++++++++-- packages/browserstack-service/src/service.ts | 14 +++++++++++++- 3 files changed, 33 insertions(+), 4 deletions(-) diff --git a/packages/browserstack-service/src/cli/eventDispatcher.ts b/packages/browserstack-service/src/cli/eventDispatcher.ts index 6fb2def6..75385d58 100644 --- a/packages/browserstack-service/src/cli/eventDispatcher.ts +++ b/packages/browserstack-service/src/cli/eventDispatcher.ts @@ -1,3 +1,7 @@ +import util from 'node:util' + +import { BStackLogger } from './cliLogger.js' + /** * EventDispatcher - Singleton class for event handling */ @@ -42,7 +46,14 @@ class EventDispatcher { async notifyObserver(event: string, args: unknown) { if (this.observers[event]) { for (const callback of this.observers[event]) { - await callback(args) + // Per-observer boundary. Without it one product module's failure aborts the loop, + // so every module registered after it silently stops receiving that state, and the + // exception escapes into whatever framework hook raised it. + try { + await callback(args) + } catch (error) { + BStackLogger.error(`notifyObserver: observer failed for ${event}, continuing: ${util.format(error)}`) + } } return } diff --git a/packages/browserstack-service/src/cli/modules/testHubModule.ts b/packages/browserstack-service/src/cli/modules/testHubModule.ts index f621b00e..5ca2dab8 100644 --- a/packages/browserstack-service/src/cli/modules/testHubModule.ts +++ b/packages/browserstack-service/src/cli/modules/testHubModule.ts @@ -51,12 +51,18 @@ export default class TestHubModule extends BaseModule { return TestHubModule.MODULE_NAME } - onBeforeTest(args: Record) { + /** + * Awaited, not fire-and-forget: sendTestSessionEvent re-throws after logging, and an + * un-awaited rejection escapes as an unhandled rejection that no caller can contain — on + * cucumber it surfaces inside the user's own Before hook via WDIO's hook domain and fails the + * scenario. Awaiting hands the rejection to eventDispatcher's per-observer boundary. + */ + async onBeforeTest(args: Record) { this.logger.debug('onBeforeTest: Called after test hook from cli configured module!!!') const autoInstace = AutomationFramework.getTrackedInstance() as AutomationFrameworkInstance const instances = [autoInstace] args.autoInstance = instances - this.sendTestSessionEvent(args) + await this.sendTestSessionEvent(args) } onAllTestEvents(args: Record) { diff --git a/packages/browserstack-service/src/service.ts b/packages/browserstack-service/src/service.ts index accb55ad..b7499303 100644 --- a/packages/browserstack-service/src/service.ts +++ b/packages/browserstack-service/src/service.ts @@ -245,6 +245,19 @@ export default class BrowserstackService implements Services.ServiceInstance { BStackLogger.error(`[Accessibility Test Run] Error in service class before function: ${err}`) } + /** + * The driver, its session id and its capabilities reach the AutomationFramework + * instance ONLY through this state, and every cli/module resolves them from there. + * It used to be raised inside the `shouldProcessEventForTesthub` block, so an + * Automate-only run (every TestHub product off) registered no driver at all: + * automateModule's sessionMap stayed empty and the session was left unmarked, + * where legacy marked it. Session tracking does not depend on TestHub, so the + * raise is gated on the CLI being up and nothing else. + */ + if (BrowserstackCLI.getInstance().isRunning()) { + await BrowserstackCLI.getInstance().getAutomationFramework()!.trackEvent(AutomationFrameworkState.CREATE, HookState.POST, { browser: this._browser, hubUrl: this._config.hostname }) + } + if (shouldProcessEventForTesthub('')) { patchConsoleLogs() @@ -255,7 +268,6 @@ export default class BrowserstackService implements Services.ServiceInstance { this._options ) if (BrowserstackCLI.getInstance().isRunning()) { - await BrowserstackCLI.getInstance().getAutomationFramework()!.trackEvent(AutomationFrameworkState.CREATE, HookState.POST, { browser: this._browser, hubUrl: this._config.hostname }) this._insightsHandler.setGitConfigPath() /** * SDK-6277: register the command/result listeners in the CLI/binary flow too, From cbbb4259ed51779db1716314b3e1edd679dfa9bb Mon Sep 17 00:00:00 2001 From: AdityaHirapara Date: Tue, 15 Sep 2026 18:55:28 +0530 Subject: [PATCH 06/10] fix(cli): emit cucumber's feature-start before the session-name REST call beforeFeature awaited _setSessionName before reaching onFeatureStart, so any rejection of that session-update call aborted the rest of the hook and the CLI framework never saw the feature. Observability then lost the feature path for that feature (root file_path empty). Reordering is safe because the two statements are independent: onFeatureStart only assigns local bookkeeping from its own arguments and raises no wire event, and _setSessionName sets nothing it reads. Cucumber-only hook, so mocha and jasmine are unaffected. Co-Authored-By: Claude Opus 5 (1M context) --- packages/browserstack-service/src/service.ts | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/packages/browserstack-service/src/service.ts b/packages/browserstack-service/src/service.ts index b7499303..6e78e8a2 100644 --- a/packages/browserstack-service/src/service.ts +++ b/packages/browserstack-service/src/service.ts @@ -775,9 +775,13 @@ export default class BrowserstackService implements Services.ServiceInstance { @PerformanceTester.Measure(PERFORMANCE_SDK_EVENTS.EVENTS.SDK_HOOK, { hookType: 'beforeFeature' }) async beforeFeature(uri: string, feature: Feature) { this._suiteTitle = feature.name + // Ordered ahead of the two awaits deliberately: onFeatureStart is local bookkeeping that + // raises no wire event, but _setSessionName issues a session-update REST call. Sequenced + // behind it, any rejection (a turboscale 401, or any 4xx/5xx) aborts the rest of the hook + // and Observability silently loses the feature path for that feature. + this._cliCucumberFramework()?.onFeatureStart(uri, feature) await this._setSessionName(feature.name) await this._setAnnotation(`Feature: ${feature.name}`) - this._cliCucumberFramework()?.onFeatureStart(uri, feature) // The legacy InsightsHandler -> Listener -> api/v1/batch transport is gated only on // TESTOPS_BUILD_COMPLETED and BROWSERSTACK_TESTHUB_JWT, both of which the CLI flow itself // sets. Left unguarded it keeps POSTing scenario events under the binary-issued JWT on top From 2ae72ed9fc2f72582354acdd243aaa03eb6a3d22 Mon Sep 17 00:00:00 2001 From: AdityaHirapara Date: Thu, 17 Sep 2026 21:59:14 +0530 Subject: [PATCH 07/10] fix(cli): apply sessionNameFormat on the CLI flow MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit sessionNameFormat is a function, so JSON.stringify drops it and it never reaches the binary. automateModule, which is written to apply it, therefore fell back to the raw suite title and PUT that over the correctly formatted name the SDK had already written. Carry a sessionNameFormatProvided boolean across the boundary instead, and have automateModule defer its name write when a formatter exists, leaving the SDK — which holds the live function — as the naming authority. beforeSuite now also names mocha on the CLI flow in that case, where previously nothing applied the format at all. Nothing changes when sessionNameFormat is unset. The guard sits at the single write site, so lastTestName, the cucumber result key and percy's session-name read are unaffected. Co-Authored-By: Claude Opus 5 --- .../src/@types/bstack-service-types.d.ts | 3 +++ packages/browserstack-service/src/cli/cliUtils.ts | 3 ++- .../browserstack-service/src/cli/modules/automateModule.ts | 7 ++++++- packages/browserstack-service/src/service.ts | 5 ++++- 4 files changed, 15 insertions(+), 3 deletions(-) diff --git a/packages/browserstack-service/src/@types/bstack-service-types.d.ts b/packages/browserstack-service/src/@types/bstack-service-types.d.ts index 9d27c065..d3454edc 100644 --- a/packages/browserstack-service/src/@types/bstack-service-types.d.ts +++ b/packages/browserstack-service/src/@types/bstack-service-types.d.ts @@ -35,6 +35,9 @@ declare global { suiteTitle: string, testTitle?: string ) => string + // A function cannot survive JSON, so sessionNameFormat never reaches the binary. This + // boolean does, and tells the module that the SDK holds the only usable formatter. + sessionNameFormatProvided?: boolean } interface GRRUrls { diff --git a/packages/browserstack-service/src/cli/cliUtils.ts b/packages/browserstack-service/src/cli/cliUtils.ts index 9944df42..a6ce51e0 100644 --- a/packages/browserstack-service/src/cli/cliUtils.ts +++ b/packages/browserstack-service/src/cli/cliUtils.ts @@ -70,7 +70,8 @@ export class CLIUtils { skipSessionStatus: isFalse(modifiedOpts.setSessionStatus), sessionNameOmitTestTitle: modifiedOpts.sessionNameOmitTestTitle || false, sessionNamePrependTopLevelSuiteTitle: modifiedOpts.sessionNamePrependTopLevelSuiteTitle || false, - sessionNameFormat: modifiedOpts.sessionNameFormat || '' + sessionNameFormat: modifiedOpts.sessionNameFormat || '', + sessionNameFormatProvided: !!modifiedOpts.sessionNameFormat } const commonBstackOptions = (() => { diff --git a/packages/browserstack-service/src/cli/modules/automateModule.ts b/packages/browserstack-service/src/cli/modules/automateModule.ts index 1cc38716..2df1d397 100644 --- a/packages/browserstack-service/src/cli/modules/automateModule.ts +++ b/packages/browserstack-service/src/cli/modules/automateModule.ts @@ -292,7 +292,12 @@ export default class AutomateModule extends BaseModule { // An empty name means nothing ever named this session — a BeforeAll that failed // before any feature loaded, so beforeFeature never ran. Legacy makes no naming // call at all in that state; PUTting '' would be an API call it never made. - if (!testContextOptions.skipSessionName && sessionData.lastTestName) { + // sessionNameFormat is a function and is dropped by JSON, so this module can + // never reproduce the user's format — it would PUT the raw title over the correctly + // formatted name the SDK already wrote. Defer whenever a formatter was configured. + // Bookkeeping above is untouched: lastTestName still keys testResults and percy. + if (!testContextOptions.skipSessionName && sessionData.lastTestName && + !testContextOptions.sessionNameFormatProvided) { await this.markSessionName(sessionId, sessionData.lastTestName, { user: userName, key: accessKey }) } diff --git a/packages/browserstack-service/src/service.ts b/packages/browserstack-service/src/service.ts index 6e78e8a2..7bf2217a 100644 --- a/packages/browserstack-service/src/service.ts +++ b/packages/browserstack-service/src/service.ts @@ -381,7 +381,10 @@ export default class BrowserstackService implements Services.ServiceInstance { this._accessibilityHandler?.setSuiteFile(suite.file) if (suite.title && suite.title !== 'Jasmine__TopLevel__Suite') { - if (!BrowserstackCLI.getInstance().isRunning() || this._config.framework !== 'mocha'){ + // On CLI, naming belongs to automateModule — except when sessionNameFormat is set, + // which only this side can apply (the function cannot cross the gRPC/JSON boundary). + if (!BrowserstackCLI.getInstance().isRunning() || this._config.framework !== 'mocha' + || this._options.sessionNameFormat){ await this._setSessionName(suite.title) } } From 3d71cc02343c509d75e4c1ba2607b30c84704793 Mon Sep 17 00:00:00 2001 From: AdityaHirapara Date: Thu, 17 Sep 2026 22:20:14 +0530 Subject: [PATCH 08/10] refactor(cli): keep all session naming inside automateModule Supersedes the previous approach, which made the legacy-side _setSessionName the namer whenever sessionNameFormat was set. That split session naming across two code paths on the same flow and reached into the legacy path to do CLI work. automateModule runs in the SDK process, so it never needed the formatter to survive the gRPC/JSON round trip -- it only needed to stop reading the round-tripped copy, where function-valued keys are silently dropped. It now takes the live service options by injection and resolves sessionNameFormat from them, so every naming decision stays in one place. Injected rather than imported: cli/index.ts constructs this module, so importing it back closes an ESM cycle (it surfaced as "Class extends value is not a constructor" in wdioMochaTestFramework). Adds two tests. The first shapes testContextOptions as the binary really returns it -- with no sessionNameFormat key -- and asserts the formatted name still reaches the session; it fails against the previous code, so it pins the behaviour rather than merely passing alongside it. The second asserts the suite title is used when no formatter is configured, guarding the claim that nothing changes for the unset case. Co-Authored-By: Claude Opus 5 --- .../src/@types/bstack-service-types.d.ts | 3 -- .../browserstack-service/src/cli/cliUtils.ts | 3 +- .../browserstack-service/src/cli/index.ts | 2 +- .../src/cli/modules/automateModule.ts | 28 ++++++---- packages/browserstack-service/src/service.ts | 5 +- .../tests/cli/modules/automateModule.test.ts | 53 +++++++++++++++++++ 6 files changed, 73 insertions(+), 21 deletions(-) diff --git a/packages/browserstack-service/src/@types/bstack-service-types.d.ts b/packages/browserstack-service/src/@types/bstack-service-types.d.ts index d3454edc..9d27c065 100644 --- a/packages/browserstack-service/src/@types/bstack-service-types.d.ts +++ b/packages/browserstack-service/src/@types/bstack-service-types.d.ts @@ -35,9 +35,6 @@ declare global { suiteTitle: string, testTitle?: string ) => string - // A function cannot survive JSON, so sessionNameFormat never reaches the binary. This - // boolean does, and tells the module that the SDK holds the only usable formatter. - sessionNameFormatProvided?: boolean } interface GRRUrls { diff --git a/packages/browserstack-service/src/cli/cliUtils.ts b/packages/browserstack-service/src/cli/cliUtils.ts index a6ce51e0..9944df42 100644 --- a/packages/browserstack-service/src/cli/cliUtils.ts +++ b/packages/browserstack-service/src/cli/cliUtils.ts @@ -70,8 +70,7 @@ export class CLIUtils { skipSessionStatus: isFalse(modifiedOpts.setSessionStatus), sessionNameOmitTestTitle: modifiedOpts.sessionNameOmitTestTitle || false, sessionNamePrependTopLevelSuiteTitle: modifiedOpts.sessionNamePrependTopLevelSuiteTitle || false, - sessionNameFormat: modifiedOpts.sessionNameFormat || '', - sessionNameFormatProvided: !!modifiedOpts.sessionNameFormat + sessionNameFormat: modifiedOpts.sessionNameFormat || '' } const commonBstackOptions = (() => { diff --git a/packages/browserstack-service/src/cli/index.ts b/packages/browserstack-service/src/cli/index.ts index 31416141..101e8678 100644 --- a/packages/browserstack-service/src/cli/index.ts +++ b/packages/browserstack-service/src/cli/index.ts @@ -136,7 +136,7 @@ export class BrowserstackCLI { this.setupAutomationFramework() this.modules[WebdriverIOModule.MODULE_NAME] = new WebdriverIOModule() - this.modules[AutomateModule.MODULE_NAME] = new AutomateModule(this.browserstackConfig as Options.Testrunner) + this.modules[AutomateModule.MODULE_NAME] = new AutomateModule(this.browserstackConfig as Options.Testrunner, this.options as Record) if (startBinResponse.testhub) { this.logBuildStartErrors(startBinResponse.testhub) diff --git a/packages/browserstack-service/src/cli/modules/automateModule.ts b/packages/browserstack-service/src/cli/modules/automateModule.ts index 2df1d397..e2c0d18b 100644 --- a/packages/browserstack-service/src/cli/modules/automateModule.ts +++ b/packages/browserstack-service/src/cli/modules/automateModule.ts @@ -30,15 +30,19 @@ export default class AutomateModule extends BaseModule { logger = BStackLogger browserStackConfig: Options.Testrunner + // The live, in-process service options. Injected rather than imported: cli/index.ts constructs + // this module, so importing it back would close an ESM cycle. + private serviceOptions: Record private sessionMap: Map = new Map() static readonly MODULE_NAME = 'AutomateModule' /** * Create a new AutomateModule */ - constructor(browserStackConfig: Options.Testrunner) { + constructor(browserStackConfig: Options.Testrunner, serviceOptions: Record = {}) { super() this.browserStackConfig = browserStackConfig + this.serviceOptions = serviceOptions this.logger.info('AutomateModule: Initializing Automate Module') TestFramework.registerObserver(TestFrameworkState.TEST, HookState.PRE, this.onBeforeTest.bind(this)) TestFramework.registerObserver(TestFrameworkState.TEST, HookState.POST, this.onAfterTest.bind(this)) @@ -82,9 +86,14 @@ export default class AutomateModule extends BaseModule { } let name = suiteTitle - if (testContextOptions.sessionNameFormat) { + // Resolved from the live in-process options, NOT from testContextOptions: that config is + // round-tripped through the binary as JSON, which silently drops function-valued keys, so + // testContextOptions.sessionNameFormat is always absent. Reading it here keeps every naming + // decision inside this module instead of splitting it across the legacy path. + const sessionNameFormat = this.serviceOptions?.sessionNameFormat + if (sessionNameFormat) { const caps = AutomationFramework.getState(autoInstance, AutomationFrameworkConstants.KEY_CAPABILITIES) - name = testContextOptions.sessionNameFormat( + name = sessionNameFormat( this.browserStackConfig, caps, suiteTitle, @@ -144,9 +153,11 @@ export default class AutomateModule extends BaseModule { } let name = suiteTitle - if (testContextOptions.sessionNameFormat) { + // See onBeforeTest: the formatter exists only in-process; the round-tripped config drops it. + const sessionNameFormat = this.serviceOptions?.sessionNameFormat + if (sessionNameFormat) { const caps = AutomationFramework.getState(autoInstance, AutomationFrameworkConstants.KEY_CAPABILITIES) - name = testContextOptions.sessionNameFormat( + name = sessionNameFormat( this.browserStackConfig, caps, suiteTitle, @@ -292,12 +303,7 @@ export default class AutomateModule extends BaseModule { // An empty name means nothing ever named this session — a BeforeAll that failed // before any feature loaded, so beforeFeature never ran. Legacy makes no naming // call at all in that state; PUTting '' would be an API call it never made. - // sessionNameFormat is a function and is dropped by JSON, so this module can - // never reproduce the user's format — it would PUT the raw title over the correctly - // formatted name the SDK already wrote. Defer whenever a formatter was configured. - // Bookkeeping above is untouched: lastTestName still keys testResults and percy. - if (!testContextOptions.skipSessionName && sessionData.lastTestName && - !testContextOptions.sessionNameFormatProvided) { + if (!testContextOptions.skipSessionName && sessionData.lastTestName) { await this.markSessionName(sessionId, sessionData.lastTestName, { user: userName, key: accessKey }) } diff --git a/packages/browserstack-service/src/service.ts b/packages/browserstack-service/src/service.ts index 7bf2217a..6e78e8a2 100644 --- a/packages/browserstack-service/src/service.ts +++ b/packages/browserstack-service/src/service.ts @@ -381,10 +381,7 @@ export default class BrowserstackService implements Services.ServiceInstance { this._accessibilityHandler?.setSuiteFile(suite.file) if (suite.title && suite.title !== 'Jasmine__TopLevel__Suite') { - // On CLI, naming belongs to automateModule — except when sessionNameFormat is set, - // which only this side can apply (the function cannot cross the gRPC/JSON boundary). - if (!BrowserstackCLI.getInstance().isRunning() || this._config.framework !== 'mocha' - || this._options.sessionNameFormat){ + if (!BrowserstackCLI.getInstance().isRunning() || this._config.framework !== 'mocha'){ await this._setSessionName(suite.title) } } diff --git a/packages/browserstack-service/tests/cli/modules/automateModule.test.ts b/packages/browserstack-service/tests/cli/modules/automateModule.test.ts index a5f76212..a22a0cfb 100644 --- a/packages/browserstack-service/tests/cli/modules/automateModule.test.ts +++ b/packages/browserstack-service/tests/cli/modules/automateModule.test.ts @@ -1,6 +1,10 @@ import { describe, expect, it, vi, beforeEach } from 'vitest' import got from 'got' import AutomateModule from '../../../src/cli/modules/automateModule.js' +import TestFramework from '../../../src/cli/frameworks/testFramework.js' +import AutomationFramework from '../../../src/cli/frameworks/automationFramework.js' +import { TestFrameworkConstants } from '../../../src/cli/frameworks/constants/testFrameworkConstants.js' +import { isBrowserstackSession } from '../../../src/util.js' import type { Options } from '@wdio/types' // Mock dependencies @@ -167,6 +171,55 @@ describe('AutomateModule', () => { await expect(moduleWithSkip.onBeforeTest(mockArgs)).resolves.toBeUndefined() }) + it('applies sessionNameFormat from the injected service options', async () => { + // The formatter is a function, so JSON drops it from the config the binary echoes back. + // It reaches the module only through the injected options; testContextOptions below is + // deliberately shaped as the binary really returns it, with no sessionNameFormat key. + const sessionNameFormat = vi.fn((_config, _caps, suiteTitle, testTitle) => `FMT::${suiteTitle}::${testTitle}`) + const moduleWithFormat = new AutomateModule(mockConfig, { sessionNameFormat }) + moduleWithFormat.config = { + testContextOptions: { skipSessionName: false, skipSessionStatus: false } + } as any + + vi.mocked(isBrowserstackSession).mockReturnValue(true) + vi.mocked(AutomationFramework.getState).mockReturnValue('session-1') + + await moduleWithFormat.onBeforeTest({ + instance: {}, + test: { title: 'test title' }, + suiteTitle: 'suite title' + }) + + expect(sessionNameFormat).toHaveBeenCalled() + expect(TestFramework.setState).toHaveBeenCalledWith( + expect.anything(), + TestFrameworkConstants.KEY_AUTOMATE_SESSION_NAME, + 'FMT::suite title::test title' + ) + }) + + it('falls back to the suite title when no sessionNameFormat is configured', async () => { + const moduleNoFormat = new AutomateModule(mockConfig, {}) + moduleNoFormat.config = { + testContextOptions: { skipSessionName: false, skipSessionStatus: false } + } as any + + vi.mocked(isBrowserstackSession).mockReturnValue(true) + vi.mocked(AutomationFramework.getState).mockReturnValue('session-2') + + await moduleNoFormat.onBeforeTest({ + instance: {}, + test: { title: 'test title', fullName: 'full name' }, + suiteTitle: 'suite title' + }) + + expect(TestFramework.setState).toHaveBeenCalledWith( + expect.anything(), + TestFrameworkConstants.KEY_AUTOMATE_SESSION_NAME, + 'suite title' + ) + }) + it('should handle onAfterTest with skipSessionStatus enabled', async () => { const configWithSkip = { ...mockConfig, From 7e631ecd4357229f92a04cb1474e154a39c06246 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Fri, 18 Sep 2026 05:54:09 +0000 Subject: [PATCH 09/10] chore(changeset): auto-generate from PR template (minor) --- .changeset/pr-207.md | 7 +++++++ 1 file changed, 7 insertions(+) create mode 100644 .changeset/pr-207.md diff --git a/.changeset/pr-207.md b/.changeset/pr-207.md new file mode 100644 index 00000000..c9f76cd8 --- /dev/null +++ b/.changeset/pr-207.md @@ -0,0 +1,7 @@ +--- +"@wdio/browserstack-service": minor +--- + +- Cucumber tests on WebdriverIO now report through the BrowserStack CLI, matching the behaviour of Mocha. +- Fixed `sessionNameFormat` being ignored, so custom session names now apply on the CLI flow. +- Fixed Observability losing a feature's file path when a session-name update failed. From 00d76817e8af5dcb59a46b1a63019b362ce1aabf Mon Sep 17 00:00:00 2001 From: AdityaHirapara Date: Fri, 18 Sep 2026 22:06:09 +0530 Subject: [PATCH 10/10] refactor(cli): move the preferScenarioName rename into automateModule service.ts decided this on the CLI path and wrote the name itself, which split session naming across two code paths on the same flow -- the module named the session at TEST/PRE from the feature title, and service.ts renamed it afterwards from its own scenario list. automateModule now counts non-skipped cucumber scenarios and applies the rename at EXECUTE/POST, the first point where "exactly one ran" is knowable. service.ts carries preferScenarioName on the cucumber TEST/POST event and keeps the rename only for the legacy path. This matches the shape the v9 line already uses. The scenario is tracked before the skipSessionStatus return: setSessionStatus false opts out of the status, not of the rename. Also completes the TestFramework test mock with getState. isCucumberInstance was previously reached only through a short-circuit, so the missing mock method never surfaced; it is called unconditionally now. Co-Authored-By: Claude Opus 5 --- .../src/cli/modules/automateModule.ts | 29 +++++++++++++++-- packages/browserstack-service/src/service.ts | 12 +++---- .../tests/cli/modules/automateModule.test.ts | 32 ++++++++++++++++++- 3 files changed, 61 insertions(+), 12 deletions(-) diff --git a/packages/browserstack-service/src/cli/modules/automateModule.ts b/packages/browserstack-service/src/cli/modules/automateModule.ts index e2c0d18b..7a3edb1e 100644 --- a/packages/browserstack-service/src/cli/modules/automateModule.ts +++ b/packages/browserstack-service/src/cli/modules/automateModule.ts @@ -24,6 +24,9 @@ interface TestResult { interface SessionData { lastTestName: string testResults: Map // testName -> TestResult + scenariosRan: number // non-skipped cucumber scenarios, for preferScenarioName + lastScenarioName?: string + preferScenarioName?: boolean } export default class AutomateModule extends BaseModule { @@ -80,7 +83,7 @@ export default class AutomateModule extends BaseModule { if (testContextOptions.skipSessionName) { this.logger.info('Skipping session name update as per configuration') if (sessionId && !this.sessionMap.has(sessionId)) { - this.sessionMap.set(sessionId, { lastTestName: '', testResults: new Map() }) + this.sessionMap.set(sessionId, { lastTestName: '', testResults: new Map(), scenariosRan: 0 }) } return } @@ -110,7 +113,8 @@ export default class AutomateModule extends BaseModule { if (!existingSession) { this.sessionMap.set(sessionId, { lastTestName: name, - testResults: new Map() + testResults: new Map(), + scenariosRan: 0 }) } else { existingSession.lastTestName = name @@ -147,6 +151,17 @@ export default class AutomateModule extends BaseModule { const suiteTitle = args.suiteTitle as string const testContextOptions = this.config.testContextOptions as TestContextOptions + // Tracked before the skipSessionStatus return on purpose: `setSessionStatus: false` opts + // out of the STATUS, not of the preferScenarioName rename. + if (!skipped && this.isCucumberInstance(instace)) { + const nameData = this.sessionMap.get(sessionId) + if (nameData) { + nameData.scenariosRan++ + nameData.lastScenarioName = testTitle + nameData.preferScenarioName = isTrue(args.preferScenarioName) + } + } + if (testContextOptions.skipSessionStatus || !isBrowserstackSession(browser)) { this.logger.info('Skipping session status update as per configuration') return @@ -243,7 +258,7 @@ export default class AutomateModule extends BaseModule { // registered yet. `lastTestName` stays empty on purpose: onAfterExecute's naming // call is what consumes it, and an empty name is what beforeFeature's own // (un-gated) _setSessionName has already applied. - this.sessionMap.set(sessionId, { lastTestName: '', testResults: new Map() }) + this.sessionMap.set(sessionId, { lastTestName: '', testResults: new Map(), scenariosRan: 0 }) } const name = this.resolveHookName(instance, hookKey) @@ -303,6 +318,14 @@ export default class AutomateModule extends BaseModule { // An empty name means nothing ever named this session — a BeforeAll that failed // before any feature loaded, so beforeFeature never ran. Legacy makes no naming // call at all in that state; PUTting '' would be an API call it never made. + // preferScenarioName: cucumber names the session after the FEATURE, but when + // exactly one non-skipped scenario ran the user can ask for that scenario's name + // instead. Only decidable here — "exactly one" is not knowable while scenarios are + // still arriving. skipSessionName still wins, on the guard below. + if (sessionData.preferScenarioName && sessionData.scenariosRan === 1 && sessionData.lastScenarioName) { + sessionData.lastTestName = sessionData.lastScenarioName + } + if (!testContextOptions.skipSessionName && sessionData.lastTestName) { await this.markSessionName(sessionId, sessionData.lastTestName, { user: userName, key: accessKey }) } diff --git a/packages/browserstack-service/src/service.ts b/packages/browserstack-service/src/service.ts index 6e78e8a2..44ce0353 100644 --- a/packages/browserstack-service/src/service.ts +++ b/packages/browserstack-service/src/service.ts @@ -546,16 +546,11 @@ export default class BrowserstackService implements Services.ServiceInstance { const { preferScenarioName, setSessionName, setSessionStatus } = this._options // For Cucumber: Checks scenarios that ran (i.e. not skipped) on the session // Only 1 Scenario ran and option enabled => Redefine session name to Scenario's name + // CLI flow: automateModule owns this rename — it counts non-skipped scenarios itself + // and applies it at EXECUTE/POST, where "exactly one" first becomes knowable. This + // branch is the legacy path only. if (preferScenarioName && this._scenariosThatRan.length === 1){ this._fullTitle = this._scenariosThatRan.pop() - // On the CLI flow the _updateJob below that carries this name is gated off, and - // automateModule has already named the session from the feature name it saw at - // TEST/PRE — so without this write the option is a silent no-op. Sequenced after - // the EXECUTE/POST tracker call above, not racing it. Unreachable for mocha and - // jasmine: _scenariosThatRan is only ever pushed from afterScenario. - if (setSessionName && BrowserstackCLI.getInstance().isRunning()) { - await this._updateJob({ name: this._fullTitle }) - } } await PerformanceTester.measureWrapper(PERFORMANCE_SDK_EVENTS.AUTOMATE_EVENTS.SESSION_STATUS, async () => { @@ -872,6 +867,7 @@ export default class BrowserstackService implements Services.ServiceInstance { result: this._cucumberTestResult(world), ignoreHooksStatus: this._options.testObservabilityOptions?.ignoreHooksStatus === true, hasStepFailures: this._insightsHandler ? this._insightsHandler.hasTestStepFailures(world) : true, + preferScenarioName: this._options.preferScenarioName === true, }) return } diff --git a/packages/browserstack-service/tests/cli/modules/automateModule.test.ts b/packages/browserstack-service/tests/cli/modules/automateModule.test.ts index a22a0cfb..d398c70d 100644 --- a/packages/browserstack-service/tests/cli/modules/automateModule.test.ts +++ b/packages/browserstack-service/tests/cli/modules/automateModule.test.ts @@ -11,7 +11,8 @@ import type { Options } from '@wdio/types' vi.mock('../../../src/cli/frameworks/testFramework.js', () => ({ default: { registerObserver: vi.fn(), - setState: vi.fn() + setState: vi.fn(), + getState: vi.fn() } })) @@ -198,6 +199,35 @@ describe('AutomateModule', () => { ) }) + it('renames the session to the scenario when preferScenarioName and exactly one scenario ran', async () => { + // The rename is only decidable at EXECUTE/POST — "exactly one" is not knowable while + // scenarios are still arriving — so the module counts them and applies it there. + const mod = new AutomateModule(mockConfig, {}) + mod.config = { + userName: 'testuser', + accessKey: 'testkey', + testContextOptions: { skipSessionName: false, skipSessionStatus: false } + } as any + + vi.mocked(isBrowserstackSession).mockReturnValue(true) + vi.mocked(AutomationFramework.getState).mockReturnValue('session-pref') + vi.mocked(TestFramework.getState).mockReturnValue('cucumber') // isCucumberInstance + + await mod.onBeforeTest({ instance: {}, test: { title: 'ignored' }, suiteTitle: 'Feature title' }) + await mod.onAfterTest({ + instance: {}, + result: { error: null, passed: true }, + test: { title: 'The only scenario' }, + suiteTitle: 'Feature title', + preferScenarioName: true + }) + + const spy = vi.spyOn(mod, 'markSessionName').mockResolvedValue(undefined) + await mod.onAfterExecute() + + expect(spy).toHaveBeenCalledWith('session-pref', 'The only scenario', expect.anything()) + }) + it('falls back to the suite title when no sessionNameFormat is configured', async () => { const moduleNoFormat = new AutomateModule(mockConfig, {}) moduleNoFormat.config = {