feat(v8): platformise CucumberJS on the CLI/binary flow

.changeset/pr-207.md

@@ -0,0 +1,7 @@
+---
+"@wdio/browserstack-service": minor
+---
+
+- Cucumber tests on WebdriverIO now report through the BrowserStack CLI, matching the behaviour of Mocha.
+- Fixed `sessionNameFormat` being ignored, so custom session names now apply on the CLI flow.
+- Fixed Observability losing a feature's file path when a session-name update failed.

packages/browserstack-service/src/cli/cliUtils.ts

@@ -39,7 +39,7 @@ import APIUtils from './apiUtils.js'
 export class CLIUtils {
     static automationFrameworkDetail = {}
     static testFrameworkDetail = {}
-    static CLISupportedFrameworks = ['mocha']
+    static CLISupportedFrameworks = ['mocha', 'cucumber']
 
     static isDevelopmentEnv() {
         return process.env.BROWSERSTACK_CLI_ENV === 'development'

packages/browserstack-service/src/cli/eventDispatcher.ts

@@ -1,3 +1,7 @@
+import util from 'node:util'
+
+import { BStackLogger } from './cliLogger.js'
+
 /**
  * EventDispatcher - Singleton class for event handling
  */
@@ -42,7 +46,14 @@ class EventDispatcher {
     async notifyObserver(event: string, args: unknown) {
         if (this.observers[event]) {
             for (const callback of this.observers[event]) {
-                await callback(args)
+                // Per-observer boundary. Without it one product module's failure aborts the loop,
+                // so every module registered after it silently stops receiving that state, and the
+                // exception escapes into whatever framework hook raised it.
+                try {
+                    await callback(args)
+                } catch (error) {
+                    BStackLogger.error(`notifyObserver: observer failed for ${event}, continuing: ${util.format(error)}`)
+                }
             }
             return
         }

packages/browserstack-service/src/cli/frameworks/constants/testFrameworkConstants.ts

@@ -29,6 +29,7 @@ export const TestFrameworkConstants = {
     KEY_EVENT_STARTED_AT : 'event_started_at',
     KEY_EVENT_ENDED_AT : 'event_ended_at',
     KEY_HOOK_ID : 'hook_id',
+    KEY_HOOK_STATE : 'hook_state',
     KEY_HOOK_RESULT : 'hook_result',
     KEY_HOOK_LOGS : 'hook_logs',
     KEY_HOOK_NAME : 'hook_name',

packages/browserstack-service/src/cli/index.ts

@@ -16,6 +16,8 @@ import { BROWSERSTACK_ACCESSIBILITY, BROWSERSTACK_OBSERVABILITY, BROWSERSTACK_TE
 import type { Options } from '@wdio/types'
 import TestOpsConfig from '../testOps/testOpsConfig.js'
 import WdioMochaTestFramework from './frameworks/wdioMochaTestFramework.js'
+import WdioCucumberTestFramework from './frameworks/wdioCucumberTestFramework.js'
+import type TestFramework from './frameworks/testFramework.js'
 import WdioAutomationFramework from './frameworks/wdioAutomationFramework.js'
 import WebdriverIOModule from './modules/webdriverIOModule.js'
 import AccessibilityModule from './modules/accessibilityModule.js'
@@ -44,7 +46,7 @@ export class BrowserstackCLI {
     isChildConnected = false
     binSessionId: string | null = null
     modules: Record<string, BaseModule> = {}
-    testFramework: WdioMochaTestFramework|null = null
+    testFramework: TestFramework|null = null
     cliParams: Record<string, string> | null = null
     automationFramework: WdioAutomationFramework|null = null
     SDK_CLI_BIN_PATH: string | null = null
@@ -134,7 +136,7 @@ export class BrowserstackCLI {
         this.setupAutomationFramework()
 
         this.modules[WebdriverIOModule.MODULE_NAME] = new WebdriverIOModule()
-        this.modules[AutomateModule.MODULE_NAME] = new AutomateModule(this.browserstackConfig as Options.Testrunner)
+        this.modules[AutomateModule.MODULE_NAME] = new AutomateModule(this.browserstackConfig as Options.Testrunner, this.options as Record<string, any>)
 
         if (startBinResponse.testhub) {
             this.logBuildStartErrors(startBinResponse.testhub)
@@ -488,8 +490,15 @@ export class BrowserstackCLI {
      */
     setupTestFramework() {
         const testFrameworkDetail = CLIUtils.getTestFrameworkDetail()
-        if (testFrameworkDetail.name.toLowerCase() === 'webdriverio-mocha') {
+        switch (testFrameworkDetail.name.toLowerCase()) {
+        case 'webdriverio-mocha':
             this.testFramework = new WdioMochaTestFramework([testFrameworkDetail.name], testFrameworkDetail.version, this.binSessionId as string)
+            break
+        case 'webdriverio-cucumber':
+            this.testFramework = new WdioCucumberTestFramework([testFrameworkDetail.name], testFrameworkDetail.version, this.binSessionId as string)
+            break
+        default:
+            this.logger.debug(`setupTestFramework: no CLI test framework for name=${testFrameworkDetail.name}`)
         }
     }
 

packages/browserstack-service/src/cli/modules/accessibilityModule.ts

@@ -203,7 +203,11 @@ export default class AccessibilityModule extends BaseModule {
 
             const sessionId = AutomationFramework.getState(autoInstance, AutomationFrameworkConstants.KEY_FRAMEWORK_SESSION_ID)
             const accessibilityOptions = this.config.accessibilityOptions
-            const shouldScanTest = this.autoScanning && shouldScanTestForAccessibility(suiteTitle, test.title, accessibilityOptions as { [key: string]: any } | undefined) && this.accessibility
+            // `world` is set only by service.beforeScenario, so this stays the 3-arg substring
+            // form for mocha. Without it a cucumber scenario has no test.title and the tag-aware
+            // branch is never entered, so include/excludeTagsInTestingScope are ignored (row 48).
+            const world = args.world
+            const shouldScanTest = this.autoScanning && shouldScanTestForAccessibility(suiteTitle, test.title, accessibilityOptions as { [key: string]: any } | undefined, world, Boolean(world)) && this.accessibility
 
             this.accessibilityMap.set(sessionId, shouldScanTest)
             // Create test metadata similar to accessibility-handler

packages/browserstack-service/src/cli/modules/automateModule.ts

@@ -24,25 +24,36 @@ interface TestResult {
 interface SessionData {
     lastTestName: string
     testResults: Map<string, TestResult> // testName -> TestResult
+    scenariosRan: number // non-skipped cucumber scenarios, for preferScenarioName
+    lastScenarioName?: string
+    preferScenarioName?: boolean
 }
 
 export default class AutomateModule extends BaseModule {
 
     logger = BStackLogger
     browserStackConfig: Options.Testrunner
+    // The live, in-process service options. Injected rather than imported: cli/index.ts constructs
+    // this module, so importing it back would close an ESM cycle.
+    private serviceOptions: Record<string, any>
     private sessionMap: Map<string, SessionData> = new Map()
 
     static readonly MODULE_NAME = 'AutomateModule'
     /**
      * Create a new AutomateModule
      */
-    constructor(browserStackConfig: Options.Testrunner) {
+    constructor(browserStackConfig: Options.Testrunner, serviceOptions: Record<string, any> = {}) {
         super()
         this.browserStackConfig = browserStackConfig
+        this.serviceOptions = serviceOptions
         this.logger.info('AutomateModule: Initializing Automate Module')
         TestFramework.registerObserver(TestFrameworkState.TEST, HookState.PRE, this.onBeforeTest.bind(this))
         TestFramework.registerObserver(TestFrameworkState.TEST, HookState.POST, this.onAfterTest.bind(this))
         TestFramework.registerObserver(AutomationFrameworkState.EXECUTE, HookState.POST, this.onAfterExecute.bind(this))
+        // Build-level hooks carry no scenario result, so they reach the session verdict only
+        // through their own state. See onBuildLevelHookEnd — cucumber-gated inside the handler.
+        TestFramework.registerObserver(TestFrameworkState.BEFORE_ALL, HookState.POST, this.onBuildLevelHookEnd.bind(this, 'BEFORE_ALL'))
+        TestFramework.registerObserver(TestFrameworkState.AFTER_ALL, HookState.POST, this.onBuildLevelHookEnd.bind(this, 'AFTER_ALL'))
     }
 
     getModuleName(): string {
@@ -60,15 +71,32 @@ export default class AutomateModule extends BaseModule {
         const suiteTitle = args.suiteTitle as string
         const testContextOptions = this.config.testContextOptions as TestContextOptions
 
-        if (testContextOptions.skipSessionName || !isBrowserstackSession(browser)) {
+        if (!isBrowserstackSession(browser)) {
+            return
+        }
+
+        // `setSessionName: false` suppresses the NAME, not the registration. The session still has
+        // to enter sessionMap or onAfterExecute has nothing to status-mark, and legacy marks it
+        // either way — its after() status block gates on setSessionStatus alone. Registering with
+        // an empty lastTestName is safe because onAfterExecute's naming call is guarded on both
+        // the flag and a non-empty name, so no name can be sent from here.
+        if (testContextOptions.skipSessionName) {
             this.logger.info('Skipping session name update as per configuration')
+            if (sessionId && !this.sessionMap.has(sessionId)) {
+                this.sessionMap.set(sessionId, { lastTestName: '', testResults: new Map(), scenariosRan: 0 })
+            }
             return
         }
 
         let name = suiteTitle
-        if (testContextOptions.sessionNameFormat) {
+        // Resolved from the live in-process options, NOT from testContextOptions: that config is
+        // round-tripped through the binary as JSON, which silently drops function-valued keys, so
+        // testContextOptions.sessionNameFormat is always absent. Reading it here keeps every naming
+        // decision inside this module instead of splitting it across the legacy path.
+        const sessionNameFormat = this.serviceOptions?.sessionNameFormat
+        if (sessionNameFormat) {
             const caps = AutomationFramework.getState(autoInstance, AutomationFrameworkConstants.KEY_CAPABILITIES)
-            name = testContextOptions.sessionNameFormat(
+            name = sessionNameFormat(
                 this.browserStackConfig,
                 caps,
                 suiteTitle,
@@ -85,7 +113,8 @@ export default class AutomateModule extends BaseModule {
         if (!existingSession) {
             this.sessionMap.set(sessionId, {
                 lastTestName: name,
-                testResults: new Map()
+                testResults: new Map(),
+                scenariosRan: 0
             })
         } else {
             existingSession.lastTestName = name
@@ -98,14 +127,20 @@ export default class AutomateModule extends BaseModule {
     async onAfterTest(args: Record<string, unknown>) {
         this.logger.debug('onAfterTest: inside automate module after test hook!')
         const instace = args.instance as TestFrameworkInstance
-        const { error, passed } = args.result as { error: Error | null, passed: boolean }
+        const { error, passed, skipped } = args.result as { error: Error | null, passed: boolean, skipped?: boolean }
         const _failReasons: string[] = []
 
-        if (!passed) {
+        // A skipped cucumber scenario must not fail the session: legacy accumulates _failReasons
+        // only for _failureStatuses (failed/ambiguous/undefined/unknown), which excludes skipped.
+        // Cucumber-scoped on purpose — mocha's collapse is its own long-standing behaviour on this
+        // flow and changing it here would alter a framework already shipping on the CLI.
+        const treatAsPassed = passed || Boolean(skipped && this.isCucumberInstance(instace))
+
+        if (!treatAsPassed) {
             _failReasons.push((error && error.message) || 'Unknown Error')
         }
 
-        const status = passed ? 'passed' : 'failed'
+        const status = treatAsPassed ? 'passed' : 'failed'
         const reason = _failReasons.length > 0 ? _failReasons.join('\n') : undefined
 
         const autoInstance = AutomationFramework.getTrackedInstance()
@@ -116,15 +151,28 @@ export default class AutomateModule extends BaseModule {
         const suiteTitle = args.suiteTitle as string
         const testContextOptions = this.config.testContextOptions as TestContextOptions
 
+        // Tracked before the skipSessionStatus return on purpose: `setSessionStatus: false` opts
+        // out of the STATUS, not of the preferScenarioName rename.
+        if (!skipped && this.isCucumberInstance(instace)) {
+            const nameData = this.sessionMap.get(sessionId)
+            if (nameData) {
+                nameData.scenariosRan++
+                nameData.lastScenarioName = testTitle
+                nameData.preferScenarioName = isTrue(args.preferScenarioName)
+            }
+        }
+
         if (testContextOptions.skipSessionStatus || !isBrowserstackSession(browser)) {
             this.logger.info('Skipping session status update as per configuration')
             return
         }
 
         let name = suiteTitle
-        if (testContextOptions.sessionNameFormat) {
+        // See onBeforeTest: the formatter exists only in-process; the round-tripped config drops it.
+        const sessionNameFormat = this.serviceOptions?.sessionNameFormat
+        if (sessionNameFormat) {
             const caps = AutomationFramework.getState(autoInstance, AutomationFrameworkConstants.KEY_CAPABILITIES)
-            name = testContextOptions.sessionNameFormat(
+            name = sessionNameFormat(
                 this.browserStackConfig,
                 caps,
                 suiteTitle,
@@ -145,14 +193,103 @@ export default class AutomateModule extends BaseModule {
                 reason: reason
             }
 
-            sessionData.testResults.set(name, testResult)
+            // `name` is the session NAME, which for cucumber is the Feature title and therefore
+            // shared by every scenario in the file — keying on it collapses N scenarios into one
+            // last-write-wins entry, so a feature whose last scenario passes reports a passed
+            // session however many earlier ones failed. Mocha leaves `fullName` undefined, so its
+            // key is unchanged.
+            const resultKey = (test && test.fullName) ? String(test.fullName) : name
+            sessionData.testResults.set(resultKey, testResult)
             this.sessionMap.set(sessionId, sessionData)
         }
 
         TestFramework.setState(instace, TestFrameworkConstants.KEY_AUTOMATE_SESSION_STATUS, status)
         TestFramework.setState(instace, TestFrameworkConstants.KEY_AUTOMATE_SESSION_REASON, reason)
     }
 
+    /**
+     * A `BeforeAll` / `AfterAll` failure produces no scenario result, so it can never enter the
+     * per-test `testResults` map that onAfterExecute aggregates — a run whose BeforeAll blew up
+     * reports its session as PASSED. Legacy pushed the hook error into `_failReasons` and
+     * `after()` marked the session failed; that whole accumulation is gated
+     * `setSessionStatus && !BrowserstackCLI.isRunning()`, so it is dead while the binary is up.
+     *
+     * Cucumber-gated deliberately. `wdio_mocha` has the identical latent shape on this flow, but
+     * legacy mocha behaved the same way, so repairing it here would be an unrequested behaviour
+     * change to the one framework already working on the CLI flow.
+     */
+    async onBuildLevelHookEnd(hookKey: string, args: Record<string, unknown>) {
+        try {
+            const instance = (args?.instance as TestFrameworkInstance) || TestFramework.getTrackedInstance()
+            if (!instance || !this.isCucumberInstance(instance)) {
+                return
+            }
+
+            const result = args?.result as { passed?: boolean, error?: Error } | undefined
+            if (!result || result.passed) {
+                return
+            }
+
+            const testContextOptions = this.config.testContextOptions as TestContextOptions
+            if (testContextOptions?.skipSessionStatus) {
+                return
+            }
+
+            const autoInstance = AutomationFramework.getTrackedInstance()
+            const sessionId = AutomationFramework.getState(autoInstance, AutomationFrameworkConstants.KEY_FRAMEWORK_SESSION_ID)
+            if (!sessionId) {
+                this.logger.debug(`onBuildLevelHookEnd: no session id resolved for ${hookKey}; nothing to mark`)
+                return
+            }
+
+            const sessionData = this.sessionMap.get(sessionId)
+            // Keyed on the absence of scenario results, never on the flag alone: legacy's
+            // `ignoreHooksStatus && this._specsRan` arm needs BOTH, and with no scenario recorded
+            // it falls through to marking `failed` regardless of the flag. The count is final
+            // here — a BeforeAll failure aborts the run, and by AfterAll every scenario is in.
+            const specsRan = (sessionData?.testResults.size ?? 0) > 0
+            if (specsRan && isTrue(args?.ignoreHooksStatus)) {
+                this.logger.debug(`onBuildLevelHookEnd: ${hookKey} failed but ignoreHooksStatus is set; not failing the session`)
+                return
+            }
+
+            if (!sessionData) {
+                // A BeforeAll can fail before any scenario ran, so the session may not be
+                // registered yet. `lastTestName` stays empty on purpose: onAfterExecute's naming
+                // call is what consumes it, and an empty name is what beforeFeature's own
+                // (un-gated) _setSessionName has already applied.
+                this.sessionMap.set(sessionId, { lastTestName: '', testResults: new Map(), scenariosRan: 0 })
+            }
+
+            const name = this.resolveHookName(instance, hookKey)
+            this.sessionMap.get(sessionId)!.testResults.set(name, {
+                testName: name,
+                status: 'failed',
+                reason: (result.error && result.error.message) || 'Hook failed'
+            })
+            this.logger.info(`onBuildLevelHookEnd: recorded ${hookKey} failure against session ${sessionId}`)
+        } catch (error) {
+            this.logger.error(`Exception in automate onBuildLevelHookEnd: ${error}`)
+        }
+    }
+
+    private isCucumberInstance(instance: TestFrameworkInstance): boolean {
+        const frameworkName = String(TestFramework.getState(instance, TestFrameworkConstants.KEY_TEST_FRAMEWORK_NAME) || '')
+        return frameworkName.toLowerCase().includes('cucumber')
+    }
+
+    /** The hook's reported name (`BEFORE_ALL for <feature>`), so the session reason names the hook. */
+    private resolveHookName(instance: TestFrameworkInstance, hookKey: string): string {
+        try {
+            const finished = TestFramework.getState(instance, TestFrameworkConstants.KEY_HOOKS_FINISHED) as Map<string, Record<string, unknown>[]> | undefined
+            const hooks = finished?.get(hookKey)
+            const hookName = hooks?.length ? hooks[hooks.length - 1][TestFrameworkConstants.KEY_HOOK_NAME] : undefined
+            return (hookName as string) || hookKey
+        } catch {
+            return hookKey
+        }
+    }
+
     async onAfterExecute() {
         this.logger.debug('onAfterExecute: inside automate module after execute hook!')
 
@@ -178,7 +315,18 @@ export default class AutomateModule extends BaseModule {
                     }
                 }
 
-                if (!testContextOptions.skipSessionName) {
+                // An empty name means nothing ever named this session — a BeforeAll that failed
+                // before any feature loaded, so beforeFeature never ran. Legacy makes no naming
+                // call at all in that state; PUTting '' would be an API call it never made.
+                // preferScenarioName: cucumber names the session after the FEATURE, but when
+                // exactly one non-skipped scenario ran the user can ask for that scenario's name
+                // instead. Only decidable here — "exactly one" is not knowable while scenarios are
+                // still arriving. skipSessionName still wins, on the guard below.
+                if (sessionData.preferScenarioName && sessionData.scenariosRan === 1 && sessionData.lastScenarioName) {
+                    sessionData.lastTestName = sessionData.lastScenarioName
+                }
+
+                if (!testContextOptions.skipSessionName && sessionData.lastTestName) {
                     await this.markSessionName(sessionId, sessionData.lastTestName, { user: userName, key: accessKey })
                 }