diff --git a/.changeset/pr-207.md b/.changeset/pr-207.md new file mode 100644 index 00000000..c9f76cd8 --- /dev/null +++ b/.changeset/pr-207.md @@ -0,0 +1,7 @@ +--- +"@wdio/browserstack-service": minor +--- + +- Cucumber tests on WebdriverIO now report through the BrowserStack CLI, matching the behaviour of Mocha. +- Fixed `sessionNameFormat` being ignored, so custom session names now apply on the CLI flow. +- Fixed Observability losing a feature's file path when a session-name update failed. diff --git a/packages/browserstack-service/src/cli/cliUtils.ts b/packages/browserstack-service/src/cli/cliUtils.ts index 423ea915..9944df42 100644 --- a/packages/browserstack-service/src/cli/cliUtils.ts +++ b/packages/browserstack-service/src/cli/cliUtils.ts @@ -39,7 +39,7 @@ import APIUtils from './apiUtils.js' export class CLIUtils { static automationFrameworkDetail = {} static testFrameworkDetail = {} - static CLISupportedFrameworks = ['mocha'] + static CLISupportedFrameworks = ['mocha', 'cucumber'] static isDevelopmentEnv() { return process.env.BROWSERSTACK_CLI_ENV === 'development' diff --git a/packages/browserstack-service/src/cli/eventDispatcher.ts b/packages/browserstack-service/src/cli/eventDispatcher.ts index 6fb2def6..75385d58 100644 --- a/packages/browserstack-service/src/cli/eventDispatcher.ts +++ b/packages/browserstack-service/src/cli/eventDispatcher.ts @@ -1,3 +1,7 @@ +import util from 'node:util' + +import { BStackLogger } from './cliLogger.js' + /** * EventDispatcher - Singleton class for event handling */ @@ -42,7 +46,14 @@ class EventDispatcher { async notifyObserver(event: string, args: unknown) { if (this.observers[event]) { for (const callback of this.observers[event]) { - await callback(args) + // Per-observer boundary. Without it one product module's failure aborts the loop, + // so every module registered after it silently stops receiving that state, and the + // exception escapes into whatever framework hook raised it. + try { + await callback(args) + } catch (error) { + BStackLogger.error(`notifyObserver: observer failed for ${event}, continuing: ${util.format(error)}`) + } } return } diff --git a/packages/browserstack-service/src/cli/frameworks/constants/testFrameworkConstants.ts b/packages/browserstack-service/src/cli/frameworks/constants/testFrameworkConstants.ts index e3213521..1dd45e24 100644 --- a/packages/browserstack-service/src/cli/frameworks/constants/testFrameworkConstants.ts +++ b/packages/browserstack-service/src/cli/frameworks/constants/testFrameworkConstants.ts @@ -29,6 +29,7 @@ export const TestFrameworkConstants = { KEY_EVENT_STARTED_AT : 'event_started_at', KEY_EVENT_ENDED_AT : 'event_ended_at', KEY_HOOK_ID : 'hook_id', + KEY_HOOK_STATE : 'hook_state', KEY_HOOK_RESULT : 'hook_result', KEY_HOOK_LOGS : 'hook_logs', KEY_HOOK_NAME : 'hook_name', diff --git a/packages/browserstack-service/src/cli/frameworks/wdioCucumberTestFramework.ts b/packages/browserstack-service/src/cli/frameworks/wdioCucumberTestFramework.ts new file mode 100644 index 00000000..2b225160 --- /dev/null +++ b/packages/browserstack-service/src/cli/frameworks/wdioCucumberTestFramework.ts @@ -0,0 +1,651 @@ +import { v4 as uuidv4 } from 'uuid' +import path from 'node:path' + +import TestFramework from './testFramework.js' +import { TestFrameworkState } from '../states/testFrameworkState.js' +import { HookState } from '../states/hookState.js' +import TestFrameworkInstance from '../instances/testFrameworkInstance.js' +import TrackedInstance from '../instances/trackedInstance.js' +import { CLIUtils } from '../cliUtils.js' +import { TestFrameworkConstants } from './constants/testFrameworkConstants.js' +import { BStackLogger as logger } from '../cliLogger.js' +import { TEST_ANALYTICS_ID } from '../../constants.js' +import { getScenarioExamples, removeAnsiColors } from '../../util.js' + +import type { Frameworks } from '@wdio/types' +import type { CucumberHook, Feature, FeatureChild, ITestCaseHookParameter, Pickle, Scenario, Step } from '../../cucumber-types.js' + +/** + * Wire keys the binary's WebdriverIO-cucumber module consumes that have no entry in + * TestFrameworkConstants. Kept local rather than appended to that file, which mocha shares. + */ +const KEY_TEST_DURATION = 'test_duration' +const KEY_BDD_META_INFO = 'bdd_meta_info' +const KEY_TEST_SKIPPED_CASCADE = 'test_skipped_cascade' +const KEY_HOOK_SCOPE = 'hook_scope' +const KEY_HOOK_RETRIES = 'hook_retries' +const KEY_HOOK_DURATION = 'hook_duration' + +type CucumberHookType = 'BEFORE_ALL' | 'AFTER_ALL' | 'BEFORE_EACH' | 'AFTER_EACH' + +const HOOK_STATES: Record = { + BEFORE_ALL: TestFrameworkState.BEFORE_ALL, + AFTER_ALL: TestFrameworkState.AFTER_ALL, + BEFORE_EACH: TestFrameworkState.BEFORE_EACH, + AFTER_EACH: TestFrameworkState.AFTER_EACH, +} + +interface StepMeta { + id: string + text: string + keyword: string + started_at?: string + finished_at?: string + result?: string + duration?: unknown + failure?: string +} + +/** + * CLI test framework for `framework: 'cucumber'` under WebdriverIO. + * + * Extends the BASE TestFramework, never WdioMochaTestFramework: WDIO does not call + * `beforeTest`/`afterTest` or deliver titled hooks for cucumber, so mocha's INIT_TEST / TEST / + * hook-boundary semantics have no source here. + * + * A scenario raises TEST/PRE at `beforeScenario` and TEST/POST at `afterScenario`. That is not a + * preference: the binary's WDIO language index dispatches only on `TEST` and `^(BEFORE_|AFTER_)`, + * so an additional state would be silently discarded on the far side. + */ +export default class WdioCucumberTestFramework extends TestFramework { + static KEY_HOOK_LAST_STARTED = 'test_hook_last_started' + static KEY_HOOK_LAST_FINISHED = 'test_hook_last_finished' + + /** + * The bookkeeping hook classification is derived from. A cucumber hook invocation carries no + * title and `BeforeAll`/`AfterAll` pass no hook object at all, so `util.ts → getHookType()` + * (Mocha quoted-title matching) can only return 'unknown' or throw on the property access. + * + * `stepDepth` is deliberately NOT reset per scenario. Legacy pushes to and pops from + * `_cucumberData.steps` with no per-scenario reset, so a step that never reaches `afterStep` + * leaves the depth above zero for the rest of the run and every later AFTER_EACH is + * classified null and dropped. That is v8 behaviour and therefore the parity target + * (pre-existing-bugs PB-V8-1); the per-scenario reset exists only on the v9 line. + */ + private cucumberData: { + feature?: Feature + uri?: string + scenario?: Pickle + scenariosStarted: boolean + stepsStarted: boolean + stepDepth: number + } = { scenariosStarted: false, stepsStarted: false, stepDepth: 0 } + + /** + * Steps of the scenario in flight, mirroring legacy's per-scenario `_tests[uniqueId].steps`. + * Re-allocated (never cleared in place) at each scenario start, so a payload already built + * from the previous scenario cannot observe this one's steps through a retained reference. + */ + private scenarioSteps: StepMeta[] = [] + + /** The hook started and not yet finished on this worker — legacy's `_currentHook`. */ + private openHook: { key: string, hookId: string } | null = null + + /** + * Set when a hook finish arrived with no recorded start. Legacy throws at that point and the + * wrapper swallows it, which drops the HookRunFinished *and* the BEFORE_ALL cascade in the + * same call; this flag reproduces the second half. + */ + private lastHookFinishOrphaned = false + + constructor(testFrameworks: string[], testFrameworkVersions: Record, binSessionId: string) { + super(testFrameworks, testFrameworkVersions, binSessionId) + logger.debug('WdioCucumberTestFramework: constructed') + } + + /** + * Feature bookkeeping. Raises no state — cucumber has no feature-level wire event, and + * `beforeSuite`/`afterSuite` are not part of its WDIO surface. + */ + onFeatureStart(uri: string, feature: Feature) { + logger.debug(`WdioCucumberTestFramework.onFeatureStart: uri=${uri} feature=${feature?.name}`) + this.cucumberData.scenariosStarted = false + this.cucumberData.feature = feature + this.cucumberData.uri = uri + } + + /** Step bookkeeping. Steps travel inside the scenario payload's BDD meta, never as events. */ + onStepStart(step: Frameworks.PickleStep) { + this.cucumberData.stepsStarted = true + this.cucumberData.stepDepth++ + this.scenarioSteps.push({ + id: step.id, + text: step.text, + keyword: step.keyword, + started_at: (new Date()).toISOString(), + }) + } + + onStepEnd(step: Frameworks.PickleStep, result: Frameworks.PickleResult) { + // Math.max mirrors Array.pop() on an empty array, which legacy relies on. + this.cucumberData.stepDepth = Math.max(0, this.cucumberData.stepDepth - 1) + const stepMeta = this.scenarioSteps.find(item => item.id === step.id) + if (!stepMeta) { + return + } + stepMeta.finished_at = (new Date()).toISOString() + stepMeta.result = result.passed ? 'PASSED' : 'FAILED' + stepMeta.duration = result.duration + if (result.error) { + stepMeta.failure = removeAnsiColors(result.error) + } + } + + /** + * Classify a cucumber hook invocation from the bookkeeping state — legacy's + * `getCucumberHookType()` algorithm. + * + * Returns null for a step-scoped hook (`BeforeStep`/`AfterStep`), which is never reported; + * reporting one would change the dashboard hook count. + */ + classifyHookType(test: CucumberHook | undefined): CucumberHookType | null { + if (!test) { + return this.cucumberData.scenariosStarted ? 'AFTER_ALL' : 'BEFORE_ALL' + } + if (!this.cucumberData.stepsStarted) { + return 'BEFORE_EACH' + } + if (this.cucumberData.stepDepth > 0) { + return null + } + return 'AFTER_EACH' + } + + /** The TestFrameworkState a cucumber hook maps to, or null when it is never reported. */ + classifyHookState(test: CucumberHook | undefined): State | null { + const hookType = this.classifyHookType(test) + return hookType ? HOOK_STATES[hookType] : null + } + + /** + * Whether a failed BEFORE_ALL should run the skip cascade. False when the finish that just + * arrived had no recorded start — see `lastHookFinishOrphaned`. + */ + shouldCascadeSkippedScenarios(): boolean { + return !this.lastHookFinishOrphaned + } + + /** ` for `; the separator is a literal ' for '. */ + private hookName(hookType: CucumberHookType): string { + switch (hookType) { + case 'BEFORE_EACH': + case 'AFTER_EACH': + return `${hookType} for ${this.cucumberData.scenario?.name}` + case 'BEFORE_ALL': + case 'AFTER_ALL': + return `${hookType} for ${this.cucumberData.feature?.name}` + } + } + + /** + * The ABSOLUTE feature path. `beforeFeature`'s uri is already absolute; path.resolve only + * normalises it. The binary re-bases this itself — `path.relative(session.pathProject, …)` + * for file_name/location and against the git root for vc_filepath — so sending a + * pre-relativised value makes both fields depend on the binary's own cwd (SDK-7233). + */ + private featurePath(): string | undefined { + const uri = this.cucumberData.uri + return uri ? path.resolve(process.cwd(), uri) : undefined + } + + private featureFilePathEntries() { + const featurePath = this.featurePath() + return { + [TestFrameworkConstants.KEY_TEST_FILE_PATH]: featurePath, + [TestFrameworkConstants.KEY_TEST_LOCATION]: featurePath, + } + } + + async trackEvent(testFrameworkState: State, hookState: State, args: Record = {}) { + logger.debug(`WdioCucumberTestFramework.trackEvent: testFrameworkState=${testFrameworkState} hookState=${hookState}`) + await super.trackEvent(testFrameworkState, hookState, args) + + const instance = this.resolveInstance(testFrameworkState, hookState) + if (!instance) { + // Console output before the first scenario has nothing to attach to; legacy drops it + // just as silently (appendTestItemLog needs a test or hook uuid and has neither). + const detail = `trackEvent: no instance for testFrameworkState=${testFrameworkState} hookState=${hookState}` + if (testFrameworkState === TestFrameworkState.LOG) { + logger.debug(detail) + } else { + logger.error(detail) + } + return + } + + const shortState = testFrameworkState.toString().split('.')[1] + const isHook = CLIUtils.matchHookRegex(shortState) + + try { + if (isHook && hookState === HookState.PRE) { + instance.updateMultipleEntries({ + [TestFrameworkConstants.KEY_HOOK_ID]: uuidv4(), + }) + } + + if (testFrameworkState === TestFrameworkState.TEST) { + if (hookState === HookState.PRE) { + this.loadScenarioData(instance, args.world as ITestCaseHookParameter) + instance.updateMultipleEntries({ + [TestFrameworkConstants.KEY_TEST_STARTED_AT]: new Date().toISOString(), + }) + } else if (hookState === HookState.POST) { + instance.updateMultipleEntries({ + [TestFrameworkConstants.KEY_TEST_ENDED_AT]: new Date().toISOString(), + }) + this.loadScenarioResult(instance, args) + } + } else if (testFrameworkState === TestFrameworkState.LOG) { + this.loadLogEntry(instance, args.logEntry as Record) + } + + if (isHook && !this.trackHookEvents(instance, shortState, hookState, args)) { + // Orphaned hook finish: legacy emits nothing for it at all. + return + } + } catch (error) { + logger.error(`trackEvent: Error in tracking events: ${error} hookState=${hookState} testFrameworkState=${testFrameworkState}`) + } + + args.instance = instance + await this.runHooks(instance, testFrameworkState, hookState, args) + } + + /** + * One instance per scenario. `TestFramework.instances` is keyed on + * `sha256(CLIUtils.getCurrentInstanceName())` = `sha256(':')`, and every + * `cli/modules/*` reads it through the single `TestFramework.getTrackedInstance()` lookup, so + * that key is the only one to register under. + * + * WDIO forks a worker process per spec and registers its own `beforeScenario`/`afterScenario` + * as cucumber Before/After hooks that cucumber awaits sequentially, so the previous scenario + * has always finished before the next one mints its instance — the slot cannot be clobbered + * mid-scenario the way mocha's two instance-creation triggers can. + */ + private resolveInstance(testFrameworkState: State, hookState: State): TestFrameworkInstance | null { + let instance = TestFramework.getTrackedInstance() + const isHook = CLIUtils.matchHookRegex(testFrameworkState.toString().split('.')[1]) + + if (testFrameworkState === TestFrameworkState.TEST && hookState === HookState.PRE) { + this.trackWdioCucumberInstance(testFrameworkState) + } else if (isHook && hookState === HookState.PRE && !instance) { + // BEFORE_ALL runs before any scenario exists. Every other hook reuses the live + // instance: WDIO's beforeScenario is registered before the user's Before hooks and + // its afterScenario runs after their After hooks, so an EACH hook always lands on + // its own scenario's instance and an ALL hook on the last scenario's — which is the + // uuid legacy stamps as test_run_id. + this.trackWdioCucumberInstance(testFrameworkState) + } + + instance = TestFramework.getTrackedInstance() + if (!instance) { + logger.debug(`resolveInstance: no instance for testFrameworkState=${testFrameworkState} hookState=${hookState}`) + return null + } + this.updateInstanceState(instance, testFrameworkState, hookState) + return instance + } + + private trackWdioCucumberInstance(testFrameworkState: State) { + const target = CLIUtils.getCurrentInstanceName() + const trackedContext = TrackedInstance.createContext(target) + + const instance = new TestFrameworkInstance( + trackedContext, + this.getTestFrameworks(), + this.getTestFrameworksVersions(), + testFrameworkState, + HookState.NONE + ) + + const frameworkName = this.getTestFrameworks()[0] + const testUuid = uuidv4() + + instance.updateMultipleEntries({ + [TestFrameworkConstants.KEY_TEST_FRAMEWORK_NAME]: frameworkName, + [TestFrameworkConstants.KEY_TEST_FRAMEWORK_VERSION]: this.getTestFrameworksVersions()[frameworkName], + [TestFrameworkConstants.KEY_TEST_LOGS]: [], + [TestFrameworkConstants.KEY_HOOKS_FINISHED]: new Map(), + [TestFrameworkConstants.KEY_HOOKS_STARTED]: new Map(), + [TestFrameworkConstants.KEY_TEST_UUID]: testUuid, + [TestFrameworkConstants.KEY_TEST_RESULT]: TestFrameworkConstants.DEFAULT_TEST_RESULT, + }) + + // Read by the Web-A11y and App-A11y scan paths to stamp the scan with this test run. + process.env[TEST_ANALYTICS_ID] = testUuid + + TestFramework.setTrackedInstance(trackedContext, instance) + logger.debug(`trackWdioCucumberInstance: contextId=${trackedContext.getId()} target=${target} testUuid=${testUuid}`) + } + + /** Scenario identity. The asymmetries between the fields are deliberate. */ + private loadScenarioData(instance: TestFrameworkInstance, world: ITestCaseHookParameter) { + if (!world?.pickle) { + logger.error('loadScenarioData: no pickle on the world object; scenario identity will be empty') + return + } + const pickle = world.pickle + const feature = world.gherkinDocument?.feature + this.cucumberData.scenario = pickle + this.cucumberData.scenariosStarted = true + this.cucumberData.stepsStarted = false + // stepDepth is NOT reset here — see the cucumberData doc comment (PB-V8-1). + this.scenarioSteps = [] + + const examples = getScenarioExamples(world) + // One space before '(' and ', ' between cells, no trailing space. + const qualifiedName = examples + ? pickle.name + ' (' + examples.join(', ') + ')' + : pickle.name + + instance.updateMultipleEntries({ + // The RAW pickle name, deliberately without the examples qualifier that name/scope + // carry: the binary maps this to `identifier`, and collapsing the two would fold + // every row of a Scenario Outline into one dashboard test. + [TestFrameworkConstants.KEY_TEST_ID]: pickle.name, + [TestFrameworkConstants.KEY_TEST_NAME]: qualifiedName, + [TestFrameworkConstants.KEY_TEST_SCOPE]: qualifiedName, + [TestFrameworkConstants.KEY_TEST_SCOPES]: [feature?.name || ''], + // Step source is never reported for cucumber, unlike the mocha path's test.body. + [TestFrameworkConstants.KEY_TEST_CODE]: null, + // Gherkin tag text INCLUDING the leading '@', source order, no dedupe, no casing + // change. `.map` allocates a new array; the pickle's own collection is never mutated. + [TestFrameworkConstants.KEY_TEST_TAGS]: pickle.tags.map(({ name }: { name: string }) => name), + ...this.featureFilePathEntries(), + [KEY_BDD_META_INFO]: this.buildBddMetaInfo(pickle, feature, examples), + }) + } + + private buildBddMetaInfo(pickle: Pickle, feature: Feature | undefined, examples: string[] | undefined) { + return { + feature: { + name: feature?.name, + path: this.featurePath(), + description: feature?.description, + }, + scenario: { name: pickle.name }, + steps: this.scenarioSteps.map(step => ({ ...step })), + examples: examples ?? [], + } + } + + /** + * Scenario result. + * + * KEY_TEST_RESULT_AT is set even though `testHubModule.onAllTestEvents()` cannot actually + * defer a completion on this line — its `sendTestFrameworkEvent()` call sits outside the + * else-if chain and runs regardless. Setting it keeps `test_deferred` off the wire and the + * "dropping due to lack of results" line out of the log. + */ + private loadScenarioResult(instance: TestFrameworkInstance, args: Record) { + const world = args.world as ITestCaseHookParameter | undefined + const pickle = world?.pickle ?? this.cucumberData.scenario + const feature = world?.gherkinDocument?.feature ?? this.cucumberData.feature + + const updates: Record = { + [TestFrameworkConstants.KEY_TEST_RESULT_AT]: new Date().toISOString(), + } + + if (pickle) { + updates[KEY_BDD_META_INFO] = this.buildBddMetaInfo(pickle, feature, getScenarioExamples(world as ITestCaseHookParameter)) + } + + const result = world?.result + if (result) { + let testResult = result.status.toLowerCase() + if (testResult !== 'passed' && testResult !== 'failed') { + // UNKNOWN / UNDEFINED / AMBIGUOUS / PENDING / SKIPPED all collapse to skipped. + testResult = 'skipped' + } + + // A scenario that failed only in a hook reports passed to Observability when the user + // declared ignoreHooksStatus. `hasStepFailures` is read from the same step store + // service.afterScenario() consults, so the o11y result and the session status cannot + // disagree; absent that store, legacy treats the failure as a step failure. + const hasStepFailures = args.hasStepFailures === undefined ? true : args.hasStepFailures === true + if (args.ignoreHooksStatus === true && testResult === 'failed' && !hasStepFailures) { + testResult = 'passed' + } + + updates[TestFrameworkConstants.KEY_TEST_RESULT] = testResult + // Cucumber's own protobuf Duration, never an ended_at - started_at delta. + updates[KEY_TEST_DURATION] = result.duration + ? result.duration.seconds * 1000 + result.duration.nanos / 1000000 + : undefined + + if (testResult === 'failed') { + const message = result.message + // A ONE-element backtrace; the mocha path sends message + stack. + updates[TestFrameworkConstants.KEY_TEST_FAILURE] = [ + { backtrace: [message ? removeAnsiColors(message) : 'unknown'] } + ] + updates[TestFrameworkConstants.KEY_TEST_FAILURE_REASON] = message ? removeAnsiColors(message) : message + if (message) { + updates[TestFrameworkConstants.KEY_TEST_FAILURE_TYPE] = message.match(/AssertionError/) + ? 'AssertionError' + : 'UnhandledError' + } + } + } + + instance.updateMultipleEntries(updates) + this.cucumberData.scenario = undefined + } + + /** + * One detached instance per scenario the feature never reached, for the BEFORE_ALL cascade + * (Rule-nested scenarios included). + * + * Detached is load-bearing: these are not registered via `setTrackedInstance`, so neither the + * live instance nor `process.env[TEST_ANALYTICS_ID]` is disturbed, and the caller sends them + * straight to TestHub. Routing them through the observers would rename the Automate session, + * stop the accessibility scan and run a Percy teardown per row, none of which legacy does. + */ + buildSkippedScenarioInstances(): TestFrameworkInstance[] { + const feature = this.cucumberData.feature + if (!feature) { + logger.debug('buildSkippedScenarioInstances: no feature recorded; nothing to cascade') + return [] + } + + const scenarios: Scenario[] = [] + for (const child of (feature.children || []) as FeatureChild[]) { + if (child.rule) { + for (const ruleChild of (child.rule.children || [])) { + if (ruleChild.scenario) { + scenarios.push(ruleChild.scenario) + } + } + } else if (child.scenario) { + scenarios.push(child.scenario) + } + } + + return scenarios.map(scenario => this.buildSkippedScenarioInstance(scenario, feature)) + } + + private buildSkippedScenarioInstance(scenario: Scenario, feature: Feature): TestFrameworkInstance { + const now = new Date().toISOString() + const instance = new TestFrameworkInstance( + TrackedInstance.createContext(CLIUtils.getCurrentInstanceName()), + this.getTestFrameworks(), + this.getTestFrameworksVersions(), + TestFrameworkState.TEST, + HookState.POST + ) + + const frameworkName = this.getTestFrameworks()[0] + instance.updateMultipleEntries({ + [TestFrameworkConstants.KEY_TEST_FRAMEWORK_NAME]: frameworkName, + [TestFrameworkConstants.KEY_TEST_FRAMEWORK_VERSION]: this.getTestFrameworksVersions()[frameworkName], + [TestFrameworkConstants.KEY_TEST_LOGS]: [], + [TestFrameworkConstants.KEY_HOOKS_STARTED]: new Map(), + [TestFrameworkConstants.KEY_HOOKS_FINISHED]: new Map(), + [TestFrameworkConstants.KEY_TEST_UUID]: uuidv4(), + // The feature never ran, so no Examples row was selected and there is nothing to + // qualify: the raw scenario name is name, scope and identifier alike. No tags either + // — legacy's cascade payload has no world to read them from. + [TestFrameworkConstants.KEY_TEST_ID]: scenario.name, + [TestFrameworkConstants.KEY_TEST_NAME]: scenario.name, + [TestFrameworkConstants.KEY_TEST_SCOPE]: scenario.name, + [TestFrameworkConstants.KEY_TEST_SCOPES]: [feature.name || ''], + [TestFrameworkConstants.KEY_TEST_CODE]: null, + [TestFrameworkConstants.KEY_TEST_RESULT]: 'skipped', + [TestFrameworkConstants.KEY_TEST_STARTED_AT]: now, + [TestFrameworkConstants.KEY_TEST_ENDED_AT]: now, + [TestFrameworkConstants.KEY_TEST_RESULT_AT]: now, + ...this.featureFilePathEntries(), + [KEY_TEST_SKIPPED_CASCADE]: true, + [KEY_BDD_META_INFO]: { + feature: { name: feature.name, path: this.featurePath(), description: feature.description }, + scenario: { name: scenario.name }, + steps: (scenario.steps || []).map((step: Step) => ({ + id: step.id, + text: step.text, + keyword: step.keyword, + result: 'skipped', + })), + examples: [], + }, + }) + + return instance + } + + /** + * Whether the scenario in flight failed in a STEP rather than only in a hook. Kept for + * callers that have no access to the legacy step store. + */ + hasStepFailures(): boolean { + return this.scenarioSteps.some(step => step.result === 'FAILED') + } + + /** + * Route a console log to the row it belongs on: the open hook's uuid while a hook is in + * flight and unfinished, otherwise the scenario's. + * + * The record always goes into KEY_TEST_LOGS, because the send path + * (`testHubModule.onAllTestEvents`) collects the test logs plus the last FINISHED hook's — a + * record parked on a still-open hook's own array would never be picked up. Routing rides on + * KEY_HOOK_ID, which `sendLogCreatedEvent` reads in preference to the test uuid. + * + * Not delegated to `WdioMochaTestFramework.loadLogEntries()`: its hook check is + * `matchHookRegex(instance.getCurrentTestState().toString())`, which tests the fully-qualified + * `TestFrameworkState.BEFORE_ALL` against an anchored `^(BEFORE_|AFTER_)` and never matches. + */ + private loadLogEntry(instance: TestFrameworkInstance, logEntry: Record) { + if (!logEntry) { + return + } + const { level, message, timestamp, kind } = logEntry + const logRecord: Record = { + // SDK-6277 forwards a saveScreenshot() result through this same state with + // kind: 'TEST_SCREENSHOT'; hardcoding KIND_LOG would reclassify it as a log line. + kind: (kind as string) ?? TestFrameworkConstants.KIND_LOG, + message: Buffer.from(message as string), + level, + timestamp, + } + + if (this.openHook) { + logRecord[TestFrameworkConstants.KEY_HOOK_ID] = this.openHook.hookId + // The uuid alone is not enough: the binary reads the log's own testFrameworkState to + // decide hook_run_uuid vs test_run_uuid, and at flush time that is 'LOG'. Carrying + // the hook's state on the record is what makes the pairing survive (row 24). + logRecord[TestFrameworkConstants.KEY_HOOK_STATE] = this.openHook.key + } + + const entries = TestFramework.getState(instance, TestFrameworkConstants.KEY_TEST_LOGS) as unknown[] + entries.push(logRecord) + instance.updateMultipleEntries({ + [TestFrameworkConstants.KEY_TEST_LOGS]: entries, + }) + } + + /** + * Hook lifecycle. Entries are keyed by the short state name, matching how the binary looks + * them up via `event.test_hooks_started[request.testFrameworkState]`. + * + * @returns false when a finish arrived with no recorded start, so the caller emits nothing. + */ + private trackHookEvents(instance: TestFrameworkInstance, key: string, hookState: State, args: Record): boolean { + const hooksStarted = TestFramework.getState(instance, TestFrameworkConstants.KEY_HOOKS_STARTED) as Map + if (!hooksStarted.has(key)) { + hooksStarted.set(key, []) + } + const hooksFinished = TestFramework.getState(instance, TestFrameworkConstants.KEY_HOOKS_FINISHED) as Map + if (!hooksFinished.has(key)) { + hooksFinished.set(key, []) + } + + const updates: Record = { + [TestFrameworkConstants.KEY_HOOKS_STARTED]: hooksStarted, + [TestFrameworkConstants.KEY_HOOKS_FINISHED]: hooksFinished, + } + + if (hookState === HookState.PRE) { + const hookId = (TestFramework.getState(instance, TestFrameworkConstants.KEY_HOOK_ID) || '') as string + const hook: Record = { + key, + [TestFrameworkConstants.KEY_HOOK_ID]: hookId, + [TestFrameworkConstants.KEY_HOOK_RESULT]: TestFrameworkConstants.DEFAULT_HOOK_RESULT, + [TestFrameworkConstants.KEY_EVENT_STARTED_AT]: new Date().toISOString(), + [TestFrameworkConstants.KEY_HOOK_LOGS]: [], + [TestFrameworkConstants.KEY_HOOK_NAME]: this.hookName(key as CucumberHookType), + // A hook's scope is the FEATURE name for all four types. `event.test_scope` is the + // examples-qualified scenario name, and an ALL hook has no scenario data at all, + // so the binary reads this key off the hook record first. + [KEY_HOOK_SCOPE]: this.cucumberData.feature?.name, + ...this.featureFilePathEntries(), + } + hooksStarted.get(key)?.push(hook) + updates[WdioCucumberTestFramework.KEY_HOOK_LAST_STARTED] = key + this.openHook = { key, hookId } + this.lastHookFinishOrphaned = false + instance.updateMultipleEntries(updates) + logger.debug(`trackHookEvents: hook started hookState=${key} name=${hook[TestFrameworkConstants.KEY_HOOK_NAME]}`) + return true + } + + const hooksList = hooksStarted.get(key) || [] + if (hooksList.length === 0) { + logger.warn(`trackHookEvents: hook finish for hookState=${key} has no recorded start — dropping the finish and any cascade`) + this.openHook = null + this.lastHookFinishOrphaned = true + instance.updateMultipleEntries(updates) + return false + } + + const hook = hooksList.pop() as Record + const hookResult = args.result as Frameworks.TestResult | undefined + if (hookResult) { + // passed / failed only — the hook path has no 'skipped' arm, unlike the scenario path. + // Leaving hook_result at 'pending' makes the binary coerce a finished hook to + // 'passed', so a failing before-hook would show green while the runner exits 1. + hook[TestFrameworkConstants.KEY_HOOK_RESULT] = hookResult.passed ? 'passed' : 'failed' + // WDIO reports hook duration as plain ms on the result, not as cucumber's Duration. + hook[KEY_HOOK_RETRIES] = hookResult.retries + hook[KEY_HOOK_DURATION] = hookResult.duration + } else { + logger.warn(`trackHookEvents: no result on the hook finish for '${key}'; result stays pending`) + } + hook[TestFrameworkConstants.KEY_EVENT_ENDED_AT] = new Date().toISOString() + hooksFinished.get(key)?.push(hook) + updates[WdioCucumberTestFramework.KEY_HOOK_LAST_FINISHED] = key + this.openHook = null + this.lastHookFinishOrphaned = false + instance.updateMultipleEntries(updates) + logger.debug(`trackHookEvents: hook finished hookState=${key} result=${hook[TestFrameworkConstants.KEY_HOOK_RESULT]}`) + return true + } +} diff --git a/packages/browserstack-service/src/cli/index.ts b/packages/browserstack-service/src/cli/index.ts index 6b8b447a..101e8678 100644 --- a/packages/browserstack-service/src/cli/index.ts +++ b/packages/browserstack-service/src/cli/index.ts @@ -16,6 +16,8 @@ import { BROWSERSTACK_ACCESSIBILITY, BROWSERSTACK_OBSERVABILITY, BROWSERSTACK_TE import type { Options } from '@wdio/types' import TestOpsConfig from '../testOps/testOpsConfig.js' import WdioMochaTestFramework from './frameworks/wdioMochaTestFramework.js' +import WdioCucumberTestFramework from './frameworks/wdioCucumberTestFramework.js' +import type TestFramework from './frameworks/testFramework.js' import WdioAutomationFramework from './frameworks/wdioAutomationFramework.js' import WebdriverIOModule from './modules/webdriverIOModule.js' import AccessibilityModule from './modules/accessibilityModule.js' @@ -44,7 +46,7 @@ export class BrowserstackCLI { isChildConnected = false binSessionId: string | null = null modules: Record = {} - testFramework: WdioMochaTestFramework|null = null + testFramework: TestFramework|null = null cliParams: Record | null = null automationFramework: WdioAutomationFramework|null = null SDK_CLI_BIN_PATH: string | null = null @@ -134,7 +136,7 @@ export class BrowserstackCLI { this.setupAutomationFramework() this.modules[WebdriverIOModule.MODULE_NAME] = new WebdriverIOModule() - this.modules[AutomateModule.MODULE_NAME] = new AutomateModule(this.browserstackConfig as Options.Testrunner) + this.modules[AutomateModule.MODULE_NAME] = new AutomateModule(this.browserstackConfig as Options.Testrunner, this.options as Record) if (startBinResponse.testhub) { this.logBuildStartErrors(startBinResponse.testhub) @@ -488,8 +490,15 @@ export class BrowserstackCLI { */ setupTestFramework() { const testFrameworkDetail = CLIUtils.getTestFrameworkDetail() - if (testFrameworkDetail.name.toLowerCase() === 'webdriverio-mocha') { + switch (testFrameworkDetail.name.toLowerCase()) { + case 'webdriverio-mocha': this.testFramework = new WdioMochaTestFramework([testFrameworkDetail.name], testFrameworkDetail.version, this.binSessionId as string) + break + case 'webdriverio-cucumber': + this.testFramework = new WdioCucumberTestFramework([testFrameworkDetail.name], testFrameworkDetail.version, this.binSessionId as string) + break + default: + this.logger.debug(`setupTestFramework: no CLI test framework for name=${testFrameworkDetail.name}`) } } diff --git a/packages/browserstack-service/src/cli/modules/accessibilityModule.ts b/packages/browserstack-service/src/cli/modules/accessibilityModule.ts index e568f14d..833909dd 100644 --- a/packages/browserstack-service/src/cli/modules/accessibilityModule.ts +++ b/packages/browserstack-service/src/cli/modules/accessibilityModule.ts @@ -203,7 +203,11 @@ export default class AccessibilityModule extends BaseModule { const sessionId = AutomationFramework.getState(autoInstance, AutomationFrameworkConstants.KEY_FRAMEWORK_SESSION_ID) const accessibilityOptions = this.config.accessibilityOptions - const shouldScanTest = this.autoScanning && shouldScanTestForAccessibility(suiteTitle, test.title, accessibilityOptions as { [key: string]: any } | undefined) && this.accessibility + // `world` is set only by service.beforeScenario, so this stays the 3-arg substring + // form for mocha. Without it a cucumber scenario has no test.title and the tag-aware + // branch is never entered, so include/excludeTagsInTestingScope are ignored (row 48). + const world = args.world + const shouldScanTest = this.autoScanning && shouldScanTestForAccessibility(suiteTitle, test.title, accessibilityOptions as { [key: string]: any } | undefined, world, Boolean(world)) && this.accessibility this.accessibilityMap.set(sessionId, shouldScanTest) // Create test metadata similar to accessibility-handler diff --git a/packages/browserstack-service/src/cli/modules/automateModule.ts b/packages/browserstack-service/src/cli/modules/automateModule.ts index b5d7b48f..7a3edb1e 100644 --- a/packages/browserstack-service/src/cli/modules/automateModule.ts +++ b/packages/browserstack-service/src/cli/modules/automateModule.ts @@ -24,25 +24,36 @@ interface TestResult { interface SessionData { lastTestName: string testResults: Map // testName -> TestResult + scenariosRan: number // non-skipped cucumber scenarios, for preferScenarioName + lastScenarioName?: string + preferScenarioName?: boolean } export default class AutomateModule extends BaseModule { logger = BStackLogger browserStackConfig: Options.Testrunner + // The live, in-process service options. Injected rather than imported: cli/index.ts constructs + // this module, so importing it back would close an ESM cycle. + private serviceOptions: Record private sessionMap: Map = new Map() static readonly MODULE_NAME = 'AutomateModule' /** * Create a new AutomateModule */ - constructor(browserStackConfig: Options.Testrunner) { + constructor(browserStackConfig: Options.Testrunner, serviceOptions: Record = {}) { super() this.browserStackConfig = browserStackConfig + this.serviceOptions = serviceOptions this.logger.info('AutomateModule: Initializing Automate Module') TestFramework.registerObserver(TestFrameworkState.TEST, HookState.PRE, this.onBeforeTest.bind(this)) TestFramework.registerObserver(TestFrameworkState.TEST, HookState.POST, this.onAfterTest.bind(this)) TestFramework.registerObserver(AutomationFrameworkState.EXECUTE, HookState.POST, this.onAfterExecute.bind(this)) + // Build-level hooks carry no scenario result, so they reach the session verdict only + // through their own state. See onBuildLevelHookEnd — cucumber-gated inside the handler. + TestFramework.registerObserver(TestFrameworkState.BEFORE_ALL, HookState.POST, this.onBuildLevelHookEnd.bind(this, 'BEFORE_ALL')) + TestFramework.registerObserver(TestFrameworkState.AFTER_ALL, HookState.POST, this.onBuildLevelHookEnd.bind(this, 'AFTER_ALL')) } getModuleName(): string { @@ -60,15 +71,32 @@ export default class AutomateModule extends BaseModule { const suiteTitle = args.suiteTitle as string const testContextOptions = this.config.testContextOptions as TestContextOptions - if (testContextOptions.skipSessionName || !isBrowserstackSession(browser)) { + if (!isBrowserstackSession(browser)) { + return + } + + // `setSessionName: false` suppresses the NAME, not the registration. The session still has + // to enter sessionMap or onAfterExecute has nothing to status-mark, and legacy marks it + // either way — its after() status block gates on setSessionStatus alone. Registering with + // an empty lastTestName is safe because onAfterExecute's naming call is guarded on both + // the flag and a non-empty name, so no name can be sent from here. + if (testContextOptions.skipSessionName) { this.logger.info('Skipping session name update as per configuration') + if (sessionId && !this.sessionMap.has(sessionId)) { + this.sessionMap.set(sessionId, { lastTestName: '', testResults: new Map(), scenariosRan: 0 }) + } return } let name = suiteTitle - if (testContextOptions.sessionNameFormat) { + // Resolved from the live in-process options, NOT from testContextOptions: that config is + // round-tripped through the binary as JSON, which silently drops function-valued keys, so + // testContextOptions.sessionNameFormat is always absent. Reading it here keeps every naming + // decision inside this module instead of splitting it across the legacy path. + const sessionNameFormat = this.serviceOptions?.sessionNameFormat + if (sessionNameFormat) { const caps = AutomationFramework.getState(autoInstance, AutomationFrameworkConstants.KEY_CAPABILITIES) - name = testContextOptions.sessionNameFormat( + name = sessionNameFormat( this.browserStackConfig, caps, suiteTitle, @@ -85,7 +113,8 @@ export default class AutomateModule extends BaseModule { if (!existingSession) { this.sessionMap.set(sessionId, { lastTestName: name, - testResults: new Map() + testResults: new Map(), + scenariosRan: 0 }) } else { existingSession.lastTestName = name @@ -98,14 +127,20 @@ export default class AutomateModule extends BaseModule { async onAfterTest(args: Record) { this.logger.debug('onAfterTest: inside automate module after test hook!') const instace = args.instance as TestFrameworkInstance - const { error, passed } = args.result as { error: Error | null, passed: boolean } + const { error, passed, skipped } = args.result as { error: Error | null, passed: boolean, skipped?: boolean } const _failReasons: string[] = [] - if (!passed) { + // A skipped cucumber scenario must not fail the session: legacy accumulates _failReasons + // only for _failureStatuses (failed/ambiguous/undefined/unknown), which excludes skipped. + // Cucumber-scoped on purpose — mocha's collapse is its own long-standing behaviour on this + // flow and changing it here would alter a framework already shipping on the CLI. + const treatAsPassed = passed || Boolean(skipped && this.isCucumberInstance(instace)) + + if (!treatAsPassed) { _failReasons.push((error && error.message) || 'Unknown Error') } - const status = passed ? 'passed' : 'failed' + const status = treatAsPassed ? 'passed' : 'failed' const reason = _failReasons.length > 0 ? _failReasons.join('\n') : undefined const autoInstance = AutomationFramework.getTrackedInstance() @@ -116,15 +151,28 @@ export default class AutomateModule extends BaseModule { const suiteTitle = args.suiteTitle as string const testContextOptions = this.config.testContextOptions as TestContextOptions + // Tracked before the skipSessionStatus return on purpose: `setSessionStatus: false` opts + // out of the STATUS, not of the preferScenarioName rename. + if (!skipped && this.isCucumberInstance(instace)) { + const nameData = this.sessionMap.get(sessionId) + if (nameData) { + nameData.scenariosRan++ + nameData.lastScenarioName = testTitle + nameData.preferScenarioName = isTrue(args.preferScenarioName) + } + } + if (testContextOptions.skipSessionStatus || !isBrowserstackSession(browser)) { this.logger.info('Skipping session status update as per configuration') return } let name = suiteTitle - if (testContextOptions.sessionNameFormat) { + // See onBeforeTest: the formatter exists only in-process; the round-tripped config drops it. + const sessionNameFormat = this.serviceOptions?.sessionNameFormat + if (sessionNameFormat) { const caps = AutomationFramework.getState(autoInstance, AutomationFrameworkConstants.KEY_CAPABILITIES) - name = testContextOptions.sessionNameFormat( + name = sessionNameFormat( this.browserStackConfig, caps, suiteTitle, @@ -145,7 +193,13 @@ export default class AutomateModule extends BaseModule { reason: reason } - sessionData.testResults.set(name, testResult) + // `name` is the session NAME, which for cucumber is the Feature title and therefore + // shared by every scenario in the file — keying on it collapses N scenarios into one + // last-write-wins entry, so a feature whose last scenario passes reports a passed + // session however many earlier ones failed. Mocha leaves `fullName` undefined, so its + // key is unchanged. + const resultKey = (test && test.fullName) ? String(test.fullName) : name + sessionData.testResults.set(resultKey, testResult) this.sessionMap.set(sessionId, sessionData) } @@ -153,6 +207,89 @@ export default class AutomateModule extends BaseModule { TestFramework.setState(instace, TestFrameworkConstants.KEY_AUTOMATE_SESSION_REASON, reason) } + /** + * A `BeforeAll` / `AfterAll` failure produces no scenario result, so it can never enter the + * per-test `testResults` map that onAfterExecute aggregates — a run whose BeforeAll blew up + * reports its session as PASSED. Legacy pushed the hook error into `_failReasons` and + * `after()` marked the session failed; that whole accumulation is gated + * `setSessionStatus && !BrowserstackCLI.isRunning()`, so it is dead while the binary is up. + * + * Cucumber-gated deliberately. `wdio_mocha` has the identical latent shape on this flow, but + * legacy mocha behaved the same way, so repairing it here would be an unrequested behaviour + * change to the one framework already working on the CLI flow. + */ + async onBuildLevelHookEnd(hookKey: string, args: Record) { + try { + const instance = (args?.instance as TestFrameworkInstance) || TestFramework.getTrackedInstance() + if (!instance || !this.isCucumberInstance(instance)) { + return + } + + const result = args?.result as { passed?: boolean, error?: Error } | undefined + if (!result || result.passed) { + return + } + + const testContextOptions = this.config.testContextOptions as TestContextOptions + if (testContextOptions?.skipSessionStatus) { + return + } + + const autoInstance = AutomationFramework.getTrackedInstance() + const sessionId = AutomationFramework.getState(autoInstance, AutomationFrameworkConstants.KEY_FRAMEWORK_SESSION_ID) + if (!sessionId) { + this.logger.debug(`onBuildLevelHookEnd: no session id resolved for ${hookKey}; nothing to mark`) + return + } + + const sessionData = this.sessionMap.get(sessionId) + // Keyed on the absence of scenario results, never on the flag alone: legacy's + // `ignoreHooksStatus && this._specsRan` arm needs BOTH, and with no scenario recorded + // it falls through to marking `failed` regardless of the flag. The count is final + // here — a BeforeAll failure aborts the run, and by AfterAll every scenario is in. + const specsRan = (sessionData?.testResults.size ?? 0) > 0 + if (specsRan && isTrue(args?.ignoreHooksStatus)) { + this.logger.debug(`onBuildLevelHookEnd: ${hookKey} failed but ignoreHooksStatus is set; not failing the session`) + return + } + + if (!sessionData) { + // A BeforeAll can fail before any scenario ran, so the session may not be + // registered yet. `lastTestName` stays empty on purpose: onAfterExecute's naming + // call is what consumes it, and an empty name is what beforeFeature's own + // (un-gated) _setSessionName has already applied. + this.sessionMap.set(sessionId, { lastTestName: '', testResults: new Map(), scenariosRan: 0 }) + } + + const name = this.resolveHookName(instance, hookKey) + this.sessionMap.get(sessionId)!.testResults.set(name, { + testName: name, + status: 'failed', + reason: (result.error && result.error.message) || 'Hook failed' + }) + this.logger.info(`onBuildLevelHookEnd: recorded ${hookKey} failure against session ${sessionId}`) + } catch (error) { + this.logger.error(`Exception in automate onBuildLevelHookEnd: ${error}`) + } + } + + private isCucumberInstance(instance: TestFrameworkInstance): boolean { + const frameworkName = String(TestFramework.getState(instance, TestFrameworkConstants.KEY_TEST_FRAMEWORK_NAME) || '') + return frameworkName.toLowerCase().includes('cucumber') + } + + /** The hook's reported name (`BEFORE_ALL for `), so the session reason names the hook. */ + private resolveHookName(instance: TestFrameworkInstance, hookKey: string): string { + try { + const finished = TestFramework.getState(instance, TestFrameworkConstants.KEY_HOOKS_FINISHED) as Map[]> | undefined + const hooks = finished?.get(hookKey) + const hookName = hooks?.length ? hooks[hooks.length - 1][TestFrameworkConstants.KEY_HOOK_NAME] : undefined + return (hookName as string) || hookKey + } catch { + return hookKey + } + } + async onAfterExecute() { this.logger.debug('onAfterExecute: inside automate module after execute hook!') @@ -178,7 +315,18 @@ export default class AutomateModule extends BaseModule { } } - if (!testContextOptions.skipSessionName) { + // An empty name means nothing ever named this session — a BeforeAll that failed + // before any feature loaded, so beforeFeature never ran. Legacy makes no naming + // call at all in that state; PUTting '' would be an API call it never made. + // preferScenarioName: cucumber names the session after the FEATURE, but when + // exactly one non-skipped scenario ran the user can ask for that scenario's name + // instead. Only decidable here — "exactly one" is not knowable while scenarios are + // still arriving. skipSessionName still wins, on the guard below. + if (sessionData.preferScenarioName && sessionData.scenariosRan === 1 && sessionData.lastScenarioName) { + sessionData.lastTestName = sessionData.lastScenarioName + } + + if (!testContextOptions.skipSessionName && sessionData.lastTestName) { await this.markSessionName(sessionId, sessionData.lastTestName, { user: userName, key: accessKey }) } diff --git a/packages/browserstack-service/src/cli/modules/testHubModule.ts b/packages/browserstack-service/src/cli/modules/testHubModule.ts index d9df9059..8228d8de 100644 --- a/packages/browserstack-service/src/cli/modules/testHubModule.ts +++ b/packages/browserstack-service/src/cli/modules/testHubModule.ts @@ -56,7 +56,13 @@ export default class TestHubModule extends BaseModule { return (TestFramework.getState(instance, TestFrameworkConstants.KEY_TEST_UUID) as string | undefined) || instance.getRef() } - onBeforeTest(args: Record) { + /** + * Awaited, not fire-and-forget: sendTestSessionEvent re-throws after logging, and an + * un-awaited rejection escapes as an unhandled rejection that no caller can contain — on + * cucumber it surfaces inside the user's own Before hook via WDIO's hook domain and fails the + * scenario. Awaiting hands the rejection to eventDispatcher's per-observer boundary. + */ + async onBeforeTest(args: Record) { this.logger.debug('onBeforeTest: Called after test hook from cli configured module!!!') const instance = args.instance as TestFrameworkInstance try { @@ -69,7 +75,7 @@ export default class TestHubModule extends BaseModule { const autoInstace = AutomationFramework.getTrackedInstance() as AutomationFrameworkInstance const instances = [autoInstace] args.autoInstance = instances - this.sendTestSessionEvent(args) + await this.sendTestSessionEvent(args) } onAllTestEvents(args: Record) { @@ -264,11 +270,16 @@ export default class TestHubModule extends BaseModule { executionContext } for (const logEntry of logEntries) { + // The uuid below may be a HOOK's, but the state is read at flush time and is + // always LOG — and the binary picks hook_run_uuid vs test_run_uuid off exactly + // this field (`^(BEFORE_|AFTER_)`), so a hook log would arrive labelled as a + // test's. A framework that knows the hook state stamps it on the record. + const entryHookState = logEntry[TestFrameworkConstants.KEY_HOOK_STATE] as string | undefined // eslint-disable-next-line camelcase const logData: LogCreatedEventRequest_LogEntry = { testFrameworkName, testFrameworkVersion, - testFrameworkState, + testFrameworkState: entryHookState || testFrameworkState, uuid: logEntry[TestFrameworkConstants.KEY_HOOK_ID] || TestFramework.getState(instance, TestFrameworkConstants.KEY_TEST_UUID), kind: logEntry.kind as string, message: logEntry.message as Uint8Array, diff --git a/packages/browserstack-service/src/service.ts b/packages/browserstack-service/src/service.ts index c14c85cf..44ce0353 100644 --- a/packages/browserstack-service/src/service.ts +++ b/packages/browserstack-service/src/service.ts @@ -46,6 +46,8 @@ import type AutomationFrameworkInstance from './cli/instances/automationFramewor import util from 'node:util' import APIUtils from './cli/apiUtils.js' import { CLIUtils } from './cli/cliUtils.js' +import WdioCucumberTestFramework from './cli/frameworks/wdioCucumberTestFramework.js' +import type TestHubModule from './cli/modules/testHubModule.js' export default class BrowserstackService implements Services.ServiceInstance { private _sessionBaseUrl = `${APIUtils.BROWSERSTACK_AUTOMATE_API_URL}/automate/sessions` @@ -243,6 +245,19 @@ export default class BrowserstackService implements Services.ServiceInstance { BStackLogger.error(`[Accessibility Test Run] Error in service class before function: ${err}`) } + /** + * The driver, its session id and its capabilities reach the AutomationFramework + * instance ONLY through this state, and every cli/module resolves them from there. + * It used to be raised inside the `shouldProcessEventForTesthub` block, so an + * Automate-only run (every TestHub product off) registered no driver at all: + * automateModule's sessionMap stayed empty and the session was left unmarked, + * where legacy marked it. Session tracking does not depend on TestHub, so the + * raise is gated on the CLI being up and nothing else. + */ + if (BrowserstackCLI.getInstance().isRunning()) { + await BrowserstackCLI.getInstance().getAutomationFramework()!.trackEvent(AutomationFrameworkState.CREATE, HookState.POST, { browser: this._browser, hubUrl: this._config.hostname }) + } + if (shouldProcessEventForTesthub('')) { patchConsoleLogs() @@ -253,7 +268,6 @@ export default class BrowserstackService implements Services.ServiceInstance { this._options ) if (BrowserstackCLI.getInstance().isRunning()) { - await BrowserstackCLI.getInstance().getAutomationFramework()!.trackEvent(AutomationFrameworkState.CREATE, HookState.POST, { browser: this._browser, hubUrl: this._config.hostname }) this._insightsHandler.setGitConfigPath() /** * SDK-6277: register the command/result listeners in the CLI/binary flow too, @@ -385,7 +399,16 @@ export default class BrowserstackService implements Services.ServiceInstance { // HookRunStarted/HookRunFinished never reach the dashboard. if (BrowserstackCLI.getInstance().isRunning()) { const framework = BrowserstackCLI.getInstance().getTestFramework() - if (framework) { + if (framework instanceof WdioCucumberTestFramework) { + // A cucumber hook invocation carries no title and BeforeAll/AfterAll pass no hook + // object at all, so getHookType — Mocha quoted-title matching — throws on the + // property access here. The framework classifies from its own bookkeeping and + // returns null for the step-scoped hooks that are deliberately never reported. + const hookFrameworkState = framework.classifyHookState(test as CucumberHook|undefined) + if (hookFrameworkState) { + await framework.trackEvent(hookFrameworkState, HookState.PRE, { test }) + } + } else if (framework) { const hookFrameworkState = TestFrameworkState[getHookType((test as Frameworks.Test).title) as keyof typeof TestFrameworkState] if (hookFrameworkState) { await framework.trackEvent(hookFrameworkState, HookState.PRE, { test }) @@ -413,6 +436,30 @@ export default class BrowserstackService implements Services.ServiceInstance { // CLI flow: mirror beforeHook — close the hook via the TestFramework tracker (gRPC). if (BrowserstackCLI.getInstance().isRunning()) { const framework = BrowserstackCLI.getInstance().getTestFramework() + if (framework instanceof WdioCucumberTestFramework) { + // Cucumber's taxonomy, not Mocha's titles — see beforeHook. + const hookFrameworkState = framework.classifyHookState(test as CucumberHook|undefined) + if (hookFrameworkState) { + // ignoreHooksStatus travels with the event because automateModule's build-level + // hook observer is the only surviving authority on a hook-only session verdict + // — after()'s own three-way logic is gated off while the binary is up. + await framework.trackEvent(hookFrameworkState, HookState.POST, { + test, + result, + ignoreHooksStatus: this._options.testObservabilityOptions?.ignoreHooksStatus === true, + }) + } + // The cascade fires on BEFORE_ALL only — never BEFORE_EACH or AFTER_EACH — and + // not at all when the finish had no recorded start: legacy throws at that point + // and loses the cascade along with the hook event. reportSuiteSkipped() is not + // reachable from here; it walks a Mocha suite tree and drives a four-event + // sequence whose INIT_TEST and LOG_REPORT states cucumber never emits. + if (hookFrameworkState === TestFrameworkState.BEFORE_ALL && result && !result.passed + && framework.shouldCascadeSkippedScenarios()) { + await this._reportCucumberScenariosSkipped(framework) + } + return + } if (framework) { const hookFrameworkState = TestFrameworkState[getHookType((test as Frameworks.Test).title) as keyof typeof TestFrameworkState] if (hookFrameworkState) { @@ -499,6 +546,9 @@ export default class BrowserstackService implements Services.ServiceInstance { const { preferScenarioName, setSessionName, setSessionStatus } = this._options // For Cucumber: Checks scenarios that ran (i.e. not skipped) on the session // Only 1 Scenario ran and option enabled => Redefine session name to Scenario's name + // CLI flow: automateModule owns this rename — it counts non-skipped scenarios itself + // and applies it at EXECUTE/POST, where "exactly one" first becomes knowable. This + // branch is the legacy path only. if (preferScenarioName && this._scenariosThatRan.length === 1){ this._fullTitle = this._scenariosThatRan.pop() } @@ -619,12 +669,121 @@ export default class BrowserstackService implements Services.ServiceInstance { * For CucumberJS */ + private _cliCucumberFramework(): WdioCucumberTestFramework|null { + if (!BrowserstackCLI.getInstance().isRunning()) { + return null + } + const framework = BrowserstackCLI.getInstance().getTestFramework() + return framework instanceof WdioCucumberTestFramework ? framework : null + } + + /** + * A `Frameworks.Test`-shaped view of a scenario, for the cli/modules that read `args.test`. + * automateModule.onBeforeTest is the first observer on TEST/PRE and dereferences `test.title` + * unguarded, so an absent view takes down every later observer for that key — including the + * one that emits the scenario event. + * + * `title` is left undefined and `fullName` populated deliberately. Legacy calls + * `_setSessionName(feature.name)` with NO test argument, so `sessionNameFormat` receives + * `undefined` as its fourth argument and the mocha arm (`test && !test.fullName`, which + * activates sessionNamePrependTopLevelSuiteTitle / sessionNameOmitTestTitle) is never taken. + * Both properties reproduce that. + */ + private _cucumberTestView(world: ITestCaseHookParameter): Frameworks.Test { + return { + title: undefined, + fullName: world.pickle?.name || this._suiteTitle || 'unknown scenario', + parent: this._suiteTitle ?? '', + file: world.gherkinDocument?.uri, + } as unknown as Frameworks.Test + } + + /** + * A `Frameworks.TestResult`-shaped view of a scenario result, for automateModule's session + * marking — which on the CLI flow is the only session-status authority, since `after()`'s + * accumulation is gated off while the binary is up. + * + * Which statuses fail the SESSION is `_failureStatuses`, not the passed/failed collapse the + * o11y result applies: UNDEFINED / AMBIGUOUS / UNKNOWN fail the session while reporting to + * Observability as `skipped`, and PENDING joins them only under `cucumberOpts.strict`. + */ + private _cucumberTestResult(world: ITestCaseHookParameter): Frameworks.TestResult { + const status = world.result?.status?.toLowerCase() + const ignoreHooksStatus = this._options.testObservabilityOptions?.ignoreHooksStatus === true + const hasStepFailures = this._insightsHandler ? this._insightsHandler.hasTestStepFailures(world) : true + const hookOnlyFailure = ignoreHooksStatus && status === 'failed' && !hasStepFailures + + const passed = status === 'passed' || hookOnlyFailure + const failed = !passed && status !== undefined && this._failureStatuses.includes(status) + + // A status that fails only via `_failureStatuses` carries no `world.result.message`, so + // the reason is synthesised exactly as afterScenario() does for `_failReasons`. + let error: Error | undefined + if (failed) { + error = new Error(world.result?.message || (status === 'pending' + ? `Some steps/hooks are pending for scenario "${world.pickle.name}"` + : 'Unknown Error')) + } else if (world.result?.message) { + error = new Error(world.result.message) + } + + return { + passed, + skipped: !passed && !failed, + error, + duration: 0, + retries: { attempts: 0, limit: 0 }, + } as unknown as Frameworks.TestResult + } + + /** + * Cucumber abandons the whole feature when a `BeforeAll` fails, so every scenario it never + * reached (Rule-nested included) is reported skipped rather than vanishing. + * + * Sent straight to TestHub rather than through `trackEvent()`: legacy called + * `listener.testFinished()` directly, so driving the observers per row would rename the + * Automate session, stop the accessibility scan and run a Percy teardown for each one. Both + * halves are sent because TestHub's v2 pipeline creates the row from the START event. + */ + private async _reportCucumberScenariosSkipped(framework: WdioCucumberTestFramework) { + try { + const testHubModule = BrowserstackCLI.getInstance().modules.TestHubModule as TestHubModule | undefined + if (!testHubModule) { + BStackLogger.debug('BEFORE_ALL cascade: TestHub module not loaded; skipped scenarios will not be reported') + return + } + + const instances = framework.buildSkippedScenarioInstances() + for (const instance of instances) { + instance.setCurrentTestState(TestFrameworkState.TEST) + instance.setCurrentHookState(HookState.PRE) + await testHubModule.sendTestFrameworkEvent({ instance }) + instance.setCurrentHookState(HookState.POST) + await testHubModule.sendTestFrameworkEvent({ instance }) + } + BStackLogger.debug(`BEFORE_ALL cascade: reported ${instances.length} scenario(s) as skipped`) + } catch (error) { + BStackLogger.debug(`Exception reporting the BEFORE_ALL skip cascade: ${util.format(error)}`) + } + } + @PerformanceTester.Measure(PERFORMANCE_SDK_EVENTS.EVENTS.SDK_HOOK, { hookType: 'beforeFeature' }) async beforeFeature(uri: string, feature: Feature) { this._suiteTitle = feature.name + // Ordered ahead of the two awaits deliberately: onFeatureStart is local bookkeeping that + // raises no wire event, but _setSessionName issues a session-update REST call. Sequenced + // behind it, any rejection (a turboscale 401, or any 4xx/5xx) aborts the rest of the hook + // and Observability silently loses the feature path for that feature. + this._cliCucumberFramework()?.onFeatureStart(uri, feature) await this._setSessionName(feature.name) await this._setAnnotation(`Feature: ${feature.name}`) - await this._insightsHandler?.beforeFeature(uri, feature) + // The legacy InsightsHandler -> Listener -> api/v1/batch transport is gated only on + // TESTOPS_BUILD_COMPLETED and BROWSERSTACK_TESTHUB_JWT, both of which the CLI flow itself + // sets. Left unguarded it keeps POSTing scenario events under the binary-issued JWT on top + // of whatever the tracker emits, reporting every scenario twice under two different uuids. + if (!BrowserstackCLI.getInstance().isRunning()) { + await this._insightsHandler?.beforeFeature(uri, feature) + } } /** @@ -634,9 +793,29 @@ export default class BrowserstackService implements Services.ServiceInstance { @PerformanceTester.Measure(PERFORMANCE_SDK_EVENTS.EVENTS.SDK_HOOK, { hookType: 'beforeScenario' }) async beforeScenario (world: ITestCaseHookParameter) { this._currentTest = world + const scenarioName = world.pickle.name || 'unknown scenario' + + // The scenario IS the unit of work, so it raises TEST/PRE — the state every cli/modules + // observer subscribes to. WDIO never calls beforeTest for cucumber, so there is no other + // moment at which the modules could be driven. + const cliFramework = this._cliCucumberFramework() + if (cliFramework) { + // accessibilityModule now owns the per-scenario scan on this flow (it observes the + // TEST/PRE raised just below). Leaving the legacy handler live too would run + // sendTestStopEvent twice per scenario against the same TEST_ANALYTICS_ID, and the + // handler is half-initialised here anyway — service.before() skips its before(), so + // its _sessionId is unset and its browser patching never happened. + await cliFramework.trackEvent(TestFrameworkState.TEST, HookState.PRE, { + world, + test: this._cucumberTestView(world), + suiteTitle: this._suiteTitle, + }) + await this._setAnnotation(`Scenario: ${scenarioName}`) + return + } + await this._accessibilityHandler?.beforeScenario(world) await this._insightsHandler?.beforeScenario(world) - const scenarioName = world.pickle.name || 'unknown scenario' await this._setAnnotation(`Scenario: ${scenarioName}`) } @@ -675,22 +854,44 @@ export default class BrowserstackService implements Services.ServiceInstance { } } + const cliFramework = this._cliCucumberFramework() + if (cliFramework) { + // hasStepFailures is read from the same step store this method's own + // ignoreHooksStatus branch consults, so the o11y result and the session status cannot + // disagree. percyModule.onAfterTest and accessibilityModule.onAfterTest observe the + // TEST/POST raised here, which is why the legacy handlers are not also called. + await cliFramework.trackEvent(TestFrameworkState.TEST, HookState.POST, { + world, + test: this._cucumberTestView(world), + suiteTitle: this._suiteTitle, + result: this._cucumberTestResult(world), + ignoreHooksStatus: this._options.testObservabilityOptions?.ignoreHooksStatus === true, + hasStepFailures: this._insightsHandler ? this._insightsHandler.hasTestStepFailures(world) : true, + preferScenarioName: this._options.preferScenarioName === true, + }) + return + } + await this._accessibilityHandler?.afterScenario(world) await this._insightsHandler?.afterScenario(world) - if (!BrowserstackCLI.getInstance().isRunning()) { - await this._percyHandler?.afterScenario() - } + await this._percyHandler?.afterScenario() } @PerformanceTester.Measure(PERFORMANCE_SDK_EVENTS.EVENTS.SDK_HOOK, { hookType: 'beforeStep' }) async beforeStep (step: Frameworks.PickleStep, scenario: Pickle) { + // NOT CLI-guarded, unlike the scenario hooks: beforeStep/afterStep emit nothing, they only + // build the step list that afterScenario reads back via hasTestStepFailures() to separate + // step failures from hook failures. That read is not CLI-gated and feeds the process exit + // code, so guarding these would make every step failure invisible under ignoreHooksStatus. await this._insightsHandler?.beforeStep(step, scenario) + this._cliCucumberFramework()?.onStepStart(step) await this._setAnnotation(`Step: ${step.keyword}${step.text}`) } @PerformanceTester.Measure(PERFORMANCE_SDK_EVENTS.EVENTS.SDK_HOOK, { hookType: 'afterStep' }) async afterStep (step: Frameworks.PickleStep, scenario: Pickle, result: Frameworks.PickleResult) { await this._insightsHandler?.afterStep(step, scenario, result) + this._cliCucumberFramework()?.onStepEnd(step, result) } @PerformanceTester.Measure(PERFORMANCE_SDK_EVENTS.EVENTS.SDK_HOOK, { hookType: 'onReload' }) diff --git a/packages/browserstack-service/tests/cli/frameworks/wdioCucumberTestFramework.test.ts b/packages/browserstack-service/tests/cli/frameworks/wdioCucumberTestFramework.test.ts new file mode 100644 index 00000000..4cd79278 --- /dev/null +++ b/packages/browserstack-service/tests/cli/frameworks/wdioCucumberTestFramework.test.ts @@ -0,0 +1,266 @@ +import { describe, expect, it, beforeEach, vi } from 'vitest' + +import WdioCucumberTestFramework from '../../../src/cli/frameworks/wdioCucumberTestFramework.js' +import TestFramework from '../../../src/cli/frameworks/testFramework.js' +import { TestFrameworkState } from '../../../src/cli/states/testFrameworkState.js' +import { HookState } from '../../../src/cli/states/hookState.js' +import { TestFrameworkConstants } from '../../../src/cli/frameworks/constants/testFrameworkConstants.js' +import type TestFrameworkInstance from '../../../src/cli/instances/testFrameworkInstance.js' + +vi.mock('../../../src/bstackLogger.js') + +const FEATURE_URI = '/abs/project/features/login.feature' + +const feature = (children: unknown[] = []) => ({ + name: 'Login feature', + description: 'Login feature description', + children, +} as any) + +const pickle = (name: string, tags: string[] = [], astNodeIds: string[] = ['scenario-1']) => ({ + name, + uri: 'features/login.feature', + astNodeIds, + tags: tags.map(t => ({ name: t })), +} as any) + +const world = (p: any, f: any, result?: any) => ({ + pickle: p, + gherkinDocument: { uri: 'features/login.feature', feature: f }, + result, +} as any) + +const step = (id: string, text = 'I do a thing', keyword = 'Given ') => ({ id, text, keyword } as any) + +const newFramework = () => new WdioCucumberTestFramework(['WebdriverIO-cucumber'], { 'WebdriverIO-cucumber': '8.50.0' }, 'bin-session') + +const liveInstance = () => TestFramework.getTrackedInstance() as TestFrameworkInstance +const dataOf = (instance: TestFrameworkInstance) => Object.fromEntries(instance.getAllData()) + +describe('WdioCucumberTestFramework', () => { + let framework: WdioCucumberTestFramework + + beforeEach(() => { + TestFramework.instances.clear() + framework = newFramework() + }) + + describe('hook classification (state machine, never a title)', () => { + it('classifies an absent hook object as BEFORE_ALL before any scenario and AFTER_ALL after one', async () => { + framework.onFeatureStart(FEATURE_URI, feature()) + expect(framework.classifyHookType(undefined)).toBe('BEFORE_ALL') + + await framework.trackEvent(TestFrameworkState.TEST, HookState.PRE, { world: world(pickle('S'), feature()) }) + expect(framework.classifyHookType(undefined)).toBe('AFTER_ALL') + }) + + it('classifies a hook object as BEFORE_EACH before the first step and AFTER_EACH after the last', async () => { + framework.onFeatureStart(FEATURE_URI, feature()) + await framework.trackEvent(TestFrameworkState.TEST, HookState.PRE, { world: world(pickle('S'), feature()) }) + + expect(framework.classifyHookType({ id: 'h', hookId: 'h1' } as any)).toBe('BEFORE_EACH') + + framework.onStepStart(step('s1')) + framework.onStepEnd(step('s1'), { passed: true } as any) + expect(framework.classifyHookType({ id: 'h', hookId: 'h1' } as any)).toBe('AFTER_EACH') + }) + + it('classifies a step-scoped hook as null so it is never reported', async () => { + framework.onFeatureStart(FEATURE_URI, feature()) + await framework.trackEvent(TestFrameworkState.TEST, HookState.PRE, { world: world(pickle('S'), feature()) }) + + framework.onStepStart(step('s1')) + expect(framework.classifyHookType({ id: 'h', hookId: 'h1' } as any)).toBeNull() + expect(framework.classifyHookState({ id: 'h', hookId: 'h1' } as any)).toBeNull() + }) + + // PB-V8-1: legacy never resets _cucumberData.steps per scenario, so one missed afterStep + // silently drops every later AFTER_EACH. Reproducing it is parity; the v9 line resets. + it('does NOT reset the step depth per scenario, so a missed step end poisons later AFTER_EACH', async () => { + framework.onFeatureStart(FEATURE_URI, feature()) + await framework.trackEvent(TestFrameworkState.TEST, HookState.PRE, { world: world(pickle('S1'), feature()) }) + framework.onStepStart(step('s1')) + // no onStepEnd — the step never completed + + await framework.trackEvent(TestFrameworkState.TEST, HookState.PRE, { world: world(pickle('S2'), feature()) }) + framework.onStepStart(step('s2')) + framework.onStepEnd(step('s2'), { passed: true } as any) + + expect(framework.classifyHookType({ id: 'h', hookId: 'h1' } as any)).toBeNull() + }) + }) + + describe('scenario identity', () => { + it('keeps test_id raw while name and scope carry the examples qualifier', async () => { + const outlineFeature = feature([{ + scenario: { + id: 'scenario-1', + examples: [{ tableBody: [{ id: 'row-1', cells: [{ value: 'alpha' }, { value: 'one' }] }] }], + }, + }]) + framework.onFeatureStart(FEATURE_URI, outlineFeature) + await framework.trackEvent(TestFrameworkState.TEST, HookState.PRE, { + world: world(pickle('Add to cart', [], ['scenario-1', 'row-1']), outlineFeature), + }) + + const data = dataOf(liveInstance()) + expect(data[TestFrameworkConstants.KEY_TEST_NAME]).toBe('Add to cart (alpha, one)') + expect(data[TestFrameworkConstants.KEY_TEST_SCOPE]).toBe('Add to cart (alpha, one)') + expect(data[TestFrameworkConstants.KEY_TEST_ID]).toBe('Add to cart') + }) + + it('reports a one-element scopes array, a null test_code and the feature path absolute', async () => { + framework.onFeatureStart(FEATURE_URI, feature()) + await framework.trackEvent(TestFrameworkState.TEST, HookState.PRE, { world: world(pickle('Add to cart'), feature()) }) + + const data = dataOf(liveInstance()) as Record + expect(data[TestFrameworkConstants.KEY_TEST_SCOPES]).toEqual(['Login feature']) + expect(data[TestFrameworkConstants.KEY_TEST_CODE]).toBeNull() + expect(data.bdd_meta_info.feature.path).toBe(FEATURE_URI) + }) + + it('sends tags with the leading @, in source order, duplicates kept, without mutating the pickle', async () => { + const p = pickle('Add to cart', ['@smoke', '@smoke', '@regression']) + framework.onFeatureStart(FEATURE_URI, feature()) + await framework.trackEvent(TestFrameworkState.TEST, HookState.PRE, { world: world(p, feature()) }) + + expect(dataOf(liveInstance())[TestFrameworkConstants.KEY_TEST_TAGS]).toEqual(['@smoke', '@smoke', '@regression']) + expect(p.tags).toEqual([{ name: '@smoke' }, { name: '@smoke' }, { name: '@regression' }]) + }) + }) + + describe('scenario result', () => { + const finish = async (fw: WdioCucumberTestFramework, result: any, args: Record = {}) => { + fw.onFeatureStart(FEATURE_URI, feature()) + await fw.trackEvent(TestFrameworkState.TEST, HookState.PRE, { world: world(pickle('S'), feature()) }) + const instance = liveInstance() + await fw.trackEvent(TestFrameworkState.TEST, HookState.POST, { world: world(pickle('S'), feature(), result), ...args }) + return dataOf(instance) as Record + } + + it('collapses every non passed/failed status to skipped and stamps the result timestamp', async () => { + const data = await finish(framework, { status: 'UNDEFINED', duration: { seconds: 1, nanos: 500000000 } }) + expect(data[TestFrameworkConstants.KEY_TEST_RESULT]).toBe('skipped') + expect(data[TestFrameworkConstants.KEY_TEST_RESULT_AT]).toBeTruthy() + }) + + it("derives test_duration from cucumber's Duration, not from the timestamp delta", async () => { + const data = await finish(framework, { status: 'PASSED', duration: { seconds: 2, nanos: 250000000 } }) + expect(data.test_duration).toBe(2250) + }) + + it('reports a one-element backtrace and the AssertionError failure type', async () => { + const data = await finish(framework, { status: 'FAILED', message: 'AssertionError: nope', duration: { seconds: 0, nanos: 0 } }) + expect(data[TestFrameworkConstants.KEY_TEST_FAILURE]).toEqual([{ backtrace: ['AssertionError: nope'] }]) + expect(data[TestFrameworkConstants.KEY_TEST_FAILURE_TYPE]).toBe('AssertionError') + }) + + it('overrides a hook-only failure to passed only when ignoreHooksStatus is declared', async () => { + const failed = { status: 'FAILED', message: 'boom', duration: { seconds: 0, nanos: 0 } } + + const overridden = await finish(newFramework(), failed, { ignoreHooksStatus: true, hasStepFailures: false }) + expect(overridden[TestFrameworkConstants.KEY_TEST_RESULT]).toBe('passed') + + TestFramework.instances.clear() + const notOverridden = await finish(newFramework(), failed, { ignoreHooksStatus: true, hasStepFailures: true }) + expect(notOverridden[TestFrameworkConstants.KEY_TEST_RESULT]).toBe('failed') + }) + }) + + describe('hook lifecycle', () => { + const openScenario = async () => { + framework.onFeatureStart(FEATURE_URI, feature()) + await framework.trackEvent(TestFrameworkState.TEST, HookState.PRE, { world: world(pickle('Add to cart'), feature()) }) + return liveInstance() + } + + it('names the hook after the scenario for EACH and the feature for ALL, and scopes both to the feature', async () => { + const instance = await openScenario() + await framework.trackEvent(TestFrameworkState.BEFORE_EACH, HookState.PRE, { test: { id: 'h', hookId: 'h1' } }) + + const started = dataOf(instance)[TestFrameworkConstants.KEY_HOOKS_STARTED] as Map + const hook = started.get('BEFORE_EACH')![0] + expect(hook[TestFrameworkConstants.KEY_HOOK_NAME]).toBe('BEFORE_EACH for Add to cart') + expect(hook.hook_scope).toBe('Login feature') + expect(hook.test_file_path).toBe(FEATURE_URI) + expect(hook[TestFrameworkConstants.KEY_TEST_TAGS]).toBeUndefined() + }) + + it('records passed/failed on the finish rather than leaving hook_result pending', async () => { + const instance = await openScenario() + await framework.trackEvent(TestFrameworkState.BEFORE_EACH, HookState.PRE, { test: { id: 'h', hookId: 'h1' } }) + await framework.trackEvent(TestFrameworkState.BEFORE_EACH, HookState.POST, { + test: { id: 'h', hookId: 'h1' }, + result: { passed: false, duration: 12, retries: { attempts: 0, limit: 0 } }, + }) + + const finished = dataOf(instance)[TestFrameworkConstants.KEY_HOOKS_FINISHED] as Map + const hook = finished.get('BEFORE_EACH')![0] + expect(hook[TestFrameworkConstants.KEY_HOOK_RESULT]).toBe('failed') + expect(hook.hook_duration).toBe(12) + expect(hook.hook_retries).toEqual({ attempts: 0, limit: 0 }) + }) + + it('drops an orphaned hook finish and suppresses the cascade that would have followed it', async () => { + const instance = await openScenario() + await framework.trackEvent(TestFrameworkState.BEFORE_ALL, HookState.POST, { + test: undefined, + result: { passed: false }, + }) + + const finished = dataOf(instance)[TestFrameworkConstants.KEY_HOOKS_FINISHED] as Map + expect(finished.get('BEFORE_ALL')).toEqual([]) + expect(framework.shouldCascadeSkippedScenarios()).toBe(false) + }) + }) + + describe('BEFORE_ALL skip cascade', () => { + it('builds one detached instance per scenario, Rule-nested rows included', () => { + const cascadeFeature = feature([ + { scenario: { name: 'Plain one', steps: [step('s1')] } }, + { rule: { children: [{ scenario: { name: 'Rule nested', steps: [step('s2')] } }] } }, + ]) + framework.onFeatureStart(FEATURE_URI, cascadeFeature) + + const instances = framework.buildSkippedScenarioInstances() + expect(instances.map(i => dataOf(i)[TestFrameworkConstants.KEY_TEST_NAME])).toEqual(['Plain one', 'Rule nested']) + + const first = dataOf(instances[0]) as Record + expect(first[TestFrameworkConstants.KEY_TEST_ID]).toBe('Plain one') + expect(first[TestFrameworkConstants.KEY_TEST_SCOPE]).toBe('Plain one') + expect(first[TestFrameworkConstants.KEY_TEST_RESULT]).toBe('skipped') + expect(first.test_skipped_cascade).toBe(true) + expect(first[TestFrameworkConstants.KEY_TEST_TAGS]).toBeUndefined() + expect(first.bdd_meta_info.steps).toEqual([{ id: 's1', text: 'I do a thing', keyword: 'Given ', result: 'skipped' }]) + + // Detached: the live tracked instance is untouched. + expect(TestFramework.getTrackedInstance()).toBeUndefined() + }) + }) + + describe('log routing', () => { + it('stamps the open hook uuid while a hook is unfinished and the test uuid otherwise', async () => { + framework.onFeatureStart(FEATURE_URI, feature()) + await framework.trackEvent(TestFrameworkState.TEST, HookState.PRE, { world: world(pickle('S'), feature()) }) + const instance = liveInstance() + + await framework.trackEvent(TestFrameworkState.BEFORE_EACH, HookState.PRE, { test: { id: 'h', hookId: 'h1' } }) + await framework.trackEvent(TestFrameworkState.LOG, HookState.POST, { + logEntry: { level: 'INFO', message: 'inside the hook', timestamp: 'now' }, + }) + await framework.trackEvent(TestFrameworkState.BEFORE_EACH, HookState.POST, { + test: { id: 'h', hookId: 'h1' }, result: { passed: true }, + }) + await framework.trackEvent(TestFrameworkState.LOG, HookState.POST, { + logEntry: { level: 'INFO', message: 'inside the step', timestamp: 'now', kind: TestFrameworkConstants.KIND_SCREENSHOT }, + }) + + const logs = dataOf(instance)[TestFrameworkConstants.KEY_TEST_LOGS] as Record[] + expect(logs[0][TestFrameworkConstants.KEY_HOOK_ID]).toBeTruthy() + expect(logs[0].kind).toBe(TestFrameworkConstants.KIND_LOG) + expect(logs[1][TestFrameworkConstants.KEY_HOOK_ID]).toBeUndefined() + // SDK-6277: an incoming screenshot kind must survive rather than be flattened to a log. + expect(logs[1].kind).toBe(TestFrameworkConstants.KIND_SCREENSHOT) + }) + }) +}) diff --git a/packages/browserstack-service/tests/cli/modules/automateModule.test.ts b/packages/browserstack-service/tests/cli/modules/automateModule.test.ts index a5f76212..d398c70d 100644 --- a/packages/browserstack-service/tests/cli/modules/automateModule.test.ts +++ b/packages/browserstack-service/tests/cli/modules/automateModule.test.ts @@ -1,13 +1,18 @@ import { describe, expect, it, vi, beforeEach } from 'vitest' import got from 'got' import AutomateModule from '../../../src/cli/modules/automateModule.js' +import TestFramework from '../../../src/cli/frameworks/testFramework.js' +import AutomationFramework from '../../../src/cli/frameworks/automationFramework.js' +import { TestFrameworkConstants } from '../../../src/cli/frameworks/constants/testFrameworkConstants.js' +import { isBrowserstackSession } from '../../../src/util.js' import type { Options } from '@wdio/types' // Mock dependencies vi.mock('../../../src/cli/frameworks/testFramework.js', () => ({ default: { registerObserver: vi.fn(), - setState: vi.fn() + setState: vi.fn(), + getState: vi.fn() } })) @@ -167,6 +172,84 @@ describe('AutomateModule', () => { await expect(moduleWithSkip.onBeforeTest(mockArgs)).resolves.toBeUndefined() }) + it('applies sessionNameFormat from the injected service options', async () => { + // The formatter is a function, so JSON drops it from the config the binary echoes back. + // It reaches the module only through the injected options; testContextOptions below is + // deliberately shaped as the binary really returns it, with no sessionNameFormat key. + const sessionNameFormat = vi.fn((_config, _caps, suiteTitle, testTitle) => `FMT::${suiteTitle}::${testTitle}`) + const moduleWithFormat = new AutomateModule(mockConfig, { sessionNameFormat }) + moduleWithFormat.config = { + testContextOptions: { skipSessionName: false, skipSessionStatus: false } + } as any + + vi.mocked(isBrowserstackSession).mockReturnValue(true) + vi.mocked(AutomationFramework.getState).mockReturnValue('session-1') + + await moduleWithFormat.onBeforeTest({ + instance: {}, + test: { title: 'test title' }, + suiteTitle: 'suite title' + }) + + expect(sessionNameFormat).toHaveBeenCalled() + expect(TestFramework.setState).toHaveBeenCalledWith( + expect.anything(), + TestFrameworkConstants.KEY_AUTOMATE_SESSION_NAME, + 'FMT::suite title::test title' + ) + }) + + it('renames the session to the scenario when preferScenarioName and exactly one scenario ran', async () => { + // The rename is only decidable at EXECUTE/POST — "exactly one" is not knowable while + // scenarios are still arriving — so the module counts them and applies it there. + const mod = new AutomateModule(mockConfig, {}) + mod.config = { + userName: 'testuser', + accessKey: 'testkey', + testContextOptions: { skipSessionName: false, skipSessionStatus: false } + } as any + + vi.mocked(isBrowserstackSession).mockReturnValue(true) + vi.mocked(AutomationFramework.getState).mockReturnValue('session-pref') + vi.mocked(TestFramework.getState).mockReturnValue('cucumber') // isCucumberInstance + + await mod.onBeforeTest({ instance: {}, test: { title: 'ignored' }, suiteTitle: 'Feature title' }) + await mod.onAfterTest({ + instance: {}, + result: { error: null, passed: true }, + test: { title: 'The only scenario' }, + suiteTitle: 'Feature title', + preferScenarioName: true + }) + + const spy = vi.spyOn(mod, 'markSessionName').mockResolvedValue(undefined) + await mod.onAfterExecute() + + expect(spy).toHaveBeenCalledWith('session-pref', 'The only scenario', expect.anything()) + }) + + it('falls back to the suite title when no sessionNameFormat is configured', async () => { + const moduleNoFormat = new AutomateModule(mockConfig, {}) + moduleNoFormat.config = { + testContextOptions: { skipSessionName: false, skipSessionStatus: false } + } as any + + vi.mocked(isBrowserstackSession).mockReturnValue(true) + vi.mocked(AutomationFramework.getState).mockReturnValue('session-2') + + await moduleNoFormat.onBeforeTest({ + instance: {}, + test: { title: 'test title', fullName: 'full name' }, + suiteTitle: 'suite title' + }) + + expect(TestFramework.setState).toHaveBeenCalledWith( + expect.anything(), + TestFrameworkConstants.KEY_AUTOMATE_SESSION_NAME, + 'suite title' + ) + }) + it('should handle onAfterTest with skipSessionStatus enabled', async () => { const configWithSkip = { ...mockConfig,