From 2a40941159f42343ae327a592a79040c8688f4de Mon Sep 17 00:00:00 2001 From: Henri Bergius Date: Sat, 10 Oct 2026 15:43:09 +1300 Subject: [PATCH 1/7] Modernize codebase to ESM with zero-build and native test runner - Convert all sources to ES Modules, with node: prefixes on stdlib imports, an exports map, and engines requiring Node.js 20+ - Replace Bluebird and callback-style APIs with native Promises and async/await - Replace isomorphic-fetch with native fetch, commander with util.parseArgs, and slug with a local slugifier - Reimplement expectation operators on node:assert, dropping Chai - Keep Node-specific imports (fs, child_process) lazy so the library stays bundleable for browsers - Replace grunt-yaml with scripts/build-schemas.js, which also emits a browser-loadable schema/schemas.js module - Replace Mocha/Chai/grunt/karma/eslint test stack with node:test and node:assert; add a JavaScript mock FBP runtime so end-to-end tests run without the Python example runtime - Remove the Mocha runner integrations (fbpspec.mocha.run and fbp-spec-mocha compat runtime), the webpack/babel browser build, and the Karma browser test harness - Add Biome linting/formatting via .editorconfig, and tsc type checking with JSDoc types (tsconfig.json, types.d.ts) --- .editorconfig | 9 + .eslintrc.json | 3 - .gitignore | 8 +- .npmignore | 6 - Gruntfile.js | 155 --- bin/fbp-spec | 4 +- bin/fbp-spec-mocha | 2 - biome.json | 13 + browser/CNAME | 1 - examples/index.js | 48 - index.js | 7 +- karma.config.js | 29 - lib/cli.js | 370 +++++--- lib/common.js | 137 ++- lib/expectation.js | 262 ++++- lib/index.js | 29 +- lib/mocha.js | 110 --- lib/mochacompat.js | 436 --------- lib/protocol.js | 255 ++--- lib/runner.js | 896 ++++++++++-------- lib/subprocess.js | 125 +-- lib/testsuite.js | 280 +++--- lib/websocket.js | 73 -- package.json | 88 +- schema/base.json | 2 +- schema/expectation.json | 2 +- schema/expectations.json | 2 +- schema/index.js | 45 +- schema/outputdata.json | 2 +- schema/schemas.js | 219 +++++ schema/testcase.json | 2 +- schema/testsfile.json | 2 +- schema/testsuite.json | 2 +- schema/testsuites.json | 2 +- scripts/build-schemas.js | 39 + spec/.eslintrc.json | 21 - spec/cli.js | 62 -- spec/cli.test.js | 207 ++++ spec/common.test.js | 108 +++ spec/examples.js | 148 --- spec/examples.test.js | 134 +++ spec/expectation.test.js | 174 ++++ spec/fixtures/.gitignore | 2 - spec/fixtures/dummy-runtime.js | 256 +++++ .../fixtures/mochacases/bdd-nested-passing.js | 19 - .../fixtures/mochacases/bdd-simple-failing.js | 12 - .../fixtures/mochacases/bdd-simple-passing.js | 12 - spec/import.js | 11 - spec/import.test.js | 23 + spec/json-schema.json | 280 +++--- spec/mocha.js | 10 - spec/mochacompat.js | 192 ---- spec/runner.test.js | 164 ++++ spec/schema.test.js | 39 + spec/testsuite.test.js | 159 ++++ spec/validate-schemas.js | 38 - tsconfig.json | 25 + types.d.ts | 146 +++ webpack.config.js | 34 - webpack.entry.js | 18 - 60 files changed, 3349 insertions(+), 2610 deletions(-) create mode 100644 .editorconfig delete mode 100644 .eslintrc.json delete mode 100644 .npmignore delete mode 100644 Gruntfile.js delete mode 100755 bin/fbp-spec-mocha create mode 100644 biome.json delete mode 100644 browser/CNAME delete mode 100644 examples/index.js delete mode 100644 karma.config.js delete mode 100644 lib/mocha.js delete mode 100644 lib/mochacompat.js delete mode 100644 lib/websocket.js create mode 100644 schema/schemas.js create mode 100644 scripts/build-schemas.js delete mode 100644 spec/.eslintrc.json delete mode 100644 spec/cli.js create mode 100644 spec/cli.test.js create mode 100644 spec/common.test.js delete mode 100644 spec/examples.js create mode 100644 spec/examples.test.js create mode 100644 spec/expectation.test.js delete mode 100644 spec/fixtures/.gitignore create mode 100644 spec/fixtures/dummy-runtime.js delete mode 100644 spec/fixtures/mochacases/bdd-nested-passing.js delete mode 100644 spec/fixtures/mochacases/bdd-simple-failing.js delete mode 100644 spec/fixtures/mochacases/bdd-simple-passing.js delete mode 100644 spec/import.js create mode 100644 spec/import.test.js delete mode 100644 spec/mocha.js delete mode 100644 spec/mochacompat.js create mode 100644 spec/runner.test.js create mode 100644 spec/schema.test.js create mode 100644 spec/testsuite.test.js delete mode 100644 spec/validate-schemas.js create mode 100644 tsconfig.json create mode 100644 types.d.ts delete mode 100644 webpack.config.js delete mode 100644 webpack.entry.js diff --git a/.editorconfig b/.editorconfig new file mode 100644 index 0000000..2c0e19c --- /dev/null +++ b/.editorconfig @@ -0,0 +1,9 @@ +root = true + +[*] +end_of_line = lf +insert_final_newline = true +charset = utf-8 +indent_style = space +indent_size = 2 +trim_trailing_whitespace = true diff --git a/.eslintrc.json b/.eslintrc.json deleted file mode 100644 index 7cde01d..0000000 --- a/.eslintrc.json +++ /dev/null @@ -1,3 +0,0 @@ -{ - "extends": "airbnb-base" -} diff --git a/.gitignore b/.gitignore index 25dd414..0356290 100644 --- a/.gitignore +++ b/.gitignore @@ -1,12 +1,8 @@ *~ node_modules/ -package-lock.json *.includes *.config *.autosave build/ -browser/ -components/ -examples/bundle* -spec/result.xml -.grunt/ +.flowtrace/ +package-lock.json diff --git a/.npmignore b/.npmignore deleted file mode 100644 index 01626bf..0000000 --- a/.npmignore +++ /dev/null @@ -1,6 +0,0 @@ -build/ -browser/ -examples/bundle* -spec/result.xml -protocol-examples/ -.grunt/ diff --git a/Gruntfile.js b/Gruntfile.js deleted file mode 100644 index fbb0f31..0000000 --- a/Gruntfile.js +++ /dev/null @@ -1,155 +0,0 @@ -const path = require('path'); -const fetch = require('isomorphic-fetch'); -const webpackConfig = require('./webpack.config.js'); - -module.exports = function () { - // Project configuration - const pkg = this.file.readJSON('package.json'); - - this.initConfig({ - pkg, - - // Schemas - yaml: { - schemas: { - files: [{ - expand: true, - cwd: 'schemata/', - src: '*.yaml', - dest: 'schema/', - }, - ], - }, - }, - - // Building for browser - webpack: { - build: webpackConfig, - }, - - watch: { - src: { - files: [ - 'src/**/*', - 'examples/**/*', - 'spec/**/*', - ], - tasks: 'test', - options: { - livereload: true, - }, - }, - }, - - exec: { - runtime: { - command: 'python2 protocol-examples/python/runtime.py --port 3334', - }, - }, - - // Coding standards - yamllint: { - schemas: ['schemata/*.yaml'], - examples: ['examples/*.yml'], - }, - - // Tests - mochaTest: { - nodejs: { - src: ['spec/*.js'], - options: { - reporter: 'spec', - grep: process.env.TESTS, - }, - }, - }, - - downloadfile: { - files: [ - { url: 'https://noflojs.org/noflo-browser/everything.html', dest: 'browser/spec/fixtures' }, - { url: 'https://noflojs.org/noflo-browser/everything.js', dest: 'browser/spec/fixtures' }, - ], - }, - - // BDD tests on browser - karma: { - unit: { - configFile: 'karma.config.js', - }, - }, - - // Deploying - copy: { - ui: { - files: [{ - expand: true, cwd: './ui/', src: '*', dest: './browser/', - }], - }, - }, - }); - - // Grunt plugins used for building - this.loadNpmTasks('grunt-yaml'); - this.loadNpmTasks('grunt-webpack'); - this.loadNpmTasks('grunt-contrib-watch'); - - // Grunt plugins used for testing - this.loadNpmTasks('grunt-yamllint'); - this.loadNpmTasks('grunt-mocha-test'); - this.loadNpmTasks('grunt-karma'); - this.loadNpmTasks('grunt-exec'); - - this.registerTask('examples:bundle', () => { - // eslint-disable-next-line global-require - const examples = require('./examples'); - return examples.bundle(); - }); - - // Grunt plugins used for deploying - this.loadNpmTasks('grunt-contrib-copy'); - - // Our local tasks - const grunt = this; - this.registerMultiTask('downloadfile', 'Download a file', function () { - const callback = this.async(); - const promises = this.data.map((conf) => fetch(conf.url) - .then((res) => res.text()).then((content) => { - const filename = path.basename(conf.url); - const location = path.join(conf.dest, path.sep, filename); - grunt.file.write(location, content); - console.log(`Wrote ${conf.url} to ${location}`); - return true; - })); - return Promise.all(promises) - .then(() => callback(), - (err) => callback(err)); - }); - - this.registerTask('build', 'Build', (target = 'all') => { - this.task.run('yaml'); - if (['all', 'browser'].includes(target)) { - this.task.run('webpack'); - this.task.run('examples:bundle'); - this.task.run('copy:ui'); - } - }); - - this.registerTask('test', 'Build and run tests', (target = 'all') => { - this.task.run('yamllint'); - this.task.run(`build:${target}`); - if (['all', 'nodejs'].includes(target)) { - this.task.run('mochaTest'); - } - if (['all', 'browser'].includes(target)) { - this.task.run('downloadfile'); - this.task.run('karma'); - } - }); - - this.registerTask('default', ['test']); - - this.registerTask('dev', 'Developing', (target = 'all') => { - this.task.run(`test:${target}`); - this.task.run('watch'); - }); -}; diff --git a/bin/fbp-spec b/bin/fbp-spec index 27002de..0b4af79 100755 --- a/bin/fbp-spec +++ b/bin/fbp-spec @@ -1,2 +1,4 @@ #!/usr/bin/env node -require('../lib/cli').main(); +import { main } from '../lib/cli.js'; + +main(); diff --git a/bin/fbp-spec-mocha b/bin/fbp-spec-mocha deleted file mode 100755 index fee3573..0000000 --- a/bin/fbp-spec-mocha +++ /dev/null @@ -1,2 +0,0 @@ -#!/usr/bin/env node -require('../lib/mochacompat').main(); diff --git a/biome.json b/biome.json new file mode 100644 index 0000000..edd39d7 --- /dev/null +++ b/biome.json @@ -0,0 +1,13 @@ +{ + "$schema": "https://biomejs.dev/schemas/2.5.15/schema.json", + "files": { + "includes": ["**", "!schema/schemas.js", "!schema/*.json", "!package-lock.json"] + }, + "assist": { + "actions": { + "source": { + "organizeImports": "off" + } + } + } +} diff --git a/browser/CNAME b/browser/CNAME deleted file mode 100644 index 185fcc0..0000000 --- a/browser/CNAME +++ /dev/null @@ -1 +0,0 @@ -fbpspec.run diff --git a/examples/index.js b/examples/index.js deleted file mode 100644 index bb5ee74..0000000 --- a/examples/index.js +++ /dev/null @@ -1,48 +0,0 @@ -const yaml = require('js-yaml'); - -const path = require('path'); - -const examplesDir = path.join(__dirname); - -const listExamples = function () { - // eslint-disable-next-line global-require - const files = require('fs').readdirSync(examplesDir); - const result = []; - files.forEach((f) => { - if (f.indexOf('.yaml') !== -1) { - result.push(f); - } - }); - return result; -}; -const getExample = function (name) { - const p = path.join(examplesDir, name); - // eslint-disable-next-line global-require - const content = require('fs').readFileSync(p, { encoding: 'utf-8' }); - const results = []; - yaml.loadAll(content, (doc) => results.push(doc)); - if (results.length === 1) { - return results[0]; - } - return results; -}; - -const buildExampleBundle = function () { - const examples = {}; - listExamples().forEach((name) => { - examples[name] = getExample(name); - }); - const contents = JSON.stringify(examples, null, 2); - - let dest = `${examplesDir}/bundle.js`; - // eslint-disable-next-line global-require - require('fs').writeFileSync(dest, `window.fbpspec_examples = ${contents};`); - console.log(`wrote ${dest}`); - - dest = `${examplesDir}/bundle.json`; - // eslint-disable-next-line global-require - require('fs').writeFileSync(dest, contents); - return console.log(`wrote ${dest}`); -}; - -exports.bundle = buildExampleBundle; diff --git a/index.js b/index.js index b58e228..4ec7fec 100644 --- a/index.js +++ b/index.js @@ -1 +1,6 @@ -module.exports = require('./lib/index'); +/** + * fbp-spec - a runtime-independent test framework for Flow Based Programming. + * @module fbp-spec + */ + +export * from "./lib/index.js"; diff --git a/karma.config.js b/karma.config.js deleted file mode 100644 index 3a3ff74..0000000 --- a/karma.config.js +++ /dev/null @@ -1,29 +0,0 @@ -module.exports = (config) => { - const configuration = { - basePath: process.cwd(), - frameworks: [ - 'mocha', - 'chai', - ], - reporters: [ - 'mocha', - ], - files: [ - 'browser/fbp-spec.js', - 'examples/bundle.js', - 'spec/import.js', - 'spec/examples.js', - { - pattern: 'browser/spec/fixtures/everything.*', - included: false, - served: true, - watched: true, - }, - ], - browsers: ['ChromeHeadless'], - logLevel: config.LOG_WARN, - singleRun: true, - }; - - config.set(configuration); -}; diff --git a/lib/cli.js b/lib/cli.js index 3b03bd2..f8c9d3c 100644 --- a/lib/cli.js +++ b/lib/cli.js @@ -1,182 +1,212 @@ -const program = require('commander'); -const debug = require('debug')('fbp-spec:cli'); -const { stat, mkdir, writeFile } = require('fs'); -const { resolve, relative } = require('path'); -const slug = require('slug'); -const testsuite = require('./testsuite'); -const runner = require('./runner'); -const subprocess = require('./subprocess'); +/** + * Command-line interface for running fbp-spec testsuites against an FBP runtime. + * @module lib/cli + */ +import { mkdir, stat, writeFile } from "node:fs/promises"; +import { relative, resolve } from "node:path"; +import { parseArgs } from "node:util"; +import debugLib from "debug"; + +import { getComponentSuites, Runner, runAll } from "./runner.js"; +import { start } from "./subprocess.js"; +import { getSuites } from "./testsuite.js"; + +const debug = debugLib("fbp-spec:cli"); + +/** @typedef {import("./testsuite.js").Suite} Suite */ + +/** @type {string[]} */ const traces = []; -// # Main -function parse(args) { - const actionHandler = (suites, opts) => { - const o = opts; - o.suites = suites; - }; +const usage = `Usage: fbp-spec [options] + +Run fbp-spec testsuites against an FBP runtime. + +Options: + --address Address of runtime to connect to (default: ws://localhost:3569) + --secret Runtime secret + --command Command to launch runtime under test + --start-timeout Time to wait for runtime to start (default: 10) + --command-timeout Max time for a FBP command (default: 3) + --component-tests Load component tests from runtime + --trace Store a replayable Flowtrace for each test run + --help Show this help +`; - program - .storeOptionsAsProperties(false) - .passCommandToAction(false) - .arguments('') - .action(actionHandler) - .option('--address ', 'Address of runtime to connect to', String, 'ws://localhost:3569') - .option('--secret ', 'Runtime secret', String, null) - .option('--command ', 'Command to launch runtime under test', String, null) - .option('--start-timeout ', 'Time to wait for runtime to start', Number, 10) - .option('--command-timeout ', 'Max time for a FBP command', Number, 3) - .option('--component-tests', 'Load component tests from runtime') - .option('--trace', 'Store a replayable Flowtrace for each test run') - .parse(args); - - return program; +/** + * Parse command-line arguments into runner options. + * @param {string[]} [args] Arguments to parse, defaults to process.argv + * @returns {{ options: { address: string, secret?: string, command?: string, startTimeout: number, commandTimeout: number, componentTests: boolean, trace: boolean, suites: string[] }, help: boolean }} Parsed options + */ +export function parse(args = process.argv.slice(2)) { + let parsed; + try { + parsed = parseArgs({ + args, + options: { + address: { type: "string", default: "ws://localhost:3569" }, + secret: { type: "string" }, + command: { type: "string" }, + "start-timeout": { type: "string", default: "10" }, + "command-timeout": { type: "string", default: "3" }, + "component-tests": { type: "boolean", default: false }, + trace: { type: "boolean", default: false }, + help: { type: "boolean", default: false }, + }, + allowPositionals: true, + strict: true, + }); + } catch (err) { + const message = err instanceof Error ? err.message : String(err); + throw new Error(`${message}\n\n${usage}`); + } + + const { values, positionals } = parsed; + const options = { + address: values.address, + secret: values.secret, + command: values.command, + startTimeout: Number(values["start-timeout"]), + commandTimeout: Number(values["command-timeout"]), + componentTests: values["component-tests"], + trace: values.trace, + suites: positionals, + }; + return { options, help: values.help }; } -function startRuntime(options, callback) { +/** + * Start the runtime-under-test as a subprocess, if requested. + * @param {{ command?: string }} options Parsed options + * @returns {Promise} The child process, or null + */ +async function startRuntime(options) { if (!options.command) { // we're not responsible for starting it - callback(null); - return; + return null; } - const subprocessOptions = {}; - subprocess.start(options.command, subprocessOptions, callback); + return start(options.command, {}); } -function hasErrors(suites) { +/** + * Whether any of the testcases have errors. + * @param {Suite[]} suites Suites to check + * @returns {boolean} True if there are failures + */ +export function hasErrors(suites) { let failures = 0; - suites.forEach((s) => { - s.cases.forEach((c) => { - if (c.error) { failures += 1; } - }); - }); + for (const s of suites) { + for (const c of s.cases ?? []) { + if (c.error) { + failures += 1; + } + } + } return failures > 0; } -function ensureTracedir(options, callback) { +/** + * Ensure the .flowtrace directory exists, if tracing is enabled. + * @param {{ trace: boolean }} options Parsed options + * @returns {Promise} + */ +async function ensureTracedir(options) { if (!options.trace) { - callback(null); return; } - const tracePath = resolve(process.cwd(), './.flowtrace'); - stat(tracePath, (err) => { - if (!err) { - callback(null); - return; - } - mkdir(tracePath, callback); - }); + const tracePath = resolve(process.cwd(), "./.flowtrace"); + try { + await stat(tracePath); + } catch { + await mkdir(tracePath); + } } -function storeTrace(options, traceId, testcaseName, trace) { +/** + * Store a Flowtrace on disk, and remember its path for the summary output. + * @param {{ trace: boolean }} options Parsed options + * @param {string} traceId ID of the graph the trace was recorded for + * @param {string} testcaseName Name of the testcase the trace was recorded for + * @param {object} trace The Flowtrace contents + * @returns {Promise} + */ +async function storeTrace(options, traceId, testcaseName, trace) { if (!options.trace) { return; } - const date = new Date().toISOString().substr(0, 10); - const fileName = slug(`${date}-${traceId}-${testcaseName}`); + const date = new Date().toISOString().slice(0, 10); + const fileName = `${date}-${traceId}-${testcaseName}` + .toLowerCase() + .replace(/[^a-z0-9]+/g, "-") + .replace(/^-+|-+$/g, ""); const tracePath = resolve(process.cwd(), `./.flowtrace/${fileName}.json`); - writeFile(tracePath, JSON.stringify(trace, null, 2), (err) => { - if (err) { - console.error(err); - } - traces.push(tracePath); - }); + await writeFile(tracePath, JSON.stringify(trace, null, 2)); + traces.push(tracePath); } -function runOptions(options, onUpdate, callback) { - let child = null; - let allSuites = []; - - function cleanReturn(err) { - if (child) { - child.kill(); - } - callback(err, allSuites); - } - +/** + * Run the given suites against the runtime, starting and stopping it as needed. + * @param {{ address: string, secret?: string, command?: string, startTimeout: number, commandTimeout: number, componentTests: boolean, trace: boolean, suites: string[] }} options Parsed options + * @param {(suites: Suite[]) => void} [onUpdate] Called when test state is updated + * @returns {Promise} All run suites, with results filled in + */ +export async function runOptions(options, onUpdate = () => {}) { const def = { - protocol: 'websocket', + protocol: "websocket", address: options.address, - secret: options.secret || '', + secret: options.secret || "", }; - debug('runtime info', def); + debug("runtime info", def); - const runnerOptions = { + const runnerOptions = /** @type {import("./runner.js").RunnerOptions} */ ({ connectTimeout: options.startTimeout * 1000, commandTimeout: options.commandTimeout * 1000, - }; + }); if (options.trace) { - runnerOptions.traceCallback = (graphId, testcaseName, trace) => { + runnerOptions.traceCallback = (graphId, testcaseName, trace) => storeTrace(options, graphId, testcaseName, trace); - }; } - const ru = new runner.Runner(def, runnerOptions); - ensureTracedir(options, (dirErr) => { - if (dirErr) { - cleanReturn(dirErr); - return; - } + const ru = new Runner(def, runnerOptions); + await ensureTracedir(options); - child = startRuntime(options, (err) => { - if (err) { - cleanReturn(err); - return; - } + let child = null; + try { + child = await startRuntime(options); + await ru.connect(); - ru.connect((connectErr) => { - if (connectErr) { - cleanReturn(connectErr); - return; - } - - testsuite.getSuites(options.suites, (getSuitesErr, suites) => { - if (getSuitesErr) { - cleanReturn(getSuitesErr); - return; - } - allSuites = allSuites.concat(suites); - - if (options.componentTests) { - // TODO: move this into runAll?? - runner.getComponentSuites(ru, (getComponentSuitesErr, componentSuites) => { - if (getComponentSuitesErr) { - cleanReturn(getComponentSuitesErr); - return; - } - allSuites = allSuites.concat(componentSuites); - runner.runAll(ru, allSuites, onUpdate, (runErr) => { - if (runErr) { - cleanReturn(runErr); - return; - } - ru.disconnect((disconnectErr) => cleanReturn(disconnectErr)); - }); - }); - return; - } - runner.runAll(ru, allSuites, onUpdate, (runErr) => { - if (runErr) { - cleanReturn(runErr); - return; - } - ru.disconnect((disconnectErr) => cleanReturn(disconnectErr)); - }); - }); - }); - }); - }); + let suites = await getSuites(options.suites); + if (options.componentTests) { + // TODO: move this into runAll?? + suites = suites.concat(await getComponentSuites(ru)); + } + await runAll(ru, suites, onUpdate); + await ru.disconnect(); + return suites; + } finally { + child?.kill(); + } } -function testStatusText(suites) { +/** + * Produce human-readable status text for the suites. + * @param {Suite[]} suites Suites with results + * @returns {string[]} Lines of status text + */ +export function testStatusText(suites) { const results = []; - const ident = ' '; - suites.forEach((s) => { - s.cases.forEach((c) => { - if ((c.passed == null) || c.shown) { return; } - if (!s.titleshown) { results.push(`${s.name}`); } + const ident = " "; + for (const s of suites) { + for (const c of s.cases ?? []) { + if (c.passed == null || c.shown) { + continue; + } + if (!s.titleshown) { + results.push(`${s.name}`); + } // bit hacky, mutates suites const suite = s; @@ -184,36 +214,64 @@ function testStatusText(suites) { suite.titleshown = true; testCase.shown = true; - let res = c.passed ? '✓' : `✗ Error: ${c.error}`; - if (c.skip) { res = `SKIP: ${c.skip}`; } + const res = c.passed ? "✓" : `✗ Error: ${c.error}`; results.push(`${ident}${c.name}\n${ident + ident}${c.assertion}: ${res}`); - }); - }); + } + } return results; } -function main() { - const cmd = parse(process.argv); - const options = cmd.opts(); +/** + * Main entry point of the fbp-spec command-line tool. + * @returns {Promise} Exits the process with status 0 or 2 + */ +export async function main() { + let options; + let help; + try { + const parsed = parse(); + ({ options, help } = parsed); + } catch (err) { + console.error(err instanceof Error ? err.message : err); + process.exitCode = 2; + return; + } + if (help) { + console.log(usage); + return; + } + if (!options.suites.length) { + console.error("Error: no suites specified\n"); + console.log(usage); + process.exitCode = 2; + return; + } + /** + * Print incremental test results. + * @param {Suite[]} suites Suites with results + * @returns {void} + */ function onUpdate(suites) { const r = testStatusText(suites); - console.log(r.join('\n')); + console.log(r.join("\n")); } - runOptions(options, onUpdate, (err, suites) => { - if (err) { throw err; } + let suites; + try { + suites = await runOptions(options, onUpdate); + } catch (err) { + console.error(err); + process.exitCode = 2; + return; + } - if (traces.length) { - console.log('\nFlowtraces produced:'); - traces.forEach((tracePath) => { - console.log(`- ${relative(process.cwd(), tracePath)}`); - }); + if (traces.length) { + console.log("\nFlowtraces produced:"); + for (const tracePath of traces) { + console.log(`- ${relative(process.cwd(), tracePath)}`); } + } - const exitStatus = hasErrors(suites) ? 2 : 0; - process.exit(exitStatus); - }); + process.exitCode = hasErrors(suites) ? 2 : 0; } - -exports.main = main; diff --git a/lib/common.js b/lib/common.js index 26fc4c0..b1dc9ba 100644 --- a/lib/common.js +++ b/lib/common.js @@ -1,56 +1,109 @@ -const Promise = require('bluebird'); +/** + * General-purpose helpers used across fbp-spec. + * @module lib/common + */ -exports.isBrowser = function isBrowser() { - if ((typeof process !== 'undefined') && process.execPath && process.execPath.match(/node|iojs/)) { +/** + * Whether we are running in a browser-like environment (no Node.js process). + * @returns {boolean} True if running in a browser + */ +export function isBrowser() { + if ( + typeof process !== "undefined" && + process.execPath && + process.execPath.match(/node|iojs/) + ) { return false; } return true; -}; - -exports.randomString = function randomString(n) { - let text = ''; - const possible = 'ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789'; - for (let i = 0, end = n, asc = end >= 0; asc ? i < end : i > end; asc ? i += 1 : i -= 1) { - const idx = Math.floor(Math.random() * possible.length); - text += possible.charAt(idx); +} + +const ALPHABET = + "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789"; + +/** + * Generate a random ASCII string, using Web Crypto when available. + * @param {number} n Number of characters to generate + * @returns {string} Random string of length n + */ +export function randomString(n) { + const bytes = new Uint8Array(n); + globalThis.crypto.getRandomValues(bytes); + let text = ""; + for (let i = 0; i < n; i += 1) { + text += ALPHABET.charAt(bytes[i] % ALPHABET.length); } return text; -}; +} -exports.asyncSeries = function asyncSeries(items, func, callback) { - const runItems = items.slice(0); +/** + * Run an async function over items sequentially, collecting results. + * The first failure aborts the sequence and rejects. + * @template T, R + * @param {T[]} items Items to process + * @param {(item: T) => Promise} func Async function applied to each item + * @returns {Promise} Results in the same order as the items + */ +export async function asyncSeries(items, func) { const results = []; - function next() { - if (runItems.length === 0) { - callback(null, results); - return; - } - const item = runItems.shift(); - func(item, (err, result) => { - if (err) { - callback(err); - return; - } - results.push(result); - next(); - }); + for (const item of items) { + results.push(await func(item)); // eslint-disable-line no-await-in-loop } - next(); -}; + return results; +} -exports.isArray = Array.isArray || ((value) => ({}).toString.call(value) === '[object Array]'); +/** + * Whether the given value is an Array. + * @param {*} value Value to check + * @returns {boolean} True if value is an array + */ +export function isArray(value) { + return Array.isArray(value); +} -exports.startsWith = (str, sub) => str.indexOf(sub) === 0; +/** + * Whether the given string starts with the given substring. + * @param {string} str String to check + * @param {string} sub Expected prefix + * @returns {boolean} True if str starts with sub + */ +export function startsWith(str, sub) { + return str.indexOf(sub) === 0; +} -// Based on http://stackoverflow.com/a/38225011/1967571 -const rejectDelayer = function rejectDelayer(delay) { - return (reason) => new Promise((resolve, reject) => setTimeout(reject.bind(null, reason), delay)); -}; +/** + * Wait for the given number of milliseconds. + * @param {number} delay Delay in milliseconds + * @returns {Promise} Resolves after the delay + */ +function delayPromise(delay) { + return new Promise((resolve) => { + setTimeout(resolve, delay); + }); +} -exports.retryUntil = function retryUntil(attempt, test, delay = 500, max = 5) { - let p = Promise.reject(new Error('retry starter')); - for (let i = 0, end = max, asc = end >= 0; asc ? i <= end : i >= end; asc ? i += 1 : i -= 1) { - p = p.catch(attempt).then(test).catch(rejectDelayer(delay)); +/** + * Repeatedly call attempt, until the test() predicate on its result passes. + * @template T + * @param {() => Promise} attempt Async function producing a value + * @param {(value: T) => Promise} test Async predicate; should reject on the value if not acceptable yet + * @param {number} [delay=500] Time to wait between attempts, in milliseconds + * @param {number} [max=5] Maximum number of attempts + * @returns {Promise} The first result that passes the test + */ +export async function retryUntil(attempt, test, delay = 500, max = 5) { + let lastError = null; + for (let i = 0; i <= max; i += 1) { + try { + const result = await attempt(); + await test(result); + return result; + } catch (error) { + lastError = error; + } + if (i < max) { + await delayPromise(delay); + } } - return p; -}; + throw lastError; +} diff --git a/lib/expectation.js b/lib/expectation.js index 7c9a1de..1f477ca 100644 --- a/lib/expectation.js +++ b/lib/expectation.js @@ -1,100 +1,270 @@ -const debug = require('debug')('fbp-spec:expectation'); -const chai = require('chai'); -const JSONPath = require('jsonpath'); +/** + * A library of standard expectation operators, used to check actual output data. + * Throws an Error when an expectation is violated, with details about the issue. + * @module lib/expectation + */ -const common = require('./common'); +import assert from "node:assert"; +import debugLib from "debug"; +import JSONPath from "jsonpath"; -// # A library of standard expectation operators -// -// Should throw AssertionError if expectation is violated, -// with details about the issue +import { isArray } from "./common.js"; + +const debug = debugLib("fbp-spec:expectation"); + +/** + * Produce an AssertionError-compatible error with a message. + * @param {string} message Description of the violated expectation + * @returns {Error} The error to throw + */ +function fail(message) { + return new assert.AssertionError({ + message, + actual: "", + expected: "", + operator: "==", + }); +} + +/** + * The library of standard expectation operators. + * Each operator takes the actual value and the expected value, + * and throws an Error if the expectation is violated. + * @type {Record void>} + */ const operators = { + /** + * Check the type of the actual value. + * @param {*} actual The received value + * @param {string} expected Expected type name + * @returns {void} + */ type(actual, expected) { - return chai.expect(actual).to.be.a(expected); + assert.equal( + typeof actual, + expected, + `expected ${JSON.stringify(actual)} to be of type ${expected}`, + ); }, + /** + * Deep equality check. + * @param {*} actual The received value + * @param {*} expected The expected value + * @returns {void} + */ equals(actual, expected) { - return chai.expect(actual).to.eql(expected); + if (!deepEqual(actual, expected)) { + throw fail( + `expected ${JSON.stringify(actual)} to equal ${JSON.stringify(expected)}`, + ); + } }, + /** + * Check that actual is above (greater than) expected. + * @param {number} actual The received value + * @param {number} expected The value to be above + * @returns {void} + */ above(actual, expected) { - return chai.expect(actual).to.be.above(expected); + if (!(actual > expected)) { + throw fail(`expected ${actual} to be above ${expected}`); + } }, + /** + * Check that actual is below (less than) expected. + * @param {number} actual The received value + * @param {number} expected The value to be below + * @returns {void} + */ below(actual, expected) { - return chai.expect(actual).to.be.below(expected); + if (!(actual < expected)) { + throw fail(`expected ${actual} to be below ${expected}`); + } }, + /** + * Check that the actual object has exactly the given keys. + * @param {*} actual The received value + * @param {string[]} expected Key names to expect + * @returns {void} + */ haveKeys(actual, expected) { - return chai.expect(actual).to.have.keys(expected); + const keys = Object.keys(actual ?? {}).sort(); + const wanted = [...expected].sort(); + if (JSON.stringify(keys) !== JSON.stringify(wanted)) { + throw fail( + `expected ${JSON.stringify(actual)} to have keys ${JSON.stringify(wanted)}, got ${JSON.stringify(keys)}`, + ); + } }, + /** + * Check that the actual object includes the given keys. + * @param {*} actual The received value + * @param {string[]} expected Key names to expect to be present + * @returns {void} + */ includeKeys(actual, expected) { - return chai.expect(actual).to.include.keys(expected); + const keys = Object.keys(actual ?? {}); + for (const key of expected) { + if (!keys.includes(key)) { + throw fail(`expected ${JSON.stringify(actual)} to include key ${key}`); + } + } }, + /** + * Check that the actual value contains the expected value. + * For strings checks substring, for arrays checks for a deep-equal member, + * for objects checks for a deep-equal subset. + * @param {*} actual The received value + * @param {*} expected Value expected to be contained + * @returns {void} + */ contains(actual, expected) { - return chai.expect(actual).to.contain(expected); + if (typeof actual === "string") { + if (!actual.includes(expected)) { + throw fail(`expected '${actual}' to contain '${expected}'`); + } + return; + } + if (isArray(actual)) { + const list = /** @type {*[]} */ (actual); + if (!list.some((member) => deepEqual(member, expected))) { + throw fail( + `expected ${JSON.stringify(actual)} to contain ${JSON.stringify(expected)}`, + ); + } + return; + } + if (actual != null && typeof actual === "object") { + const obj = /** @type {Record} */ (actual); + for (const key of Object.keys(expected)) { + if (!deepEqual(obj[key], expected[key])) { + throw fail( + `expected ${JSON.stringify(actual)} to contain { '${key}': ${JSON.stringify(expected[key])} }`, + ); + } + } + return; + } + throw fail(`cannot check contains on ${JSON.stringify(actual)}`); }, + /** + * Check that the actual value is not an error-like object. + * @param {*} actual The received value + * @returns {void} + */ noterror(actual) { - if (actual != null ? actual.message : undefined) { throw actual; } + if (actual?.message) { + throw actual; + } }, }; -// returns a predicate function -// (data) -> +/** + * Structurally compare two values for equality. + * @param {*} a First value + * @param {*} b Second value + * @returns {boolean} True if the values are deeply equal + */ +function deepEqual(a, b) { + try { + assert.deepStrictEqual(a, b); + return true; + } catch { + return false; + } +} + +/** + * Find the expectation operator referenced in an expectation object, + * and return it wrapped in a predicate function. + * @param {Record} expectation Expectation object, like { equals: true } or { path, equals } + * @returns {(data: *) => void} Predicate function, throws on violated expectation + */ function findOperator(expectation) { const ops = Object.keys(expectation); - for (let i = 0; i < ops.length; i += 1) { - const opname = ops[i]; + for (const opname of ops) { const expectValue = expectation[opname]; - if (opname !== 'path') { + if (opname !== "path") { const op = operators[opname]; if (op) { - const predicate = (data) => op(data, expectValue); + const predicate = /** @type {(data: *) => void} */ ( + (data) => op(data, expectValue) + ); predicate.toString = () => `${opname} ${expectValue}`; return predicate; } } } - throw new Error(`fbp-spec: No operator matching ${Object.keys(expectation)}. Available: ${Object.keys(operators)}`); + throw new Error( + `fbp-spec: No operator matching ${Object.keys(expectation)}. Available: ${Object.keys(operators)}`, + ); } +/** + * Extract the data values the expectation applies to. + * Uses the JSONPath in expectation.path when present, otherwise the data itself. + * @param {Record} expectation Expectation object, possibly with a .path JSONPath + * @param {*} data The data to match against + * @returns {*[]} Matching data values + */ function extractMatches(expectation, data) { let matches; if (expectation.path) { - debug('extracting JSONPath from', expectation.path, data); + debug("extracting JSONPath from", expectation.path, data); matches = JSONPath.query(data, expectation.path); - if (!matches.length) { throw new Error(`expected JSONPath '${expectation.path}' to match data in ${JSON.stringify(data)}`); } + if (!matches.length) { + throw new Error( + `expected JSONPath '${expectation.path}' to match data in ${JSON.stringify(data)}`, + ); + } } else { matches = [data]; } - debug('matching against', matches); + debug("matching against", matches); return matches; } -exports.expect = function expect(expects, portsdata) { +/** + * Check actual output data against a set of expectations. + * Throws an Error on the first violated expectation. + * @param {*|Array>} expects One expectation, or a sequence of expectations + * @param {Record} portsdata Data received on the outports, keyed by port name + * @returns {void} + */ +export function expect(expects, portsdata) { // can have one or more set of expectations of messages - if (!common.isArray(expects)) { - expect([expects], portsdata); - return; - } + const expectationSets = isArray(expects) ? expects : [expects]; - expects.forEach((e) => { + for (const e of expectationSets) { // each message expectation can match on ports - Object.keys(e).forEach((port) => { + for (const port of Object.keys(e)) { // for each matching port, there can be one or more assertions on the value - const expectations = common.isArray(e[port]) ? e[port] : [e[port]]; - expectations.forEach((expectation) => { - debug('checking port for expectation', port, expectation); + const expectations = isArray(e[port]) ? e[port] : [e[port]]; + for (const expectation of expectations) { + debug("checking port for expectation", port, expectation); const data = portsdata[port]; - if (typeof data === 'undefined') { + if (typeof data === "undefined") { throw new Error(`No data received on port ${port}`); } const predicate = findOperator(expectation); const matches = extractMatches(expectation, data); - matches.forEach((m) => { - debug('checking against predicate', m, predicate.toString()); - predicate(m); - }); - }); - }); - }); -}; + for (const match of matches) { + debug("checking against predicate", match, predicate.toString()); + predicate(match); + } + } + } + } +} -exports.noError = (maybeErr) => chai.expect(maybeErr).to.not.exist; +/** + * Assert that the given value is not an error. + * @param {*} maybeErr Value that should not be an error + * @returns {void} + */ +export function noError(maybeErr) { + if (maybeErr != null) { + throw maybeErr; + } +} diff --git a/lib/index.js b/lib/index.js index d062787..e3214e1 100644 --- a/lib/index.js +++ b/lib/index.js @@ -1,19 +1,12 @@ -function getSchema(id) { - const schemaId = id.replace('.json', ''); - return module.exports.schema[schemaId]; -} +/** + * Public API of fbp-spec. + * @module lib/index + */ -module.exports = { - // eslint-disable-next-line global-require - runner: require('./runner'), - // eslint-disable-next-line global-require - subprocess: require('./subprocess'), - // eslint-disable-next-line global-require - mocha: require('./mocha'), - // eslint-disable-next-line global-require - testsuite: require('./testsuite'), - // eslint-disable-next-line global-require,import/no-unresolved - schema: require('../schema'), - getSchema, - ui: {}, -}; +import * as schema from "../schema/index.js"; +import { getSchema } from "../schema/index.js"; +import * as runner from "./runner.js"; +import * as subprocess from "./subprocess.js"; +import * as testsuite from "./testsuite.js"; + +export { getSchema, runner, schema, subprocess, testsuite }; diff --git a/lib/mocha.js b/lib/mocha.js deleted file mode 100644 index e118859..0000000 --- a/lib/mocha.js +++ /dev/null @@ -1,110 +0,0 @@ -/* eslint-env node, mocha */ -// # Run fbp-spec testcases using Mocha as a runner/frontend -// # Intended to allow reuse of Mocha reporters, or -// # to quickly add a couple of fbp-spec cases to a predominantly Mocha-based testsuite -// # See also ./mochacompat.js - -const debug = require('debug')('fbp-spec:mocha'); - -const runnerModule = require('./runner'); - -const { - Runner, -} = runnerModule; -const testsuite = require('./testsuite'); -const expectation = require('./expectation'); -const subprocess = require('./subprocess'); - -function runSuite(runner, suite) { - const suiteDescribe = suite.skip ? describe.skip : describe; - suiteDescribe(`${suite.name}`, () => { - beforeEach(function (done) { - if (suite.timeout != null) { - this.timeout(suite.timeout); - } - runner.setupSuite(suite, done); - }); - afterEach((done) => runner.teardownSuite(suite, done)); - - suite.cases.forEach((testcase) => { - const caseDescribe = testcase.skip ? describe.skip : describe; - caseDescribe(testcase.name, () => it(testcase.assertion, function (done) { - if (suite.timeout != null) { - this.timeout(suite.timeout); - } - if (testcase.timeout != null) { - this.timeout(testcase.timeout); - } - - runnerModule.runTestAndCheck(runner, testcase, (err, result) => { - if (err) { - done(err); - return; - } - done(result.error); - }); - })); - }); - }); -} - -// # run() -// Must be be ran using Mocha, -// it is responsible for setting up the "describe", and "it" functions -exports.run = function run(rt, tests, options) { - // default pretty high to give time for runtime to start - let suites; - const opts = options; - if ((options.starttimeout == null)) { opts.starttimeout = 5000; } - if ((options.fixturetimeout == null)) { opts.fixturetimeout = 2000; } - - const runnerOptions = { - connectTimeout: options.starttimeout, - commandTimeout: options.commandtimeout, - }; - const runner = new Runner(rt, runnerOptions); - try { - suites = testsuite.getSuitesSync(tests); - } catch (e) { - console.log('Unable to get suites:', e); - throw e; - } - let process = null; - - const start = function (callback) { - if (!rt.command) { - callback(null); - return; - } - const subprocessOptions = {}; - process = subprocess.start(rt.command, subprocessOptions, callback); - }; - const stop = function (callback) { - if (process) { process.kill(); } - callback(null); - }; - - before(function (done) { - this.timeout(options.starttimeout + 500); - start((err) => { - debug('started', err); - expectation.noError(err); - runner.connect(done); - }); - }); - after((done) => { - stop((err) => { - debug('stopped', err); - expectation.noError(err); - runner.disconnect(done); - }); - }); - - suites.forEach((suite) => { - if ((suite.timeout == null)) { - const s = suite; - s.timeout = options.fixturetimeout; - } - runSuite(runner, suite); - }); -}; diff --git a/lib/mochacompat.js b/lib/mochacompat.js deleted file mode 100644 index f39417f..0000000 --- a/lib/mochacompat.js +++ /dev/null @@ -1,436 +0,0 @@ -/* eslint-env browser, node */ -// # Run Mocha testcases using fbp-spec as a runner/frontend -// # Intended to allow existing Mocha testcases to be seen and executed -// # from a FBP protocol client like Flowhub, without requiring them -// # to be rewritten as fbp-spec tests -// # This is especially useful to allow partial and gradual migration of existing test suites -// # See also ./mocha.coffee, which can be used in combination - -// Partially based on example code from https://github.com/mochajs/mocha/wiki/Third-party-UIs - -const http = require('http'); -const Mocha = require('mocha'); -const crypto = require('crypto'); -const debug = require('debug')('fbp-spec:mochacompat'); - -// FIXME: split out transport interface of noflo-runtime-*, use that directly -const websocket = require('./websocket'); - -const testsuite = require('./testsuite'); - -function loadTests(files) { - const options = {}; - const mocha = new Mocha(options); - - files.forEach((f) => { - const resolved = require.resolve(f); - delete require.cache[resolved]; // clear module cache - mocha.addFile(f); - }); - mocha.loadFiles(); - return mocha; -} - -// similar to mocha.run(), but files must be loaded beforehand -function runTests(mocha, progress, callback) { - const { - suite, - } = mocha; - const { - options, - } = mocha; - options.files = mocha.files; - const runner = new Mocha.Runner(suite, options.delay); - function registerReporter() { - runner.on('pass', (test) => progress(null, test)); - runner.on('fail', (test, err) => progress(err, test)); - } - - mocha.reporter(registerReporter, {}); - // eslint-disable-next-line no-underscore-dangle - const reporter = new mocha._reporter(runner, options); - - runner.ignoreLeaks = options.ignoreLeaks !== false; - runner.fullStackTrace = options.fullStackTrace; - runner.asyncOnly = options.asyncOnly; - runner.allowUncaught = options.allowUncaught; - if (options.grep) { - runner.grep(options.grep, options.invert); - } - if (options.globals) { - runner.globals(options.globals); - } - if (options.growl) { - // eslint-disable-next-line no-underscore-dangle - mocha._growl(runner, reporter); - } - if (options.useColors != null) { - Mocha.reporters.Base.useColors = options.useColors; - } - Mocha.reporters.Base.inlineDiffs = options.useInlineDiffs; - - function done(failures) { - if (reporter.done) { - reporter.done(failures, callback); - return; - } - callback(failures); - } - - runner.run(done); -} - -function testId(fullname) { - const hash = crypto.createHash('sha256'); - hash.update(fullname); - return hash.digest('hex').substr(0, 10); -} - -function loadSuite(fbpSuite, suite) { - suite.tests.forEach((testcase) => { - // FIXME: make the full name recursive - const fullName = fbpSuite.name + testcase.parent.title + testcase.title; - // FIXME: salt so it does not collide - const id = testId(fullName); - const c = testcase; - // eslint-disable-next-line no-underscore-dangle - c._fbpid = id; - const fbpCase = { - name: testcase.parent.title, - assertion: testcase.title, - _id: id, - inputs: { - test: id, - }, - expect: { - error: { - noterror: null, - }, - }, - }; - - fbpSuite.cases.push(fbpCase); - }); - - // load recursively - suite.suites.forEach((sub) => { - loadSuite(fbpSuite, sub); - }); -} - -function buildFbpSpecs(mocha) { - const specs = []; - - const top = mocha.suite; - top.suites.forEach((suite) => { - // console.log 's', suite - - const fbpSuite = testsuite.create({ - name: `${suite.title} (Mocha tests)`, - fixture: { - type: 'fbp', - data: `\ -# @runtime fbp-spec-mocha - -INPORT=runTest.IN:TEST -OUTPORT=runTest.ERROR:ERROR - -runTest(mocha/LoadTestCase) OUT -> IN verifyResult(mocha/CheckResults)\ -`, - }, - }); - - loadSuite(fbpSuite, suite); - specs.push(fbpSuite); - }); - - return specs; -} - -function dumpSpecs(suites) { - // eslint-disable-next-line global-require - const yaml = (typeof window !== 'undefined' && window.jsyaml) ? window.jsyaml : require('js-yaml'); - - let str = ''; - const delimiter = '---\n'; - suites.forEach((s) => { - str += `${yaml.dump(s)}`; - if (suites.length > 1) { str += delimiter; } - }); - - return str; -} - -function discoverHost(preferredIface) { - // eslint-disable-next-line global-require - const os = require('os'); // node.js only - - const ifaces = os.networkInterfaces(); - let address; - let intAddress; - - function filter(connection) { - if (connection.family !== 'IPv4') { - return; - } - if (connection.internal) { - intAddress = connection.address; - } else { - ({ - address, - } = connection); - } - } - - if ((typeof preferredIface === 'string') && Array.from(ifaces).includes(preferredIface)) { - ifaces[preferredIface].forEach(filter); - } else { - Object.keys(ifaces).forEach((device) => { - ifaces[device].forEach(filter); - }); - } - return address || intAddress; -} - -const knownUnsupportedCommands = () => false; - -function fbpComponentName(s) { - // TODO: use topic/filename? - return `fbp-spec-mocha/${s.name}`; -} - -function fbpComponentFromSpec(s) { - // component:component - return { - name: fbpComponentName(s), - subgraph: false, - inPorts: [], - outPorts: [], - }; -} - -function fbpSourceFromSpec(s) { - // component:source message, :getsource response - const serialized = dumpSpecs([s]); - return { - name: fbpComponentName(s), - code: '', - language: 'whitespace', - tests: serialized, - }; -} - -function handleFbpCommand( - originalState, - runtime, - mocha, - specs, - protocol, - command, - payload, - context, -) { - const state = originalState; - function updateStatus(news, event) { - if (news.started != null) { state.started = news.started; } - if (news.running != null) { state.running = news.running; } - const runtimeState = { started: state.started, running: state.running, graph: payload.graph }; - debug('update status', runtimeState); - runtime.send('network', event, runtimeState, context); - } - - // sendEvent = (e) -> - // runtime.send e.protocol, e.command, e.payload, context - function ackMessage() { - // reply with same message as we got in - runtime.send(protocol, command, payload, context); - } - - function checkSendCurrentTest() { - const completed = state.completedTests[state.currentTest]; - debug('checking for', state.currentTest, (completed != null)); - if (completed) { - const m = { - graph: state.graph, - event: 'data', - port: 'error', - payload: completed.err, - }; - runtime.send('runtime', 'packet', m, context); - delete state.completedTests[state.currentTest]; - } - } - - // collect results of completed tests - function testDone(err, test) { - // eslint-disable-next-line no-underscore-dangle - debug('test completed', test._fbpid, err, Object.keys(state.completedTests).length); - // eslint-disable-next-line no-underscore-dangle - state.completedTests[test._fbpid] = { test, err }; - checkSendCurrentTest(); - } - - // # Runtime - if ((protocol === 'runtime') && (command === 'getruntime')) { - const capabilities = [ - 'protocol:graph', // read-only from client - 'protocol:component', // read-only from client - 'protocol:network', - 'protocol:runtime', - 'component:getsource', - ]; - const info = { - type: 'fbp-spec-mocha', - version: '0.5', - capabilities, - allCapabilities: capabilities, - graph: 'default/main', // HACK, so Flowhub will ask for our graph - }; - runtime.send('runtime', 'runtime', info, context); - // sendGraphs mytrace, send, (err) -> # XXX: right place? - // ignored - } if ((protocol === 'runtime') && (command === 'packet')) { - debug('test message', payload, state.running); - - if ((payload.port !== 'test') || (payload.event !== 'data')) { - debug('unexpected test message format'); - return; - } - - state.currentTest = payload.payload; - - checkSendCurrentTest(); // we might have completed it already - - if (!state.running) { - runTests(mocha, testDone, () => updateStatus({ running: false }, 'status')); - - updateStatus({ running: true }, 'status'); - } - - // # Graph - } else if ((protocol === 'graph') && (command === 'addnode')) { - ackMessage(); - } else if ((protocol === 'graph') && (command === 'addedge')) { - ackMessage(); - } else if ((protocol === 'graph') && (command === 'addinport')) { - ackMessage(); - } else if ((protocol === 'graph') && (command === 'addoutport')) { - ackMessage(); - } else if ((protocol === 'graph') && (command === 'clear')) { - state.graph = payload.id; - debug('new graph', state.graph); - ackMessage(); - - // # Network - } else if ((protocol === 'network') && (command === 'getstatus')) { - runtime.send('network', 'status', state, context); - } else if ((protocol === 'network') && (command === 'start')) { - debug('FBP network start'); - updateStatus({ started: true, running: false, graph: payload.graph }, 'started'); - } else if ((protocol === 'network') && (command === 'stop')) { - debug('FBP network stop'); - updateStatus({ started: false, running: false, graph: payload.graph }, 'stopped'); - - // # Component - } else if ((protocol === 'component') && (command === 'list')) { - // one fake component per Mocha suite - specs.forEach((s) => { - runtime.send('component', 'component', fbpComponentFromSpec(s), context); - }); - runtime.send('component', 'componentsready', specs.length, context); - } else if ((protocol === 'component') && (command === 'getsource')) { - // one fake component per Mocha suite - let found = null; - specs.forEach((s) => { - const componentName = fbpComponentName(s); - if (componentName === payload.name) { - found = s; - } - }); - debug('component getsource', `'${payload.name}'`, found != null ? found.name : undefined); - if (found) { - runtime.send('component', 'source', fbpSourceFromSpec(found), context); - } - } else if (knownUnsupportedCommands(protocol, command)) { - // ignored - } else { - debug('Warning: Unknown FBP protocol message', protocol, command); - } -} - -// # Commandline things -function normalizeOptions(options) { - let match; - let discovered; - if (options.host === 'autodetect') { - discovered = discoverHost(); - } else if (/autodetect\(([a-z0-9]+)\)/.exec(options.host)) { - discovered = discoverHost(match[1]); - } - - return { - ...options, - host: discovered || options.host, - port: options.port || 3333, - }; -} - -function parse(args) { - // eslint-disable-next-line global-require - const program = require('commander'); - - const actionHandler = (suites, opts) => { - const o = opts; - o.suites = suites; - }; - - // TODO: take list of files as input instead, to be more mocha compatible - program - .storeOptionsAsProperties(false) - .passCommandToAction(false) - .arguments('') - .action(actionHandler) - .option('--host ', 'Hostname we serve on, for live-url', String, 'autodetect') - .option('--port ', 'Command to launch runtime under test', Number, 3333) - .parse(args); - - return program; -} - -exports.setup = function setup(options, callback) { - const opts = normalizeOptions(options); - - const mocha = loadTests(opts.files); - const specs = buildFbpSpecs(mocha); - - const state = { - started: false, - running: false, - currentTest: null, - graph: null, - specs, - completedTests: {}, // 'id' -> { err: ?Error, test: MochaTest } - }; - const httpServer = new http.Server(); - const runtime = websocket(httpServer, {}); - runtime.receive = (protocol, command, payload, context) => { - handleFbpCommand(state, runtime, mocha, specs, protocol, command, payload, context); - }; - - httpServer.listen(opts.port, (err) => callback(err, state, httpServer)); -}; - -exports.main = function main() { - const cmd = parse(process.argv); - const options = cmd.opts(); - - exports.setup(options, (err, state) => { - if (err) { throw err; } - console.log(`fbp-spec-mocha started on ws://${options.host}:${options.port}`); - console.log(`found ${state.specs.length} test suites`); - }); -}; - -if (!module.parent) { - exports.main(); -} diff --git a/lib/protocol.js b/lib/protocol.js index 9b0692a..b13b04f 100644 --- a/lib/protocol.js +++ b/lib/protocol.js @@ -1,126 +1,161 @@ -// FBP protocol dependent code - -const fbpGraph = require('fbp-graph'); -const debug = require('debug')('fbp-spec:protocol'); -const Promise = require('bluebird'); -const common = require('./common'); - -exports.sendGraph = function sendGraph(client, graph, callback) { - const main = false; // this is a component? +/** + * FBP protocol dependent code: high-level operations on top of an fbp-client instance. + * @module lib/protocol + */ + +import debugLib from "debug"; +import fbpGraph from "fbp-graph"; + +import { randomString } from "./common.js"; + +const debug = debugLib("fbp-spec:protocol"); + +/** + * A graph JSON payload, as sent over the FBP protocol or parsed from FBP notation. + * @typedef {{ name?: string, properties?: Record, inports?: Record, outports?: Record, processes?: Record, connections?: *[] }} GraphPayload + */ + +/** + * A component as advertised by a runtime. + * @typedef {{ name: string, description?: string, subgraph?: boolean, inPorts: { id: string, [key: string]: * }[], outPorts: { id: string, [key: string]: * }[], [key: string]: * }} RuntimeComponent + */ + +/** + * Send a graph to the runtime, and return the ID it was registered under. + * @param {import("fbp-client").FbpClient} client Connected fbp-client instance + * @param {GraphPayload|import("fbp-graph").Graph} graph Graph to send, either a fbp-graph instance or graph JSON payload + * @param {boolean} [main=false] Whether this is the main graph of the runtime + * @returns {Promise} The graph ID assigned on the runtime + */ +export async function sendGraph(client, graph, main = false) { if (!graph) { - callback(new Error('Graph not defined')); - return; + throw new Error("Graph not defined"); } - let graphId = graph.name || graph.properties.id; - if (!graphId) { graphId = `fixture.${common.randomString(10)}`; } - const g = graph; - g.name = graphId; + let graphId = graph.name || graph.properties?.id; + if (!graphId) { + graphId = `fixture.${randomString(10)}`; + } + graph.name = graphId; if (!(graph instanceof fbpGraph.Graph)) { - // fbp-client operates on fbp-graph instances - fbpGraph.graph.loadJSON(graph, (err, loadedGraph) => { - if (err) { - callback(err); - return; - } - exports.sendGraph(client, loadedGraph, callback); - }); - return; + // fbp-client operates on fbp-graph instances; loadJSON always returns a Promise + const loadedGraph = await fbpGraph.graph.loadJSON(graph); + return sendGraph(client, loadedGraph, main); } - debug('sendgraph', graphId); - - Promise.resolve() - .then(() => client.protocol.graph.send(graph, main)) - .then(() => graphId) - .nodeify(callback); -}; + debug("sendgraph", graphId); + await client.protocol.graph.send(graph, main); + return graphId; +} -exports.startNetwork = function startNetwork(client, graphId, callback) { - debug('startnetwork', graphId); +/** + * Start the network for a graph on the runtime. + * @param {import("fbp-client").FbpClient} client Connected fbp-client instance + * @param {string} graphId ID of the graph to start + * @returns {Promise} Resolves when the network has started + */ +export async function startNetwork(client, graphId) { + debug("startnetwork", graphId); // TODO: Register start with recorder - Promise.resolve() - .then(() => client.protocol.network.start({ - graph: graphId, - })) - .nodeify(callback); -}; - -exports.stopNetwork = function stopNetwork(client, graphId, callback) { - debug('stopnetwork', graphId); + await client.protocol.network.start({ graph: graphId }); +} + +/** + * Stop the network for a graph on the runtime. + * @param {import("fbp-client").FbpClient} client Connected fbp-client instance + * @param {string} graphId ID of the graph to stop + * @returns {Promise} Resolves when the network has stopped + */ +export async function stopNetwork(client, graphId) { + debug("stopnetwork", graphId); // TODO: Register stop with recorder - Promise.resolve() - .then(() => client.protocol.network.stop({ - graph: graphId, - })) - .nodeify(callback); -}; - -exports.sendPackets = function sendPackets(client, graphId, packets, callback) { - debug('sendpackets', graphId, packets); - - Promise.all(Object.keys(packets).map((port) => client.protocol.runtime.packet({ - event: 'data', - port, - payload: packets[port], - graph: graphId, - }))) - .nodeify(callback); -}; - -exports.getComponents = function getComponents(client, callback) { - debug('get components'); - - Promise.resolve() - .then(() => client.protocol.component.list()) - .then((componentList) => { - const components = {}; - componentList.forEach((component) => { - components[component.name] = component; - }); - return components; - }) - .nodeify(callback); -}; - -exports.getCapabilities = function getCapabilities(client, callback) { + await client.protocol.network.stop({ graph: graphId }); +} + +/** + * Send input data packets to the input ports of a graph. + * @param {import("fbp-client").FbpClient} client Connected fbp-client instance + * @param {string} graphId ID of the graph to send to + * @param {Record} packets Data to send, keyed by port name + * @returns {Promise} Resolves when all packets have been sent + */ +export async function sendPackets(client, graphId, packets) { + debug("sendpackets", graphId, packets); + + await Promise.all( + Object.keys(packets).map((port) => + client.protocol.runtime.packet({ + event: "data", + port, + payload: packets[port], + graph: graphId, + }), + ), + ); +} + +/** + * Get the components available on the runtime. + * @param {import("fbp-client").FbpClient} client Connected fbp-client instance + * @returns {Promise>} Components keyed by name + */ +export async function getComponents(client) { + debug("get components"); + + const componentList = await client.protocol.component.list(); + /** @type {Record} */ + const components = {}; + for (const component of componentList) { + components[component.name] = component; + } + return components; +} + +/** + * Get the capabilities advertised by the runtime. + * @param {import("fbp-client").FbpClient} client Connected fbp-client instance + * @returns {Promise} Capability names + */ +export async function getCapabilities(client) { const def = client.definition; - if (def && def.capabilities && def.capabilities.length) { + if (def?.capabilities?.length) { // We already know the capabilities - callback(null, def.capabilities); - return; + return def.capabilities; } - Promise.resolve() - .then(() => client.protocol.runtime.getruntime()) - .then((definition) => definition.capabilities) - .nodeify(callback); -}; - -exports.getComponentTests = function getComponentTests(client, callback) { - debug('get component tests'); - - Promise.resolve() - .then(() => client.protocol.component.list()) - .then((components) => Promise.all(components.map((component) => client - .protocol.component.getsource({ - name: component.name, - }).then( + const definition = await client.protocol.runtime.getruntime(); + return definition.capabilities ?? []; +} + +/** + * Get the fbp-spec tests attached to components on the runtime. + * @param {import("fbp-client").FbpClient} client Connected fbp-client instance + * @returns {Promise>} YAML test sources, keyed by component name + */ +export async function getComponentTests(client) { + debug("get component tests"); + + const components = await client.protocol.component.list(); + const sources = await Promise.all( + components.map((component) => + client.protocol.component.getsource({ name: component.name }).then( (source) => source, - () => ({ - tests: null, - }), - )))) - .then((sources) => { - const tests = {}; - sources.forEach((source) => { - if (!source.tests) { return; } - const name = source.library ? `${source.library}/${source.name}` : source.name; - tests[name] = source.tests; - }); - return tests; - }) - .nodeify(callback); -}; + () => ({ tests: null }), + ), + ), + ); + /** @type {Record} */ + const tests = {}; + for (const source of sources) { + if (!source.tests) { + continue; + } + const name = source.library + ? `${source.library}/${source.name}` + : source.name; + tests[name] = source.tests; + } + return tests; +} diff --git a/lib/runner.js b/lib/runner.js index 2019959..b13e887 100644 --- a/lib/runner.js +++ b/lib/runner.js @@ -1,28 +1,71 @@ -const fbp = require('fbp'); -const fbpClient = require('fbp-client'); -const debug = require('debug')('fbp-spec:runner'); -const debugReceived = require('debug')('fbp-spec:runner:received'); -const Promise = require('bluebird'); -const { FlowtraceRecorder } = require('flowtrace'); -const common = require('./common'); -const protocol = require('./protocol'); -const testsuite = require('./testsuite'); -const expectation = require('./expectation'); - +/** + * Runs fbp-spec testsuites against an FBP runtime. + * @module lib/runner + */ + +import debugLib from "debug"; +import fbp from "fbp"; +import fbpClient from "fbp-client"; +import flowtrace from "flowtrace"; + +import { asyncSeries, isArray, retryUntil } from "./common.js"; +import { expect as checkExpectation } from "./expectation.js"; +import * as protocol from "./protocol.js"; +import * as testsuite from "./testsuite.js"; + +const debug = debugLib("fbp-spec:runner"); +const debugReceived = debugLib("fbp-spec:runner:received"); +const { FlowtraceRecorder } = flowtrace; + +/** + * Options for configuring a Runner. + * @typedef {object} RunnerOptions + * @property {number} [connectTimeout] Time to wait for the connection to come up, in milliseconds + * @property {number} [commandTimeout] Maximum time to wait for a command response, in milliseconds + * @property {((graphId: string, testcaseName: string, trace: object) => void)} [traceCallback] + * Called with a recorded Flowtrace for each test, when tracing is enabled + * @property {object} [parentElement] DOM element to attach iframe runtimes to (browser only) + */ + +/** + * One entry in a test sequence: inputs sent, expected outputs, and the actual outputs. + * @typedef {{ inputs: *, expect: *, actual?: * }} TestSequenceEntry + */ + +/** + * Anything that can run a testcase; satisfied by Runner. + * @typedef {{ runTest(testcase: object): Promise }} RunnerLike + */ + +/** + * Log all protocol messages received from a client. + * @param {import("fbp-client").FbpClient} client Client to listen on + * @returns {void} + */ function debugReceivedMessages(client) { - client.on('signal', ({ proto, command, payload }) => debugReceived(proto, command, payload)); + client.on("signal", (message) => { + const { proto, command, payload } = + /** @type {{ proto: string, command: string, payload: * }} */ (message); + debugReceived(proto, command, payload); + }); } -function synthesizeTopicFixture(topic, components, callback) { - debug('synthesizing fixture', topic); +/** + * Build a fixture graph that exposes all ports of the topic component. + * @param {string} topic Component name to use as fixture + * @param {Record} components Components available on the runtime + * @returns {import("./protocol.js").GraphPayload} A graph JSON payload + */ +function synthesizeTopicFixture(topic, components) { + debug("synthesizing fixture", topic); // export each of the ports for topic component // return a FBP graph? const component = components[topic]; if (!component) { - callback(new Error(`Could not find component for topic: ${topic}`)); - return; + throw new Error(`Could not find component for topic: ${topic}`); } + /** @type {import("./protocol.js").GraphPayload & { properties: Record, inports: Record, outports: Record, processes: Record, connections: *[] }} */ const graph = { properties: {}, inports: {}, @@ -31,385 +74,474 @@ function synthesizeTopicFixture(topic, components, callback) { connections: [], }; - const processName = 'testee'; + const processName = "testee"; graph.processes[processName] = { component: topic }; - component.inPorts.forEach((port) => { + for (const port of component.inPorts) { const portName = port.id; graph.inports[portName] = { process: processName, port: portName, }; - }); - component.outPorts.forEach((port) => { + } + for (const port of component.outPorts) { const portName = port.id; graph.outports[portName] = { process: processName, port: portName, }; - }); + } // Sanity checking if this is usable as a fixture if (Object.keys(graph.outports).length < 1) { - callback(new Error(`Component '${topic}' used as fixture has no outports`)); - return; + throw new Error(`Component '${topic}' used as fixture has no outports`); } if (Object.keys(graph.inports).length < 1) { - callback(new Error(`Component '${topic}' used as fixture has no inports`)); - return; + throw new Error(`Component '${topic}' used as fixture has no inports`); } - callback(null, graph); + return graph; } -// @context should have .components = {} property -function getFixtureGraph(context, suite, callback) { +/** + * Determine the fixture graph for a suite, loading components from the runtime if needed. + * @param {{ components: Record, client: import("fbp-client").FbpClient }} context + * Object holding the runtime context; the .components property is populated as needed + * @param {import("./testsuite.js").Suite} suite Testsuite to determine fixture for + * @returns {Promise} Graph JSON payload + */ +async function getFixtureGraph(context, suite) { // TODO: follow runtime for component changes - const hasComponents = (s) => (s.components != null) && (typeof s.components === 'object') && Object.keys(s.components).length; + const hasComponents = + /** @type {(s: { components?: Record }) => boolean} */ ( + (s) => + s.components != null && + typeof s.components === "object" && + Object.keys(s.components).length + ); - function updateComponents(cb) { - if (hasComponents(context)) { - cb(null); - return; - } - protocol.getComponents(context.client, (err, components) => { - if (err) { - cb(err); - return; - } - context.components = components; - cb(null); - }); + if (!hasComponents(context)) { + context.components = await protocol.getComponents(context.client); } - updateComponents((err) => { - if (err) { - callback(err); - return; - } + if (!suite.fixture) { + return synthesizeTopicFixture(suite.topic ?? "", context.components); + } - if (!suite.fixture) { - synthesizeTopicFixture(suite.topic, context.components, callback); - return; + if (suite.fixture.type === "json") { + let graph; + try { + graph = /** @type {import("./protocol.js").GraphPayload} */ ( + JSON.parse(suite.fixture.data) + ); + } catch (error) { + const err = /** @type {Error} */ (error); + throw new Error(`Could not parse JSON fixture: ${err.message}`); } - - if (suite.fixture.type === 'json') { - let graph; - try { - graph = JSON.parse(suite.fixture.data); - } catch (error) { - callback(error); - return; - } - callback(null, graph); - return; + return graph; + } + if (suite.fixture.type === "fbp") { + let graph; + try { + graph = /** @type {import("./protocol.js").GraphPayload} */ ( + fbp.parse(suite.fixture.data) + ); + } catch (error) { + const err = /** @type {Error} */ (error); + throw new Error(`Could not parse FBP fixture: ${err.message}`); } - if (suite.fixture.type === 'fbp') { - let graph; - try { - graph = fbp.parse(suite.fixture.data); - } catch (error) { - callback(error); - return; - } - if (!graph.properties) { - graph.properties = {}; - } - callback(null, graph); - return; + if (!graph.properties) { + graph.properties = {}; } - callback(new Error(`Unknown fixture type ${suite.fixture.type}`)); - }); + return graph; + } + throw new Error(`Unknown fixture type ${suite.fixture.type}`); } -function sendMessageAndWait(client, currentGraph, inputData, expectData, callback) { - const observer = client.observe((s) => (s.protocol === 'runtime') && (s.command === 'packet') && (s.payload.graph === currentGraph)); - +/** + * Send input data to the graph, and wait for the expected output packets to arrive. + * @param {import("fbp-client").FbpClient} client Connected client + * @param {string} currentGraph ID of the graph under test + * @param {Record} inputData Input data keyed by port name + * @param {Record} expectData Expected output data keyed by port name + * @returns {Promise>} Data received on the outports + */ +async function sendMessageAndWait(client, currentGraph, inputData, expectData) { + const observer = client.observe( + (s) => + s.protocol === "runtime" && + s.command === "packet" && + s.payload.graph === currentGraph, + ); + + /** + * Convert observed signals into received data, keyed by port. + * @param {{ payload: Record }[]} signals Observed signals + * @returns {Record} Data keyed by port name + */ function signalsToReceived(signals) { + /** @type {Record} */ const received = {}; - signals.forEach((signal) => { + for (const signal of signals) { received[signal.payload.port] = signal.payload.payload; - }); + } return received; } + /** + * Check whether a signal means we have received all the expected output. + * @param {{ protocol: string, command: string, payload: Record }} s Signal to check + * @returns {boolean} True when complete + */ function checkSuccess(s) { // We're only interested in response packets - if ((s.protocol !== 'runtime') || (s.command !== 'packet')) { return false; } + if (s.protocol !== "runtime" || s.command !== "packet") { + return false; + } // Check that is for the current graph under test - if (s.payload.graph !== currentGraph) { return false; } + if (s.payload.graph !== currentGraph) { + return false; + } // We're only interested in data IPs, not brackets - if (s.payload.event !== 'data') { return false; } + if (s.payload.event !== "data") { + return false; + } // Get signals received until this message - const receivedSignals = observer.signals.slice(0, observer.signals.indexOf(s) + 1); + const receivedSignals = observer.signals.slice( + 0, + observer.signals.indexOf(s) + 1, + ); const received = signalsToReceived(receivedSignals); - const result = (Object.keys(received).length === Object.keys(expectData).length); + const result = + Object.keys(received).length === Object.keys(expectData).length; return result; } + /** + * Check whether a signal means the test has failed. + * @param {{ protocol: string, command: string, payload: Record }} s Signal to check + * @returns {boolean} True when failed + */ function checkFailure(s) { - if ((s.protocol === 'network') && (s.command === 'error')) { + if (s.protocol === "network" && s.command === "error") { // network:error should imply failed test - if (s.payload.graph && (s.payload.graph !== currentGraph)) { return false; } + if (s.payload.graph && s.payload.graph !== currentGraph) { + return false; + } return true; } - if ((s.protocol === 'network') && (s.command === 'processerror')) { + if (s.protocol === "network" && s.command === "processerror") { // network:processerror should imply failed test - if (s.payload.graph && (s.payload.graph !== currentGraph)) { return false; } + if (s.payload.graph && s.payload.graph !== currentGraph) { + return false; + } return true; } - if ((s.protocol === 'runtime') && (s.command === 'packet')) { + if (s.protocol === "runtime" && s.command === "packet") { // Output packet, see if it is an unexpected error // Check that is for the current graph under test - if (s.payload.graph !== currentGraph) { return false; } + if (s.payload.graph !== currentGraph) { + return false; + } // We're only interested in data IPs, not brackets - if (s.payload.event !== 'data') { return false; } + if (s.payload.event !== "data") { + return false; + } // We only care for packets to error port - if (s.payload.port !== 'error') { return false; } + if (s.payload.port !== "error") { + return false; + } // We only care if spec isn't expecting errors - if (typeof expectData.error !== 'undefined') { return false; } + if (typeof expectData.error !== "undefined") { + return false; + } return true; } return false; } // send input packets - const sendPackets = Promise.promisify(protocol.sendPackets); - Promise.resolve() - .then(() => sendPackets(client, currentGraph, inputData)) - .then(() => observer.until(checkSuccess, checkFailure)) - .then((signals) => signalsToReceived(signals)) - .nodeify(callback); + await protocol.sendPackets(client, currentGraph, inputData); + const signals = await observer.until(checkSuccess, checkFailure); + return signalsToReceived(signals); } +/** + * Whether a suite has any testcases that are not skipped. + * @param {import("./testsuite.js").Suite} suite Testsuite + * @returns {boolean} True if the suite needs setup + */ function needsSetup(suite) { - const notSkipped = suite.cases.filter((c) => !c.skip); + const notSkipped = (suite.cases ?? []).filter((c) => !c.skip); return notSkipped.length > 0; } -class Runner { +/** + * Runs testsuites against a connected FBP runtime. + */ +export class Runner { + /** + * @param {object|import("fbp-client").FbpClient} client Either a runtime definition, + * or an already-connected fbp-client instance + * @param {RunnerOptions} [options] Runner options; timeouts are in milliseconds + */ constructor(client, options = {}) { - this.client = client; + /** @type {import("fbp-client").FbpClient} */ + this.client = /** @type {import("fbp-client").FbpClient} */ (client); this.currentGraphId = null; + /** @type {Record} */ this.components = {}; - this.parentElement = null; + this.parentElement = /** @type {object|null} */ (null); + /** @type {RunnerOptions} */ this.options = options; + /** @type {import("flowtrace").FlowtraceRecorder|null} */ this.recorder = null; - if ((this.options.connectTimeout == null)) { this.options.connectTimeout = 5 * 1000; } - if ((this.options.commandTimeout == null)) { this.options.commandTimeout = 3 * 1000; } + if (this.options.connectTimeout == null) { + this.options.connectTimeout = 5 * 1000; + } + if (this.options.commandTimeout == null) { + this.options.commandTimeout = 3 * 1000; + } } - prepareClient(callback) { - if ((this.client.protocol != null) && (this.client.address != null)) { + /** + * Ensure this.client is a connected fbp-client instance. + * @returns {Promise} Connected client + */ + async prepareClient() { + const isDefinition = "protocol" in this.client && "address" in this.client; + if (isDefinition) { // is a runtime definition - Promise.resolve() - .then(() => fbpClient(this.client, { commandTimeout: this.options.commandTimeout })) - .then((client) => { - this.client = client; - - if (this.parentElement && (client.definition.protocol === 'iframe')) { - // We need to set up the parent element in this case - client.transport.setParentElement(this.parentElement); - } - - return client; - }) - .nodeify(callback); - return; + const client = await fbpClient(this.client, { + commandTimeout: this.options.commandTimeout, + }); + + if (this.parentElement && client.definition.protocol === "iframe") { + // We need to set up the parent element in this case + client.transport.setParentElement(this.parentElement); + } + + this.client = client; + return client; } // This is a client instance - callback(null, this.client); + return this.client; } // TODO: check the runtime capabilities before continuing - connect(callback) { - debug('connect'); + /** + * Connect to the runtime, retrying until the connection is up. + * @returns {Promise} Resolves when connected and capabilities are checked + */ + async connect() { + debug("connect"); + + const client = await this.prepareClient(); + debugReceivedMessages(client); + client.on("network", (message) => { + const { command, payload } = + /** @type {{ command: string, payload: { message?: string } }} */ ( + message + ); + if (command === "output" && payload.message) { + console.log(payload.message); + } + }); - this.prepareClient((err) => { - if (err) { - callback(err); + const timeBetweenAttempts = 500; + const attempts = Math.floor( + (this.options.connectTimeout ?? 5000) / timeBetweenAttempts, + ); + const isOnline = async () => { + const connected = this.client.isConnected(); + if (connected) { return; } - - debugReceivedMessages(this.client); - this.client.on('network', ({ command, payload }) => { - if ((command === 'output') && payload.message) { - console.log(payload.message); - } - }); - - const timeBetweenAttempts = 500; - const attempts = Math.floor(this.options.connectTimeout / timeBetweenAttempts); - const isOnline = () => { - const connected = this.client.isConnected(); - if (connected) { - return Promise.resolve(); - } - return Promise.reject(new Error('Not connected to runtime')); - }; - const tryConnect = () => { - debug('trying to connect'); - return this.client.connect(); - }; - common.retryUntil(tryConnect, isOnline, timeBetweenAttempts, attempts) - .then(() => this.checkCapabilities(['protocol:graph', 'protocol:network', 'protocol:runtime'])) - .nodeify(callback); - }); + throw new Error("Not connected to runtime"); + }; + const tryConnect = async () => { + debug("trying to connect"); + return this.client.connect(); + }; + await retryUntil(tryConnect, isOnline, timeBetweenAttempts, attempts); + await this.checkCapabilities([ + "protocol:graph", + "protocol:network", + "protocol:runtime", + ]); } - disconnect(callback) { - debug('disconnect'); + /** + * Disconnect from the runtime, if connected. + * @returns {Promise} Resolves when disconnected + */ + async disconnect() { + debug("disconnect"); - if (!(this.client != null ? this.client.isConnected() : undefined)) { - callback(); + if (!this.client?.isConnected()) { return; } - Promise.resolve() - .then(() => this.client.disconnect()) - .nodeify(callback); + await this.client.disconnect(); } - checkCapabilities(capabilities) { + /** + * Check that the runtime provides the given capabilities. + * @param {string[]} capabilities Capability names required + * @returns {Promise} Rejects if a capability is missing + */ + async checkCapabilities(capabilities) { if (!this.client.isConnected()) { - return Promise.reject(new Error('Not connected to runtime')); + throw new Error("Not connected to runtime"); } - if (!this.client.definition - || !this.client.definition.capabilities - || !this.client.definition.capabilities.length) { - return Promise.reject(new Error('Runtime provides no capabilities')); + if (!this.client.definition?.capabilities?.length) { + throw new Error("Runtime provides no capabilities"); } - for (let i = 0; i < capabilities.length; i += 1) { - const capability = capabilities[i]; - if (this.client.definition.capabilities.indexOf(capability) === -1) { - return Promise.reject(new Error(`Runtime doesn't provide ${capability}`)); + const caps = this.client.definition?.capabilities ?? []; + for (const capability of capabilities) { + if (caps.indexOf(capability) === -1) { + throw new Error(`Runtime doesn't provide ${capability}`); } } - return Promise.resolve(); } - setupSuite(suite, callback) { + /** + * Set up the fixture graph for a suite, and start its network. + * @param {import("./testsuite.js").Suite} suite Testsuite to set up + * @returns {Promise} Resolves when the network is running + */ + async setupSuite(suite) { debug(`setup suite "${suite.name}"`); if (this.options.traceCallback && !this.recorder) { this.recorder = new FlowtraceRecorder(this.client); } if (!needsSetup(suite)) { - callback(null); return; } if (!this.client.isConnected()) { - callback(new Error('Disconnected from runtime')); - return; + throw new Error("Disconnected from runtime"); } - getFixtureGraph(this, suite, (err, graph) => { - if (err) { - callback(err); - return; - } - protocol.sendGraph(this.client, graph, (sendErr, graphId) => { - if (sendErr) { - callback(sendErr); - return; - } - this.currentGraphId = graphId; - if (this.recorder) { - this.recorder.start(graphId) - .then(() => { - protocol.startNetwork(this.client, graphId, callback); - }, callback); - return; - } - protocol.startNetwork(this.client, graphId, callback); - }); - }); + const graph = await getFixtureGraph(this, suite); + const graphId = await protocol.sendGraph(this.client, graph); + this.currentGraphId = graphId; + if (this.recorder) { + await this.recorder.start(graphId); + } + await protocol.startNetwork(this.client, graphId); } - teardownSuite(suite, callback) { + /** + * Stop the network for a suite, and stop any tracing. + * @param {import("./testsuite.js").Suite} suite Testsuite to tear down + * @returns {Promise} Resolves when the network is stopped + */ + async teardownSuite(suite) { debug(`teardown suite "${suite.name}"`); if (this.recorder && this.currentGraphId) { - // FIXME: This returns a Promise that we should wait for - this.recorder.stop(this.currentGraphId); + await this.recorder.stop(this.currentGraphId); } if (!needsSetup(suite)) { - callback(null); return; } if (!this.client.isConnected()) { - callback(new Error('Disconnected from runtime')); - return; + throw new Error("Disconnected from runtime"); } if (!this.currentGraphId) { // Graph was never successfully set up, so no need to stop - callback(); return; } // FIXME: also remove the graph. Ideally using a 'destroy' message in FBP protocol - protocol.stopNetwork(this.client, this.currentGraphId, callback); + await protocol.stopNetwork(this.client, this.currentGraphId); } - runTest(testcase, callback) { + /** + * Run a single testcase, recording the actual outputs in the sequence. + * @param {import("./testsuite.js").Testcase} testcase Testcase to run + * @returns {Promise} The input/expect sequence with actuals filled in + */ + async runTest(testcase) { debug(`runtest "${testcase.name}"`); if (!this.client.isConnected()) { - callback(new Error('Disconnected from runtime')); - return; + throw new Error("Disconnected from runtime"); + } + if (!this.currentGraphId) { + // Suite has not been set up + throw new Error("No graph is set up for the suite"); } // XXX: normalize and validate in testsuite.js instead? - const inputs = common.isArray(testcase.inputs) ? testcase.inputs : [testcase.inputs]; - const expects = common.isArray(testcase.expect) ? testcase.expect : [testcase.expect]; + const inputs = /** @type {Record|Record[]} */ ( + testcase.inputs + ); + const expects = /** @type {Record|Record[]} */ ( + testcase.expect + ); + const inputList = /** @type {Record[]} */ ( + isArray(inputs) ? inputs : [inputs] + ); + const expectList = /** @type {Record[]} */ ( + isArray(expects) ? expects : [expects] + ); + /** @type {TestSequenceEntry[]} */ const sequence = []; - for (let i = 0, end = inputs.length, asc = end >= 0; - asc ? i < end : i > end; - asc ? i += 1 : i -= 1) { + for (let i = 0; i < inputList.length; i += 1) { sequence.push({ - inputs: inputs[i], - expect: expects[i], + inputs: inputList[i], + expect: expectList[i], }); } - const sendWait = (data, cb) => sendMessageAndWait( - this.client, - this.currentGraphId, - data.inputs, - data.expect, - cb, - ); - common.asyncSeries(sequence, sendWait, (err, actuals) => { - if (this.options.traceCallback && this.currentGraphId && this.recorder) { - this.options.traceCallback( + const actuals = []; + for (const data of sequence) { + actuals.push( + await sendMessageAndWait( + // eslint-disable-line no-await-in-loop + this.client, this.currentGraphId, - testcase.name, - this.recorder.dump(this.currentGraphId), - ); - } - if (err) { - callback(err); - return; - } - actuals.forEach((r, idx) => { - sequence[idx].actual = r; - }); - callback(null, sequence); + data.inputs, + data.expect, + ), + ); + } + + if (this.options.traceCallback && this.currentGraphId && this.recorder) { + this.options.traceCallback( + this.currentGraphId, + testcase.name ?? "", + this.recorder.dump(this.currentGraphId), + ); + } + actuals.forEach((r, idx) => { + sequence[idx].actual = r; }); + return sequence; } } -// TODO: should this go into expectation? -function checkResults(results) { +/** + * Check actual test results against the expectations. + * @param {Array} results + * Input/expect/actual sequence as produced by Runner#runTest + * @returns {{ passed: boolean, error?: Error }} Result of the check + */ +export function checkResults(results) { + /** @type {{ passed: boolean, error?: Error }} */ let result; const actuals = results.filter((r) => r.actual != null); const expects = results.filter((r) => r.expect != null); if (actuals.length < expects.length) { result = { passed: false, - error: new Error(`Only got ${actuals.length} output messages out of ${expects.length}`), + error: new Error( + `Only got ${actuals.length} output messages out of ${expects.length}`, + ), }; return result; } @@ -417,10 +549,10 @@ function checkResults(results) { const mappedResults = results.map((res) => { res.error = null; try { - expectation.expect(res.expect, res.actual); + checkExpectation(res.expect, res.actual); } catch (e) { // FIXME: only catch actual AssertionErrors - res.error = e; + res.error = /** @type {Error} */ (e); } return res; }); @@ -429,168 +561,188 @@ function checkResults(results) { if (failures.length === 0) { result = { passed: true }; } else if (expects.length === 1) { - result = { error: failures[0].error }; - } else if ((expects.length > 1) && (failures.length === 1)) { + result = { + passed: false, + error: failures[0].error ?? new Error("unknown expectation failure"), + }; + } else if (expects.length > 1 && failures.length === 1) { const index = mappedResults.findIndex((r) => r.error); const failed = mappedResults[index]; - result = { error: new Error(`Expectation ${index} of sequence failed: ${failed.error.message}`) }; + result = { + passed: false, + error: new Error( + `Expectation ${index} of sequence failed: ${failed.error?.message}`, + ), + }; } else { - const errors = mappedResults.map((r) => (r.error != null ? r.error.message : undefined) || ''); - result = { error: new Error(`Multiple failures in sequence: ${errors}`) }; + const errors = mappedResults.map( + (r) => (r.error != null ? r.error.message : undefined) || "", + ); + result = { + passed: false, + error: new Error(`Multiple failures in sequence: ${errors}`), + }; } return result; } -function runTestAndCheck(runner, testcase, callback) { +/** + * Run a single testcase and check its results. + * @param {RunnerLike} runner Runner with a set up suite + * @param {import("./testsuite.js").Testcase} testcase Testcase to run + * @returns {Promise<{ passed: boolean, error?: Error }>} Test result + */ +export async function runTestAndCheck(runner, testcase) { if (testcase.skip) { // TODO: pass some skipped state? its indirectly in .skip though - callback(null, { passed: true }); - return; + return { passed: true }; } // XXX: normalize and validate in testsuite.js instead? - const inputs = common.isArray(testcase.inputs) ? testcase.inputs : [testcase.inputs]; - const expects = common.isArray(testcase.expect) ? testcase.expect : [testcase.expect]; - if (inputs.length !== expects.length) { - callback(null, { + const inputs = /** @type {Record|Record[]} */ ( + testcase.inputs + ); + const expects = /** @type {Record|Record[]} */ ( + testcase.expect + ); + const inputList = /** @type {Record[]} */ ( + isArray(inputs) ? inputs : [inputs] + ); + const expectList = /** @type {Record[]} */ ( + isArray(expects) ? expects : [expects] + ); + if (inputList.length !== expectList.length) { + return { passed: false, - error: new Error(`Test sequence length mismatch. Got ${inputs.length} inputs and ${expects.length} expectations`), - }); - return; + error: new Error( + `Test sequence length mismatch. Got ${inputList.length} inputs and ${expectList.length} expectations`, + ), + }; } - runner.runTest(testcase, (err, results) => { - let result; - if (err) { - // Map error to a test failure - result = { - passed: false, - error: err, - }; - callback(null, result); - return; - } - result = checkResults(results); - if (result.error) { - result.passed = false; - } - callback(null, result); - }); + /** @type {TestSequenceEntry[]} */ + let results; + try { + results = await runner.runTest(testcase); + } catch (err) { + // Map error to a test failure + return { + passed: false, + error: /** @type {Error} */ (err), + }; + } + + const result = checkResults(results); + return result; } -function runSuite(runner, suite, runTest, callback) { +/** + * Run all testcases in a suite. + * @param {Runner} runner Runner to run with + * @param {import("./testsuite.js").Suite} suite Testsuite to run + * @param {(testcase: import("./testsuite.js").Testcase) => Promise} runTest Async function running and checking one testcase + * @returns {Promise} The suite + */ +export async function runSuite(runner, suite, runTest) { if (suite.skip) { // TODO: pass some skipped state? its indirectly in .skip though - callback(null, suite); - return; + return suite; } - runner.setupSuite(suite, (err) => { - debug('setup suite', err); - if (err) { - callback(err, suite); - return; - } - - common.asyncSeries(suite.cases, runTest, (runErr) => { - debug('testrun complete', runErr); - if (runErr) { - runner.teardownSuite(suite, (teardownErr) => { - debug('teardown suite', teardownErr); - callback(runErr, suite); - }); - return; - } + try { + await runner.setupSuite(suite); + } catch (err) { + debug("setup suite", err); + throw err; + } - runner.teardownSuite(suite, (teardownErr) => { - debug('teardown suite', teardownErr); - callback(teardownErr, suite); - }); - }); - }); + try { + await asyncSeries(suite.cases ?? [], runTest); + } finally { + await runner + .teardownSuite(suite) + .catch((err) => debug("teardown suite", err)); + } + return suite; } -function loadComponentSuites(componentTests, callback) { - function loadYAML(source, cb) { - testsuite.loadYAML(source, (err, suite) => { - if (err) { - // ignore, could be non fbp-spec test - // TODO: include tests type in FBP protocol, so we know whether this is error or legit - cb(null, null); - return; - } - cb(null, suite); - }); - } +/** + * Load and parse the fbp-spec tests attached to components. + * Non-fbp-spec test sources are ignored. + * @param {Record} componentTests YAML test sources, keyed by component name + * @returns {Promise} Parsed testsuites + */ +export async function loadComponentSuites(componentTests) { const tests = Object.keys(componentTests).map((name) => componentTests[name]); - common.asyncSeries(tests, loadYAML, (err, suites) => { - if (err) { - callback(err); - return; + const suites = await asyncSeries(tests, (source) => { + try { + return Promise.resolve(testsuite.loadYAML(source)); + } catch { + // ignore, could be non fbp-spec test + // TODO: include tests type in FBP protocol, so we know whether this is error or legit + return Promise.resolve(null); } - callback(null, suites.filter((s) => s !== null).reduce((a, b) => a.concat(b), [])); }); + return suites.filter((s) => s !== null).reduce((a, b) => a.concat(b), []); } -exports.getComponentSuites = function getComponentSuites(runner, callback) { - protocol.getCapabilities(runner.client, (err, caps) => { - if (err) { - callback(err); - return; +/** + * Get the testsuites attached to components on the runtime. + * Requires the runtime to support the 'component:getsource' capability. + * @param {Runner} runner Runner connected to the runtime + * @returns {Promise} Testsuites + */ +export async function getComponentSuites(runner) { + const caps = await protocol.getCapabilities(runner.client); + if (caps.indexOf("component:getsource") === -1) { + return []; + } + + const tests = await protocol.getComponentTests(runner.client); + const suites = await loadComponentSuites(tests); + debug("get component suites", Object.keys(tests).length, suites.length); + return suites; +} + +/** + * Run all the given suites, updating the testcase objects in-place + * with .passed and .error states as tests are run. + * @param {Runner} runner Runner to run with + * @param {import("./testsuite.js").Suite[]} suites Suites to run + * @param {(suites: import("./testsuite.js").Suite[]) => void} [updateCallback] Called when test state is updated + * @returns {Promise} Resolves when all suites have run + */ +export async function runAll(runner, suites, updateCallback = () => {}) { + debug( + "running suites", + suites.map((s) => s.name), + ); + + /** + * Run and check a single testcase, storing the result on the testcase. + * @param {import("./testsuite.js").Testcase} testcase Testcase to run + * @returns {Promise} + */ + async function runTest(testcase) { + const results = await runTestAndCheck(runner, testcase); + // ignore error to not bail out early + const resultRecord = /** @type {Record} */ (results); + for (const key of Object.keys(resultRecord)) { + testcase[key] = resultRecord[key]; } - if (caps.indexOf('component:getsource') === -1) { - callback(null, []); - return; + if (testcase.error instanceof Error) { + testcase.error = testcase.error.message; } - - protocol.getComponentTests(runner.client, (testsErr, tests) => { - if (testsErr) { - callback(testsErr); - return; - } - loadComponentSuites(tests, (loadErr, suites) => { - if (loadErr) { - callback(loadErr); - return; - } - debug('get component suites', tests.length, suites.length); - callback(null, suites); - }); - }); - }); -}; - -// will update each of the testcases in @suites -// with .passed and .error states as tests are ran -function runAll(runner, suites, updateCallback, doneCallback) { - function runTest(testcase, callback) { - function done(error) { - updateCallback(suites); - callback(error); - } - - runTestAndCheck(runner, testcase, (err, results) => { - // ignore error to not bail out early - const tcase = testcase; - Object.keys(results).forEach((key) => { - const val = results[key]; - tcase[key] = val; - }); - if (testcase.error) { - tcase.error = testcase.error.message; - } - debug(`ran test "${testcase.name}"`, testcase.passed, err); - done(null); - }); + debug(`ran test "${testcase.name}"`, testcase.passed); + updateCallback(suites); } - const runOneSuite = (suite, cb) => runSuite(runner, suite, runTest, cb); - - debug('running suites', suites.map((s) => s.name)); - common.asyncSeries(suites, runOneSuite, (err) => doneCallback(err)); + for (const suite of suites) { + try { + await runSuite(runner, suite, runTest); // eslint-disable-line no-await-in-loop + } catch (err) { + debug("suite failed", err); + } + } } - -exports.Runner = Runner; -exports.runAll = runAll; -exports.runTestAndCheck = runTestAndCheck; diff --git a/lib/subprocess.js b/lib/subprocess.js index 43f5534..06b380f 100644 --- a/lib/subprocess.js +++ b/lib/subprocess.js @@ -1,63 +1,66 @@ -const debug = require('debug')('fbp-spec:subprocess'); - -exports.start = function start(command, options, callback) { - let childProcess; - try { - // eslint-disable-next-line global-require - childProcess = require('child_process'); - } catch (error) { - callback(error); - return null; - } - - if ((options.timeout == null)) { - const opts = options; - opts.timeout = 300; - } - - let started = false; - let stderr = ''; - - // FIXME: using sh to interpret command will Unix-specific - const prog = 'sh'; - const args = ['-c', command]; - const child = childProcess.spawn(prog, args); - - debug('spawned', `'${prog} ${args.join(' ')}'`); - debug('waiting for output'); - - child.on('error', (err) => callback(err)); - - child.stdout.on('data', (data) => { - const dataString = data.toString(); - debug('sub stdout', dataString); - if (!started) { - debug('got output, transitioning to started'); - started = true; - // give process some time to open port - setTimeout(callback, 100); - } - }); +/** + * Start runtimes-under-test as subprocesses. + * @module lib/subprocess + */ - child.stderr.on('data', (data) => { - const dataString = data.toString(); - stderr += dataString; - debug('sub stderr', dataString); - if (!started) { - debug('got stderr, failing'); - started = true; - callback(new Error(`Subprocess wrote on stderr: '${stderr}'`)); - } - }); +import debugLib from "debug"; + +const debug = debugLib("fbp-spec:subprocess"); + +/** + * Start a runtime as a subprocess, using sh to interpret the command. + * Considers the process started once it produces output, or after a timeout. + * Uses a dynamic import so that this module can also be bundled for browsers. + * @param {string} command Shell command to launch the runtime + * @param {{ timeout?: number }} [options] Options; timeout in ms before assuming startup succeeded + * @returns {Promise} The spawned child process + */ +export async function start(command, options = {}) { + const timeout = options.timeout ?? 300; + const childProcess = await import("node:child_process"); + + return new Promise((resolve, reject) => { + // FIXME: using sh to interpret command will Unix-specific + const prog = "sh"; + const args = ["-c", command]; + const child = childProcess.spawn(prog, args); + + debug("spawned", `'${prog} ${args.join(" ")}'`); + debug("waiting for output"); - setTimeout(() => { - if (!started) { - debug('timeout waiting for output, assuming started'); - started = true; - callback(null); - } - }, - options.timeout); - - return child; -}; + let started = false; + let stderr = ""; + + child.on("error", (err) => reject(err)); + + child.stdout.on("data", (data) => { + const dataString = data.toString(); + debug("sub stdout", dataString); + if (!started) { + debug("got output, transitioning to started"); + started = true; + // give process some time to open port + setTimeout(() => resolve(child), 100); + } + }); + + child.stderr.on("data", (data) => { + const dataString = data.toString(); + stderr += dataString; + debug("sub stderr", dataString); + if (!started) { + debug("got stderr, failing"); + started = true; + reject(new Error(`Subprocess wrote on stderr: '${stderr}'`)); + } + }); + + setTimeout(() => { + if (!started) { + debug("timeout waiting for output, assuming started"); + started = true; + resolve(child); + } + }, timeout); + }); +} diff --git a/lib/testsuite.js b/lib/testsuite.js index 33e95e7..ae08dfc 100644 --- a/lib/testsuite.js +++ b/lib/testsuite.js @@ -1,165 +1,159 @@ -/* eslint-env browser, node */ -const fetch = require('isomorphic-fetch'); -const path = require('path'); -const common = require('./common'); -// eslint-disable-next-line import/no-unresolved -const schemas = require('../schema'); - -exports.validate = function validate(obj) { - // eslint-disable-next-line global-require - const tv4 = (typeof window !== 'undefined' && window.tv4) ? window.tv4 : require('tv4'); - if (!tv4) { - throw new Error('TV4 not available, unable to validate'); - } - +/** + * Loading, normalizing and validating of fbp-spec testsuites. + * @module lib/testsuite + */ + +import { extname, join } from "node:path"; +import debugLib from "debug"; +import { loadAll } from "js-yaml"; +import tv4 from "tv4"; + +import schemas from "../schema/index.js"; +import { isArray, startsWith } from "./common.js"; + +const debug = debugLib("fbp-spec:testsuite"); + +/** + * A single test case within a testsuite. + * @typedef {object} TestcaseProps + * @property {string} [name] Name of this testcase + * @property {string} [assertion] Description of what this testcase asserts + * @property {Record|Array>} [inputs] Inputs sent to fixture inports, or a sequence of such + * @property {Record|Array>} [expect] Expected data on outports, or a sequence of such + * @property {string} [skip] Reason to skip this testcase + * @property {number} [timeout] Maximum time to run this testcase for, in milliseconds + * @property {boolean} [passed] Set to the test result by the runner + * @property {string|Error} [error] Set to the test error message, if the test failed + * @property {boolean} [shown] Internal: whether testcase result has been shown in CLI output + * @property {*} [actual] Set to the actual output data by the runner + */ + +/** + * A testcase, with free-form additional properties from the YAML source. + * @typedef {TestcaseProps & Record} Testcase + */ + +/** + * A fbp-spec testsuite: a fixture plus a set of testcases. + * @typedef {object} SuiteProps + * @property {string} [name] Name of the suite; defaults to the topic + * @property {string} [topic] Name of the component or graph under test + * @property {{ type: string, data: string }} [fixture] Fixture graph definition + * @property {Testcase[]} [cases] Testcases to run against the fixture + * @property {string} [skip] Reason to skip the whole suite + * @property {number} [timeout] Maximum time to run the suite for, in milliseconds + * @property {boolean} [titleshown] Internal: whether suite title has been shown in CLI output + */ + +/** + * A testsuite, with free-form additional properties from the YAML source. + * @typedef {SuiteProps & Record} Suite + */ + +/** + * JSON Schema definition for fbp-spec data. + * @typedef {{ id: string, title?: string, description?: string, [key: string]: * }} Schema + */ + +/** + * Validate an fbp-spec object against the fbp-spec schemas. + * @param {*} obj Object to validate, typically a testsuite or testsfile structure + * @returns {{ passed: boolean, errors: object[], missing: string[], valid: boolean }} Validation results + */ +export function validate(obj) { tv4.reset(); - Object.keys(schemas).forEach((name) => { + for (const name of Object.keys(schemas)) { const schema = schemas[name]; tv4.addSchema(schema.id, schema); - }); - - const results = tv4.validateMultiple(obj, 'testsfile.json'); - if (results.missing.length) { - results.passed = false; } + + const results = tv4.validateMultiple(obj, "testsfile.json"); + results.passed = results.valid && results.missing.length === 0; return results; -}; +} -function normalize(suite) { - // Default name to topic - const s = suite; +/** + * Apply defaults to a testsuite object. + * Defaults name to the topic, and cases to an empty list. + * @param {Suite} base Testsuite definition, as parsed from YAML + * @returns {Suite} Normalized testsuite + */ +export function create(base) { + const suite = base; if (!suite.name) { - s.name = suite.topic; + suite.name = suite.topic; } if (!suite.cases) { - s.cases = []; + suite.cases = []; } - return suite; } -exports.create = (base) => normalize(base); - -exports.loadYAML = function loadYAML(data, callback) { - const suites = []; - try { - // eslint-disable-next-line global-require - const yaml = (typeof window !== 'undefined' && window.jsyaml) ? window.jsyaml : require('js-yaml'); - yaml.loadAll(data, (doc) => suites.push(normalize(doc))); - } catch (e) { - callback(e); - return; - } - callback(null, suites); -}; - -exports.loadYAMLSync = function loadYAMLSync(data) { - // eslint-disable-next-line global-require - const yaml = (typeof window !== 'undefined' && window.jsyaml) ? window.jsyaml : require('js-yaml'); - const suites = []; - yaml.loadAll(data, (doc) => suites.push(normalize(doc))); - return suites; -}; - -function loadHTTP(url, callback) { - fetch(url) - .then((res) => res.text()) - .then((source) => new Promise((resolve, reject) => { - exports.loadYAML(source, (err, suites) => { - if (err) { - reject(err); - return; - } - resolve(suites); - }); - })) - .then((suites) => { - callback(null, suites); - }, callback); +/** + * Parse YAML containing one or more testsuites. + * @param {string} data YAML source + * @returns {Suite[]} Normalized testsuites + */ +export function loadYAML(data) { + const docs = /** @type {Suite[]} */ (loadAll(data)); + return docs.map((doc) => create(doc)); } -function getFileSync(test) { - // eslint-disable-next-line global-require - const fs = require('fs'); - - const stat = fs.statSync(test); - if (stat.isDirectory()) { - const files = fs.readdirSync(test); - const filePaths = files - .filter((f) => path.extname(f) === '.yaml') - .map((f) => path.join(test, f)); - return exports.getSuitesSync(filePaths); +/** + * Read testsuites from YAML files, directories of YAML files, or HTTP URLs. + * @param {string|string[]} tests File or directory path, or HTTP(S) URL, or a list of such + * @returns {Promise} Normalized testsuites + */ +export async function getSuites(tests) { + const list = /** @type {string[]} */ (isArray(tests) ? tests : [tests]); + const suitesList = []; + for (const test of list) { + suitesList.push(await loadLocation(test)); // eslint-disable-line no-await-in-loop } - const c = fs.readFileSync(test, 'utf-8'); - return exports.loadYAMLSync(c); + return suitesList.reduce((a, b) => a.concat(b), []); } -function getFile(test, callback) { - // eslint-disable-next-line global-require - const fs = require('fs'); - - fs.stat(test, (statErr, stat) => { - if (statErr) { - callback(statErr); - return; - } - if (stat.isDirectory()) { - fs.readdir(test, (dirErr, files) => { - if (dirErr) { - callback(dirErr); - return; - } - const filePaths = files - .filter((f) => path.extname(f) === '.yaml') - .map((f) => path.join(test, f)); - exports.getSuites(filePaths, callback); - }); - return; - } - fs.readFile(test, 'utf-8', (readErr, yaml) => { - if (readErr) { - callback(readErr); - return; - } - exports.loadYAML(yaml, callback); - }); - }); +/** + * Load suites from a single file path, directory, or URL. + * @param {string} test Location to load from + * @returns {Promise} Normalized testsuites + */ +async function loadLocation(test) { + if (startsWith(test, "http://") || startsWith(test, "https://")) { + return loadHTTP(test); + } + return getFile(test); } -// FIXME: get rid of this... -exports.getSuitesSync = function getSuitesSync(tests) { - if (!common.isArray(tests)) { - return exports.getSuitesSync([tests]); - } +/** + * Fetch a YAML testsfile over HTTP(S), and parse it. + * @param {string} url URL to fetch from + * @returns {Promise} Normalized testsuites + */ +async function loadHTTP(url) { + debug("loading suites from", url); + const res = await fetch(url); + const source = await res.text(); + return loadYAML(source); +} - let suites = []; - tests.forEach((test) => { - suites = suites.concat(getFileSync(test)); - }); - return suites; -}; - -exports.getSuites = function getSuites(tests, callback) { - if (!common.isArray(tests)) { - exports.getSuites([tests], callback); - return; +/** + * Read a testsfile from disk. + * Directories are scanned for `.yaml` files. + * Uses a dynamic import so that this module can also be bundled for browsers. + * @param {string} test File or directory path + * @returns {Promise} Normalized testsuites + */ +async function getFile(test) { + const fs = await import("node:fs"); + const info = await fs.promises.stat(test); + if (info.isDirectory()) { + const files = await fs.promises.readdir(test); + const filePaths = files + .filter((f) => extname(f) === ".yaml") + .map((f) => join(test, f)); + return getSuites(filePaths); } - - const loadTest = (test, cb) => { - if (common.startsWith(test, 'http://' || common.startsWith('https://'))) { - loadHTTP(test, cb); - return; - } - getFile(test, cb); - }; - - common.asyncSeries(tests, loadTest, (err, suitesList) => { - if (err) { - callback(err); - return; - } - // flatten list - const suites = suitesList.reduce((a, b) => a.concat(b), []); - callback(null, suites); - }); -}; + const yaml = await fs.promises.readFile(test, "utf-8"); + return loadYAML(yaml); +} diff --git a/lib/websocket.js b/lib/websocket.js deleted file mode 100644 index 3a1bf55..0000000 --- a/lib/websocket.js +++ /dev/null @@ -1,73 +0,0 @@ -// eslint-disable-next-line import/no-extraneous-dependencies -const WebSocketServer = require('websocket').server; - -// XXX: Copied from https://github.com/noflo/noflo-runtime-websocket/blob/master/runtime/network.js -// should probably reuse it as-is - -class WebSocketRuntime { - constructor(options = {}) { - this.options = options; - this.connections = []; - } - - send(protocol, topic, payload, context) { - if (!context.connection || !context.connection.connected) { - return; - } - if ((topic === 'error') && payload instanceof Error) { - this.send(protocol, topic, { - message: payload.message, - stack: payload.stack, - }, context); - return; - } - context.connection.sendUTF(JSON.stringify({ - protocol, - command: topic, - payload, - })); - } - - sendAll(protocol, topic, payload) { - this.connections.forEach((connection) => { - this.send(protocol, topic, payload, { - connection, - }); - }); - } -} - -module.exports = (httpServer, options) => { - const wsServer = new WebSocketServer({ httpServer }); - const runtime = new WebSocketRuntime(options); - - function handleMessage(message, connection) { - if (message.type === 'utf8') { - let contents; - try { - contents = JSON.parse(message.utf8Data); - } catch (e) { - if (e.stack) { - console.error(e.stack); - } else { - console.error(`Error: ${e.toString()}`); - } - return; - } - runtime.receive(contents.protocol, contents.command, contents.payload, { connection }); - } - } - - wsServer.on('request', (request) => { - const subProtocol = request.requestedProtocols.indexOf('noflo') !== -1 ? 'noflo' : null; - const connection = request.accept(subProtocol, request.origin); - runtime.connections.push(connection); - connection.on('message', (message) => handleMessage(message, connection)); - return connection.on('close', () => { - if (runtime.connections.indexOf(connection) === -1) { return; } - runtime.connections.splice(runtime.connections.indexOf(connection), 1); - }); - }); - - return runtime; -}; diff --git a/package.json b/package.json index 51500a2..20566fc 100644 --- a/package.json +++ b/package.json @@ -13,64 +13,52 @@ "bugs": "http://github.com/flowbased/fbp-spec/issues", "homepage": "http://flowbased.org", "license": "MIT", + "type": "module", "main": "index.js", + "exports": { + ".": "./index.js", + "./schema": "./schema/index.js", + "./runner": "./lib/runner.js", + "./testsuite": "./lib/testsuite.js", + "./expectation": "./lib/expectation.js", + "./package.json": "./package.json" + }, + "bin": { + "fbp-spec": "./bin/fbp-spec" + }, "engines": { - "node": ">=6.0.0" + "node": ">=20.0.0" }, + "files": [ + "bin", + "lib", + "schema", + "index.js" + ], "scripts": { - "build": "grunt build", - "pretest": "eslint *.js lib/*.js spec/*.js", - "test": "grunt test --stack" - }, - "bin": { - "fbp-spec": "./bin/fbp-spec", - "fbp-spec-mocha": "./bin/fbp-spec-mocha" + "build": "node scripts/build-schemas.js", + "lint": "npx @biomejs/biome check --use-editorconfig=true lib/ schema/ scripts/ spec/ examples/ index.js bin/", + "format": "npx @biomejs/biome check --use-editorconfig=true --write lib/ schema/ scripts/ spec/ examples/ index.js bin/", + "typecheck": "tsc", + "test": "npm run lint && npm run typecheck && node --test \"spec/*.test.js\"" }, "dependencies": { - "bluebird": "^3.5.0", - "chai": "~4.2.0", - "commander": "~6.2.0", - "debug": "~4.3.1", - "fbp": "^1.7.0", - "fbp-client": "^0.4.1", + "debug": "^4.4.3", + "fbp": "^1.8.0", + "fbp-client": "^0.4.3", "fbp-graph": "^0.7.0", - "flowtrace": "^0.1.2", - "isomorphic-fetch": "^3.0.0", - "js-yaml": "~4.0.0", - "jsonpath": "^1.0.2", - "mocha": "~8.2.0", - "slug": "^4.0.2", - "tv4": "~1.3.0" + "flowtrace": "^0.1.10", + "js-yaml": "^5.4.3", + "jsonpath": "^1.3.0", + "tv4": "^1.3.0" }, "devDependencies": { - "@babel/core": "^7.11.5", - "@babel/preset-env": "^7.11.5", - "babel-loader": "^8.1.0", - "buffer": "^6.0.1", - "es6-shim": "^0.35.3", - "eslint": "^7.11.0", - "eslint-config-airbnb-base": "^14.2.0", - "eslint-plugin-chai": "0.0.1", - "eslint-plugin-import": "^2.22.1", - "eslint-plugin-mocha": "^8.0.0", - "events": "^3.2.0", - "grunt": "^1.0.1", - "grunt-contrib-copy": "^1.0.0", - "grunt-contrib-watch": "^1.0.0", - "grunt-exec": "^3.0.0", - "grunt-karma": "^4.0.0", - "grunt-mocha-test": "^0.13.2", - "grunt-webpack": "^4.0.2", - "grunt-yaml": "^0.4.2", - "grunt-yamllint": "https://github.com/gruntjs-updater/grunt-yamllint.git#8e7c04a", - "karma": "^5.2.3", - "karma-chai": "^0.1.0", - "karma-chrome-launcher": "^3.1.0", - "karma-mocha": "^2.0.1", - "karma-mocha-reporter": "^2.2.5", - "path-browserify": "^1.0.1", - "url": "^0.11.0", - "webpack": "^5.1.3", - "websocket": "^1.0.26" + "@biomejs/biome": "^2.5.15", + "@types/debug": "^4.1.13", + "@types/jsonpath": "^0.2.4", + "@types/node": "^26.6.5", + "@types/websocket": "^1.0.10", + "typescript": "^7.0.2", + "websocket": "^1.0.35" } } diff --git a/schema/base.json b/schema/base.json index bcefb98..619a911 100644 --- a/schema/base.json +++ b/schema/base.json @@ -10,4 +10,4 @@ "example": "products.book[2]" } } -} \ No newline at end of file +} diff --git a/schema/expectation.json b/schema/expectation.json index 01814eb..05e5bba 100644 --- a/schema/expectation.json +++ b/schema/expectation.json @@ -42,4 +42,4 @@ "description": "String or array contains substring/item" } } -} \ No newline at end of file +} diff --git a/schema/expectations.json b/schema/expectations.json index 3269564..26e5e8c 100644 --- a/schema/expectations.json +++ b/schema/expectations.json @@ -7,4 +7,4 @@ "items": { "$ref": "expectation.json" } -} \ No newline at end of file +} diff --git a/schema/index.js b/schema/index.js index 4984f82..1080bce 100644 --- a/schema/index.js +++ b/schema/index.js @@ -1,13 +1,34 @@ -/* eslint global-require: "off" */ -try { - module.exports = { - base: require('./base.json'), - testcase: require('./testcase.json'), - testsuite: require('./testsuite.json'), - testsuites: require('./testsuites.json'), - expectations: require('./expectations.json'), - testsfile: require('./testsfile.json'), - }; -} catch (e) { - console.log('fbp-spec: Failed to load schemas', e); +/** + * JSON Schema definitions for the fbp-spec data format. + * Built from the YAML sources in `schemata/` by `npm run build`. + * @module schema/index + */ + +/** + * JSON Schema definition for fbp-spec data. + * @typedef {{ id: string, title?: string, description?: string, [key: string]: * }} Schema + */ + +import schemas from "./schemas.js"; + +/** @type {Record} */ +const allSchemas = schemas; +export default allSchemas; +export const { + base, + testcase, + testsuite, + testsuites, + expectations, + testsfile, +} = allSchemas; + +/** + * Get a single schema by name. + * @param {string} name Schema name, with or without `.json` suffix + * @returns {Schema|undefined} The schema, if found + */ +export function getSchema(name) { + const schemaId = name.replace(".json", ""); + return allSchemas[schemaId]; } diff --git a/schema/outputdata.json b/schema/outputdata.json index caa2275..74ce837 100644 --- a/schema/outputdata.json +++ b/schema/outputdata.json @@ -14,4 +14,4 @@ } ] } -} \ No newline at end of file +} diff --git a/schema/schemas.js b/schema/schemas.js new file mode 100644 index 0000000..47697b4 --- /dev/null +++ b/schema/schemas.js @@ -0,0 +1,219 @@ +// Generated by scripts/build-schemas.js from schemata/*.yaml. Do not edit. +export default { + "base": { + "id": "base.json", + "$schema": "http://json-schema.org/draft-04/schema", + "title": "Base definitions", + "definitions": { + "JsonPath": { + "type": "string", + "description": "JsonPath, see http://goessner.net/articles/JsonPath/", + "format": "jsonpath", + "example": "products.book[2]" + } + } + }, + "expectation": { + "id": "expectation.json", + "$schema": "http://json-schema.org/draft-04/schema", + "title": "Expectation", + "description": "An expectation on data.", + "type": "object", + "properties": { + "path": { + "$ref": "base.json#/definitions/JsonPath", + "description": "JSONPath to run messages data through. Result is passed to assertion predicate." + }, + "equals": { + "description": "Strict equality" + }, + "above": { + "description": "Greater than (>)", + "type": "number" + }, + "below": { + "description": "Less than than (<)", + "type": "number" + }, + "type": { + "description": "JSON type", + "type": "string" + }, + "haveKeys": { + "description": "Object has only these keys", + "type": "array", + "items": { + "type": "string" + } + }, + "includeKeys": { + "description": "Object has at least these keys", + "type": "array", + "items": { + "type": "string" + } + }, + "contains": { + "description": "String or array contains substring/item" + } + } + }, + "expectations": { + "id": "testsuites.json", + "$schema": "http://json-schema.org/draft-04/schema", + "title": "Expectation list", + "description": "Array of Expectation", + "type": "array", + "items": { + "$ref": "expectation.json" + } + }, + "outputdata": { + "id": "outputdata.json", + "$schema": "http://json-schema.org/draft-04/schema", + "title": "Output data", + "description": "Data expected in a testcase.\nEach key refers to a port, and the value is a description of the expected message on that port.\nThere can be one expectation (check) on each message, or an array of them.\n", + "type": "object", + "items": { + "oneOf": [ + { + "$ref": "expectation.json" + }, + { + "$ref": "expectations.json" + } + ] + } + }, + "testcase": { + "id": "testcase.json", + "$schema": "http://json-schema.org/draft-04/schema", + "title": "Testcase", + "description": "A single test case", + "type": "object", + "properties": { + "name": { + "type": "string", + "description": "Name of this testcase/scenario" + }, + "assertion": { + "type": "string", + "description": "Description of what this testcase asserts" + }, + "inputs": { + "description": "The inputs send to the fixture input ports, or a sequence of such inputs", + "oneOf": [ + { + "type": "object" + }, + { + "type": "array", + "items": { + "type": "object" + } + } + ] + }, + "expect": { + "description": "Expected data on fixture output ports.\nCan either be an object, or a sequence of such objects.\n", + "oneOf": [ + { + "$ref": "outputdata.json" + }, + { + "type": "array", + "items": [ + { + "$ref": "outputdata.json" + } + ] + } + ] + }, + "timeout": { + "type": "number", + "description": "Timeout (in milliseconds). Overrides that set in the suite", + "default": 2000 + }, + "skip": { + "type": "string", + "description": "Flag to skip testcase.\nShould be a description of the reason for skipping.\n" + } + }, + "required": [ + "name", + "assertion", + "inputs", + "expect" + ] + }, + "testsfile": { + "id": "testsfile.json", + "$schema": "http://json-schema.org/draft-04/schema", + "title": "Tests file", + "description": "One or more suites", + "oneOf": [ + { + "$ref": "testsuites.json" + }, + { + "$ref": "testsuite.json" + } + ] + }, + "testsuite": { + "id": "testsuite.json", + "$schema": "http://json-schema.org/draft-04/schema", + "title": "Test suite", + "description": "A collection of test cases", + "type": [ + "object" + ], + "properties": { + "topic": { + "type": "string", + "description": "The component/graph under test", + "example": "MyComponent3" + }, + "name": { + "type": "string", + "description": "Name of this test suite. Topic is used if no name is provided.", + "default": "" + }, + "fixture": { + "type": "object", + "description": "The graph fixture to run for each test case" + }, + "cases": { + "type": "array", + "description": "The test cases", + "items": { + "$ref": "testcase.json" + }, + "default": [] + }, + "timeout": { + "type": "number", + "description": "Timeout (in milliseconds) for setup and default for each test case.", + "default": 2000 + }, + "skip": { + "type": "string", + "description": "Flag to skip test suite (and all containing test cases).\nShould be a description of the reason for skipping.\n" + } + }, + "required": [ + "topic" + ] + }, + "testsuites": { + "id": "testsuites.json", + "$schema": "http://json-schema.org/draft-04/schema", + "title": "TestSuite list", + "description": "Array of TestSuite", + "type": "array", + "items": { + "$ref": "testsuite.json" + } + } +}; diff --git a/schema/testcase.json b/schema/testcase.json index 9cf419f..5c7e8ce 100644 --- a/schema/testcase.json +++ b/schema/testcase.json @@ -59,4 +59,4 @@ "inputs", "expect" ] -} \ No newline at end of file +} diff --git a/schema/testsfile.json b/schema/testsfile.json index 93aa608..250020b 100644 --- a/schema/testsfile.json +++ b/schema/testsfile.json @@ -11,4 +11,4 @@ "$ref": "testsuite.json" } ] -} \ No newline at end of file +} diff --git a/schema/testsuite.json b/schema/testsuite.json index 2c6957b..809bd82 100644 --- a/schema/testsuite.json +++ b/schema/testsuite.json @@ -42,4 +42,4 @@ "required": [ "topic" ] -} \ No newline at end of file +} diff --git a/schema/testsuites.json b/schema/testsuites.json index 35d0291..63011a3 100644 --- a/schema/testsuites.json +++ b/schema/testsuites.json @@ -7,4 +7,4 @@ "items": { "$ref": "testsuite.json" } -} \ No newline at end of file +} diff --git a/scripts/build-schemas.js b/scripts/build-schemas.js new file mode 100644 index 0000000..547f904 --- /dev/null +++ b/scripts/build-schemas.js @@ -0,0 +1,39 @@ +/** + * Build JSON Schema artifacts from the YAML sources in schemata/. + * + * Outputs: + * - schema/.json: one JSON Schema document per source file + * - schema/schemas.js: an ES module exporting all schemas, for runtime loading + * without filesystem access (also usable in browsers) + * + * Replaces the old grunt-yaml build step. Run via `npm run build`. + */ + +import { readdir, readFile, writeFile } from "node:fs/promises"; +import { join } from "node:path"; +import { load } from "js-yaml"; + +const schemataDir = join(import.meta.dirname, "..", "schemata"); +const schemaDir = join(import.meta.dirname, "..", "schema"); + +const files = (await readdir(schemataDir)) + .filter((f) => f.endsWith(".yaml")) + .sort(); + +/** @type {Record} */ +const schemas = {}; +for (const file of files) { + const name = file.replace(/\.yaml$/, ""); + const source = await readFile(join(schemataDir, file), "utf-8"); + const doc = /** @type {object} */ (load(source)); + schemas[name] = doc; + + const jsonDest = join(schemaDir, `${name}.json`); + await writeFile(jsonDest, `${JSON.stringify(doc, null, 2)}\n`); + console.log(`built schema/${name}.json`); +} + +const jsDest = join(schemaDir, "schemas.js"); +const jsSource = `// Generated by scripts/build-schemas.js from schemata/*.yaml. Do not edit.\nexport default ${JSON.stringify(schemas, null, 2)};\n`; +await writeFile(jsDest, jsSource); +console.log("built schema/schemas.js"); diff --git a/spec/.eslintrc.json b/spec/.eslintrc.json deleted file mode 100644 index 6dfc3b1..0000000 --- a/spec/.eslintrc.json +++ /dev/null @@ -1,21 +0,0 @@ -{ - "extends": "airbnb-base", - "plugins": [ - "chai", - "mocha" - ], - "env": { - "browser": true, - "node": true, - "mocha": true - }, - "rules": { - "func-names": 0 - }, - "globals": { - "baseDir": false, - "chai": false, - "noflo": false, - "window": false - } -} diff --git a/spec/cli.js b/spec/cli.js deleted file mode 100644 index c75759c..0000000 --- a/spec/cli.js +++ /dev/null @@ -1,62 +0,0 @@ -const chai = require('chai'); -const childProcess = require('child_process'); -const path = require('path'); - -const fbpSpec = function (suite, callback) { - const prog = './bin/fbp-spec'; - const args = [ - suite, - '--command', 'python2 protocol-examples/python/runtime.py', - ]; - const options = {}; - return childProcess.execFile(prog, args, options, callback); -}; - -const example = (name) => path.join('examples', name); - -const countTestcases = function (str) { - let count = 0; - const passChars = ['✓', '✗']; - for (let i = 0; i < str.length; i += 1) { - const char = str[i]; - if (passChars.indexOf(char) !== -1) { - count += 1; - } - } - return count; -}; - -const pyTimeout = 3000; -describe('fbp-spec', () => { - describe('with failing testcases', () => it('should exit with non-zero code', function (done) { - this.timeout(pyTimeout); - fbpSpec(example('simple-failing.yaml'), (err) => { - chai.expect(err).to.be.an('error'); - chai.expect(err.code).to.not.equal(0); - chai.expect(err.message).to.contain('Command failed'); - done(); - }); - })); - - describe('with passing testcases', () => it('should exit with 0 code', function (done) { - this.timeout(pyTimeout); - fbpSpec(example('simple-passing.yaml'), (err) => { - if (err) { - done(err); - return; - } - done(); - }); - })); - - describe('with multiple suites and some failing cases', () => it('should run all testcases', function (done) { - this.timeout(pyTimeout); - fbpSpec(example('multisuite-failandpass.yaml'), (err, stdout) => { - chai.expect(err).to.be.an('error'); - chai.expect(countTestcases(stdout)).to.equal(4); - chai.expect(stdout).to.contain('sending a boolean with wrong expect'); - chai.expect(stdout).to.contain('should repeat the same'); - done(); - }); - })); -}); diff --git a/spec/cli.test.js b/spec/cli.test.js new file mode 100644 index 0000000..77a29fc --- /dev/null +++ b/spec/cli.test.js @@ -0,0 +1,207 @@ +import assert from "node:assert/strict"; +import { execFile } from "node:child_process"; +import { after, before, describe, it } from "node:test"; +import { fileURLToPath } from "node:url"; +import { hasErrors, parse, testStatusText } from "../lib/cli.js"; +import { startDummyRuntime } from "./fixtures/dummy-runtime.js"; + +const binPath = fileURLToPath(new URL("../bin/fbp-spec", import.meta.url)); +/** @param {string} name */ +const examplePath = (name) => + fileURLToPath(new URL(`../examples/${name}`, import.meta.url)); + +/** + * Run the fbp-spec CLI as a subprocess. + * @param {string[]} args CLI arguments + * @returns {Promise<{ code: number, stdout: string, stderr: string }>} Process result + */ +function runCli(args) { + return new Promise((resolve) => { + execFile( + process.execPath, + [binPath, ...args], + { encoding: "utf-8", timeout: 30000 }, + (err, stdout, stderr) => { + resolve({ + code: typeof err?.code === "number" ? err.code : 0, + stdout, + stderr, + }); + }, + ); + }); +} + +describe("CLI", () => { + /** @type {Awaited>} */ + let runtime; + + before(async () => { + runtime = await startDummyRuntime(); + }); + after(async () => { + await runtime.close(); + }); + + describe("parse", () => { + it("should parse options and positionals", () => { + const { options, help } = parse([ + "--address", + "ws://localhost:1234", + "--secret", + "s3cret", + "--start-timeout", + "2", + "--command-timeout", + "5", + "--component-tests", + "--trace", + "examples/simple-passing.yaml", + ]); + assert.equal(help, false); + assert.equal(options.address, "ws://localhost:1234"); + assert.equal(options.secret, "s3cret"); + assert.equal(options.startTimeout, 2); + assert.equal(options.commandTimeout, 5); + assert.equal(options.componentTests, true); + assert.equal(options.trace, true); + assert.deepEqual(options.suites, ["examples/simple-passing.yaml"]); + }); + it("should apply defaults", () => { + const { options } = parse(["suite.yaml"]); + assert.equal(options.address, "ws://localhost:3569"); + assert.equal(options.startTimeout, 10); + assert.equal(options.commandTimeout, 3); + assert.equal(options.componentTests, false); + assert.equal(options.trace, false); + assert.deepEqual(options.suites, ["suite.yaml"]); + }); + it("should support multiple suite files", () => { + const { options } = parse(["a.yaml", "b.yaml"]); + assert.deepEqual(options.suites, ["a.yaml", "b.yaml"]); + }); + it("should reject unknown options", () => { + assert.throws(() => parse(["--nope", "a.yaml"]), /Unknown option/); + }); + }); + + describe("hasErrors", () => { + it("should detect failing testcases", () => { + assert.equal(hasErrors([{ cases: [{ passed: true }, {}] }]), false); + assert.equal( + hasErrors([{ cases: [{ passed: true }, { error: "boom" }] }]), + true, + ); + }); + }); + + describe("testStatusText", () => { + it("should render testcase results", () => { + const lines = testStatusText([ + { + name: "My suite", + cases: [ + { name: "case a", assertion: "should work", passed: true }, + { + name: "case b", + assertion: "should fail", + passed: false, + error: "expected 1 to equal 2", + }, + { + name: "case c", + assertion: "should skip", + passed: true, + skip: "no runtime", + }, + ], + }, + ]); + assert.equal(lines.length, 4); + assert.match(lines[0], /My suite/); + assert.match(lines[1], /case a/); + assert.match(lines[1], /✓/); + assert.match(lines[2], /case b/); + assert.match(lines[2], /✗ Error: expected 1 to equal 2/); + }); + }); + + describe("against a runtime (end-to-end)", () => { + it("should exit 0 for passing suites", async () => { + const result = await runCli([ + examplePath("simple-passing.yaml"), + "--address", + runtime.address, + "--start-timeout", + "2", + ]); + assert.equal(result.code, 0, `stderr: ${result.stderr}`); + assert.match(result.stdout, /should repeat the same: ✓/); + }); + it("should exit non-zero for failing suites", async () => { + const result = await runCli([ + examplePath("simple-failing.yaml"), + "--address", + runtime.address, + "--start-timeout", + "2", + ]); + assert.equal(result.code, 2, `stderr: ${result.stderr}`); + assert.match( + result.stdout, + /should fail: ✗ Error: expected true to equal false/, + ); + }); + it("should run all testcases across multiple suites", async () => { + const result = await runCli([ + examplePath("multisuite-failandpass.yaml"), + "--address", + runtime.address, + "--start-timeout", + "2", + ]); + assert.equal(result.code, 2); + assert.match(result.stdout, /sending a boolean with wrong expect/); + assert.match(result.stdout, /should repeat the same/); + assert.equal((result.stdout.match(/✓/g) ?? []).length, 2); + assert.equal((result.stdout.match(/✗/g) ?? []).length, 2); + }); + it("should load component tests with --component-tests", async () => { + const result = await runCli([ + examplePath("simple-passing.yaml"), + "--address", + runtime.address, + "--start-timeout", + "2", + "--component-tests", + ]); + assert.equal(result.code, 0, `stderr: ${result.stderr}`); + assert.match(result.stdout, /DummyComponent tests/); + assert.match(result.stdout, /should repeat the same: ✓/); + }); + it("should start the runtime with --command", async () => { + const runtimePath = fileURLToPath( + new URL("./fixtures/dummy-runtime.js", import.meta.url), + ); + const result = await runCli([ + examplePath("simple-passing.yaml"), + "--command", + `node ${runtimePath} 33577`, + "--address", + "ws://127.0.0.1:33577", + "--start-timeout", + "10", + ]); + assert.equal( + result.code, + 0, + `stdout: ${result.stdout}\nstderr: ${result.stderr}`, + ); + }); + it("should print usage and exit non-zero without suites", async () => { + const result = await runCli(["--address", runtime.address]); + assert.equal(result.code, 2); + assert.match(result.stderr, /no suites specified/); + }); + }); +}); diff --git a/spec/common.test.js b/spec/common.test.js new file mode 100644 index 0000000..28e1422 --- /dev/null +++ b/spec/common.test.js @@ -0,0 +1,108 @@ +import assert from "node:assert/strict"; +import { describe, it } from "node:test"; + +import { + asyncSeries, + isArray, + isBrowser, + randomString, + retryUntil, + startsWith, +} from "../lib/common.js"; + +describe("common", () => { + describe("isBrowser", () => { + it("should detect Node.js", () => { + assert.equal(isBrowser(), false); + }); + }); + + describe("randomString", () => { + it("should generate string of requested length", () => { + assert.equal(randomString(10).length, 10); + assert.equal(randomString(0).length, 0); + }); + it("should only contain ASCII alphanumerics", () => { + assert.match(randomString(100), /^[A-Za-z0-9]+$/); + }); + it("should produce different strings", () => { + assert.notEqual(randomString(20), randomString(20)); + }); + }); + + describe("asyncSeries", () => { + it("should run items in order and collect results", async () => { + /** @type {number[]} */ + const order = []; + const results = await asyncSeries([1, 2, 3], async (item) => { + order.push(item); + return item * 2; + }); + assert.deepEqual(order, [1, 2, 3]); + assert.deepEqual(results, [2, 4, 6]); + }); + it("should reject on first error", async () => { + await assert.rejects( + asyncSeries([1, 2, 3], async (item) => { + if (item === 2) { + throw new Error("boom"); + } + return item; + }), + /boom/, + ); + }); + }); + + describe("isArray", () => { + it("should detect arrays", () => { + assert.equal(isArray([]), true); + assert.equal(isArray([1, 2]), true); + assert.equal(isArray({}), false); + assert.equal(isArray("string"), false); + assert.equal(isArray(null), false); + }); + }); + + describe("startsWith", () => { + it("should check string prefixes", () => { + assert.equal(startsWith("foobar", "foo"), true); + assert.equal(startsWith("foobar", "bar"), false); + assert.equal(startsWith("foobar", "foobar"), true); + assert.equal(startsWith("", "x"), false); + }); + }); + + describe("retryUntil", () => { + it("should return first passing result", async () => { + let attempts = 0; + const result = await retryUntil( + async () => { + attempts += 1; + return attempts; + }, + async (value) => { + if (value < 3) { + throw new Error("not yet"); + } + }, + 1, + 5, + ); + assert.equal(result, 3); + }); + it("should reject after max attempts", async () => { + await assert.rejects( + retryUntil( + async () => false, + async () => { + throw new Error("nope"); + }, + 1, + 2, + ), + /nope/, + ); + }); + }); +}); diff --git a/spec/examples.js b/spec/examples.js deleted file mode 100644 index 1ab93af..0000000 --- a/spec/examples.js +++ /dev/null @@ -1,148 +0,0 @@ -const startRuntime = function (fbpspec, runner, info, callback) { - let runtime = null; - if (info.command) { - runtime = fbpspec.subprocess.start(info.command, {}, callback); - } else if (info.protocol === 'iframe') { - const parent = document.createElement('div'); - document.body.appendChild(parent); - const r = runner; - r.parentElement = parent; - callback(null); - } else { - callback(null); - } - - return runtime; -}; - -const stopRuntime = function (runtime) { - if (runtime) { - runtime.kill(); - } -}; - -const setupAndRun = (fbpspec, runner, suite, testcase, callback) => { - runner.setupSuite(suite, (err) => { - if (err) { - callback(err); - return; - } - fbpspec.runner.runTestAndCheck(runner, testcase, (runErr, r) => { - if (runErr) { - callback(runErr); - return; - } - const results = r; - runner.teardownSuite(suite, (e) => callback(e, results)); - }); - }); -}; - -describe('Examples', () => { - const isBrowser = () => !((typeof process !== 'undefined' && process !== null) && process.execPath && process.execPath.match(/node|iojs/)); - // eslint-disable-next-line global-require - const chai = require('chai'); - // eslint-disable-next-line import/no-unresolved,global-require - const fbpspec = isBrowser() ? require('fbp-spec') : require('..'); - let examples; - let runtimeInfo; - if (isBrowser()) { - examples = window.fbpspec_examples; - runtimeInfo = { - protocol: 'iframe', - address: '/base/browser/spec/fixtures/everything.html?fbp_noload=true&fbp_protocol=iframe', - }; - } else { - // eslint-disable-next-line global-require,import/no-unresolved - examples = require('../examples/bundle.json'); - runtimeInfo = { - protocol: 'websocket', - address: 'ws://localhost:3335', - command: 'python2 protocol-examples/python/runtime.py --port 3335', - }; - } - - let runner = null; - let runtime = null; - before(function (done) { - this.timeout(6000); - runner = new fbpspec.runner.Runner(runtimeInfo); - runtime = startRuntime(fbpspec, runner, runtimeInfo, (err) => { - if (err) { - done(err); - return; - } - runner.connect(done); - }); - }); - after((done) => { - stopRuntime(runtime); - runner.disconnect(done); - }); - - Object.keys(examples).forEach((name) => { - let example = null; - describe(`${name}`, () => { - before((done) => { - try { - example = examples[name]; - done(); - } catch (e) { - done(e); - } - }); - - it('should validate against schema', function () { - if (isBrowser()) { - this.skip(); - return; - } - const results = fbpspec.testsuite.validate(example); - chai.expect(results.errors).to.eql([]); - chai.expect(results.missing).to.eql([]); - chai.expect(results.valid).to.equal(true); - }); - - describe('testcases', () => { - if (!example) { example = []; } - const suites = Array.isArray(example) ? example.slice(0) : [example]; - suites.forEach((suite) => suite.cases.forEach((testcase) => describe(`${testcase.name}`, () => { - let itOrSkip = testcase.skip ? it.skip : it; - if (isBrowser() && (suite.topic === 'DummyComponent')) { - // These tests only work with the Python runtime - itOrSkip = it.skip; - } - - if (testcase.assertion === 'should pass') { - itOrSkip('should pass', function (done) { - this.timeout(10000); - setupAndRun(fbpspec, runner, suite, testcase, (err, results) => { - if (err) { - done(err); - return; - } - chai.expect(results.error).to.be.a('null'); - chai.expect(results.passed).to.equal(true); - done(); - }); - }); - } if (testcase.assertion === 'should fail') { - itOrSkip('should fail', function (done) { - this.timeout(10000); - setupAndRun(fbpspec, runner, suite, testcase, (err, results) => { - if (err) { - done(err); - return; - } - chai.expect(results.error, 'missing error').be.an('error'); - chai.expect(results.error.message).to.contain('expect'); - chai.expect(results.passed).equal(false); - done(); - }); - }); - } - }))); - }); - }); - }); -}); diff --git a/spec/examples.test.js b/spec/examples.test.js new file mode 100644 index 0000000..84e3ea7 --- /dev/null +++ b/spec/examples.test.js @@ -0,0 +1,134 @@ +import assert from "node:assert/strict"; +import { readdirSync } from "node:fs"; +import { readFile } from "node:fs/promises"; +import { after, before, describe, it } from "node:test"; +import { + getComponentSuites, + Runner, + runAll, + runTestAndCheck, +} from "../lib/runner.js"; +import { getSuites, loadYAML, validate } from "../lib/testsuite.js"; +import { startDummyRuntime } from "./fixtures/dummy-runtime.js"; + +describe("Examples (end-to-end with dummy runtime)", () => { + /** @type {Awaited>} */ + let runtime; + /** @type {Runner} */ + let runner; + /** @typedef {import("../lib/testsuite.js").Suite} Suite */ + + before(async () => { + runtime = await startDummyRuntime(); + runner = new Runner( + { protocol: "websocket", address: runtime.address }, + { connectTimeout: 2000 }, + ); + await runner.connect(); + }); + after(async () => { + await runner.disconnect(); + await runtime.close(); + }); + + /** + * Set up a suite, run one testcase, and tear down. + * @param {object} suite Testsuite + * @param {object} testcase Testcase + * @returns {Promise<{ passed: boolean, error?: Error }>} Test result + */ + async function setupAndRun(suite, testcase) { + await runner.setupSuite(suite); + try { + return await runTestAndCheck(runner, testcase); + } finally { + await runner.teardownSuite(suite); + } + } + + const examplesDir = new URL("../examples/", import.meta.url); + const exampleFiles = readdirSync(examplesDir).filter((f) => + f.endsWith(".yaml"), + ); + + for (const file of exampleFiles) { + describe(file, () => { + /** @type {Suite[]} */ + let suites; + + before(async () => { + const source = await readFile(new URL(file, examplesDir), "utf-8"); + suites = loadYAML(source); + }); + + it("suites should validate against schema", () => { + assert.equal(suites.length > 0, true); + const results = validate(suites); + assert.deepEqual(results.errors, []); + assert.deepEqual(results.missing, []); + assert.equal(results.valid, true); + }); + + it("testcases should pass or fail as declared", async () => { + for (const suite of suites) { + for (const testcase of suite.cases ?? []) { + const result = await setupAndRun(suite, testcase); + const shouldFail = testcase.assertion === "should fail"; + if (shouldFail) { + assert.equal( + result.passed, + false, + `expected fail: ${suite.name} / ${testcase.name}`, + ); + assert.ok( + result.error instanceof Error, + "failing testcase should have an error", + ); + } else { + assert.deepEqual( + result, + { passed: true }, + `expected pass: ${suite.name} / ${testcase.name}`, + ); + } + } + } + }); + }); + } + + describe("runAll", () => { + it("should fill in results on all testcases", async () => { + const suites = await getSuites( + new URL("multisuite-failandpass.yaml", examplesDir).pathname, + ); + await runAll(runner, suites); + const cases = suites.flatMap((s) => s.cases ?? []); + assert.equal(cases.length, 4); + assert.deepEqual( + cases.map((c) => c.passed), + [false, false, true, true], + ); + for (const c of cases) { + if (!c.passed) { + assert.ok(typeof c.error === "string"); + } + } + }); + }); + + describe("component tests", () => { + it("should load and run tests attached to runtime components", async () => { + const suites = /** @type {Suite[]} */ (await getComponentSuites(runner)); + assert.equal(suites.length, 1); + const suite = suites[0]; + assert.ok(suite, "expected a suite"); + assert.equal(suite.name, "DummyComponent tests"); + await runAll(runner, suites); + assert.deepEqual( + (suite.cases ?? []).map((c) => c.passed), + [true], + ); + }); + }); +}); diff --git a/spec/expectation.test.js b/spec/expectation.test.js new file mode 100644 index 0000000..dbe337e --- /dev/null +++ b/spec/expectation.test.js @@ -0,0 +1,174 @@ +import assert from "node:assert/strict"; +import { describe, it } from "node:test"; + +import { expect, noError } from "../lib/expectation.js"; + +describe("expectation", () => { + describe("expect", () => { + it("should pass on matching expectations", () => { + expect({ out: { equals: 42 } }, { out: 42 }); + expect({ out: { equals: { a: 1 } } }, { out: { a: 1 } }); + expect({ out: { type: "number" } }, { out: 42 }); + }); + it("should throw on non-matching expectations", () => { + assert.throws( + () => expect({ out: { equals: 42 } }, { out: 41 }), + /expected 41 to equal 42/, + ); + }); + it("should throw when no data received on port", () => { + assert.throws( + () => expect({ out: { equals: 42 } }, {}), + /No data received on port out/, + ); + }); + it("should throw on unknown operator", () => { + assert.throws( + () => expect({ out: { unknownop: 42 } }, { out: 42 }), + /No operator matching/, + ); + }); + it("should support sequences of expectations", () => { + expect([{ out: { equals: 1 } }, { out: { equals: 1 } }], { out: 1 }); + }); + it("should support arrays of expectations on one port", () => { + expect( + { out: [{ type: "object" }, { haveKeys: ["a"] }] }, + { out: { a: 1 } }, + ); + assert.throws( + () => + expect( + { out: [{ type: "object" }, { haveKeys: ["b"] }] }, + { out: { a: 1 } }, + ), + /have keys/, + ); + }); + + describe("operators", () => { + it("type", () => { + expect({ out: { type: "object" } }, { out: {} }); + expect({ out: { type: "string" } }, { out: "hello" }); + assert.throws( + () => expect({ out: { type: "string" } }, { out: 42 }), + /to be of type string/, + ); + }); + it("above and below", () => { + expect({ out: { above: 41 } }, { out: 42 }); + expect({ out: { below: 43 } }, { out: 42 }); + assert.throws( + () => expect({ out: { above: 42 } }, { out: 42 }), + /to be above 42/, + ); + assert.throws( + () => expect({ out: { below: 42 } }, { out: 42 }), + /to be below 42/, + ); + }); + it("haveKeys", () => { + expect({ out: { haveKeys: ["a", "b"] } }, { out: { a: 1, b: 2 } }); + assert.throws( + () => expect({ out: { haveKeys: ["a"] } }, { out: { a: 1, b: 2 } }), + /have keys/, + ); + }); + it("includeKeys", () => { + expect({ out: { includeKeys: ["a"] } }, { out: { a: 1, b: 2 } }); + assert.throws( + () => expect({ out: { includeKeys: ["c"] } }, { out: { a: 1 } }), + /include key c/, + ); + }); + it("contains on strings", () => { + expect( + { out: { contains: "sub" } }, + { out: "a string with substring" }, + ); + assert.throws( + () => expect({ out: { contains: "nope" } }, { out: "string" }), + /to contain 'nope'/, + ); + }); + it("contains on arrays", () => { + expect({ out: { contains: { a: 1 } } }, { out: [{ a: 1 }, { b: 2 }] }); + assert.throws( + () => expect({ out: { contains: 3 } }, { out: [1, 2] }), + /to contain 3/, + ); + }); + it("contains on objects", () => { + expect({ out: { contains: { a: 1 } } }, { out: { a: 1, b: 2 } }); + assert.throws( + () => expect({ out: { contains: { c: 3 } } }, { out: { a: 1 } }), + /to contain/, + ); + }); + it("noterror", () => { + expect({ out: { noterror: null } }, { out: "value" }); + assert.throws( + () => + expect({ out: { noterror: null } }, { out: new Error("broken") }), + /broken/, + ); + }); + }); + + describe("JSONPath", () => { + it("should extract and match values", () => { + expect( + { out: { path: "$.outer.inner.foo", equals: "bar" } }, + { out: { outer: { inner: { foo: "bar" } } } }, + ); + }); + it("should match multiple values", () => { + expect( + { out: { path: "$.outer.*.foo", equals: "bar" } }, + { out: { outer: { first: { foo: "bar" }, second: { foo: "bar" } } } }, + ); + }); + it("should fail when path does not match", () => { + assert.throws( + () => + expect( + { out: { path: "$.outer.nonexist", equals: "bar" } }, + { out: { outer: { inner: { foo: "bar" } } } }, + ), + /to match data/, + ); + }); + it("should fail when a match violates the expectation", () => { + assert.throws( + () => + expect( + { out: { path: "$.outer.*.foo", equals: "bar" } }, + { + out: { + outer: { first: { foo: "bar" }, second: { foo: "bazz" } }, + }, + }, + ), + /expected "bazz" to equal "bar"/, + ); + }); + }); + }); + + describe("noError", () => { + it("should pass on nullish values", () => { + noError(null); + noError(undefined); + }); + it("should throw the error itself", () => { + const err = new Error("failure"); + assert.throws( + () => noError(err), + (e) => e === err, + ); + }); + it("should throw on other defined values", () => { + assert.throws(() => noError("value"), /value/); + }); + }); +}); diff --git a/spec/fixtures/.gitignore b/spec/fixtures/.gitignore deleted file mode 100644 index f638875..0000000 --- a/spec/fixtures/.gitignore +++ /dev/null @@ -1,2 +0,0 @@ -everything.* - diff --git a/spec/fixtures/dummy-runtime.js b/spec/fixtures/dummy-runtime.js new file mode 100644 index 0000000..3797dcb --- /dev/null +++ b/spec/fixtures/dummy-runtime.js @@ -0,0 +1,256 @@ +/** + * A minimal in-process FBP runtime, speaking the FBP runtime protocol over WebSocket. + * Mirrors the behaviour of protocol-examples/python/runtime.py (the 'python-example' + * runtime used by the fbp-spec examples): it echoes any input packet back on the + * 'out' port, and exposes a single DummyComponent. + * + * Used by the test suite so that end-to-end tests can run without Python. + * @module spec/fixtures/dummy-runtime + */ + +import http from "node:http"; +import { fileURLToPath } from "node:url"; +import { server as WebSocketServer } from "websocket"; + +/** YAML testsuite attached to DummyComponent, served via component:getsource. */ +export const DUMMY_COMPONENT_TESTS = ` +name: DummyComponent tests +topic: DummyComponent +cases: +- + name: 'sending a boolean' + assertion: 'should repeat the same' + inputs: + in: true + expect: + out: + equals: true +`; + +/** @type {Record} */ +const components = { + DummyComponent: { + name: "DummyComponent", + description: "An example component which does exactly nothing", + subgraph: false, + inPorts: [ + { + id: "portA", + type: "boolean", + description: "A boolean port", + addressable: false, + required: true, + }, + { id: "portB", type: "any" }, + ], + outPorts: [{ id: "out1", type: "string" }], + }, +}; + +/** + * Start a dummy FBP runtime listening on a port. + * @param {{ secret?: string, port?: number }} [options] Optional secret the runtime requires, and port (0 = ephemeral) + * @returns {Promise<{ address: string, port: number, close: () => Promise, httpServer: import('node:http').Server }>} Handle to the runtime + */ +export function startDummyRuntime(options = {}) { + const httpServer = http.createServer((_req, res) => { + res.writeHead(404); + res.end(); + }); + const wsServer = new WebSocketServer({ httpServer }); + + wsServer.on("request", (request) => { + const subProtocol = + request.requestedProtocols.indexOf("noflo") !== -1 ? "noflo" : null; + const connection = request.accept(subProtocol, request.origin); + + /** + * Send an FBP message to the client, stripping any runtime secret. + * @param {string} protocol FBP protocol name + * @param {string} command FBP command name + * @param {*} payload Message payload + * @returns {void} + */ + function send(protocol, command, payload) { + if (payload && typeof payload === "object" && "secret" in payload) { + const { secret, ...rest } = payload; // eslint-disable-line no-unused-vars + payload = rest; + } + connection.sendUTF(JSON.stringify({ protocol, command, payload })); + } + + /** + * Acknowledge a graph protocol message by echoing it back. + * @param {string} command FBP command name + * @param {object} payload Message payload + * @returns {void} + */ + /** + * @param {string} command FBP command name + * @param {Record} payload Message payload + * @returns {void} + */ + function handleGraph(command, payload) { + send("graph", command, payload); + } + + /** + * Handle a network protocol message. + * @param {string} command FBP command name + * @param {object} payload Message payload + * @returns {void} + */ + /** + * @param {string} command FBP command name + * @param {Record} payload Message payload + * @returns {void} + */ + function handleNetwork(command, payload) { + const graph = payload?.graph ?? null; + if (command === "getstatus") { + send("network", "status", { graph, started: false, running: false }); + return; + } + if (command === "start") { + send("network", "started", { graph, started: true, running: true }); + return; + } + if (command === "stop") { + send("network", "stopped", { graph, started: false, running: false }); + return; + } + console.warn(`Unknown network command ${command}`); + } + + /** + * Handle a runtime protocol message. + * @param {string} command FBP command name + * @param {object} payload Message payload + * @returns {void} + */ + /** + * @param {string} command FBP command name + * @param {Record} payload Message payload + * @returns {void} + */ + function handleRuntime(command, payload) { + if (command === "getruntime") { + if (options.secret && payload.secret !== options.secret) { + send("runtime", "error", { message: "Invalid secret" }); + return; + } + send("runtime", "runtime", { + type: "fbp-js-dummy", + version: "0.7", + capabilities: [ + "protocol:graph", + "protocol:component", + "protocol:network", + "protocol:runtime", + "component:getsource", + ], + }); + return; + } + if (command === "packet") { + // We don't actually run anything, just echo input back on 'out' + send("runtime", "packetsent", payload); + send("runtime", "packet", { ...payload, port: "out" }); + return; + } + console.warn(`Unknown runtime command ${command}`); + } + + /** + * Handle a component protocol message. + * @param {string} command FBP command name + * @param {object} payload Message payload + * @returns {void} + */ + /** + * @param {string} command FBP command name + * @param {Record} payload Message payload + * @returns {void} + */ + function handleComponent(command, payload) { + if (command === "list") { + for (const component of Object.values(components)) { + send("component", "component", component); + } + send("component", "componentsready", Object.keys(components).length); + return; + } + if (command === "getsource") { + const component = components[payload.name]; + if (!component) { + send("component", "error", { + message: `Component not found: ${payload.name}`, + }); + return; + } + send("component", "source", { + name: component.name, + code: "", + language: "whitespace", + tests: DUMMY_COMPONENT_TESTS, + }); + return; + } + console.warn(`Unknown component command ${command}`); + } + + connection.on("message", (message) => { + if (message.type !== "utf8") { + return; + } + let contents; + try { + contents = JSON.parse(message.utf8Data); + } catch (e) { + const err = /** @type {Error} */ (e); + console.error(`Error: ${err.toString()}`); + return; + } + switch (contents.protocol) { + case "runtime": + handleRuntime(contents.command, contents.payload); + break; + case "component": + handleComponent(contents.command, contents.payload); + break; + case "graph": + handleGraph(contents.command, contents.payload); + break; + case "network": + handleNetwork(contents.command, contents.payload); + break; + default: + console.warn(`Unknown protocol ${contents.protocol}`); + } + }); + }); + + return new Promise((resolve) => { + httpServer.listen(options.port ?? 0, "127.0.0.1", () => { + const addr = httpServer.address(); + const port = typeof addr === "object" && addr ? addr.port : 0; + resolve({ + address: `ws://127.0.0.1:${port}`, + port, + httpServer, + close: () => + new Promise((res) => { + wsServer.shutDown(); + httpServer.close(() => res()); + }), + }); + }); + }); +} + +// Allow running standalone, for use as a runtime-under-test via the CLI --command option +if (process.argv[1] && fileURLToPath(import.meta.url) === process.argv[1]) { + const port = Number(process.argv[2] ?? process.env.PORT ?? 3569); + await startDummyRuntime({ port }); + console.log(`INFO: Dummy runtime listening on port ${port}`); +} diff --git a/spec/fixtures/mochacases/bdd-nested-passing.js b/spec/fixtures/mochacases/bdd-nested-passing.js deleted file mode 100644 index 5091f1a..0000000 --- a/spec/fixtures/mochacases/bdd-nested-passing.js +++ /dev/null @@ -1,19 +0,0 @@ -/* - * decaffeinate suggestions: - * DS102: Remove unnecessary code created because of implicit returns - * Full docs: https://github.com/decaffeinate/decaffeinate/blob/master/docs/suggestions.md - */ - -const chai = require('chai'); - -describe('nested passing', () => describe('sub topic', () => { - it('should pass', (done) => { - chai.expect(42).to.equal(42); - return done(); - }); - - return describe('sub sub topic', () => it('should pass', (done) => { - chai.expect(42).to.equal(42); - return done(); - })); -})); diff --git a/spec/fixtures/mochacases/bdd-simple-failing.js b/spec/fixtures/mochacases/bdd-simple-failing.js deleted file mode 100644 index 580cf57..0000000 --- a/spec/fixtures/mochacases/bdd-simple-failing.js +++ /dev/null @@ -1,12 +0,0 @@ -/* - * decaffeinate suggestions: - * DS102: Remove unnecessary code created because of implicit returns - * Full docs: https://github.com/decaffeinate/decaffeinate/blob/master/docs/suggestions.md - */ - -const chai = require('chai'); - -describe('simple failing', () => it('should fail', (done) => { - chai.expect(42).to.equal(41); - return done(); -})); diff --git a/spec/fixtures/mochacases/bdd-simple-passing.js b/spec/fixtures/mochacases/bdd-simple-passing.js deleted file mode 100644 index 9ca678e..0000000 --- a/spec/fixtures/mochacases/bdd-simple-passing.js +++ /dev/null @@ -1,12 +0,0 @@ -/* - * decaffeinate suggestions: - * DS102: Remove unnecessary code created because of implicit returns - * Full docs: https://github.com/decaffeinate/decaffeinate/blob/master/docs/suggestions.md - */ - -const chai = require('chai'); - -describe('simple passing', () => it('should pass', (done) => { - chai.expect(42).to.equal(42); - return done(); -})); diff --git a/spec/import.js b/spec/import.js deleted file mode 100644 index 45118b2..0000000 --- a/spec/import.js +++ /dev/null @@ -1,11 +0,0 @@ -describe("require('fbp-spec')", () => { - const isBrowser = () => !((typeof process !== 'undefined' && process !== null) && process.execPath && process.execPath.match(/node|iojs/)); - // eslint-disable-next-line global-require - const chai = require('chai'); - - it('should not throw', () => { - // eslint-disable-next-line global-require,import/no-unresolved - const fbpspec = isBrowser() ? require('fbp-spec') : require('..'); - chai.expect(fbpspec).to.be.an('object'); - }); -}); diff --git a/spec/import.test.js b/spec/import.test.js new file mode 100644 index 0000000..c73694f --- /dev/null +++ b/spec/import.test.js @@ -0,0 +1,23 @@ +import assert from "node:assert/strict"; +import { describe, it } from "node:test"; + +import * as fbpSpec from "../index.js"; + +describe("require('fbp-spec')", () => { + it("should not throw", () => { + assert.equal(typeof fbpSpec, "object"); + }); + + it("should export the public API", () => { + assert.equal(typeof fbpSpec.runner.Runner, "function"); + assert.equal(typeof fbpSpec.runner.runAll, "function"); + assert.equal(typeof fbpSpec.runner.runTestAndCheck, "function"); + assert.equal(typeof fbpSpec.subprocess.start, "function"); + assert.equal(typeof fbpSpec.testsuite.loadYAML, "function"); + assert.equal(typeof fbpSpec.testsuite.validate, "function"); + assert.equal(typeof fbpSpec.schema.testcase, "object"); + const schema = fbpSpec.getSchema("testsfile.json"); + assert.ok(schema, "expected testsfile schema"); + assert.equal(schema.id, "testsfile.json"); + }); +}); diff --git a/spec/json-schema.json b/spec/json-schema.json index 85eb502..644bf82 100644 --- a/spec/json-schema.json +++ b/spec/json-schema.json @@ -1,150 +1,146 @@ { - "id": "http://json-schema.org/draft-04/schema#", - "$schema": "http://json-schema.org/draft-04/schema#", - "description": "Core schema meta-schema", + "id": "http://json-schema.org/draft-04/schema#", + "$schema": "http://json-schema.org/draft-04/schema#", + "description": "Core schema meta-schema", + "definitions": { + "schemaArray": { + "type": "array", + "minItems": 1, + "items": { "$ref": "#" } + }, + "positiveInteger": { + "type": "integer", + "minimum": 0 + }, + "positiveIntegerDefault0": { + "allOf": [{ "$ref": "#/definitions/positiveInteger" }, { "default": 0 }] + }, + "simpleTypes": { + "enum": [ + "array", + "boolean", + "integer", + "null", + "number", + "object", + "string" + ] + }, + "stringArray": { + "type": "array", + "items": { "type": "string" }, + "minItems": 1, + "uniqueItems": true + } + }, + "type": "object", + "properties": { + "id": { + "type": "string", + "format": "uri" + }, + "$schema": { + "type": "string", + "format": "uri" + }, + "title": { + "type": "string" + }, + "description": { + "type": "string" + }, + "default": {}, + "multipleOf": { + "type": "number", + "minimum": 0, + "exclusiveMinimum": true + }, + "maximum": { + "type": "number" + }, + "exclusiveMaximum": { + "type": "boolean", + "default": false + }, + "minimum": { + "type": "number" + }, + "exclusiveMinimum": { + "type": "boolean", + "default": false + }, + "maxLength": { "$ref": "#/definitions/positiveInteger" }, + "minLength": { "$ref": "#/definitions/positiveIntegerDefault0" }, + "pattern": { + "type": "string", + "format": "regex" + }, + "additionalItems": { + "anyOf": [{ "type": "boolean" }, { "$ref": "#" }], + "default": {} + }, + "items": { + "anyOf": [{ "$ref": "#" }, { "$ref": "#/definitions/schemaArray" }], + "default": {} + }, + "maxItems": { "$ref": "#/definitions/positiveInteger" }, + "minItems": { "$ref": "#/definitions/positiveIntegerDefault0" }, + "uniqueItems": { + "type": "boolean", + "default": false + }, + "maxProperties": { "$ref": "#/definitions/positiveInteger" }, + "minProperties": { "$ref": "#/definitions/positiveIntegerDefault0" }, + "required": { "$ref": "#/definitions/stringArray" }, + "additionalProperties": { + "anyOf": [{ "type": "boolean" }, { "$ref": "#" }], + "default": {} + }, "definitions": { - "schemaArray": { - "type": "array", - "minItems": 1, - "items": { "$ref": "#" } - }, - "positiveInteger": { - "type": "integer", - "minimum": 0 - }, - "positiveIntegerDefault0": { - "allOf": [ { "$ref": "#/definitions/positiveInteger" }, { "default": 0 } ] - }, - "simpleTypes": { - "enum": [ "array", "boolean", "integer", "null", "number", "object", "string" ] - }, - "stringArray": { - "type": "array", - "items": { "type": "string" }, - "minItems": 1, - "uniqueItems": true - } + "type": "object", + "additionalProperties": { "$ref": "#" }, + "default": {} }, - "type": "object", "properties": { - "id": { - "type": "string", - "format": "uri" - }, - "$schema": { - "type": "string", - "format": "uri" - }, - "title": { - "type": "string" - }, - "description": { - "type": "string" - }, - "default": {}, - "multipleOf": { - "type": "number", - "minimum": 0, - "exclusiveMinimum": true - }, - "maximum": { - "type": "number" - }, - "exclusiveMaximum": { - "type": "boolean", - "default": false - }, - "minimum": { - "type": "number" - }, - "exclusiveMinimum": { - "type": "boolean", - "default": false - }, - "maxLength": { "$ref": "#/definitions/positiveInteger" }, - "minLength": { "$ref": "#/definitions/positiveIntegerDefault0" }, - "pattern": { - "type": "string", - "format": "regex" - }, - "additionalItems": { - "anyOf": [ - { "type": "boolean" }, - { "$ref": "#" } - ], - "default": {} - }, - "items": { - "anyOf": [ - { "$ref": "#" }, - { "$ref": "#/definitions/schemaArray" } - ], - "default": {} - }, - "maxItems": { "$ref": "#/definitions/positiveInteger" }, - "minItems": { "$ref": "#/definitions/positiveIntegerDefault0" }, - "uniqueItems": { - "type": "boolean", - "default": false - }, - "maxProperties": { "$ref": "#/definitions/positiveInteger" }, - "minProperties": { "$ref": "#/definitions/positiveIntegerDefault0" }, - "required": { "$ref": "#/definitions/stringArray" }, - "additionalProperties": { - "anyOf": [ - { "type": "boolean" }, - { "$ref": "#" } - ], - "default": {} - }, - "definitions": { - "type": "object", - "additionalProperties": { "$ref": "#" }, - "default": {} - }, - "properties": { - "type": "object", - "additionalProperties": { "$ref": "#" }, - "default": {} - }, - "patternProperties": { - "type": "object", - "additionalProperties": { "$ref": "#" }, - "default": {} - }, - "dependencies": { - "type": "object", - "additionalProperties": { - "anyOf": [ - { "$ref": "#" }, - { "$ref": "#/definitions/stringArray" } - ] - } - }, - "enum": { - "type": "array", - "minItems": 1, - "uniqueItems": true - }, - "type": { - "anyOf": [ - { "$ref": "#/definitions/simpleTypes" }, - { - "type": "array", - "items": { "$ref": "#/definitions/simpleTypes" }, - "minItems": 1, - "uniqueItems": true - } - ] - }, - "allOf": { "$ref": "#/definitions/schemaArray" }, - "anyOf": { "$ref": "#/definitions/schemaArray" }, - "oneOf": { "$ref": "#/definitions/schemaArray" }, - "not": { "$ref": "#" } + "type": "object", + "additionalProperties": { "$ref": "#" }, + "default": {} + }, + "patternProperties": { + "type": "object", + "additionalProperties": { "$ref": "#" }, + "default": {} }, "dependencies": { - "exclusiveMaximum": [ "maximum" ], - "exclusiveMinimum": [ "minimum" ] + "type": "object", + "additionalProperties": { + "anyOf": [{ "$ref": "#" }, { "$ref": "#/definitions/stringArray" }] + } + }, + "enum": { + "type": "array", + "minItems": 1, + "uniqueItems": true + }, + "type": { + "anyOf": [ + { "$ref": "#/definitions/simpleTypes" }, + { + "type": "array", + "items": { "$ref": "#/definitions/simpleTypes" }, + "minItems": 1, + "uniqueItems": true + } + ] }, - "default": {} + "allOf": { "$ref": "#/definitions/schemaArray" }, + "anyOf": { "$ref": "#/definitions/schemaArray" }, + "oneOf": { "$ref": "#/definitions/schemaArray" }, + "not": { "$ref": "#" } + }, + "dependencies": { + "exclusiveMaximum": ["maximum"], + "exclusiveMinimum": ["minimum"] + }, + "default": {} } diff --git a/spec/mocha.js b/spec/mocha.js deleted file mode 100644 index 22bc598..0000000 --- a/spec/mocha.js +++ /dev/null @@ -1,10 +0,0 @@ -const fbpspec = require('..'); - -const rt = { - protocol: 'websocket', - address: 'ws://localhost:3569', - secret: 'py3k', // Optional. If needed to connect/authenticate to runtime - command: 'python2 protocol-examples/python/runtime.py', // Optional. Can be used to start runtime automatically -}; - -fbpspec.mocha.run(rt, './examples/simple-passing.yaml', { starttimeout: 1000 }); diff --git a/spec/mochacompat.js b/spec/mochacompat.js deleted file mode 100644 index 0aeae3c..0000000 --- a/spec/mochacompat.js +++ /dev/null @@ -1,192 +0,0 @@ -const chai = require('chai'); -const fbpClient = require('fbp-client'); -const mochacompat = require('../lib/mochacompat'); -const protocol = require('../lib/protocol'); -const testsuite = require('../lib/testsuite'); -const runner = require('../lib/runner'); - -const testPath = function (name) { - // eslint-disable-next-line global-require - const path = require('path'); - const test = path.join(__dirname, 'fixtures/mochacases', name); - return test; -}; - -const runtimeDefinition = function (options) { - const def = { - protocol: 'websocket', - address: `ws://localhost:${options.port || 3333}`, - }; - return def; -}; - -const setupAndConnect = function (options, callback) { - mochacompat.setup(options, (err, state, httpServer) => { - if (err) { - callback(err); - return; - } - const def = runtimeDefinition(options); - let client = null; - fbpClient(def) - .then((c) => { - client = c; - return client.connect(); - }) - .then((() => callback(null, client, def, state, httpServer)), callback); - }); -}; - -const runAllComponentTests = function (ru, callback) { - let state = null; - const onUpdate = (s) => { - state = s; - }; - ru.connect((err) => { - if (err) { - callback(err); - return; - } - runner.getComponentSuites(ru, (suiteErr, suites) => { - if (suiteErr) { - callback(suiteErr); - return; - } - runner.runAll(ru, suites, onUpdate, (runErr) => callback(runErr, state)); - }); - }); -}; - -describe('Mocha compatibility runner', () => { - let httpServer = null; - let definition = null; - let ru = null; - - afterEach((done) => { - if (httpServer) { - httpServer.close(); - httpServer = null; - } - if (ru) { - ru.disconnect((err) => { - ru = null; - done(err); - }); - return; - } - done(); - }); - - it('should implement the FBP runtime protocol', (done) => { - const options = { files: [testPath('bdd-nested-passing.js')] }; - setupAndConnect(options, (err, client, def, state, server) => { - httpServer = server; - definition = def; - if (err) { - done(err); - return; - } - chai.expect(def).to.include.keys(['protocol', 'type', 'version', 'capabilities']); - chai.expect(def.protocol).to.equal('websocket'); - chai.expect(def.type).to.equal('fbp-spec-mocha'); - chai.expect(def.version).to.equal('0.5'); - done(); - }); - }); - it('has the required FBP runtime capabilities', () => { - const c = definition.capabilities; - chai.expect(c).to.include('protocol:graph'); - chai.expect(c).to.include('protocol:component'); - chai.expect(c).to.include('protocol:network'); - chai.expect(c).to.include('component:getsource'); - }); - - describe('loading test file with nested describe()', () => it('should list each it() as separate fbp-spec testcase', (done) => { - const options = { files: [testPath('bdd-nested-passing.js')] }; - setupAndConnect(options, (err, client, def, state, server) => { - httpServer = server; - if (err) { - done(err); - return; - } - protocol.getComponentTests(client, (componentTestsErr, suites) => { - if (componentTestsErr) { - done(componentTestsErr); - return; - } - chai.expect(suites).to.be.a('object'); - const suiteNames = Object.keys(suites); - chai.expect(suiteNames).to.have.length(1); - const t = suites[suiteNames[0]]; - testsuite.loadYAML(t, (yamlErr, tests) => { - if (yamlErr) { - done(yamlErr); - return; - } - chai.expect(tests).to.have.length(1); - chai.expect(tests[0]).to.include.keys(['name', 'fixture', 'cases']); - chai.expect(tests[0].cases).to.have.length(2); - const [caseA, caseB] = tests[0].cases; - chai.expect(caseA.name).to.include('sub topic'); - chai.expect(caseB.name).to.include('sub sub topic'); - done(); - }); - }); - }); - })); - - describe('running a passing test', () => it('should recorded 1 passed test', (done) => { - const options = { files: [testPath('bdd-simple-passing.js')] }; - mochacompat.setup(options, (err, state, server) => { - httpServer = server; - - ru = new runner.Runner(runtimeDefinition(options)); - runAllComponentTests(ru, (runErr, updatedState) => { - if (runErr) { - done(runErr); - return; - } - chai.expect(updatedState).to.have.length(1); - const { - cases, - } = updatedState[0]; - chai.expect(cases).to.have.length(1); - chai.expect(cases[0].passed, 'testcase did not pass').to.equal(true); - done(); - }); - }); - })); - - describe('running a failing test', () => { - let testcase = null; - it('should recorded 1 failed test', (done) => { - const options = { files: [testPath('bdd-simple-failing.js')] }; - mochacompat.setup(options, (err, state, server) => { - httpServer = server; - ru = new runner.Runner(runtimeDefinition(options)); - runAllComponentTests(ru, (runErr, updatedState) => { - if (runErr) { - done(runErr); - return; - } - chai.expect(updatedState).to.have.length(1); - const { - cases, - } = updatedState[0]; - chai.expect(cases).to.have.length(1); - [testcase] = cases; - chai.expect(testcase.passed, 'failing testcase passed').to.equal(false); - done(); - }); - }); - }); - it('has error message of the Chai assertion', () => { - chai.expect(testcase != null ? testcase.error : undefined).to.contain('expected 42 to equal 41'); - }); - }); - - describe('suite with some skipped tests', () => { - it('skipped tests should be marked as such'); - it('non-skipped tests should be ran'); - }); -}); diff --git a/spec/runner.test.js b/spec/runner.test.js new file mode 100644 index 0000000..a19c877 --- /dev/null +++ b/spec/runner.test.js @@ -0,0 +1,164 @@ +import assert from "node:assert/strict"; +import { describe, it } from "node:test"; + +import { checkResults, Runner, runTestAndCheck } from "../lib/runner.js"; + +describe("runner", () => { + describe("checkResults", () => { + it("should pass when actual matches expect", () => { + const result = checkResults([ + { + inputs: { in: 1 }, + expect: { out: { equals: 1 } }, + actual: { out: 1 }, + }, + ]); + assert.deepEqual(result, { passed: true }); + }); + it("should fail when actual does not match expect", () => { + const result = checkResults([ + { + inputs: { in: 1 }, + expect: { out: { equals: 2 } }, + actual: { out: 1 }, + }, + ]); + assert.equal(result.passed, false); + assert.ok(result.error instanceof Error); + assert.match(result.error.message, /to equal/); + }); + it("should fail when not enough outputs received", () => { + const result = checkResults([ + { inputs: { in: 1 }, expect: { out: { equals: 1 } } }, + ]); + assert.ok(result.error instanceof Error); + assert.match(result.error.message, /Only got 0 output messages out of 1/); + }); + it("should report sequence position of failures", () => { + const result = checkResults([ + { + inputs: { in: 1 }, + expect: { out: { equals: 1 } }, + actual: { out: 1 }, + }, + { + inputs: { in: 2 }, + expect: { out: { equals: 2 } }, + actual: { out: 3 }, + }, + ]); + assert.ok(result.error instanceof Error); + assert.match(result.error.message, /Expectation 1 of sequence failed/); + }); + it("should report multiple failures", () => { + const result = checkResults([ + { + inputs: { in: 1 }, + expect: { out: { equals: 2 } }, + actual: { out: 1 }, + }, + { + inputs: { in: 2 }, + expect: { out: { equals: 3 } }, + actual: { out: 2 }, + }, + ]); + assert.ok(result.error instanceof Error); + assert.match(result.error.message, /Multiple failures in sequence/); + }); + it("should not fail when actual is null but expects are empty", () => { + const result = checkResults([ + { inputs: { in: 1 }, expect: {}, actual: { out: 1 } }, + ]); + assert.deepEqual(result, { passed: true }); + }); + }); + + describe("runTestAndCheck", () => { + /** + * A Runner stand-in with canned results. + * @param {import("../lib/runner.js").RunnerLike["runTest"]} runTest Implementation of runTest + * @returns {import("../lib/runner.js").RunnerLike} Fake runner + */ + function fakeRunner(runTest) { + return { runTest }; + } + + it("should pass skipped testcases without running them", async () => { + let ran = false; + const result = await runTestAndCheck( + fakeRunner(async () => { + ran = true; + return []; + }), + { skip: "not now" }, + ); + assert.deepEqual(result, { passed: true }); + assert.equal(ran, false); + }); + it("should fail on inputs/expect length mismatch", async () => { + const result = await runTestAndCheck( + fakeRunner(async () => []), + { + inputs: [{ in: 1 }], + expect: [{ out: { equals: 1 } }, { out: { equals: 2 } }], + }, + ); + assert.equal(result.passed, false); + assert.ok(result.error instanceof Error); + assert.match(result.error.message, /sequence length mismatch/); + }); + it("should map runner errors to test failures", async () => { + const result = await runTestAndCheck( + fakeRunner(async () => { + throw new Error("runtime went away"); + }), + { inputs: { in: 1 }, expect: { out: { equals: 1 } } }, + ); + assert.equal(result.passed, false); + assert.ok(result.error instanceof Error); + assert.match(result.error.message, /runtime went away/); + }); + it("should pass on matching results", async () => { + const result = await runTestAndCheck( + fakeRunner(async (testcase) => [ + { + inputs: /** @type {Record} */ (testcase).inputs, + expect: /** @type {Record} */ (testcase).expect, + actual: { out: 42 }, + }, + ]), + { inputs: { in: 1 }, expect: { out: { equals: 42 } } }, + ); + assert.deepEqual(result, { passed: true }); + }); + it("should stringify errors into testcase results", async () => { + const testcase = { inputs: { in: 1 }, expect: { out: { equals: 2 } } }; + const runner = fakeRunner(async () => [ + { + inputs: testcase.inputs, + expect: testcase.expect, + actual: { out: 1 }, + }, + ]); + const result = await runTestAndCheck(runner, testcase); + assert.equal(result.passed, false); + }); + }); + + describe("Runner", () => { + it("should default options", () => { + const runner = new Runner({}); + assert.equal(runner.options.connectTimeout, 5000); + assert.equal(runner.options.commandTimeout, 3000); + }); + + it("should fail to connect to an unavailable runtime", async () => { + const runner = new Runner( + { protocol: "websocket", address: "ws://127.0.0.1:1" }, + { connectTimeout: 1500 }, + ); + await assert.rejects(() => runner.connect(), Error); + }); + }); +}); diff --git a/spec/schema.test.js b/spec/schema.test.js new file mode 100644 index 0000000..204c9ce --- /dev/null +++ b/spec/schema.test.js @@ -0,0 +1,39 @@ +import assert from "node:assert/strict"; +import { readFileSync } from "node:fs"; +import { describe, it } from "node:test"; +import tv4 from "tv4"; + +import schemas from "../schema/index.js"; + +const metaSchema = JSON.parse( + readFileSync(new URL("./json-schema.json", import.meta.url), "utf-8"), +); + +describe("Schema meta validation", () => { + for (const name of Object.keys(schemas)) { + const schema = schemas[name]; + describe(`${name}.json (${schema.title || schema.description})`, () => { + it("should validate against JSON meta schema", () => { + tv4.reset(); + tv4.addSchema("http://json-schema.org/draft-04/schema", metaSchema); + tv4.addSchema(schema.id, schema); + const result = tv4.validateResult( + schema, + "http://json-schema.org/draft-04/schema", + ); + assert.equal(result.valid, true, JSON.stringify(result.errors)); + }); + }); + } +}); + +describe("Schema module", () => { + it("should provide getSchema lookup", async () => { + const { getSchema } = await import("../schema/index.js"); + const schema = getSchema("testcase.json"); + assert.ok(schema, "expected testcase schema"); + assert.equal(schema.id, "testcase.json"); + assert.equal(getSchema("testcase")?.id, "testcase.json"); + assert.equal(getSchema("nonexistent.json"), undefined); + }); +}); diff --git a/spec/testsuite.test.js b/spec/testsuite.test.js new file mode 100644 index 0000000..784198d --- /dev/null +++ b/spec/testsuite.test.js @@ -0,0 +1,159 @@ +import assert from "node:assert/strict"; +import { mkdtemp, writeFile } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { after, before, describe, it } from "node:test"; + +import { create, getSuites, loadYAML, validate } from "../lib/testsuite.js"; + +const MULTI_DOC_YAML = `--- +name: First suite +topic: DummyComponent +cases: +- + name: 'case one' + assertion: 'should pass' + inputs: + in: true + expect: + out: + equals: true +--- +name: Second suite +topic: core/Repeat +cases: [] +`; + +const PASSING_YAML = ` +name: Simple passing +topic: DummyComponent +cases: +- + name: 'sending a boolean' + assertion: 'should pass' + inputs: + in: true + expect: + out: + equals: true +`; + +describe("testsuite", () => { + describe("create", () => { + it("should default name to topic", () => { + const suite = create({ topic: "core/Repeat" }); + assert.equal(suite.name, "core/Repeat"); + assert.deepEqual(suite.cases, []); + }); + it("should keep existing name and cases", () => { + const suite = create({ + name: "Mine", + topic: "core/Repeat", + cases: [{ name: "x" }], + }); + assert.equal(suite.name, "Mine"); + assert.equal(suite.cases?.length, 1); + }); + }); + + describe("loadYAML", () => { + it("should parse single-document YAML", () => { + const suites = loadYAML(PASSING_YAML); + assert.equal(suites.length, 1); + assert.equal(suites[0].name, "Simple passing"); + assert.equal(suites[0].cases?.length, 1); + }); + it("should parse multi-document YAML", () => { + const suites = loadYAML(MULTI_DOC_YAML); + assert.equal(suites.length, 2); + assert.equal(suites[0].name, "First suite"); + assert.equal(suites[1].name, "Second suite"); + }); + it("should throw on invalid YAML", () => { + assert.throws(() => loadYAML("foo: [unclosed"), Error); + }); + }); + + describe("getSuites", () => { + /** @type {string} */ + let dir; + + before(async () => { + dir = await mkdtemp(join(tmpdir(), "fbp-spec-test-")); + await writeFile(join(dir, "passing.yaml"), PASSING_YAML); + await writeFile(join(dir, "other.yaml"), MULTI_DOC_YAML); + }); + + it("should load from a single file", async () => { + const suites = await getSuites(join(dir, "passing.yaml")); + assert.equal(suites.length, 1); + assert.equal(suites[0].name, "Simple passing"); + }); + it("should load from a list of files", async () => { + const suites = await getSuites([ + join(dir, "passing.yaml"), + join(dir, "other.yaml"), + ]); + assert.equal(suites.length, 3); + }); + it("should load from a directory", async () => { + const suites = await getSuites(dir); + assert.equal(suites.length, 3); + }); + it("should reject on missing files", async () => { + await assert.rejects(getSuites(join(dir, "missing.yaml")), Error); + }); + + describe("over HTTP", () => { + /** @type {import('node:http').Server} */ + let server; + /** @type {string} */ + let baseUrl; + + before(async () => { + const http = await import("node:http"); + server = http.createServer((_req, res) => { + res.writeHead(200, { "Content-Type": "text/yaml" }); + res.end(PASSING_YAML); + }); + /** @type {Promise} */ + const listening = new Promise((resolve) => { + server.listen({ port: 0, host: "127.0.0.1" }, resolve); + }); + await listening; + const addr = server.address(); + const port = typeof addr === "object" && addr ? addr.port : 0; + baseUrl = `http://127.0.0.1:${port}/tests.yaml`; + }); + after( + () => + new Promise((resolve) => { + server.close(resolve); + }), + ); + + it("should load suites from a URL", async () => { + const suites = await getSuites(baseUrl); + assert.equal(suites.length, 1); + assert.equal(suites[0].name, "Simple passing"); + }); + }); + }); + + describe("validate", () => { + it("should accept a valid testsfile", () => { + const results = validate(loadYAML(MULTI_DOC_YAML)); + assert.equal(results.valid, true); + assert.equal(results.passed, true); + assert.deepEqual(results.errors, []); + assert.deepEqual(results.missing, []); + }); + it("should reject invalid testcases", () => { + const results = validate([ + { name: "Broken suite", cases: [{ inputs: 42 }] }, + ]); + assert.equal(results.valid, false); + assert.ok(results.errors.length > 0); + }); + }); +}); diff --git a/spec/validate-schemas.js b/spec/validate-schemas.js deleted file mode 100644 index c62dd0f..0000000 --- a/spec/validate-schemas.js +++ /dev/null @@ -1,38 +0,0 @@ -const tv4 = require('tv4'); -const chai = require('chai'); -const path = require('path'); -const fs = require('fs'); - -const schemaPath = '../schema'; - -function loadSchema(filepath) { - const content = fs.readFileSync(filepath, { encoding: 'utf-8' }); - return JSON.parse(content); -} -function getSchema(name) { - const filepath = path.join(__dirname, schemaPath, name); - return loadSchema(filepath); -} - -describe('Schema meta validation', () => { - let schemas = fs.readdirSync(path.join(__dirname, schemaPath)); - schemas = schemas.filter((s) => s.indexOf('.json') !== -1); - - before(() => { - const metaSchema = loadSchema(path.join(__dirname, 'json-schema.json')); - tv4.addSchema('http://json-schema.org/draft-04/schema', metaSchema); - }); - after(() => { - tv4.reset(); - }); - schemas.forEach((schemaFile) => { - const schema = getSchema(schemaFile); - tv4.addSchema(schema.id, schema); - describe(`${schemaFile} (${schema.title || schema.description})`, () => { - it('should validate against JSON meta schema', () => { - const result = tv4.validateResult(schema, 'http://json-schema.org/draft-04/schema'); - chai.expect(result.valid).to.equal(true); - }); - }); - }); -}); diff --git a/tsconfig.json b/tsconfig.json new file mode 100644 index 0000000..b5a9b6f --- /dev/null +++ b/tsconfig.json @@ -0,0 +1,25 @@ +{ + "compilerOptions": { + "allowJs": true, + "checkJs": true, + "noEmit": true, + "target": "ES2022", + "lib": ["ES2022", "DOM", "DOM.Iterable"], + "module": "Node16", + "moduleResolution": "Node16", + "strict": true, + "noImplicitAny": true, + "noUnusedLocals": false, + "skipLibCheck": true, + "resolveJsonModule": true + }, + "include": [ + "index.js", + "lib/**/*.js", + "schema/**/*.js", + "scripts/**/*.js", + "spec/**/*.js", + "examples/**/*.js", + "types.d.ts" + ] +} diff --git a/types.d.ts b/types.d.ts new file mode 100644 index 0000000..8f57382 --- /dev/null +++ b/types.d.ts @@ -0,0 +1,146 @@ +/** + * Ambient type declarations for dependencies without their own type information. + * The shapes here are limited to what fbp-spec uses. + */ + +declare module "fbp" { + /** + * Parse FBP graph definition text into a graph JSON payload. + * @param source FBP graph language source + * @param options Parsing options + * @returns Graph JSON payload + */ + export function parse(source: string, options?: { caseSensitive?: boolean }): object; +} + +declare module "fbp-client" { + /** A connected FBP protocol client. */ + export interface FbpClient { + definition: { + protocol?: string; + address?: string; + capabilities?: string[]; + namespace?: string; + [key: string]: unknown; + }; + options: Record; + isConnected(): boolean; + connect(): Promise; + disconnect(): Promise; + observe(filter: (message: any) => boolean): { + signals: { payload: Record }[]; + until( + onSuccess: (message: any) => boolean, + onFailure?: (message: any) => boolean, + ): Promise<{ payload: Record }[]>; + }; + on(event: string, listener: (...args: any[]) => void): void; + transport: { + setParentElement(element: unknown): void; + [key: string]: unknown; + }; + protocol: { + graph: { + send(graph: object, main?: boolean): Promise; + clear(payload: object): Promise; + addnode(payload: object): Promise; + addedge(payload: object): Promise; + addinitial(payload: object): Promise; + addinport(payload: object): Promise; + addoutport(payload: object): Promise; + }; + network: { + start(payload: object): Promise; + stop(payload: object): Promise; + }; + runtime: { + getruntime(payload?: object): Promise<{ capabilities?: string[] }>; + packet(payload: object): Promise; + }; + component: { + list(): Promise<{ name: string; inPorts: { id: string }[]; outPorts: { id: string }[]; [key: string]: unknown }[]>; + getsource(payload: { name: string }): Promise<{ + name: string; + library?: string; + tests?: string | null; + [key: string]: unknown; + }>; + }; + }; + } + + /** + * Create a FBP protocol client for a runtime. + * @param definition Runtime definition + * @param options Client options + * @returns Client instance + */ + function fbpClient( + definition: object, + options?: { commandTimeout?: number; connectionTimeout?: number }, + ): Promise; + export default fbpClient; +} + +declare module "fbp-graph" { + /** A FBP graph instance. */ + export interface Graph { + name?: string; + properties: Record; + nodes: unknown[]; + edges: unknown[]; + initializers: unknown[]; + inports: Record; + outports: Record; + [key: string]: unknown; + } + const fbpGraph: { + Graph: new (name: string, options?: object) => Graph; + graph: { + loadJSON( + definition: object | string, + callback?: (err: Error | null, graph?: Graph) => void, + metadata?: object, + ): Promise; + loadFBP( + source: string, + callback?: (err: Error | null, graph?: Graph) => void, + metadata?: object, + caseSensitive?: boolean, + ): Promise; + }; + }; + export default fbpGraph; +} + +declare module "flowtrace" { + /** Records FBP protocol traffic as replayable Flowtraces. */ + export class FlowtraceRecorder { + constructor(client: object, options?: object); + start(graphId: string): Promise; + stop(graphId: string): Promise; + dump(graphId: string): object; + [key: string]: unknown; + } +} + +declare module "tv4" { + /** Result of a tv4 validation. */ + export interface Tv4Result { + valid: boolean; + errors: { message: string; [key: string]: unknown }[]; + missing: string[]; + passed: boolean; + } + const tv4: { + addSchema(id: string, schema: object): void; + reset(): void; + validateMultiple(data: unknown, schemaId: string): Tv4Result; + validateResult( + data: unknown, + schemaId: string, + ): { valid: boolean; errors: { message: string }[]; missing: string[] }; + [key: string]: unknown; + }; + export default tv4; +} diff --git a/webpack.config.js b/webpack.config.js deleted file mode 100644 index 7987dd3..0000000 --- a/webpack.config.js +++ /dev/null @@ -1,34 +0,0 @@ -module.exports = { - entry: './webpack.entry.js', - output: { - path: __dirname, - filename: 'browser/fbp-spec.js', - }, - module: { - rules: [ - { - test: /\.js$/, - use: [ - { - loader: 'babel-loader', - options: { - presets: ['@babel/preset-env'], - }, - }, - ], - }, - ], - }, - resolve: { - extensions: ['.js'], - fallback: { - buffer: require.resolve('buffer/'), - events: require.resolve('events/'), - fs: false, - path: require.resolve('path-browserify'), - url: require.resolve('url/'), - }, - }, - externals: { - }, -}; diff --git a/webpack.entry.js b/webpack.entry.js deleted file mode 100644 index 0799b3a..0000000 --- a/webpack.entry.js +++ /dev/null @@ -1,18 +0,0 @@ -/* eslint-env browser */ -const exported = { - // eslint-disable-next-line global-require - 'fbp-spec': require('./index'), - // eslint-disable-next-line global-require - chai: require('chai'), - // eslint-disable-next-line global-require - 'js-yaml': require('js-yaml'), -}; - -if (window) { - window.require = function (moduleName) { - if (exported[moduleName]) { - return exported[moduleName]; - } - throw new Error(`Module ${moduleName} not available`); - }; -} From 192e0507dec64006120d5fd1d71b8d9647a1ffdd Mon Sep 17 00:00:00 2001 From: Henri Bergius Date: Sat, 10 Oct 2026 15:43:24 +1300 Subject: [PATCH 2/7] Relicense under EUPL-1.2 The project has no external contributors (only the original authors), so the license is updated from MIT to the European Union Public Licence v1.2 (EUPL-1.2). --- LICENSE.md | 190 ++++++++++++++++++++++++++++++++++++++++++++++----- package.json | 2 +- 2 files changed, 174 insertions(+), 18 deletions(-) diff --git a/LICENSE.md b/LICENSE.md index fb33730..dd8b720 100644 --- a/LICENSE.md +++ b/LICENSE.md @@ -1,23 +1,179 @@ +European Union Public Licence Version 1.2 -The MIT License (MIT) +EUPL © the European Union 2007, 2016 -Copyright (c) 2015-2018 Flowhub UG +This European Union Public Licence (the ‘EUPL’) applies to the Work (as defined below) which is provided under the terms of this Licence. Any use of the Work, other than as authorised under this Licence is prohibited (to the extent such use is covered by a right of the copyright holder of the Work). -Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the "Software"), to deal -in the Software without restriction, including without limitation the rights -to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the Software is -furnished to do so, subject to the following conditions: +The Work is provided under the terms of this Licence when the Licensor (as defined below) has placed the following notice immediately following the copyright notice for the Work: -The above copyright notice and this permission notice shall be included in -all copies or substantial portions of the Software. +Licensed under the EUPL -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN -THE SOFTWARE. +or has expressed by any other means his willingness to license under the EUPL. +1. Definitions + +In this Licence, the following terms shall have the following meanings: + +— ‘Licence’: this Licence Agreement. + +— ‘The Licensor’: the natural or legal person that distributes and/or communicates the Work under the Licence. + +— ‘The Author’: the creator of the Work. + +— ‘The Work’: the software distributed or communicated by the Licensor under this Licence. + +— ‘Contribution’: any work of authorship, including the original version of the Work and any modifications to it (the ‘Modified Version’), any derivative work, any additions, deletions or substitutions, that the Distributor receives from the Contributor and that is distributed and/or communicated by the Distributor under the Licence. + +— ‘Contributor’: the Licensor or any individual or legal entity in the world who modifies, directly or indirectly, the Work under the Licence, or contributes a Modification. + +— ‘Distributor’: any individual or legal entity in the world that receives and subsequently redistributes or communicates the Work under the Licence to other natural or legal persons. + +— ‘Original Work’: the software distributed or communicated by the Licensor under the Licence before any modification, in its source and/or object code form. + +— ‘Modified Work’: the Original Work updated, improved, completed, corrected or otherwise modified by the Contributor. + +— ‘Source Code’: the human-readable form of the Work which is the most convenient for people to study and modify. + +— ‘Executable Code’: any code which has generally been compiled and which is meant to be interpreted by a computer as a result. + +— ‘You’: the user of the Work being a legal entity or natural person. + +— ‘Reproduction’: the direct or indirect transitory or permanent copying of all or part of the Work by any means and in any form, including the permanent or temporary storage of such material by any means and in any form. + +— ‘Modification’: any alteration, by You, of all or any part of the Work, whether by altering, adding to, deleting from or adapting any part of it, in any way, including the incorporation of any part of the Work into another work, and which results in a Modified Work. + +— ‘Communication’: the act by which the Work, the Modified Version or the Derivative Work is distributed or otherwise made available to third parties in any form. + +— ‘Derivative Work’: means the original work of authorship which is derived from or based upon the Original Work or any part thereof, with the exclusion of works which are mere separable combinations of the Original Work and other works not based on the Original Work. + +2. Scope + +This Licence applies to any software which the copyright holder states to be subject to this Licence, and to any subsequent Modifications of that software received from a Contributor under the terms of this Licence. + +The Licensed Work is provided under the terms of this Licence when the Licensor has placed the notice specified in the Preamble immediately following the copyright notice for the Work, or has expressed by any other means his willingness to license under the EUPL. + +3. Grant of rights + +Subject to the terms and conditions of this Licence, the Licensor hereby grants You a world-wide, royalty-free, non-exclusive, sub-licensable (as set out in Clause 10) licence for the duration of the intellectual property rights attached to the Work, to: + +a) Reproduce the Work in any form and by any means, including making multiple copies of the Work; + +b) Communicate the Work, in its original or Modified form, in Source Code or Executable Code form, to third parties; + +c) Create Derivative Works, and communicate and distribute such Derivative Works in any form; + +d) Modify the Work, thereby creating a Modified Work, and communicate and distribute the Modified Work in any form. + +These rights may be exercised worldwide, and no territorial restriction applies. + +4. Limitations on Licence + +The rights granted under Clause 3 are granted subject to the limitations and conditions set out in this Clause 4. You are not granted the right to: + +a) use the Work in a manner which is defamatory of the Author or which might damage the reputation or honour of the Author; + +b) use the trademarks, trade names or logos of the Licensor or the Author in any way, except in so far as required for reasonable and customary use in describing the origin of the Work and reproducing the content of any copyright notice; + +c) use the Work in breach of applicable law. + +5. Licensing conditions for distribution and communication of the Work + +Whenever You distribute or communicate by any means the Work, whether in its original form, Modified form, or as a Derivative Work, You must: + +a) provide the recipient with a copy of this Licence; + +b) retain all copyright, patent, trademarks and any other proprietary notices contained in the Work; + +c) ensure that the recipient receives the Source Code of the Work which You distribute or communicate, in the same medium, at no extra charge; + +d) indicate, in a conspicuous way in the Source Code, any Modifications that You have made, including the date of the Modifications; + +e) cause the Modified Work, if any, to carry prominent notices stating that You have changed the Work and the date of any change; + +f) ensure that the recipients of the Work, Modified Work or Derivative Work are granted the rights set out in this Licence. + +6. Licensing conditions for distribution and communication of a Modified Work + +Whenever You distribute or communicate a Modified Work, You must: + +a) ensure that the Modified Work is licensed under this Licence, and that the recipients of the Modified Work are granted the rights set out in this Licence; + +b) indicate, in a conspicuous way in the Source Code of the Modified Work, the parts of it which consist of code derived from the Original Work, and the parts which constitute Your Modifications; + +c) cause the Modified Work to carry prominent notices stating that it is a Modified Work, and the date of any change; + +d) license the Modified Work under this Licence, without prejudice to the clauses below concerning compatibility with other licences. + +7. Compatibility clauses + +Whenever the distribution or communication of the Work, a Modified Work or a Derivative Work is subject to compatibility requirements with the licence of other software, and the Licensor has stated that the Work is compatible with one or more of the licences listed in the Annex, then You may distribute or communicate the Work, Modified Work or Derivative Work under the terms of such compatible licence(s), provided that You comply with the conditions of those licence(s). + +8. Patent and trademark licences + +The Licensor grants You a world-wide, royalty-free, non-exclusive licence, to the extent of its intellectual property rights, under any patents it holds or controls, to make, use, sell, offer for sale, import and otherwise exploit the Work, for the duration of the relevant patents. + +Where patent rights are concerned, theLicence only covers the use of the Work in the form in which it is distributed or communicated by the Licensor. Any patent licence granted under this Clause is limited to the patents held by the Licensor which are necessarily infringed by the use of the Work as distributed or communicated by the Licensor. + +The Licensor does not grant any licence in respect of trademarks, trade names or logos, other than as expressly provided in Clause 4. + +9. Warranty + +The Work is provided by the Licensor "as is" and without warranty of any kind, whether express or implied, including but not limited to the implied warranties of satisfactory quality, fitness for a particular purpose, title and non-infringement of third party rights. + +You are solely responsible for determining whether the use of the Work is appropriate for your purposes, and for the use you make of the Work. + +10. Liability + +Except in the case of gross negligence, wilful misconduct or where liability cannot be excluded by law, the Licensor shall not be liable for any direct or indirect damages, including loss of data, loss of profits, loss of business or loss of goodwill, arising out of or in connection with the use of the Work. + +Nothing in this Licence limits the Licensor's liability where such limitation is prohibited by applicable law. + +11. Additional licences + +The Licensor may offer the Work under separate licences or terms, in addition to this Licence. Such additional licences or terms shall not affect the rights granted under this Licence, and shall not prevent the exercise of the rights granted under this Licence. + +12. Termination + +Without prejudice to the survival of any accrued rights and obligations, this Licence and the rights granted hereunder shall terminate automatically upon any breach by You of the terms and conditions of this Licence. + +Where You are in breach of this Licence, the Licensor may, at its option, notify You of the breach and allow You a period of thirty (30) days to cure the breach. If the breach is not cured within this period, the Licence terminates. + +Upon termination, You shall cease all use of the Work, and destroy all copies of the Work in Your possession or control, provided that You shall not be prevented from retaining copies required by applicable law. + +13. Sub-licensing + +You may sub-license the Work (in original or Modified form) to third parties, but only under the terms of this Licence. + +Where You distribute or communicate the Work in Executable Code form, You may additionally offer recipients a warranty, support, indemnity or other liability terms, but doing so does not extend or modify the terms of this Licence, and You remain solely responsible for any such additional commitments. + +14. Governing law + +This Licence is governed by the law of the European Union, and, supplementing it, the law of the Member State where the Licensor has his seat or habitual residence, or, if the Licensor is a legal entity, the law of the Member State where it has its registered office. + +15. Language of the Licence + +This Licence is available in all the official languages of the European Union. In the event of any conflict between the different language versions, the version in the language of the European Union in which the Licence was first expressed shall prevail, or, failing that, the English version. + +Annex — Compatible licences + +The licences on which the Work may be distributed or communicated, as compatible with the EUPL, are the following: + +— GNU General Public License (GPL) v. 2, v. 3 + +— GNU Affero General Public License (AGPL) v. 3 + +— Open Software License (OSL) v. 2.1, v. 3.0 + +— Eclipse Public License (EPL) v. 1.0 + +— CeCILL v. 2.0, v. 2.1 + +— Mozilla Public License (MPL) v. 2 + +— GNU Lesser General Public License (LGPL) v. 2.1, v. 3 + +— Creative Commons Attribution-ShareAlike v. 3.0 Unported (CC BY-SA 3.0) for works other than software + +— European Union Public Licence (EUPL) v. 1.1, v. 1.2 + +— Québec Free and Open-Source Licence — Reciprocity (LiRo) v. 1.1 diff --git a/package.json b/package.json index 20566fc..b9225ee 100644 --- a/package.json +++ b/package.json @@ -12,7 +12,7 @@ }, "bugs": "http://github.com/flowbased/fbp-spec/issues", "homepage": "http://flowbased.org", - "license": "MIT", + "license": "EUPL-1.2", "type": "module", "main": "index.js", "exports": { From 9b0f92159b4f002c4f11168de3a705841fd422de Mon Sep 17 00:00:00 2001 From: Henri Bergius Date: Sat, 10 Oct 2026 15:43:45 +1300 Subject: [PATCH 3/7] Convert changelog to Keep a Changelog format Replace CHANGES.md with CHANGELOG.md, restructured into the Keep a Changelog format, with an Unreleased section documenting the modernization. --- CHANGELOG.md | 187 +++++++++++++++++++++++++++++++++++++++++++++++++++ CHANGES.md | 123 --------------------------------- 2 files changed, 187 insertions(+), 123 deletions(-) create mode 100644 CHANGELOG.md delete mode 100644 CHANGES.md diff --git a/CHANGELOG.md b/CHANGELOG.md new file mode 100644 index 0000000..0e02f0f --- /dev/null +++ b/CHANGELOG.md @@ -0,0 +1,187 @@ +# Changelog + +All notable changes to this project will be documented in this file. + +The format is based on [Keep a Changelog](https://keepachangelog.com/), +and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). + +## Unreleased + +### Changed + +- Converted the codebase to ECMAScript Modules (ESM), with `"type": "module"` and an explicit `"exports"` map in `package.json` +- Scoped all Node.js standard library imports with the `node:` prefix, and made Node-specific imports lazy so the library can still be bundled for browsers +- Converted all callback-style and Bluebird-based asynchronous APIs to native Promises with async/await. `Runner` methods (`connect`, `disconnect`, `setupSuite`, `teardownSuite`, `runTest`), `runAll`, `runTestAndCheck`, `getComponentSuites`, `subprocess.start` and `testsuite.getSuites` now return Promises +- Replaced Bluebird with native Promises, `isomorphic-fetch` with the native `fetch` API, and `commander`/`slug` with `node:util` `parseArgs` and a local slugifier +- Replaced the Chai-based expectation operators with implementations built on `node:assert` +- Added comprehensive JSDoc type annotations, verified with TypeScript in `checkJs` mode (`npm run typecheck`) +- Switched linting and formatting to Biome, configured via `.editorconfig` +- Replaced the grunt-yaml schema build step with `scripts/build-schemas.js`, which also emits a browser-loadable `schema/schemas.js` module +- Updated the test suite to the native `node:test` runner with `node:assert`, and added a JavaScript mock FBP runtime so end-to-end tests no longer require the Python example runtime +- Updated GitHub Actions workflows to test against Node.js 20 and 22, and to publish to npm via OpenID Connect (OIDC) with provenance +- Updated the license from MIT to EUPL-1.2 + +### Removed + +- Removed the Mocha runner integrations: running fbp-spec suites inside external Mocha suites (`fbp-spec.mocha.run`), and the `fbp-spec-mocha` compatibility runtime exposing Mocha tests over the FBP protocol +- Removed the browser build: Webpack, Babel, Karma configurations and the gh-pages deployment +- Removed the deprecated Mocha, Chai, Bluebird, Commander, isomorphic-fetch, slug, tv4-in-browser and Grunt dependencies +- Removed Dependabot auto-merge workflow configuration + +## 0.8.0 - 2020-11-11 + +### Added + +- Added a `--trace` command-line flag to capture Flowtraces for fbp-spec runs. The Flowtraces will be stored into a `.flowtrace` folder under the project + +## 0.7.1 - 2020-10-22 + +### Fixed + +- Fixed issue with loading spec files + +## 0.7.0 - 2020-10-22 + +### Added + +- Added a `--component-tests` command-line flag to control whether fbp-spec loads additional component tests from the runtime + +### Changed + +- Ported from CoffeeScript to modern JavaScript + +## 0.6.8 - 2020-09-09 + +### Fixed + +- Made fbp-spec ignore failing getSource requests +- If network setup fails, teardown for the network will also be skipped + +## 0.6.7 - 2020-09-01 + +### Changed + +- Updated dependencies to include latest FBP Protocol Client and Schemas +- Updated browser build to use a modern Babel version + +## 0.6.2 - 2018-03-31 + +### Fixed + +- Fixed a race condition with synchronous networks where we expect multiple output packets + +## 0.6.1 - 2018-03-31 + +### Fixed + +- Fixed issue where received bracket IPs were causing fbp-spec to not see the actual data packet + +## 0.6.0 - 2018-03-30 + +### Changed + +- Switched FBP Protocol communications with the runtime to use the new [fbp-client](https://github.com/flowbased/fbp-client) library +- All messages to and from runtime are now validated against the FBP Protocol schema +- Permissions are now validated against capabilities advertised by the runtime +- Tests now fail if runtime breaks the connection (for example, if runtime crashes mid-run) +- Tests now fail if runtime sends a `network:error` message +- Tests now fail if runtime sends a `network:processerror` message +- Tests now fail if runtime sends a message to an exported `error` outport if message to that port was expected +- All FBP Protocol operations now time out if there is no response + +## 0.5.0 - 2017-11-17 + +### Changed + +- Updated to fbp parser 1.7 which introduces consistency validations for graphs. This can cause some tests to break due to them having incorrectly defined nodes in them + +### Fixed + +- We actually catch and show FBP parsing errors now + +## 0.4.1 - 2017-11-07 + +### Changed + +- Require ES6 support. Node.js 6+ or a modern browser. If support for older environments are needed, use Babel to transpile +- Now using CoffeeScript 2 + +## 0.3.0 - 2017-11-06 + +### Changed + +- Updated to fbp-protocol-client 0.2.x, without support for MicroFlo transport + +## 0.2.3 - 2017-10-04 + +### Fixed + +- Fixed triggering Mocha error in start/stop due to returning Promise. Regression since 0.2.2 + +## 0.2.2 - 2017-10-03 + +### Added + +- Automatically retry connection while during setup until `startTimeout` + +## 0.2.0 - 2017-02-19 + +### Changed + +- Updated to Mocha 3.x. Can break tests in some cases where Promise is returned and callback is used at same time +- Updated all dependencies to latest version + +## 0.1.16 - 2016-10-11 + +### Added + +- Several conveniences added in .fbp DSL from `fbp 1.5` + +## 0.1.8 - 2016-04-20 + +### Fixed + +- `testsuite.validate()`: Fix not catching invalid files, due to wrong schema lookup + +### Added + +- Add support for sequences of inputs/expect. Allows testing stateful components/graphs/systems + +## 0.1.7 - 2016-04-17 + +### Added + +- Make `fixture` optional. If not specified, will be auto-created from component/graph in `topic` + +## 0.1.6 - 2016-04-14 + +### Changed + +- Some improvements to experimental `fbp-spec-mocha` + +## 0.1.4 - 2016-05-07 + +### Added + +- Send `network:output` FBP messages to `console.log()`, for easier debugging of problems on runtime side +- Added experimental `fbp-spec-mocha` tool, for running Mocha tests over fbp-spec protocol + +## 0.1.3 - 2015-08-27 + +First useful release + +### Added + +- Initial YAML testformat fully defined +- Tested with NoFlo, MicroFlo and Python example runtimes +- `fbp-spec` commandline tool for running +- `fbpspec.mocha.run` for integration in a Mocha testsuite +- Simple set of UI widgets, with experimental integration in noflo-ui + +## 0.0.1 - 2015-04-17 + +Proof of concept + +### Added + +- Could run a MicroFlo testcase by talking over FBP runtime protocol diff --git a/CHANGES.md b/CHANGES.md deleted file mode 100644 index aef7e16..0000000 --- a/CHANGES.md +++ /dev/null @@ -1,123 +0,0 @@ -# 0.8.0 - released 11.11.2020 - -* Added a `--trace` command-line flag to capture Flowtraces for fbp-spec runs. The Flowtraces will be stored into a `.flowtrace` folder under the project - -# 0.7.1 - released 22.10.2020 - -* Fixed issue with loading spec files - -# 0.7.0 - released 22.10.2020 - -* Added a `--component-tests` command-line flag to control whether fbp-spec loads additional component tests from the runtime -* Ported from CoffeeScript to modern JavaScript - -# 0.6.8 - released 09.09.2020 - -* Made fbp-spec ignore failing getSource requests -* If network setup fails, teardown for the network will also be skipped - -# 0.6.7 - released 01.09.2020 - -* Updated dependencies to include latest FBP Protocol Client and Schemas -* Updated browser build to use a modern Babel version - -# 0.6.2 - released 31.03.2018 - -* Fixed a race condition with synchronous networks where we expect multiple output packets - -# 0.6.1 - released 31.03.2018 - -* Fixed issue where received bracket IPs were causing fbp-spec to not see the actual data packet - -# 0.6.0 - released 30.03.2018 - -* Switched FBP Protocol communications with the runtime to use the new [fbp-client](https://github.com/flowbased/fbp-client) library -* All messages to and from runtime are now validated against the FBP Protocol schema -* Permissions are now validated against capabilities advertised by the runtime -* Tests now fail if runtime breaks the connection (for example, if runtime crashes mid-run) -* Tests now fail if runtime sends a `network:error` message -* Tests now fail if runtime sends a `network:processerror` message -* Tests now fail if runtime sends a message to an exported `error` outport if message to that port was expected -* All FBP Protocol operations now time out if there is no response - -# 0.5.0 - released 17.11.2017 - -* Updated to fbp parser 1.7 which introduces consistency validations for graphs. This can cause some tests to break due to them having incorrectly defined nodes in them - -Bugfixes: - -* We actually catch and show FBP parsing errors now - -# 0.4.1 - released 07.11.2017 - -Breaking changes - -* Require ES6 support. Node.js 6+ or a modern browser. -If support for older environments are needed, use Babel to transpile. - -Internal changes - -* Now using CoffeeScript 2 - -# 0.3.0 - released 06.11.2017 - -* Updated to fbp-protocol-client 0.2.x, without support for MicroFlo transport - -# 0.2.3 - released 04.10.2017 - -Bugfixes - -* Fixed triggering Mocha error in start/stop due to returning Promise. Regression since 0.2.2. - -# 0.2.2 - released 03.10.2017 - -* Automatically retry connection while during setup until `startTimeout` - -# 0.2.0 - released 19.02.2017 - -* Updated to Mocha 3.x. -Can break tests in some cases where Promise is returned and callback is used at same time. - -Internal changes - -* Updated all dependencies to latest version - -# 0.1.16 - released 11.10.2016 - -Features - -* Several conveniences added in .fbp DSL from `fbp 1.5` - -# 0.1.8 - released 20.04.2016 - -* `testsuite.validate()`: Fix not catching invalid files, due to wrong schema lookup -* Add support for sequences of inputs/expect. Allows testing stateful components/graphs/systems. - -# 0.1.7 - released 17.04.2016 - -* Make `fixture` optional. If not specified, will be auto-created from component/graph in `topic` - -# 0.1.6 - released 14.04.2016 - -* Some improvements to experimental `fbp-spec-mocha` - -# 0.1.4 - released 07.05.2016 - -* Send `network:output` FBP messages to `console.log()`, for easier debugging of problems on runtime side. -* Added experimental `fbp-spec-mocha` tool, for running Mocha tests over fbp-spec protocol. - -# 0.1.3 - released 27.08.2015 - -First useful release - -* Initial YAML testformat fully defined -* Tested with NoFlo, MicroFlo and Python example runtimes -* `fbp-spec` commandline tool for running -* `fbpspec.mocha.run` for integration in a Mocha testsuite -* Simple set of UI widgets, with experimental integration in noflo-ui - -# 0.0.1 - released 17.04.2015 - -Proof of concept - -* Could run a MicroFlo testcase by talking over FBP runtime protocol From 5905d44b5e2c648f270df4ee31848d1fd3faa9ad Mon Sep 17 00:00:00 2001 From: Henri Bergius Date: Sat, 10 Oct 2026 15:43:45 +1300 Subject: [PATCH 4/7] Update README for ESM codebase - Remove claims about the Mocha runner, browser build and Flowhub integration, which have been removed - Update license reference to EUPL-1.2 - Add library usage examples using the Promise-based ESM API - Update CLI documentation for the current options --- README.md | 113 +++++++++++++++++++++++++++++------------------------- 1 file changed, 60 insertions(+), 53 deletions(-) diff --git a/README.md b/README.md index 684f092..cea789c 100644 --- a/README.md +++ b/README.md @@ -1,7 +1,7 @@ fbp-spec ======== -A runtime-independent test framework for Flow Based Programming (FBP) component and graphs, +A runtime-independent test framework for Flow Based Programming (FBP) components and graphs, using declarative, data-driven testing. One can use fbp-spec to do testing at multiple levels, @@ -11,13 +11,12 @@ each approximately corresponding to the different architectural levels of Flow B * Integration (FBP graph) * System (FBP runtime) -## Status: +## Status **In production** -* Used by several projects, including [Ingress Table](https://github.com/c-base/ingress-table) and [BIG IoT NoFlo bridge](https://github.com/flowhub/bigiot-bridge) * Tested with several FBP runtimes: [NoFlo](https://noflojs.org), [MicroFlo](https://microflo.org), [Python example](https://github.com/flowbased/protocol-examples), [MsgFlo](http://msgflo.org) -* Runners available for contious integration (CLI, Mocha) and interactively (in [Flowhub](https://flowhub.org)) +* Runner available for continuous integration (CLI) and as a library for building custom runners, interactive or automated ## Purpose & Scope @@ -30,13 +29,13 @@ The following is considered out-of-scope: ## License -The [MIT license](./LICENSE.md) +The [European Union Public Licence v1.2 (EUPL-1.2)](./LICENSE.md) # Usage ## Installing -Set up fbp-spec as an NPM dependency +Requires Node.js 20 or newer. Set up fbp-spec as an npm dependency: npm install --save-dev fbp-spec @@ -108,7 +107,7 @@ You can send data to multiple inports and check expectations on multiple ports p #### Sequence of packets For testing components with state, you can sending multiple input packets -[in sequence](https://github.com/flowbased/fbp-spec/blob/master/examples/sequences.yaml). +[in sequence](./examples/sequences.yaml). ```YAML - @@ -215,7 +214,7 @@ Instead of `equals` you can use any of the supported assertion predicates. Examp haveKeys includeKeys -For a full set of assertions, see [the schema](https://github.com/flowbased/fbp-spec/blob/master/schemata/expectation.yaml) +For a full set of assertions, see [the schema](./schemata/expectation.yaml) #### More @@ -223,7 +222,7 @@ A comprehensive set of examples can be found under [./examples](./examples). For the detailed definition of the dataformat for tests, see [schemata/](./schemata/). -## Running tests with fbp-spec commandline tool +## Running tests with the fbp-spec commandline tool The simplest and most universal way of running tests is with the `fbp-spec` commandline tool. @@ -231,59 +230,74 @@ The simplest and most universal way of running tests is with the `fbp-spec` comm MultiSuite, failing tests sending a boolean with wrong expect - should fail: ✗ Error: expected true to deeply equal false + should fail: ✗ Error: expected true to equal false sending a number with wrong expect - should fail: ✗ Error: expected 1000 to deeply equal 1003 + should fail: ✗ Error: expected 1000 to equal 1003 MultiSuite, passing tests sending a boolean should repeat the same: ✓ sending a number should repeat the same: ✓ -The `--command` options can be used to specify a command which will start the runtime under test: +The `--command` option can be used to specify a command which will start the runtime under test: - fbp-spec --command "python2 protocol-examples/python/runtime.py" + fbp-spec --command "node my-runtime.js" examples/ -It sets the exit status to non-zero on failure, so is suitable for integrating into a `Makefile` or similar. +Multiple suite files (or directories) can be given at once. +It sets the exit status to non-zero on failure, so is suitable for integrating into a `Makefile` or CI workflow. +Run `fbp-spec --help` for the full set of options. -## Running tests by integrating with Mocha +Use the `--component-tests` option to also load fbp-spec tests +attached to components on the runtime, and `--trace` to store a replayable +[Flowtrace](https://github.com/flowbased/flowtrace) for each testcase under `.flowtrace/`. -[Mocha](http://mochajs.org/) iss a popular test runner framework for JavaScript/CoffeeScript on browser and node.js. +## Running tests from a JavaScript application -Since fbp-spec communicates with your runtime over a network protocol, -you can use this also when your project is not JavaScript-based. -The Mocha runner is for instance [used in microflo-core](https://github.com/microflo/microflo-core/blob/master/spec/ComponentTests.coffee) -to test C++ components for microcontrollers & embedded devices. +The test runner is available as a zero-build ES Module, +and can be used from Node.js or bundled for the browser. -You can have your fbp-spec tests run in Mocha by calling the `fbpspec.mocha.run()` function, in a file which is -executed with the standard Mocha runner. Eg. `mocha --reporter spec tests/fbpspecs.js` +```js +import { Runner, runAll, runTestAndCheck } from 'fbp-spec'; +import * as testsuite from 'fbp-spec/testsuite'; - // fbpspecs.js - fbpspec = require('fbp-spec'); +// Load suites from a file, directory or HTTP URL +const suites = await testsuite.getSuites('./examples/simple-passing.yaml'); +// Or parse a YAML string directly, for example fetched from a project store: +// const suites = testsuite.loadYAML(yamlSource); - rt = { - protocol: "websocket", - address: "ws://localhost:3569", - secret: "py3k", // Optional. If needed to connect/authenticate to runtime - command: 'python2 protocol-examples/python/runtime.py' // Optional. Can be used to start runtime automatically - }; - fbpspec.mocha.run(rt, './examples/simple-passing.yaml', { starttimeout: 1000 }); +// Either pass a runtime definition (fbp-spec will connect), or an +// already-connected fbp-client instance +const runner = new Runner({ + protocol: 'websocket', + address: 'ws://localhost:3569', + secret: '', // optional. If needed to connect/authenticate to runtime +}); -The tests can be specified as a list of files, or directories. -You can use the standard `grep` option of Mocha to run only some tests. - -For CoffeScript example, see [./spec/mocha.js](./spec/mocha.js). +await runner.connect(); +await runAll(runner, suites, (updated) => { + // Called as test results come in; testcase objects get + // .passed and .error set in-place, suitable for rendering progress + ui.renderProgress(updated); +}); +await runner.disconnect(); +``` -## Running tests interactively in Flowhub +For finer-grained control, set up a suite and run individual testcases: -[Flowhub](http://app.flowhub.io) IDE (version 0.11 and later) has integrated support for fbp-spec. No installation is required. +```js +await runner.setupSuite(suite); +const result = await runTestAndCheck(runner, testcase); +// result: { passed: true } or { passed: false, error: Error } +await runner.teardownSuite(suite); +``` -* Open existing project, or create a new one -* Open a component, and write/copypaste in a test in the `Tests` panel -* Ensure you have a runtime set up, and connected +Test suites can be validated against the fbp-spec schemas: -When you make changes to your project (components,graphs) or tests, Flowhub will now automatically (re-)run your tests. -You can see the status in the top-right corner. Clicking on it brings up more details. +```js +import { validate } from 'fbp-spec/testsuite'; +const results = validate(suites); +// results: { valid, passed, errors, missing } +``` ## Generating tests programatically @@ -292,7 +306,7 @@ a format one directly specifies tests in, but can also be generated from other s Sometimes data-driven testing, one does a large amount of very similar tests, with multiple test-cases per set of input data. -By capturing only the unique parts of testcases in a specialied data-structure (JSON, YAML, etc), +By capturing only the unique parts of testcases in a specialized data-structure (JSON, YAML, etc), and then transforming this into standard `fbp-spec` files with some code, adding/removing cases becomes even easier. For instance in `imgflo-server`, [testcases](https://github.com/jonnor/imgflo-server/blob/master/spec/graphtests.yaml) @@ -300,18 +314,12 @@ can be defined by providing a name, an URL and a reference result (a file with n Similarly, one can generate testcases using fuzzing, schema-based, model-based or similar tools. -## Integrating test runner in an application - -The test runner code is accessible as a JavaScript library, -and can be integrated into other apps (like Flowhub does). -See examples of [commandline](./lib/cli.js) and webappp usage. - -## Add supporting for a new runtime +## Add support for a new runtime You need to implement the [FBP network protocol](https://github.com/flowbased/fbp-protocol). At least the `protocol:runtime`, `protocol:graph`, and `protocol:network` capabilities are required. -All transports supported by [fbp-protocol-client]((https://github.com/flowbased/fbp-protocol)) +All transports supported by [fbp-protocol-client](https://github.com/flowbased/fbp-protocol-client) are supported by fbp-spec, including WebSocket, WebRTC, and iframe/postMessage. fbp-spec is intended to be used with flow-based and dataflow-programming, @@ -323,5 +331,4 @@ As long as you stay compatible with the [fbp-spec testformat](./schemata/) and [FBP protocol](http://noflojs.org/documentation/protocol/), you can implement a compatible runner in any programming language. -You can consider the fbp-spec code (in CoffeeScript) as a reference implementation. - +You can consider the fbp-spec code (in JavaScript) as a reference implementation. From 1c6690547cc2ca111caab268e5fdb1989da27260 Mon Sep 17 00:00:00 2001 From: Henri Bergius Date: Sat, 10 Oct 2026 15:43:51 +1300 Subject: [PATCH 5/7] Modernize CI workflows - Test against Node.js 20 and 22, and use current action versions - Publish to npm via OpenID Connect with provenance, replacing the legacy NPM_TOKEN secret authentication - Remove the Dependabot auto-merge configuration --- .github/dependabot.yml | 11 ---------- .github/workflows/publish.yml | 40 ++++++++++------------------------- .github/workflows/test.yml | 24 ++++++--------------- 3 files changed, 17 insertions(+), 58 deletions(-) delete mode 100644 .github/dependabot.yml diff --git a/.github/dependabot.yml b/.github/dependabot.yml deleted file mode 100644 index c2e5e7b..0000000 --- a/.github/dependabot.yml +++ /dev/null @@ -1,11 +0,0 @@ -version: 2 -updates: - - package-ecosystem: npm - directory: "/" - schedule: - interval: daily - versioning-strategy: increase-if-necessary - - package-ecosystem: github-actions - directory: "/" - schedule: - interval: weekly diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index 594979a..46f75f5 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -9,43 +9,25 @@ jobs: build: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v2.3.4 - - uses: actions/setup-node@v2.1.4 + - uses: actions/checkout@v4 + - uses: actions/setup-node@v4 with: - node-version: 12 - - run: git submodule update --init - - run: sudo apt-get install -y python-dev build-essential - - run: sudo pip install setuptools wheel --upgrade - - run: pip install --system -r protocol-examples/python/requirements.txt -t protocol-examples/python - - run: npm install - - run: npm test - publish-npm: - needs: build - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v2.3.4 - - uses: actions/setup-node@v2.1.4 - with: - node-version: 12 - registry-url: https://registry.npmjs.org/ + node-version: "22" - run: npm install - run: npm run build - - run: npm publish - env: - NODE_AUTH_TOKEN: ${{secrets.NPM_TOKEN}} + - run: npm test - deploy: + publish-npm: needs: build runs-on: ubuntu-latest + permissions: + id-token: write steps: - - uses: actions/checkout@v2.3.4 - - uses: actions/setup-node@v2.1.4 + - uses: actions/checkout@v4 + - uses: actions/setup-node@v4 with: - node-version: 12 + node-version: "22" registry-url: https://registry.npmjs.org/ - run: npm install - run: npm run build - - uses: peaceiris/actions-gh-pages@v3 - with: - github_token: ${{ secrets.GITHUB_TOKEN }} - publish_dir: ./browser + - run: npm publish --provenance diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index ac4fd8a..9affd7a 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -8,29 +8,17 @@ jobs: runs-on: ubuntu-latest strategy: matrix: - node-version: [12.x] + node-version: [20.x, 22.x] steps: - - uses: actions/checkout@v2.3.4 + - uses: actions/checkout@v4 + with: + submodules: true - name: Use Node.js ${{ matrix.node-version }} - uses: actions/setup-node@v2.1.4 + uses: actions/setup-node@v4 with: node-version: ${{ matrix.node-version }} - - name: install Python runtime example - run: | - sudo apt-get install -y python-dev build-essential - sudo pip install setuptools wheel --upgrade - git submodule update --init - pip install --system -r protocol-examples/python/requirements.txt -t protocol-examples/python - run: npm install + - run: npm run build - run: npm test env: CI: true - merge-me: - name: Auto-merge dependency updates - needs: test - runs-on: ubuntu-latest - steps: - - uses: ridedott/merge-me-action@v1.8.115 - with: - GITHUB_LOGIN: 'dependabot[bot]' - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} From 232e1697e4e66a8a2eec23b2d6796b3e9226bab2 Mon Sep 17 00:00:00 2001 From: Henri Bergius Date: Sat, 10 Oct 2026 15:54:55 +1300 Subject: [PATCH 6/7] Revert license back to MIT The project has significant third-party contributions, so the proposed EUPL-1.2 relicensing is reverted and the MIT license kept. --- CHANGELOG.md | 1 - LICENSE.md | 190 +++++---------------------------------------------- README.md | 2 +- package.json | 2 +- 4 files changed, 19 insertions(+), 176 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 0e02f0f..e58c13a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -19,7 +19,6 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - Replaced the grunt-yaml schema build step with `scripts/build-schemas.js`, which also emits a browser-loadable `schema/schemas.js` module - Updated the test suite to the native `node:test` runner with `node:assert`, and added a JavaScript mock FBP runtime so end-to-end tests no longer require the Python example runtime - Updated GitHub Actions workflows to test against Node.js 20 and 22, and to publish to npm via OpenID Connect (OIDC) with provenance -- Updated the license from MIT to EUPL-1.2 ### Removed diff --git a/LICENSE.md b/LICENSE.md index dd8b720..fb33730 100644 --- a/LICENSE.md +++ b/LICENSE.md @@ -1,179 +1,23 @@ -European Union Public Licence Version 1.2 -EUPL © the European Union 2007, 2016 +The MIT License (MIT) -This European Union Public Licence (the ‘EUPL’) applies to the Work (as defined below) which is provided under the terms of this Licence. Any use of the Work, other than as authorised under this Licence is prohibited (to the extent such use is covered by a right of the copyright holder of the Work). +Copyright (c) 2015-2018 Flowhub UG -The Work is provided under the terms of this Licence when the Licensor (as defined below) has placed the following notice immediately following the copyright notice for the Work: +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: -Licensed under the EUPL +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. -or has expressed by any other means his willingness to license under the EUPL. +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. -1. Definitions - -In this Licence, the following terms shall have the following meanings: - -— ‘Licence’: this Licence Agreement. - -— ‘The Licensor’: the natural or legal person that distributes and/or communicates the Work under the Licence. - -— ‘The Author’: the creator of the Work. - -— ‘The Work’: the software distributed or communicated by the Licensor under this Licence. - -— ‘Contribution’: any work of authorship, including the original version of the Work and any modifications to it (the ‘Modified Version’), any derivative work, any additions, deletions or substitutions, that the Distributor receives from the Contributor and that is distributed and/or communicated by the Distributor under the Licence. - -— ‘Contributor’: the Licensor or any individual or legal entity in the world who modifies, directly or indirectly, the Work under the Licence, or contributes a Modification. - -— ‘Distributor’: any individual or legal entity in the world that receives and subsequently redistributes or communicates the Work under the Licence to other natural or legal persons. - -— ‘Original Work’: the software distributed or communicated by the Licensor under the Licence before any modification, in its source and/or object code form. - -— ‘Modified Work’: the Original Work updated, improved, completed, corrected or otherwise modified by the Contributor. - -— ‘Source Code’: the human-readable form of the Work which is the most convenient for people to study and modify. - -— ‘Executable Code’: any code which has generally been compiled and which is meant to be interpreted by a computer as a result. - -— ‘You’: the user of the Work being a legal entity or natural person. - -— ‘Reproduction’: the direct or indirect transitory or permanent copying of all or part of the Work by any means and in any form, including the permanent or temporary storage of such material by any means and in any form. - -— ‘Modification’: any alteration, by You, of all or any part of the Work, whether by altering, adding to, deleting from or adapting any part of it, in any way, including the incorporation of any part of the Work into another work, and which results in a Modified Work. - -— ‘Communication’: the act by which the Work, the Modified Version or the Derivative Work is distributed or otherwise made available to third parties in any form. - -— ‘Derivative Work’: means the original work of authorship which is derived from or based upon the Original Work or any part thereof, with the exclusion of works which are mere separable combinations of the Original Work and other works not based on the Original Work. - -2. Scope - -This Licence applies to any software which the copyright holder states to be subject to this Licence, and to any subsequent Modifications of that software received from a Contributor under the terms of this Licence. - -The Licensed Work is provided under the terms of this Licence when the Licensor has placed the notice specified in the Preamble immediately following the copyright notice for the Work, or has expressed by any other means his willingness to license under the EUPL. - -3. Grant of rights - -Subject to the terms and conditions of this Licence, the Licensor hereby grants You a world-wide, royalty-free, non-exclusive, sub-licensable (as set out in Clause 10) licence for the duration of the intellectual property rights attached to the Work, to: - -a) Reproduce the Work in any form and by any means, including making multiple copies of the Work; - -b) Communicate the Work, in its original or Modified form, in Source Code or Executable Code form, to third parties; - -c) Create Derivative Works, and communicate and distribute such Derivative Works in any form; - -d) Modify the Work, thereby creating a Modified Work, and communicate and distribute the Modified Work in any form. - -These rights may be exercised worldwide, and no territorial restriction applies. - -4. Limitations on Licence - -The rights granted under Clause 3 are granted subject to the limitations and conditions set out in this Clause 4. You are not granted the right to: - -a) use the Work in a manner which is defamatory of the Author or which might damage the reputation or honour of the Author; - -b) use the trademarks, trade names or logos of the Licensor or the Author in any way, except in so far as required for reasonable and customary use in describing the origin of the Work and reproducing the content of any copyright notice; - -c) use the Work in breach of applicable law. - -5. Licensing conditions for distribution and communication of the Work - -Whenever You distribute or communicate by any means the Work, whether in its original form, Modified form, or as a Derivative Work, You must: - -a) provide the recipient with a copy of this Licence; - -b) retain all copyright, patent, trademarks and any other proprietary notices contained in the Work; - -c) ensure that the recipient receives the Source Code of the Work which You distribute or communicate, in the same medium, at no extra charge; - -d) indicate, in a conspicuous way in the Source Code, any Modifications that You have made, including the date of the Modifications; - -e) cause the Modified Work, if any, to carry prominent notices stating that You have changed the Work and the date of any change; - -f) ensure that the recipients of the Work, Modified Work or Derivative Work are granted the rights set out in this Licence. - -6. Licensing conditions for distribution and communication of a Modified Work - -Whenever You distribute or communicate a Modified Work, You must: - -a) ensure that the Modified Work is licensed under this Licence, and that the recipients of the Modified Work are granted the rights set out in this Licence; - -b) indicate, in a conspicuous way in the Source Code of the Modified Work, the parts of it which consist of code derived from the Original Work, and the parts which constitute Your Modifications; - -c) cause the Modified Work to carry prominent notices stating that it is a Modified Work, and the date of any change; - -d) license the Modified Work under this Licence, without prejudice to the clauses below concerning compatibility with other licences. - -7. Compatibility clauses - -Whenever the distribution or communication of the Work, a Modified Work or a Derivative Work is subject to compatibility requirements with the licence of other software, and the Licensor has stated that the Work is compatible with one or more of the licences listed in the Annex, then You may distribute or communicate the Work, Modified Work or Derivative Work under the terms of such compatible licence(s), provided that You comply with the conditions of those licence(s). - -8. Patent and trademark licences - -The Licensor grants You a world-wide, royalty-free, non-exclusive licence, to the extent of its intellectual property rights, under any patents it holds or controls, to make, use, sell, offer for sale, import and otherwise exploit the Work, for the duration of the relevant patents. - -Where patent rights are concerned, theLicence only covers the use of the Work in the form in which it is distributed or communicated by the Licensor. Any patent licence granted under this Clause is limited to the patents held by the Licensor which are necessarily infringed by the use of the Work as distributed or communicated by the Licensor. - -The Licensor does not grant any licence in respect of trademarks, trade names or logos, other than as expressly provided in Clause 4. - -9. Warranty - -The Work is provided by the Licensor "as is" and without warranty of any kind, whether express or implied, including but not limited to the implied warranties of satisfactory quality, fitness for a particular purpose, title and non-infringement of third party rights. - -You are solely responsible for determining whether the use of the Work is appropriate for your purposes, and for the use you make of the Work. - -10. Liability - -Except in the case of gross negligence, wilful misconduct or where liability cannot be excluded by law, the Licensor shall not be liable for any direct or indirect damages, including loss of data, loss of profits, loss of business or loss of goodwill, arising out of or in connection with the use of the Work. - -Nothing in this Licence limits the Licensor's liability where such limitation is prohibited by applicable law. - -11. Additional licences - -The Licensor may offer the Work under separate licences or terms, in addition to this Licence. Such additional licences or terms shall not affect the rights granted under this Licence, and shall not prevent the exercise of the rights granted under this Licence. - -12. Termination - -Without prejudice to the survival of any accrued rights and obligations, this Licence and the rights granted hereunder shall terminate automatically upon any breach by You of the terms and conditions of this Licence. - -Where You are in breach of this Licence, the Licensor may, at its option, notify You of the breach and allow You a period of thirty (30) days to cure the breach. If the breach is not cured within this period, the Licence terminates. - -Upon termination, You shall cease all use of the Work, and destroy all copies of the Work in Your possession or control, provided that You shall not be prevented from retaining copies required by applicable law. - -13. Sub-licensing - -You may sub-license the Work (in original or Modified form) to third parties, but only under the terms of this Licence. - -Where You distribute or communicate the Work in Executable Code form, You may additionally offer recipients a warranty, support, indemnity or other liability terms, but doing so does not extend or modify the terms of this Licence, and You remain solely responsible for any such additional commitments. - -14. Governing law - -This Licence is governed by the law of the European Union, and, supplementing it, the law of the Member State where the Licensor has his seat or habitual residence, or, if the Licensor is a legal entity, the law of the Member State where it has its registered office. - -15. Language of the Licence - -This Licence is available in all the official languages of the European Union. In the event of any conflict between the different language versions, the version in the language of the European Union in which the Licence was first expressed shall prevail, or, failing that, the English version. - -Annex — Compatible licences - -The licences on which the Work may be distributed or communicated, as compatible with the EUPL, are the following: - -— GNU General Public License (GPL) v. 2, v. 3 - -— GNU Affero General Public License (AGPL) v. 3 - -— Open Software License (OSL) v. 2.1, v. 3.0 - -— Eclipse Public License (EPL) v. 1.0 - -— CeCILL v. 2.0, v. 2.1 - -— Mozilla Public License (MPL) v. 2 - -— GNU Lesser General Public License (LGPL) v. 2.1, v. 3 - -— Creative Commons Attribution-ShareAlike v. 3.0 Unported (CC BY-SA 3.0) for works other than software - -— European Union Public Licence (EUPL) v. 1.1, v. 1.2 - -— Québec Free and Open-Source Licence — Reciprocity (LiRo) v. 1.1 diff --git a/README.md b/README.md index cea789c..8f3ce1b 100644 --- a/README.md +++ b/README.md @@ -29,7 +29,7 @@ The following is considered out-of-scope: ## License -The [European Union Public Licence v1.2 (EUPL-1.2)](./LICENSE.md) +The [MIT license](./LICENSE.md) # Usage diff --git a/package.json b/package.json index b9225ee..20566fc 100644 --- a/package.json +++ b/package.json @@ -12,7 +12,7 @@ }, "bugs": "http://github.com/flowbased/fbp-spec/issues", "homepage": "http://flowbased.org", - "license": "EUPL-1.2", + "license": "MIT", "type": "module", "main": "index.js", "exports": { From 06d739447099c7e3ffd70df376d3d1ea66f16e4d Mon Sep 17 00:00:00 2001 From: Henri Bergius Date: Sat, 10 Oct 2026 15:57:38 +1300 Subject: [PATCH 7/7] Fix test script glob, and target Node.js 24 and 26 in CI - Let the shell expand the spec glob in the test script: quoted globs are only handled by node --test itself since Node 21, so the CI job on older Node failed with 'Could not find spec/*.test.js' - Test against Node.js 24 and 26 in CI, and publish on Node 26 - Drop the protocol-examples submodule checkout, as tests now use the JavaScript mock runtime - Restrict publish workflow permissions to contents: read + id-token: write --- .github/workflows/publish.yml | 5 +++-- .github/workflows/test.yml | 4 +--- package.json | 2 +- 3 files changed, 5 insertions(+), 6 deletions(-) diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index 46f75f5..63bf4f7 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -12,7 +12,7 @@ jobs: - uses: actions/checkout@v4 - uses: actions/setup-node@v4 with: - node-version: "22" + node-version: "26" - run: npm install - run: npm run build - run: npm test @@ -21,12 +21,13 @@ jobs: needs: build runs-on: ubuntu-latest permissions: + contents: read id-token: write steps: - uses: actions/checkout@v4 - uses: actions/setup-node@v4 with: - node-version: "22" + node-version: "26" registry-url: https://registry.npmjs.org/ - run: npm install - run: npm run build diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 9affd7a..33700f8 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -8,11 +8,9 @@ jobs: runs-on: ubuntu-latest strategy: matrix: - node-version: [20.x, 22.x] + node-version: [24.x, 26.x] steps: - uses: actions/checkout@v4 - with: - submodules: true - name: Use Node.js ${{ matrix.node-version }} uses: actions/setup-node@v4 with: diff --git a/package.json b/package.json index 20566fc..b6e28de 100644 --- a/package.json +++ b/package.json @@ -40,7 +40,7 @@ "lint": "npx @biomejs/biome check --use-editorconfig=true lib/ schema/ scripts/ spec/ examples/ index.js bin/", "format": "npx @biomejs/biome check --use-editorconfig=true --write lib/ schema/ scripts/ spec/ examples/ index.js bin/", "typecheck": "tsc", - "test": "npm run lint && npm run typecheck && node --test \"spec/*.test.js\"" + "test": "npm run lint && npm run typecheck && node --test spec/*.test.js" }, "dependencies": { "debug": "^4.4.3",