diff --git a/.editorconfig b/.editorconfig new file mode 100644 index 0000000..2c0e19c --- /dev/null +++ b/.editorconfig @@ -0,0 +1,9 @@ +root = true + +[*] +end_of_line = lf +insert_final_newline = true +charset = utf-8 +indent_style = space +indent_size = 2 +trim_trailing_whitespace = true diff --git a/.eslintrc.json b/.eslintrc.json deleted file mode 100644 index 7cde01d..0000000 --- a/.eslintrc.json +++ /dev/null @@ -1,3 +0,0 @@ -{ - "extends": "airbnb-base" -} diff --git a/.github/dependabot.yml b/.github/dependabot.yml deleted file mode 100644 index c2e5e7b..0000000 --- a/.github/dependabot.yml +++ /dev/null @@ -1,11 +0,0 @@ -version: 2 -updates: - - package-ecosystem: npm - directory: "/" - schedule: - interval: daily - versioning-strategy: increase-if-necessary - - package-ecosystem: github-actions - directory: "/" - schedule: - interval: weekly diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index 594979a..63bf4f7 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -9,43 +9,26 @@ jobs: build: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v2.3.4 - - uses: actions/setup-node@v2.1.4 + - uses: actions/checkout@v4 + - uses: actions/setup-node@v4 with: - node-version: 12 - - run: git submodule update --init - - run: sudo apt-get install -y python-dev build-essential - - run: sudo pip install setuptools wheel --upgrade - - run: pip install --system -r protocol-examples/python/requirements.txt -t protocol-examples/python - - run: npm install - - run: npm test - publish-npm: - needs: build - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v2.3.4 - - uses: actions/setup-node@v2.1.4 - with: - node-version: 12 - registry-url: https://registry.npmjs.org/ + node-version: "26" - run: npm install - run: npm run build - - run: npm publish - env: - NODE_AUTH_TOKEN: ${{secrets.NPM_TOKEN}} + - run: npm test - deploy: + publish-npm: needs: build runs-on: ubuntu-latest + permissions: + contents: read + id-token: write steps: - - uses: actions/checkout@v2.3.4 - - uses: actions/setup-node@v2.1.4 + - uses: actions/checkout@v4 + - uses: actions/setup-node@v4 with: - node-version: 12 + node-version: "26" registry-url: https://registry.npmjs.org/ - run: npm install - run: npm run build - - uses: peaceiris/actions-gh-pages@v3 - with: - github_token: ${{ secrets.GITHUB_TOKEN }} - publish_dir: ./browser + - run: npm publish --provenance diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index ac4fd8a..33700f8 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -8,29 +8,15 @@ jobs: runs-on: ubuntu-latest strategy: matrix: - node-version: [12.x] + node-version: [24.x, 26.x] steps: - - uses: actions/checkout@v2.3.4 + - uses: actions/checkout@v4 - name: Use Node.js ${{ matrix.node-version }} - uses: actions/setup-node@v2.1.4 + uses: actions/setup-node@v4 with: node-version: ${{ matrix.node-version }} - - name: install Python runtime example - run: | - sudo apt-get install -y python-dev build-essential - sudo pip install setuptools wheel --upgrade - git submodule update --init - pip install --system -r protocol-examples/python/requirements.txt -t protocol-examples/python - run: npm install + - run: npm run build - run: npm test env: CI: true - merge-me: - name: Auto-merge dependency updates - needs: test - runs-on: ubuntu-latest - steps: - - uses: ridedott/merge-me-action@v1.8.115 - with: - GITHUB_LOGIN: 'dependabot[bot]' - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} diff --git a/.gitignore b/.gitignore index 25dd414..0356290 100644 --- a/.gitignore +++ b/.gitignore @@ -1,12 +1,8 @@ *~ node_modules/ -package-lock.json *.includes *.config *.autosave build/ -browser/ -components/ -examples/bundle* -spec/result.xml -.grunt/ +.flowtrace/ +package-lock.json diff --git a/.npmignore b/.npmignore deleted file mode 100644 index 01626bf..0000000 --- a/.npmignore +++ /dev/null @@ -1,6 +0,0 @@ -build/ -browser/ -examples/bundle* -spec/result.xml -protocol-examples/ -.grunt/ diff --git a/CHANGELOG.md b/CHANGELOG.md new file mode 100644 index 0000000..e58c13a --- /dev/null +++ b/CHANGELOG.md @@ -0,0 +1,186 @@ +# Changelog + +All notable changes to this project will be documented in this file. + +The format is based on [Keep a Changelog](https://keepachangelog.com/), +and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). + +## Unreleased + +### Changed + +- Converted the codebase to ECMAScript Modules (ESM), with `"type": "module"` and an explicit `"exports"` map in `package.json` +- Scoped all Node.js standard library imports with the `node:` prefix, and made Node-specific imports lazy so the library can still be bundled for browsers +- Converted all callback-style and Bluebird-based asynchronous APIs to native Promises with async/await. `Runner` methods (`connect`, `disconnect`, `setupSuite`, `teardownSuite`, `runTest`), `runAll`, `runTestAndCheck`, `getComponentSuites`, `subprocess.start` and `testsuite.getSuites` now return Promises +- Replaced Bluebird with native Promises, `isomorphic-fetch` with the native `fetch` API, and `commander`/`slug` with `node:util` `parseArgs` and a local slugifier +- Replaced the Chai-based expectation operators with implementations built on `node:assert` +- Added comprehensive JSDoc type annotations, verified with TypeScript in `checkJs` mode (`npm run typecheck`) +- Switched linting and formatting to Biome, configured via `.editorconfig` +- Replaced the grunt-yaml schema build step with `scripts/build-schemas.js`, which also emits a browser-loadable `schema/schemas.js` module +- Updated the test suite to the native `node:test` runner with `node:assert`, and added a JavaScript mock FBP runtime so end-to-end tests no longer require the Python example runtime +- Updated GitHub Actions workflows to test against Node.js 20 and 22, and to publish to npm via OpenID Connect (OIDC) with provenance + +### Removed + +- Removed the Mocha runner integrations: running fbp-spec suites inside external Mocha suites (`fbp-spec.mocha.run`), and the `fbp-spec-mocha` compatibility runtime exposing Mocha tests over the FBP protocol +- Removed the browser build: Webpack, Babel, Karma configurations and the gh-pages deployment +- Removed the deprecated Mocha, Chai, Bluebird, Commander, isomorphic-fetch, slug, tv4-in-browser and Grunt dependencies +- Removed Dependabot auto-merge workflow configuration + +## 0.8.0 - 2020-11-11 + +### Added + +- Added a `--trace` command-line flag to capture Flowtraces for fbp-spec runs. The Flowtraces will be stored into a `.flowtrace` folder under the project + +## 0.7.1 - 2020-10-22 + +### Fixed + +- Fixed issue with loading spec files + +## 0.7.0 - 2020-10-22 + +### Added + +- Added a `--component-tests` command-line flag to control whether fbp-spec loads additional component tests from the runtime + +### Changed + +- Ported from CoffeeScript to modern JavaScript + +## 0.6.8 - 2020-09-09 + +### Fixed + +- Made fbp-spec ignore failing getSource requests +- If network setup fails, teardown for the network will also be skipped + +## 0.6.7 - 2020-09-01 + +### Changed + +- Updated dependencies to include latest FBP Protocol Client and Schemas +- Updated browser build to use a modern Babel version + +## 0.6.2 - 2018-03-31 + +### Fixed + +- Fixed a race condition with synchronous networks where we expect multiple output packets + +## 0.6.1 - 2018-03-31 + +### Fixed + +- Fixed issue where received bracket IPs were causing fbp-spec to not see the actual data packet + +## 0.6.0 - 2018-03-30 + +### Changed + +- Switched FBP Protocol communications with the runtime to use the new [fbp-client](https://github.com/flowbased/fbp-client) library +- All messages to and from runtime are now validated against the FBP Protocol schema +- Permissions are now validated against capabilities advertised by the runtime +- Tests now fail if runtime breaks the connection (for example, if runtime crashes mid-run) +- Tests now fail if runtime sends a `network:error` message +- Tests now fail if runtime sends a `network:processerror` message +- Tests now fail if runtime sends a message to an exported `error` outport if message to that port was expected +- All FBP Protocol operations now time out if there is no response + +## 0.5.0 - 2017-11-17 + +### Changed + +- Updated to fbp parser 1.7 which introduces consistency validations for graphs. This can cause some tests to break due to them having incorrectly defined nodes in them + +### Fixed + +- We actually catch and show FBP parsing errors now + +## 0.4.1 - 2017-11-07 + +### Changed + +- Require ES6 support. Node.js 6+ or a modern browser. If support for older environments are needed, use Babel to transpile +- Now using CoffeeScript 2 + +## 0.3.0 - 2017-11-06 + +### Changed + +- Updated to fbp-protocol-client 0.2.x, without support for MicroFlo transport + +## 0.2.3 - 2017-10-04 + +### Fixed + +- Fixed triggering Mocha error in start/stop due to returning Promise. Regression since 0.2.2 + +## 0.2.2 - 2017-10-03 + +### Added + +- Automatically retry connection while during setup until `startTimeout` + +## 0.2.0 - 2017-02-19 + +### Changed + +- Updated to Mocha 3.x. Can break tests in some cases where Promise is returned and callback is used at same time +- Updated all dependencies to latest version + +## 0.1.16 - 2016-10-11 + +### Added + +- Several conveniences added in .fbp DSL from `fbp 1.5` + +## 0.1.8 - 2016-04-20 + +### Fixed + +- `testsuite.validate()`: Fix not catching invalid files, due to wrong schema lookup + +### Added + +- Add support for sequences of inputs/expect. Allows testing stateful components/graphs/systems + +## 0.1.7 - 2016-04-17 + +### Added + +- Make `fixture` optional. If not specified, will be auto-created from component/graph in `topic` + +## 0.1.6 - 2016-04-14 + +### Changed + +- Some improvements to experimental `fbp-spec-mocha` + +## 0.1.4 - 2016-05-07 + +### Added + +- Send `network:output` FBP messages to `console.log()`, for easier debugging of problems on runtime side +- Added experimental `fbp-spec-mocha` tool, for running Mocha tests over fbp-spec protocol + +## 0.1.3 - 2015-08-27 + +First useful release + +### Added + +- Initial YAML testformat fully defined +- Tested with NoFlo, MicroFlo and Python example runtimes +- `fbp-spec` commandline tool for running +- `fbpspec.mocha.run` for integration in a Mocha testsuite +- Simple set of UI widgets, with experimental integration in noflo-ui + +## 0.0.1 - 2015-04-17 + +Proof of concept + +### Added + +- Could run a MicroFlo testcase by talking over FBP runtime protocol diff --git a/CHANGES.md b/CHANGES.md deleted file mode 100644 index aef7e16..0000000 --- a/CHANGES.md +++ /dev/null @@ -1,123 +0,0 @@ -# 0.8.0 - released 11.11.2020 - -* Added a `--trace` command-line flag to capture Flowtraces for fbp-spec runs. The Flowtraces will be stored into a `.flowtrace` folder under the project - -# 0.7.1 - released 22.10.2020 - -* Fixed issue with loading spec files - -# 0.7.0 - released 22.10.2020 - -* Added a `--component-tests` command-line flag to control whether fbp-spec loads additional component tests from the runtime -* Ported from CoffeeScript to modern JavaScript - -# 0.6.8 - released 09.09.2020 - -* Made fbp-spec ignore failing getSource requests -* If network setup fails, teardown for the network will also be skipped - -# 0.6.7 - released 01.09.2020 - -* Updated dependencies to include latest FBP Protocol Client and Schemas -* Updated browser build to use a modern Babel version - -# 0.6.2 - released 31.03.2018 - -* Fixed a race condition with synchronous networks where we expect multiple output packets - -# 0.6.1 - released 31.03.2018 - -* Fixed issue where received bracket IPs were causing fbp-spec to not see the actual data packet - -# 0.6.0 - released 30.03.2018 - -* Switched FBP Protocol communications with the runtime to use the new [fbp-client](https://github.com/flowbased/fbp-client) library -* All messages to and from runtime are now validated against the FBP Protocol schema -* Permissions are now validated against capabilities advertised by the runtime -* Tests now fail if runtime breaks the connection (for example, if runtime crashes mid-run) -* Tests now fail if runtime sends a `network:error` message -* Tests now fail if runtime sends a `network:processerror` message -* Tests now fail if runtime sends a message to an exported `error` outport if message to that port was expected -* All FBP Protocol operations now time out if there is no response - -# 0.5.0 - released 17.11.2017 - -* Updated to fbp parser 1.7 which introduces consistency validations for graphs. This can cause some tests to break due to them having incorrectly defined nodes in them - -Bugfixes: - -* We actually catch and show FBP parsing errors now - -# 0.4.1 - released 07.11.2017 - -Breaking changes - -* Require ES6 support. Node.js 6+ or a modern browser. -If support for older environments are needed, use Babel to transpile. - -Internal changes - -* Now using CoffeeScript 2 - -# 0.3.0 - released 06.11.2017 - -* Updated to fbp-protocol-client 0.2.x, without support for MicroFlo transport - -# 0.2.3 - released 04.10.2017 - -Bugfixes - -* Fixed triggering Mocha error in start/stop due to returning Promise. Regression since 0.2.2. - -# 0.2.2 - released 03.10.2017 - -* Automatically retry connection while during setup until `startTimeout` - -# 0.2.0 - released 19.02.2017 - -* Updated to Mocha 3.x. -Can break tests in some cases where Promise is returned and callback is used at same time. - -Internal changes - -* Updated all dependencies to latest version - -# 0.1.16 - released 11.10.2016 - -Features - -* Several conveniences added in .fbp DSL from `fbp 1.5` - -# 0.1.8 - released 20.04.2016 - -* `testsuite.validate()`: Fix not catching invalid files, due to wrong schema lookup -* Add support for sequences of inputs/expect. Allows testing stateful components/graphs/systems. - -# 0.1.7 - released 17.04.2016 - -* Make `fixture` optional. If not specified, will be auto-created from component/graph in `topic` - -# 0.1.6 - released 14.04.2016 - -* Some improvements to experimental `fbp-spec-mocha` - -# 0.1.4 - released 07.05.2016 - -* Send `network:output` FBP messages to `console.log()`, for easier debugging of problems on runtime side. -* Added experimental `fbp-spec-mocha` tool, for running Mocha tests over fbp-spec protocol. - -# 0.1.3 - released 27.08.2015 - -First useful release - -* Initial YAML testformat fully defined -* Tested with NoFlo, MicroFlo and Python example runtimes -* `fbp-spec` commandline tool for running -* `fbpspec.mocha.run` for integration in a Mocha testsuite -* Simple set of UI widgets, with experimental integration in noflo-ui - -# 0.0.1 - released 17.04.2015 - -Proof of concept - -* Could run a MicroFlo testcase by talking over FBP runtime protocol diff --git a/Gruntfile.js b/Gruntfile.js deleted file mode 100644 index fbb0f31..0000000 --- a/Gruntfile.js +++ /dev/null @@ -1,155 +0,0 @@ -const path = require('path'); -const fetch = require('isomorphic-fetch'); -const webpackConfig = require('./webpack.config.js'); - -module.exports = function () { - // Project configuration - const pkg = this.file.readJSON('package.json'); - - this.initConfig({ - pkg, - - // Schemas - yaml: { - schemas: { - files: [{ - expand: true, - cwd: 'schemata/', - src: '*.yaml', - dest: 'schema/', - }, - ], - }, - }, - - // Building for browser - webpack: { - build: webpackConfig, - }, - - watch: { - src: { - files: [ - 'src/**/*', - 'examples/**/*', - 'spec/**/*', - ], - tasks: 'test', - options: { - livereload: true, - }, - }, - }, - - exec: { - runtime: { - command: 'python2 protocol-examples/python/runtime.py --port 3334', - }, - }, - - // Coding standards - yamllint: { - schemas: ['schemata/*.yaml'], - examples: ['examples/*.yml'], - }, - - // Tests - mochaTest: { - nodejs: { - src: ['spec/*.js'], - options: { - reporter: 'spec', - grep: process.env.TESTS, - }, - }, - }, - - downloadfile: { - files: [ - { url: 'https://noflojs.org/noflo-browser/everything.html', dest: 'browser/spec/fixtures' }, - { url: 'https://noflojs.org/noflo-browser/everything.js', dest: 'browser/spec/fixtures' }, - ], - }, - - // BDD tests on browser - karma: { - unit: { - configFile: 'karma.config.js', - }, - }, - - // Deploying - copy: { - ui: { - files: [{ - expand: true, cwd: './ui/', src: '*', dest: './browser/', - }], - }, - }, - }); - - // Grunt plugins used for building - this.loadNpmTasks('grunt-yaml'); - this.loadNpmTasks('grunt-webpack'); - this.loadNpmTasks('grunt-contrib-watch'); - - // Grunt plugins used for testing - this.loadNpmTasks('grunt-yamllint'); - this.loadNpmTasks('grunt-mocha-test'); - this.loadNpmTasks('grunt-karma'); - this.loadNpmTasks('grunt-exec'); - - this.registerTask('examples:bundle', () => { - // eslint-disable-next-line global-require - const examples = require('./examples'); - return examples.bundle(); - }); - - // Grunt plugins used for deploying - this.loadNpmTasks('grunt-contrib-copy'); - - // Our local tasks - const grunt = this; - this.registerMultiTask('downloadfile', 'Download a file', function () { - const callback = this.async(); - const promises = this.data.map((conf) => fetch(conf.url) - .then((res) => res.text()).then((content) => { - const filename = path.basename(conf.url); - const location = path.join(conf.dest, path.sep, filename); - grunt.file.write(location, content); - console.log(`Wrote ${conf.url} to ${location}`); - return true; - })); - return Promise.all(promises) - .then(() => callback(), - (err) => callback(err)); - }); - - this.registerTask('build', 'Build', (target = 'all') => { - this.task.run('yaml'); - if (['all', 'browser'].includes(target)) { - this.task.run('webpack'); - this.task.run('examples:bundle'); - this.task.run('copy:ui'); - } - }); - - this.registerTask('test', 'Build and run tests', (target = 'all') => { - this.task.run('yamllint'); - this.task.run(`build:${target}`); - if (['all', 'nodejs'].includes(target)) { - this.task.run('mochaTest'); - } - if (['all', 'browser'].includes(target)) { - this.task.run('downloadfile'); - this.task.run('karma'); - } - }); - - this.registerTask('default', ['test']); - - this.registerTask('dev', 'Developing', (target = 'all') => { - this.task.run(`test:${target}`); - this.task.run('watch'); - }); -}; diff --git a/README.md b/README.md index 684f092..8f3ce1b 100644 --- a/README.md +++ b/README.md @@ -1,7 +1,7 @@ fbp-spec ======== -A runtime-independent test framework for Flow Based Programming (FBP) component and graphs, +A runtime-independent test framework for Flow Based Programming (FBP) components and graphs, using declarative, data-driven testing. One can use fbp-spec to do testing at multiple levels, @@ -11,13 +11,12 @@ each approximately corresponding to the different architectural levels of Flow B * Integration (FBP graph) * System (FBP runtime) -## Status: +## Status **In production** -* Used by several projects, including [Ingress Table](https://github.com/c-base/ingress-table) and [BIG IoT NoFlo bridge](https://github.com/flowhub/bigiot-bridge) * Tested with several FBP runtimes: [NoFlo](https://noflojs.org), [MicroFlo](https://microflo.org), [Python example](https://github.com/flowbased/protocol-examples), [MsgFlo](http://msgflo.org) -* Runners available for contious integration (CLI, Mocha) and interactively (in [Flowhub](https://flowhub.org)) +* Runner available for continuous integration (CLI) and as a library for building custom runners, interactive or automated ## Purpose & Scope @@ -36,7 +35,7 @@ The [MIT license](./LICENSE.md) ## Installing -Set up fbp-spec as an NPM dependency +Requires Node.js 20 or newer. Set up fbp-spec as an npm dependency: npm install --save-dev fbp-spec @@ -108,7 +107,7 @@ You can send data to multiple inports and check expectations on multiple ports p #### Sequence of packets For testing components with state, you can sending multiple input packets -[in sequence](https://github.com/flowbased/fbp-spec/blob/master/examples/sequences.yaml). +[in sequence](./examples/sequences.yaml). ```YAML - @@ -215,7 +214,7 @@ Instead of `equals` you can use any of the supported assertion predicates. Examp haveKeys includeKeys -For a full set of assertions, see [the schema](https://github.com/flowbased/fbp-spec/blob/master/schemata/expectation.yaml) +For a full set of assertions, see [the schema](./schemata/expectation.yaml) #### More @@ -223,7 +222,7 @@ A comprehensive set of examples can be found under [./examples](./examples). For the detailed definition of the dataformat for tests, see [schemata/](./schemata/). -## Running tests with fbp-spec commandline tool +## Running tests with the fbp-spec commandline tool The simplest and most universal way of running tests is with the `fbp-spec` commandline tool. @@ -231,59 +230,74 @@ The simplest and most universal way of running tests is with the `fbp-spec` comm MultiSuite, failing tests sending a boolean with wrong expect - should fail: ✗ Error: expected true to deeply equal false + should fail: ✗ Error: expected true to equal false sending a number with wrong expect - should fail: ✗ Error: expected 1000 to deeply equal 1003 + should fail: ✗ Error: expected 1000 to equal 1003 MultiSuite, passing tests sending a boolean should repeat the same: ✓ sending a number should repeat the same: ✓ -The `--command` options can be used to specify a command which will start the runtime under test: +The `--command` option can be used to specify a command which will start the runtime under test: - fbp-spec --command "python2 protocol-examples/python/runtime.py" + fbp-spec --command "node my-runtime.js" examples/ -It sets the exit status to non-zero on failure, so is suitable for integrating into a `Makefile` or similar. +Multiple suite files (or directories) can be given at once. +It sets the exit status to non-zero on failure, so is suitable for integrating into a `Makefile` or CI workflow. +Run `fbp-spec --help` for the full set of options. -## Running tests by integrating with Mocha +Use the `--component-tests` option to also load fbp-spec tests +attached to components on the runtime, and `--trace` to store a replayable +[Flowtrace](https://github.com/flowbased/flowtrace) for each testcase under `.flowtrace/`. -[Mocha](http://mochajs.org/) iss a popular test runner framework for JavaScript/CoffeeScript on browser and node.js. +## Running tests from a JavaScript application -Since fbp-spec communicates with your runtime over a network protocol, -you can use this also when your project is not JavaScript-based. -The Mocha runner is for instance [used in microflo-core](https://github.com/microflo/microflo-core/blob/master/spec/ComponentTests.coffee) -to test C++ components for microcontrollers & embedded devices. +The test runner is available as a zero-build ES Module, +and can be used from Node.js or bundled for the browser. -You can have your fbp-spec tests run in Mocha by calling the `fbpspec.mocha.run()` function, in a file which is -executed with the standard Mocha runner. Eg. `mocha --reporter spec tests/fbpspecs.js` +```js +import { Runner, runAll, runTestAndCheck } from 'fbp-spec'; +import * as testsuite from 'fbp-spec/testsuite'; - // fbpspecs.js - fbpspec = require('fbp-spec'); +// Load suites from a file, directory or HTTP URL +const suites = await testsuite.getSuites('./examples/simple-passing.yaml'); +// Or parse a YAML string directly, for example fetched from a project store: +// const suites = testsuite.loadYAML(yamlSource); - rt = { - protocol: "websocket", - address: "ws://localhost:3569", - secret: "py3k", // Optional. If needed to connect/authenticate to runtime - command: 'python2 protocol-examples/python/runtime.py' // Optional. Can be used to start runtime automatically - }; - fbpspec.mocha.run(rt, './examples/simple-passing.yaml', { starttimeout: 1000 }); +// Either pass a runtime definition (fbp-spec will connect), or an +// already-connected fbp-client instance +const runner = new Runner({ + protocol: 'websocket', + address: 'ws://localhost:3569', + secret: '', // optional. If needed to connect/authenticate to runtime +}); -The tests can be specified as a list of files, or directories. -You can use the standard `grep` option of Mocha to run only some tests. - -For CoffeScript example, see [./spec/mocha.js](./spec/mocha.js). +await runner.connect(); +await runAll(runner, suites, (updated) => { + // Called as test results come in; testcase objects get + // .passed and .error set in-place, suitable for rendering progress + ui.renderProgress(updated); +}); +await runner.disconnect(); +``` -## Running tests interactively in Flowhub +For finer-grained control, set up a suite and run individual testcases: -[Flowhub](http://app.flowhub.io) IDE (version 0.11 and later) has integrated support for fbp-spec. No installation is required. +```js +await runner.setupSuite(suite); +const result = await runTestAndCheck(runner, testcase); +// result: { passed: true } or { passed: false, error: Error } +await runner.teardownSuite(suite); +``` -* Open existing project, or create a new one -* Open a component, and write/copypaste in a test in the `Tests` panel -* Ensure you have a runtime set up, and connected +Test suites can be validated against the fbp-spec schemas: -When you make changes to your project (components,graphs) or tests, Flowhub will now automatically (re-)run your tests. -You can see the status in the top-right corner. Clicking on it brings up more details. +```js +import { validate } from 'fbp-spec/testsuite'; +const results = validate(suites); +// results: { valid, passed, errors, missing } +``` ## Generating tests programatically @@ -292,7 +306,7 @@ a format one directly specifies tests in, but can also be generated from other s Sometimes data-driven testing, one does a large amount of very similar tests, with multiple test-cases per set of input data. -By capturing only the unique parts of testcases in a specialied data-structure (JSON, YAML, etc), +By capturing only the unique parts of testcases in a specialized data-structure (JSON, YAML, etc), and then transforming this into standard `fbp-spec` files with some code, adding/removing cases becomes even easier. For instance in `imgflo-server`, [testcases](https://github.com/jonnor/imgflo-server/blob/master/spec/graphtests.yaml) @@ -300,18 +314,12 @@ can be defined by providing a name, an URL and a reference result (a file with n Similarly, one can generate testcases using fuzzing, schema-based, model-based or similar tools. -## Integrating test runner in an application - -The test runner code is accessible as a JavaScript library, -and can be integrated into other apps (like Flowhub does). -See examples of [commandline](./lib/cli.js) and webappp usage. - -## Add supporting for a new runtime +## Add support for a new runtime You need to implement the [FBP network protocol](https://github.com/flowbased/fbp-protocol). At least the `protocol:runtime`, `protocol:graph`, and `protocol:network` capabilities are required. -All transports supported by [fbp-protocol-client]((https://github.com/flowbased/fbp-protocol)) +All transports supported by [fbp-protocol-client](https://github.com/flowbased/fbp-protocol-client) are supported by fbp-spec, including WebSocket, WebRTC, and iframe/postMessage. fbp-spec is intended to be used with flow-based and dataflow-programming, @@ -323,5 +331,4 @@ As long as you stay compatible with the [fbp-spec testformat](./schemata/) and [FBP protocol](http://noflojs.org/documentation/protocol/), you can implement a compatible runner in any programming language. -You can consider the fbp-spec code (in CoffeeScript) as a reference implementation. - +You can consider the fbp-spec code (in JavaScript) as a reference implementation. diff --git a/bin/fbp-spec b/bin/fbp-spec index 27002de..0b4af79 100755 --- a/bin/fbp-spec +++ b/bin/fbp-spec @@ -1,2 +1,4 @@ #!/usr/bin/env node -require('../lib/cli').main(); +import { main } from '../lib/cli.js'; + +main(); diff --git a/bin/fbp-spec-mocha b/bin/fbp-spec-mocha deleted file mode 100755 index fee3573..0000000 --- a/bin/fbp-spec-mocha +++ /dev/null @@ -1,2 +0,0 @@ -#!/usr/bin/env node -require('../lib/mochacompat').main(); diff --git a/biome.json b/biome.json new file mode 100644 index 0000000..edd39d7 --- /dev/null +++ b/biome.json @@ -0,0 +1,13 @@ +{ + "$schema": "https://biomejs.dev/schemas/2.5.15/schema.json", + "files": { + "includes": ["**", "!schema/schemas.js", "!schema/*.json", "!package-lock.json"] + }, + "assist": { + "actions": { + "source": { + "organizeImports": "off" + } + } + } +} diff --git a/browser/CNAME b/browser/CNAME deleted file mode 100644 index 185fcc0..0000000 --- a/browser/CNAME +++ /dev/null @@ -1 +0,0 @@ -fbpspec.run diff --git a/examples/index.js b/examples/index.js deleted file mode 100644 index bb5ee74..0000000 --- a/examples/index.js +++ /dev/null @@ -1,48 +0,0 @@ -const yaml = require('js-yaml'); - -const path = require('path'); - -const examplesDir = path.join(__dirname); - -const listExamples = function () { - // eslint-disable-next-line global-require - const files = require('fs').readdirSync(examplesDir); - const result = []; - files.forEach((f) => { - if (f.indexOf('.yaml') !== -1) { - result.push(f); - } - }); - return result; -}; -const getExample = function (name) { - const p = path.join(examplesDir, name); - // eslint-disable-next-line global-require - const content = require('fs').readFileSync(p, { encoding: 'utf-8' }); - const results = []; - yaml.loadAll(content, (doc) => results.push(doc)); - if (results.length === 1) { - return results[0]; - } - return results; -}; - -const buildExampleBundle = function () { - const examples = {}; - listExamples().forEach((name) => { - examples[name] = getExample(name); - }); - const contents = JSON.stringify(examples, null, 2); - - let dest = `${examplesDir}/bundle.js`; - // eslint-disable-next-line global-require - require('fs').writeFileSync(dest, `window.fbpspec_examples = ${contents};`); - console.log(`wrote ${dest}`); - - dest = `${examplesDir}/bundle.json`; - // eslint-disable-next-line global-require - require('fs').writeFileSync(dest, contents); - return console.log(`wrote ${dest}`); -}; - -exports.bundle = buildExampleBundle; diff --git a/index.js b/index.js index b58e228..4ec7fec 100644 --- a/index.js +++ b/index.js @@ -1 +1,6 @@ -module.exports = require('./lib/index'); +/** + * fbp-spec - a runtime-independent test framework for Flow Based Programming. + * @module fbp-spec + */ + +export * from "./lib/index.js"; diff --git a/karma.config.js b/karma.config.js deleted file mode 100644 index 3a3ff74..0000000 --- a/karma.config.js +++ /dev/null @@ -1,29 +0,0 @@ -module.exports = (config) => { - const configuration = { - basePath: process.cwd(), - frameworks: [ - 'mocha', - 'chai', - ], - reporters: [ - 'mocha', - ], - files: [ - 'browser/fbp-spec.js', - 'examples/bundle.js', - 'spec/import.js', - 'spec/examples.js', - { - pattern: 'browser/spec/fixtures/everything.*', - included: false, - served: true, - watched: true, - }, - ], - browsers: ['ChromeHeadless'], - logLevel: config.LOG_WARN, - singleRun: true, - }; - - config.set(configuration); -}; diff --git a/lib/cli.js b/lib/cli.js index 3b03bd2..f8c9d3c 100644 --- a/lib/cli.js +++ b/lib/cli.js @@ -1,182 +1,212 @@ -const program = require('commander'); -const debug = require('debug')('fbp-spec:cli'); -const { stat, mkdir, writeFile } = require('fs'); -const { resolve, relative } = require('path'); -const slug = require('slug'); -const testsuite = require('./testsuite'); -const runner = require('./runner'); -const subprocess = require('./subprocess'); +/** + * Command-line interface for running fbp-spec testsuites against an FBP runtime. + * @module lib/cli + */ +import { mkdir, stat, writeFile } from "node:fs/promises"; +import { relative, resolve } from "node:path"; +import { parseArgs } from "node:util"; +import debugLib from "debug"; + +import { getComponentSuites, Runner, runAll } from "./runner.js"; +import { start } from "./subprocess.js"; +import { getSuites } from "./testsuite.js"; + +const debug = debugLib("fbp-spec:cli"); + +/** @typedef {import("./testsuite.js").Suite} Suite */ + +/** @type {string[]} */ const traces = []; -// # Main -function parse(args) { - const actionHandler = (suites, opts) => { - const o = opts; - o.suites = suites; - }; +const usage = `Usage: fbp-spec [options] + +Run fbp-spec testsuites against an FBP runtime. + +Options: + --address Address of runtime to connect to (default: ws://localhost:3569) + --secret Runtime secret + --command Command to launch runtime under test + --start-timeout Time to wait for runtime to start (default: 10) + --command-timeout Max time for a FBP command (default: 3) + --component-tests Load component tests from runtime + --trace Store a replayable Flowtrace for each test run + --help Show this help +`; - program - .storeOptionsAsProperties(false) - .passCommandToAction(false) - .arguments('') - .action(actionHandler) - .option('--address ', 'Address of runtime to connect to', String, 'ws://localhost:3569') - .option('--secret ', 'Runtime secret', String, null) - .option('--command ', 'Command to launch runtime under test', String, null) - .option('--start-timeout ', 'Time to wait for runtime to start', Number, 10) - .option('--command-timeout ', 'Max time for a FBP command', Number, 3) - .option('--component-tests', 'Load component tests from runtime') - .option('--trace', 'Store a replayable Flowtrace for each test run') - .parse(args); - - return program; +/** + * Parse command-line arguments into runner options. + * @param {string[]} [args] Arguments to parse, defaults to process.argv + * @returns {{ options: { address: string, secret?: string, command?: string, startTimeout: number, commandTimeout: number, componentTests: boolean, trace: boolean, suites: string[] }, help: boolean }} Parsed options + */ +export function parse(args = process.argv.slice(2)) { + let parsed; + try { + parsed = parseArgs({ + args, + options: { + address: { type: "string", default: "ws://localhost:3569" }, + secret: { type: "string" }, + command: { type: "string" }, + "start-timeout": { type: "string", default: "10" }, + "command-timeout": { type: "string", default: "3" }, + "component-tests": { type: "boolean", default: false }, + trace: { type: "boolean", default: false }, + help: { type: "boolean", default: false }, + }, + allowPositionals: true, + strict: true, + }); + } catch (err) { + const message = err instanceof Error ? err.message : String(err); + throw new Error(`${message}\n\n${usage}`); + } + + const { values, positionals } = parsed; + const options = { + address: values.address, + secret: values.secret, + command: values.command, + startTimeout: Number(values["start-timeout"]), + commandTimeout: Number(values["command-timeout"]), + componentTests: values["component-tests"], + trace: values.trace, + suites: positionals, + }; + return { options, help: values.help }; } -function startRuntime(options, callback) { +/** + * Start the runtime-under-test as a subprocess, if requested. + * @param {{ command?: string }} options Parsed options + * @returns {Promise} The child process, or null + */ +async function startRuntime(options) { if (!options.command) { // we're not responsible for starting it - callback(null); - return; + return null; } - const subprocessOptions = {}; - subprocess.start(options.command, subprocessOptions, callback); + return start(options.command, {}); } -function hasErrors(suites) { +/** + * Whether any of the testcases have errors. + * @param {Suite[]} suites Suites to check + * @returns {boolean} True if there are failures + */ +export function hasErrors(suites) { let failures = 0; - suites.forEach((s) => { - s.cases.forEach((c) => { - if (c.error) { failures += 1; } - }); - }); + for (const s of suites) { + for (const c of s.cases ?? []) { + if (c.error) { + failures += 1; + } + } + } return failures > 0; } -function ensureTracedir(options, callback) { +/** + * Ensure the .flowtrace directory exists, if tracing is enabled. + * @param {{ trace: boolean }} options Parsed options + * @returns {Promise} + */ +async function ensureTracedir(options) { if (!options.trace) { - callback(null); return; } - const tracePath = resolve(process.cwd(), './.flowtrace'); - stat(tracePath, (err) => { - if (!err) { - callback(null); - return; - } - mkdir(tracePath, callback); - }); + const tracePath = resolve(process.cwd(), "./.flowtrace"); + try { + await stat(tracePath); + } catch { + await mkdir(tracePath); + } } -function storeTrace(options, traceId, testcaseName, trace) { +/** + * Store a Flowtrace on disk, and remember its path for the summary output. + * @param {{ trace: boolean }} options Parsed options + * @param {string} traceId ID of the graph the trace was recorded for + * @param {string} testcaseName Name of the testcase the trace was recorded for + * @param {object} trace The Flowtrace contents + * @returns {Promise} + */ +async function storeTrace(options, traceId, testcaseName, trace) { if (!options.trace) { return; } - const date = new Date().toISOString().substr(0, 10); - const fileName = slug(`${date}-${traceId}-${testcaseName}`); + const date = new Date().toISOString().slice(0, 10); + const fileName = `${date}-${traceId}-${testcaseName}` + .toLowerCase() + .replace(/[^a-z0-9]+/g, "-") + .replace(/^-+|-+$/g, ""); const tracePath = resolve(process.cwd(), `./.flowtrace/${fileName}.json`); - writeFile(tracePath, JSON.stringify(trace, null, 2), (err) => { - if (err) { - console.error(err); - } - traces.push(tracePath); - }); + await writeFile(tracePath, JSON.stringify(trace, null, 2)); + traces.push(tracePath); } -function runOptions(options, onUpdate, callback) { - let child = null; - let allSuites = []; - - function cleanReturn(err) { - if (child) { - child.kill(); - } - callback(err, allSuites); - } - +/** + * Run the given suites against the runtime, starting and stopping it as needed. + * @param {{ address: string, secret?: string, command?: string, startTimeout: number, commandTimeout: number, componentTests: boolean, trace: boolean, suites: string[] }} options Parsed options + * @param {(suites: Suite[]) => void} [onUpdate] Called when test state is updated + * @returns {Promise} All run suites, with results filled in + */ +export async function runOptions(options, onUpdate = () => {}) { const def = { - protocol: 'websocket', + protocol: "websocket", address: options.address, - secret: options.secret || '', + secret: options.secret || "", }; - debug('runtime info', def); + debug("runtime info", def); - const runnerOptions = { + const runnerOptions = /** @type {import("./runner.js").RunnerOptions} */ ({ connectTimeout: options.startTimeout * 1000, commandTimeout: options.commandTimeout * 1000, - }; + }); if (options.trace) { - runnerOptions.traceCallback = (graphId, testcaseName, trace) => { + runnerOptions.traceCallback = (graphId, testcaseName, trace) => storeTrace(options, graphId, testcaseName, trace); - }; } - const ru = new runner.Runner(def, runnerOptions); - ensureTracedir(options, (dirErr) => { - if (dirErr) { - cleanReturn(dirErr); - return; - } + const ru = new Runner(def, runnerOptions); + await ensureTracedir(options); - child = startRuntime(options, (err) => { - if (err) { - cleanReturn(err); - return; - } + let child = null; + try { + child = await startRuntime(options); + await ru.connect(); - ru.connect((connectErr) => { - if (connectErr) { - cleanReturn(connectErr); - return; - } - - testsuite.getSuites(options.suites, (getSuitesErr, suites) => { - if (getSuitesErr) { - cleanReturn(getSuitesErr); - return; - } - allSuites = allSuites.concat(suites); - - if (options.componentTests) { - // TODO: move this into runAll?? - runner.getComponentSuites(ru, (getComponentSuitesErr, componentSuites) => { - if (getComponentSuitesErr) { - cleanReturn(getComponentSuitesErr); - return; - } - allSuites = allSuites.concat(componentSuites); - runner.runAll(ru, allSuites, onUpdate, (runErr) => { - if (runErr) { - cleanReturn(runErr); - return; - } - ru.disconnect((disconnectErr) => cleanReturn(disconnectErr)); - }); - }); - return; - } - runner.runAll(ru, allSuites, onUpdate, (runErr) => { - if (runErr) { - cleanReturn(runErr); - return; - } - ru.disconnect((disconnectErr) => cleanReturn(disconnectErr)); - }); - }); - }); - }); - }); + let suites = await getSuites(options.suites); + if (options.componentTests) { + // TODO: move this into runAll?? + suites = suites.concat(await getComponentSuites(ru)); + } + await runAll(ru, suites, onUpdate); + await ru.disconnect(); + return suites; + } finally { + child?.kill(); + } } -function testStatusText(suites) { +/** + * Produce human-readable status text for the suites. + * @param {Suite[]} suites Suites with results + * @returns {string[]} Lines of status text + */ +export function testStatusText(suites) { const results = []; - const ident = ' '; - suites.forEach((s) => { - s.cases.forEach((c) => { - if ((c.passed == null) || c.shown) { return; } - if (!s.titleshown) { results.push(`${s.name}`); } + const ident = " "; + for (const s of suites) { + for (const c of s.cases ?? []) { + if (c.passed == null || c.shown) { + continue; + } + if (!s.titleshown) { + results.push(`${s.name}`); + } // bit hacky, mutates suites const suite = s; @@ -184,36 +214,64 @@ function testStatusText(suites) { suite.titleshown = true; testCase.shown = true; - let res = c.passed ? '✓' : `✗ Error: ${c.error}`; - if (c.skip) { res = `SKIP: ${c.skip}`; } + const res = c.passed ? "✓" : `✗ Error: ${c.error}`; results.push(`${ident}${c.name}\n${ident + ident}${c.assertion}: ${res}`); - }); - }); + } + } return results; } -function main() { - const cmd = parse(process.argv); - const options = cmd.opts(); +/** + * Main entry point of the fbp-spec command-line tool. + * @returns {Promise} Exits the process with status 0 or 2 + */ +export async function main() { + let options; + let help; + try { + const parsed = parse(); + ({ options, help } = parsed); + } catch (err) { + console.error(err instanceof Error ? err.message : err); + process.exitCode = 2; + return; + } + if (help) { + console.log(usage); + return; + } + if (!options.suites.length) { + console.error("Error: no suites specified\n"); + console.log(usage); + process.exitCode = 2; + return; + } + /** + * Print incremental test results. + * @param {Suite[]} suites Suites with results + * @returns {void} + */ function onUpdate(suites) { const r = testStatusText(suites); - console.log(r.join('\n')); + console.log(r.join("\n")); } - runOptions(options, onUpdate, (err, suites) => { - if (err) { throw err; } + let suites; + try { + suites = await runOptions(options, onUpdate); + } catch (err) { + console.error(err); + process.exitCode = 2; + return; + } - if (traces.length) { - console.log('\nFlowtraces produced:'); - traces.forEach((tracePath) => { - console.log(`- ${relative(process.cwd(), tracePath)}`); - }); + if (traces.length) { + console.log("\nFlowtraces produced:"); + for (const tracePath of traces) { + console.log(`- ${relative(process.cwd(), tracePath)}`); } + } - const exitStatus = hasErrors(suites) ? 2 : 0; - process.exit(exitStatus); - }); + process.exitCode = hasErrors(suites) ? 2 : 0; } - -exports.main = main; diff --git a/lib/common.js b/lib/common.js index 26fc4c0..b1dc9ba 100644 --- a/lib/common.js +++ b/lib/common.js @@ -1,56 +1,109 @@ -const Promise = require('bluebird'); +/** + * General-purpose helpers used across fbp-spec. + * @module lib/common + */ -exports.isBrowser = function isBrowser() { - if ((typeof process !== 'undefined') && process.execPath && process.execPath.match(/node|iojs/)) { +/** + * Whether we are running in a browser-like environment (no Node.js process). + * @returns {boolean} True if running in a browser + */ +export function isBrowser() { + if ( + typeof process !== "undefined" && + process.execPath && + process.execPath.match(/node|iojs/) + ) { return false; } return true; -}; - -exports.randomString = function randomString(n) { - let text = ''; - const possible = 'ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789'; - for (let i = 0, end = n, asc = end >= 0; asc ? i < end : i > end; asc ? i += 1 : i -= 1) { - const idx = Math.floor(Math.random() * possible.length); - text += possible.charAt(idx); +} + +const ALPHABET = + "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789"; + +/** + * Generate a random ASCII string, using Web Crypto when available. + * @param {number} n Number of characters to generate + * @returns {string} Random string of length n + */ +export function randomString(n) { + const bytes = new Uint8Array(n); + globalThis.crypto.getRandomValues(bytes); + let text = ""; + for (let i = 0; i < n; i += 1) { + text += ALPHABET.charAt(bytes[i] % ALPHABET.length); } return text; -}; +} -exports.asyncSeries = function asyncSeries(items, func, callback) { - const runItems = items.slice(0); +/** + * Run an async function over items sequentially, collecting results. + * The first failure aborts the sequence and rejects. + * @template T, R + * @param {T[]} items Items to process + * @param {(item: T) => Promise} func Async function applied to each item + * @returns {Promise} Results in the same order as the items + */ +export async function asyncSeries(items, func) { const results = []; - function next() { - if (runItems.length === 0) { - callback(null, results); - return; - } - const item = runItems.shift(); - func(item, (err, result) => { - if (err) { - callback(err); - return; - } - results.push(result); - next(); - }); + for (const item of items) { + results.push(await func(item)); // eslint-disable-line no-await-in-loop } - next(); -}; + return results; +} -exports.isArray = Array.isArray || ((value) => ({}).toString.call(value) === '[object Array]'); +/** + * Whether the given value is an Array. + * @param {*} value Value to check + * @returns {boolean} True if value is an array + */ +export function isArray(value) { + return Array.isArray(value); +} -exports.startsWith = (str, sub) => str.indexOf(sub) === 0; +/** + * Whether the given string starts with the given substring. + * @param {string} str String to check + * @param {string} sub Expected prefix + * @returns {boolean} True if str starts with sub + */ +export function startsWith(str, sub) { + return str.indexOf(sub) === 0; +} -// Based on http://stackoverflow.com/a/38225011/1967571 -const rejectDelayer = function rejectDelayer(delay) { - return (reason) => new Promise((resolve, reject) => setTimeout(reject.bind(null, reason), delay)); -}; +/** + * Wait for the given number of milliseconds. + * @param {number} delay Delay in milliseconds + * @returns {Promise} Resolves after the delay + */ +function delayPromise(delay) { + return new Promise((resolve) => { + setTimeout(resolve, delay); + }); +} -exports.retryUntil = function retryUntil(attempt, test, delay = 500, max = 5) { - let p = Promise.reject(new Error('retry starter')); - for (let i = 0, end = max, asc = end >= 0; asc ? i <= end : i >= end; asc ? i += 1 : i -= 1) { - p = p.catch(attempt).then(test).catch(rejectDelayer(delay)); +/** + * Repeatedly call attempt, until the test() predicate on its result passes. + * @template T + * @param {() => Promise} attempt Async function producing a value + * @param {(value: T) => Promise} test Async predicate; should reject on the value if not acceptable yet + * @param {number} [delay=500] Time to wait between attempts, in milliseconds + * @param {number} [max=5] Maximum number of attempts + * @returns {Promise} The first result that passes the test + */ +export async function retryUntil(attempt, test, delay = 500, max = 5) { + let lastError = null; + for (let i = 0; i <= max; i += 1) { + try { + const result = await attempt(); + await test(result); + return result; + } catch (error) { + lastError = error; + } + if (i < max) { + await delayPromise(delay); + } } - return p; -}; + throw lastError; +} diff --git a/lib/expectation.js b/lib/expectation.js index 7c9a1de..1f477ca 100644 --- a/lib/expectation.js +++ b/lib/expectation.js @@ -1,100 +1,270 @@ -const debug = require('debug')('fbp-spec:expectation'); -const chai = require('chai'); -const JSONPath = require('jsonpath'); +/** + * A library of standard expectation operators, used to check actual output data. + * Throws an Error when an expectation is violated, with details about the issue. + * @module lib/expectation + */ -const common = require('./common'); +import assert from "node:assert"; +import debugLib from "debug"; +import JSONPath from "jsonpath"; -// # A library of standard expectation operators -// -// Should throw AssertionError if expectation is violated, -// with details about the issue +import { isArray } from "./common.js"; + +const debug = debugLib("fbp-spec:expectation"); + +/** + * Produce an AssertionError-compatible error with a message. + * @param {string} message Description of the violated expectation + * @returns {Error} The error to throw + */ +function fail(message) { + return new assert.AssertionError({ + message, + actual: "", + expected: "", + operator: "==", + }); +} + +/** + * The library of standard expectation operators. + * Each operator takes the actual value and the expected value, + * and throws an Error if the expectation is violated. + * @type {Record void>} + */ const operators = { + /** + * Check the type of the actual value. + * @param {*} actual The received value + * @param {string} expected Expected type name + * @returns {void} + */ type(actual, expected) { - return chai.expect(actual).to.be.a(expected); + assert.equal( + typeof actual, + expected, + `expected ${JSON.stringify(actual)} to be of type ${expected}`, + ); }, + /** + * Deep equality check. + * @param {*} actual The received value + * @param {*} expected The expected value + * @returns {void} + */ equals(actual, expected) { - return chai.expect(actual).to.eql(expected); + if (!deepEqual(actual, expected)) { + throw fail( + `expected ${JSON.stringify(actual)} to equal ${JSON.stringify(expected)}`, + ); + } }, + /** + * Check that actual is above (greater than) expected. + * @param {number} actual The received value + * @param {number} expected The value to be above + * @returns {void} + */ above(actual, expected) { - return chai.expect(actual).to.be.above(expected); + if (!(actual > expected)) { + throw fail(`expected ${actual} to be above ${expected}`); + } }, + /** + * Check that actual is below (less than) expected. + * @param {number} actual The received value + * @param {number} expected The value to be below + * @returns {void} + */ below(actual, expected) { - return chai.expect(actual).to.be.below(expected); + if (!(actual < expected)) { + throw fail(`expected ${actual} to be below ${expected}`); + } }, + /** + * Check that the actual object has exactly the given keys. + * @param {*} actual The received value + * @param {string[]} expected Key names to expect + * @returns {void} + */ haveKeys(actual, expected) { - return chai.expect(actual).to.have.keys(expected); + const keys = Object.keys(actual ?? {}).sort(); + const wanted = [...expected].sort(); + if (JSON.stringify(keys) !== JSON.stringify(wanted)) { + throw fail( + `expected ${JSON.stringify(actual)} to have keys ${JSON.stringify(wanted)}, got ${JSON.stringify(keys)}`, + ); + } }, + /** + * Check that the actual object includes the given keys. + * @param {*} actual The received value + * @param {string[]} expected Key names to expect to be present + * @returns {void} + */ includeKeys(actual, expected) { - return chai.expect(actual).to.include.keys(expected); + const keys = Object.keys(actual ?? {}); + for (const key of expected) { + if (!keys.includes(key)) { + throw fail(`expected ${JSON.stringify(actual)} to include key ${key}`); + } + } }, + /** + * Check that the actual value contains the expected value. + * For strings checks substring, for arrays checks for a deep-equal member, + * for objects checks for a deep-equal subset. + * @param {*} actual The received value + * @param {*} expected Value expected to be contained + * @returns {void} + */ contains(actual, expected) { - return chai.expect(actual).to.contain(expected); + if (typeof actual === "string") { + if (!actual.includes(expected)) { + throw fail(`expected '${actual}' to contain '${expected}'`); + } + return; + } + if (isArray(actual)) { + const list = /** @type {*[]} */ (actual); + if (!list.some((member) => deepEqual(member, expected))) { + throw fail( + `expected ${JSON.stringify(actual)} to contain ${JSON.stringify(expected)}`, + ); + } + return; + } + if (actual != null && typeof actual === "object") { + const obj = /** @type {Record} */ (actual); + for (const key of Object.keys(expected)) { + if (!deepEqual(obj[key], expected[key])) { + throw fail( + `expected ${JSON.stringify(actual)} to contain { '${key}': ${JSON.stringify(expected[key])} }`, + ); + } + } + return; + } + throw fail(`cannot check contains on ${JSON.stringify(actual)}`); }, + /** + * Check that the actual value is not an error-like object. + * @param {*} actual The received value + * @returns {void} + */ noterror(actual) { - if (actual != null ? actual.message : undefined) { throw actual; } + if (actual?.message) { + throw actual; + } }, }; -// returns a predicate function -// (data) -> +/** + * Structurally compare two values for equality. + * @param {*} a First value + * @param {*} b Second value + * @returns {boolean} True if the values are deeply equal + */ +function deepEqual(a, b) { + try { + assert.deepStrictEqual(a, b); + return true; + } catch { + return false; + } +} + +/** + * Find the expectation operator referenced in an expectation object, + * and return it wrapped in a predicate function. + * @param {Record} expectation Expectation object, like { equals: true } or { path, equals } + * @returns {(data: *) => void} Predicate function, throws on violated expectation + */ function findOperator(expectation) { const ops = Object.keys(expectation); - for (let i = 0; i < ops.length; i += 1) { - const opname = ops[i]; + for (const opname of ops) { const expectValue = expectation[opname]; - if (opname !== 'path') { + if (opname !== "path") { const op = operators[opname]; if (op) { - const predicate = (data) => op(data, expectValue); + const predicate = /** @type {(data: *) => void} */ ( + (data) => op(data, expectValue) + ); predicate.toString = () => `${opname} ${expectValue}`; return predicate; } } } - throw new Error(`fbp-spec: No operator matching ${Object.keys(expectation)}. Available: ${Object.keys(operators)}`); + throw new Error( + `fbp-spec: No operator matching ${Object.keys(expectation)}. Available: ${Object.keys(operators)}`, + ); } +/** + * Extract the data values the expectation applies to. + * Uses the JSONPath in expectation.path when present, otherwise the data itself. + * @param {Record} expectation Expectation object, possibly with a .path JSONPath + * @param {*} data The data to match against + * @returns {*[]} Matching data values + */ function extractMatches(expectation, data) { let matches; if (expectation.path) { - debug('extracting JSONPath from', expectation.path, data); + debug("extracting JSONPath from", expectation.path, data); matches = JSONPath.query(data, expectation.path); - if (!matches.length) { throw new Error(`expected JSONPath '${expectation.path}' to match data in ${JSON.stringify(data)}`); } + if (!matches.length) { + throw new Error( + `expected JSONPath '${expectation.path}' to match data in ${JSON.stringify(data)}`, + ); + } } else { matches = [data]; } - debug('matching against', matches); + debug("matching against", matches); return matches; } -exports.expect = function expect(expects, portsdata) { +/** + * Check actual output data against a set of expectations. + * Throws an Error on the first violated expectation. + * @param {*|Array>} expects One expectation, or a sequence of expectations + * @param {Record} portsdata Data received on the outports, keyed by port name + * @returns {void} + */ +export function expect(expects, portsdata) { // can have one or more set of expectations of messages - if (!common.isArray(expects)) { - expect([expects], portsdata); - return; - } + const expectationSets = isArray(expects) ? expects : [expects]; - expects.forEach((e) => { + for (const e of expectationSets) { // each message expectation can match on ports - Object.keys(e).forEach((port) => { + for (const port of Object.keys(e)) { // for each matching port, there can be one or more assertions on the value - const expectations = common.isArray(e[port]) ? e[port] : [e[port]]; - expectations.forEach((expectation) => { - debug('checking port for expectation', port, expectation); + const expectations = isArray(e[port]) ? e[port] : [e[port]]; + for (const expectation of expectations) { + debug("checking port for expectation", port, expectation); const data = portsdata[port]; - if (typeof data === 'undefined') { + if (typeof data === "undefined") { throw new Error(`No data received on port ${port}`); } const predicate = findOperator(expectation); const matches = extractMatches(expectation, data); - matches.forEach((m) => { - debug('checking against predicate', m, predicate.toString()); - predicate(m); - }); - }); - }); - }); -}; + for (const match of matches) { + debug("checking against predicate", match, predicate.toString()); + predicate(match); + } + } + } + } +} -exports.noError = (maybeErr) => chai.expect(maybeErr).to.not.exist; +/** + * Assert that the given value is not an error. + * @param {*} maybeErr Value that should not be an error + * @returns {void} + */ +export function noError(maybeErr) { + if (maybeErr != null) { + throw maybeErr; + } +} diff --git a/lib/index.js b/lib/index.js index d062787..e3214e1 100644 --- a/lib/index.js +++ b/lib/index.js @@ -1,19 +1,12 @@ -function getSchema(id) { - const schemaId = id.replace('.json', ''); - return module.exports.schema[schemaId]; -} +/** + * Public API of fbp-spec. + * @module lib/index + */ -module.exports = { - // eslint-disable-next-line global-require - runner: require('./runner'), - // eslint-disable-next-line global-require - subprocess: require('./subprocess'), - // eslint-disable-next-line global-require - mocha: require('./mocha'), - // eslint-disable-next-line global-require - testsuite: require('./testsuite'), - // eslint-disable-next-line global-require,import/no-unresolved - schema: require('../schema'), - getSchema, - ui: {}, -}; +import * as schema from "../schema/index.js"; +import { getSchema } from "../schema/index.js"; +import * as runner from "./runner.js"; +import * as subprocess from "./subprocess.js"; +import * as testsuite from "./testsuite.js"; + +export { getSchema, runner, schema, subprocess, testsuite }; diff --git a/lib/mocha.js b/lib/mocha.js deleted file mode 100644 index e118859..0000000 --- a/lib/mocha.js +++ /dev/null @@ -1,110 +0,0 @@ -/* eslint-env node, mocha */ -// # Run fbp-spec testcases using Mocha as a runner/frontend -// # Intended to allow reuse of Mocha reporters, or -// # to quickly add a couple of fbp-spec cases to a predominantly Mocha-based testsuite -// # See also ./mochacompat.js - -const debug = require('debug')('fbp-spec:mocha'); - -const runnerModule = require('./runner'); - -const { - Runner, -} = runnerModule; -const testsuite = require('./testsuite'); -const expectation = require('./expectation'); -const subprocess = require('./subprocess'); - -function runSuite(runner, suite) { - const suiteDescribe = suite.skip ? describe.skip : describe; - suiteDescribe(`${suite.name}`, () => { - beforeEach(function (done) { - if (suite.timeout != null) { - this.timeout(suite.timeout); - } - runner.setupSuite(suite, done); - }); - afterEach((done) => runner.teardownSuite(suite, done)); - - suite.cases.forEach((testcase) => { - const caseDescribe = testcase.skip ? describe.skip : describe; - caseDescribe(testcase.name, () => it(testcase.assertion, function (done) { - if (suite.timeout != null) { - this.timeout(suite.timeout); - } - if (testcase.timeout != null) { - this.timeout(testcase.timeout); - } - - runnerModule.runTestAndCheck(runner, testcase, (err, result) => { - if (err) { - done(err); - return; - } - done(result.error); - }); - })); - }); - }); -} - -// # run() -// Must be be ran using Mocha, -// it is responsible for setting up the "describe", and "it" functions -exports.run = function run(rt, tests, options) { - // default pretty high to give time for runtime to start - let suites; - const opts = options; - if ((options.starttimeout == null)) { opts.starttimeout = 5000; } - if ((options.fixturetimeout == null)) { opts.fixturetimeout = 2000; } - - const runnerOptions = { - connectTimeout: options.starttimeout, - commandTimeout: options.commandtimeout, - }; - const runner = new Runner(rt, runnerOptions); - try { - suites = testsuite.getSuitesSync(tests); - } catch (e) { - console.log('Unable to get suites:', e); - throw e; - } - let process = null; - - const start = function (callback) { - if (!rt.command) { - callback(null); - return; - } - const subprocessOptions = {}; - process = subprocess.start(rt.command, subprocessOptions, callback); - }; - const stop = function (callback) { - if (process) { process.kill(); } - callback(null); - }; - - before(function (done) { - this.timeout(options.starttimeout + 500); - start((err) => { - debug('started', err); - expectation.noError(err); - runner.connect(done); - }); - }); - after((done) => { - stop((err) => { - debug('stopped', err); - expectation.noError(err); - runner.disconnect(done); - }); - }); - - suites.forEach((suite) => { - if ((suite.timeout == null)) { - const s = suite; - s.timeout = options.fixturetimeout; - } - runSuite(runner, suite); - }); -}; diff --git a/lib/mochacompat.js b/lib/mochacompat.js deleted file mode 100644 index f39417f..0000000 --- a/lib/mochacompat.js +++ /dev/null @@ -1,436 +0,0 @@ -/* eslint-env browser, node */ -// # Run Mocha testcases using fbp-spec as a runner/frontend -// # Intended to allow existing Mocha testcases to be seen and executed -// # from a FBP protocol client like Flowhub, without requiring them -// # to be rewritten as fbp-spec tests -// # This is especially useful to allow partial and gradual migration of existing test suites -// # See also ./mocha.coffee, which can be used in combination - -// Partially based on example code from https://github.com/mochajs/mocha/wiki/Third-party-UIs - -const http = require('http'); -const Mocha = require('mocha'); -const crypto = require('crypto'); -const debug = require('debug')('fbp-spec:mochacompat'); - -// FIXME: split out transport interface of noflo-runtime-*, use that directly -const websocket = require('./websocket'); - -const testsuite = require('./testsuite'); - -function loadTests(files) { - const options = {}; - const mocha = new Mocha(options); - - files.forEach((f) => { - const resolved = require.resolve(f); - delete require.cache[resolved]; // clear module cache - mocha.addFile(f); - }); - mocha.loadFiles(); - return mocha; -} - -// similar to mocha.run(), but files must be loaded beforehand -function runTests(mocha, progress, callback) { - const { - suite, - } = mocha; - const { - options, - } = mocha; - options.files = mocha.files; - const runner = new Mocha.Runner(suite, options.delay); - function registerReporter() { - runner.on('pass', (test) => progress(null, test)); - runner.on('fail', (test, err) => progress(err, test)); - } - - mocha.reporter(registerReporter, {}); - // eslint-disable-next-line no-underscore-dangle - const reporter = new mocha._reporter(runner, options); - - runner.ignoreLeaks = options.ignoreLeaks !== false; - runner.fullStackTrace = options.fullStackTrace; - runner.asyncOnly = options.asyncOnly; - runner.allowUncaught = options.allowUncaught; - if (options.grep) { - runner.grep(options.grep, options.invert); - } - if (options.globals) { - runner.globals(options.globals); - } - if (options.growl) { - // eslint-disable-next-line no-underscore-dangle - mocha._growl(runner, reporter); - } - if (options.useColors != null) { - Mocha.reporters.Base.useColors = options.useColors; - } - Mocha.reporters.Base.inlineDiffs = options.useInlineDiffs; - - function done(failures) { - if (reporter.done) { - reporter.done(failures, callback); - return; - } - callback(failures); - } - - runner.run(done); -} - -function testId(fullname) { - const hash = crypto.createHash('sha256'); - hash.update(fullname); - return hash.digest('hex').substr(0, 10); -} - -function loadSuite(fbpSuite, suite) { - suite.tests.forEach((testcase) => { - // FIXME: make the full name recursive - const fullName = fbpSuite.name + testcase.parent.title + testcase.title; - // FIXME: salt so it does not collide - const id = testId(fullName); - const c = testcase; - // eslint-disable-next-line no-underscore-dangle - c._fbpid = id; - const fbpCase = { - name: testcase.parent.title, - assertion: testcase.title, - _id: id, - inputs: { - test: id, - }, - expect: { - error: { - noterror: null, - }, - }, - }; - - fbpSuite.cases.push(fbpCase); - }); - - // load recursively - suite.suites.forEach((sub) => { - loadSuite(fbpSuite, sub); - }); -} - -function buildFbpSpecs(mocha) { - const specs = []; - - const top = mocha.suite; - top.suites.forEach((suite) => { - // console.log 's', suite - - const fbpSuite = testsuite.create({ - name: `${suite.title} (Mocha tests)`, - fixture: { - type: 'fbp', - data: `\ -# @runtime fbp-spec-mocha - -INPORT=runTest.IN:TEST -OUTPORT=runTest.ERROR:ERROR - -runTest(mocha/LoadTestCase) OUT -> IN verifyResult(mocha/CheckResults)\ -`, - }, - }); - - loadSuite(fbpSuite, suite); - specs.push(fbpSuite); - }); - - return specs; -} - -function dumpSpecs(suites) { - // eslint-disable-next-line global-require - const yaml = (typeof window !== 'undefined' && window.jsyaml) ? window.jsyaml : require('js-yaml'); - - let str = ''; - const delimiter = '---\n'; - suites.forEach((s) => { - str += `${yaml.dump(s)}`; - if (suites.length > 1) { str += delimiter; } - }); - - return str; -} - -function discoverHost(preferredIface) { - // eslint-disable-next-line global-require - const os = require('os'); // node.js only - - const ifaces = os.networkInterfaces(); - let address; - let intAddress; - - function filter(connection) { - if (connection.family !== 'IPv4') { - return; - } - if (connection.internal) { - intAddress = connection.address; - } else { - ({ - address, - } = connection); - } - } - - if ((typeof preferredIface === 'string') && Array.from(ifaces).includes(preferredIface)) { - ifaces[preferredIface].forEach(filter); - } else { - Object.keys(ifaces).forEach((device) => { - ifaces[device].forEach(filter); - }); - } - return address || intAddress; -} - -const knownUnsupportedCommands = () => false; - -function fbpComponentName(s) { - // TODO: use topic/filename? - return `fbp-spec-mocha/${s.name}`; -} - -function fbpComponentFromSpec(s) { - // component:component - return { - name: fbpComponentName(s), - subgraph: false, - inPorts: [], - outPorts: [], - }; -} - -function fbpSourceFromSpec(s) { - // component:source message, :getsource response - const serialized = dumpSpecs([s]); - return { - name: fbpComponentName(s), - code: '', - language: 'whitespace', - tests: serialized, - }; -} - -function handleFbpCommand( - originalState, - runtime, - mocha, - specs, - protocol, - command, - payload, - context, -) { - const state = originalState; - function updateStatus(news, event) { - if (news.started != null) { state.started = news.started; } - if (news.running != null) { state.running = news.running; } - const runtimeState = { started: state.started, running: state.running, graph: payload.graph }; - debug('update status', runtimeState); - runtime.send('network', event, runtimeState, context); - } - - // sendEvent = (e) -> - // runtime.send e.protocol, e.command, e.payload, context - function ackMessage() { - // reply with same message as we got in - runtime.send(protocol, command, payload, context); - } - - function checkSendCurrentTest() { - const completed = state.completedTests[state.currentTest]; - debug('checking for', state.currentTest, (completed != null)); - if (completed) { - const m = { - graph: state.graph, - event: 'data', - port: 'error', - payload: completed.err, - }; - runtime.send('runtime', 'packet', m, context); - delete state.completedTests[state.currentTest]; - } - } - - // collect results of completed tests - function testDone(err, test) { - // eslint-disable-next-line no-underscore-dangle - debug('test completed', test._fbpid, err, Object.keys(state.completedTests).length); - // eslint-disable-next-line no-underscore-dangle - state.completedTests[test._fbpid] = { test, err }; - checkSendCurrentTest(); - } - - // # Runtime - if ((protocol === 'runtime') && (command === 'getruntime')) { - const capabilities = [ - 'protocol:graph', // read-only from client - 'protocol:component', // read-only from client - 'protocol:network', - 'protocol:runtime', - 'component:getsource', - ]; - const info = { - type: 'fbp-spec-mocha', - version: '0.5', - capabilities, - allCapabilities: capabilities, - graph: 'default/main', // HACK, so Flowhub will ask for our graph - }; - runtime.send('runtime', 'runtime', info, context); - // sendGraphs mytrace, send, (err) -> # XXX: right place? - // ignored - } if ((protocol === 'runtime') && (command === 'packet')) { - debug('test message', payload, state.running); - - if ((payload.port !== 'test') || (payload.event !== 'data')) { - debug('unexpected test message format'); - return; - } - - state.currentTest = payload.payload; - - checkSendCurrentTest(); // we might have completed it already - - if (!state.running) { - runTests(mocha, testDone, () => updateStatus({ running: false }, 'status')); - - updateStatus({ running: true }, 'status'); - } - - // # Graph - } else if ((protocol === 'graph') && (command === 'addnode')) { - ackMessage(); - } else if ((protocol === 'graph') && (command === 'addedge')) { - ackMessage(); - } else if ((protocol === 'graph') && (command === 'addinport')) { - ackMessage(); - } else if ((protocol === 'graph') && (command === 'addoutport')) { - ackMessage(); - } else if ((protocol === 'graph') && (command === 'clear')) { - state.graph = payload.id; - debug('new graph', state.graph); - ackMessage(); - - // # Network - } else if ((protocol === 'network') && (command === 'getstatus')) { - runtime.send('network', 'status', state, context); - } else if ((protocol === 'network') && (command === 'start')) { - debug('FBP network start'); - updateStatus({ started: true, running: false, graph: payload.graph }, 'started'); - } else if ((protocol === 'network') && (command === 'stop')) { - debug('FBP network stop'); - updateStatus({ started: false, running: false, graph: payload.graph }, 'stopped'); - - // # Component - } else if ((protocol === 'component') && (command === 'list')) { - // one fake component per Mocha suite - specs.forEach((s) => { - runtime.send('component', 'component', fbpComponentFromSpec(s), context); - }); - runtime.send('component', 'componentsready', specs.length, context); - } else if ((protocol === 'component') && (command === 'getsource')) { - // one fake component per Mocha suite - let found = null; - specs.forEach((s) => { - const componentName = fbpComponentName(s); - if (componentName === payload.name) { - found = s; - } - }); - debug('component getsource', `'${payload.name}'`, found != null ? found.name : undefined); - if (found) { - runtime.send('component', 'source', fbpSourceFromSpec(found), context); - } - } else if (knownUnsupportedCommands(protocol, command)) { - // ignored - } else { - debug('Warning: Unknown FBP protocol message', protocol, command); - } -} - -// # Commandline things -function normalizeOptions(options) { - let match; - let discovered; - if (options.host === 'autodetect') { - discovered = discoverHost(); - } else if (/autodetect\(([a-z0-9]+)\)/.exec(options.host)) { - discovered = discoverHost(match[1]); - } - - return { - ...options, - host: discovered || options.host, - port: options.port || 3333, - }; -} - -function parse(args) { - // eslint-disable-next-line global-require - const program = require('commander'); - - const actionHandler = (suites, opts) => { - const o = opts; - o.suites = suites; - }; - - // TODO: take list of files as input instead, to be more mocha compatible - program - .storeOptionsAsProperties(false) - .passCommandToAction(false) - .arguments('') - .action(actionHandler) - .option('--host ', 'Hostname we serve on, for live-url', String, 'autodetect') - .option('--port ', 'Command to launch runtime under test', Number, 3333) - .parse(args); - - return program; -} - -exports.setup = function setup(options, callback) { - const opts = normalizeOptions(options); - - const mocha = loadTests(opts.files); - const specs = buildFbpSpecs(mocha); - - const state = { - started: false, - running: false, - currentTest: null, - graph: null, - specs, - completedTests: {}, // 'id' -> { err: ?Error, test: MochaTest } - }; - const httpServer = new http.Server(); - const runtime = websocket(httpServer, {}); - runtime.receive = (protocol, command, payload, context) => { - handleFbpCommand(state, runtime, mocha, specs, protocol, command, payload, context); - }; - - httpServer.listen(opts.port, (err) => callback(err, state, httpServer)); -}; - -exports.main = function main() { - const cmd = parse(process.argv); - const options = cmd.opts(); - - exports.setup(options, (err, state) => { - if (err) { throw err; } - console.log(`fbp-spec-mocha started on ws://${options.host}:${options.port}`); - console.log(`found ${state.specs.length} test suites`); - }); -}; - -if (!module.parent) { - exports.main(); -} diff --git a/lib/protocol.js b/lib/protocol.js index 9b0692a..b13b04f 100644 --- a/lib/protocol.js +++ b/lib/protocol.js @@ -1,126 +1,161 @@ -// FBP protocol dependent code - -const fbpGraph = require('fbp-graph'); -const debug = require('debug')('fbp-spec:protocol'); -const Promise = require('bluebird'); -const common = require('./common'); - -exports.sendGraph = function sendGraph(client, graph, callback) { - const main = false; // this is a component? +/** + * FBP protocol dependent code: high-level operations on top of an fbp-client instance. + * @module lib/protocol + */ + +import debugLib from "debug"; +import fbpGraph from "fbp-graph"; + +import { randomString } from "./common.js"; + +const debug = debugLib("fbp-spec:protocol"); + +/** + * A graph JSON payload, as sent over the FBP protocol or parsed from FBP notation. + * @typedef {{ name?: string, properties?: Record, inports?: Record, outports?: Record, processes?: Record, connections?: *[] }} GraphPayload + */ + +/** + * A component as advertised by a runtime. + * @typedef {{ name: string, description?: string, subgraph?: boolean, inPorts: { id: string, [key: string]: * }[], outPorts: { id: string, [key: string]: * }[], [key: string]: * }} RuntimeComponent + */ + +/** + * Send a graph to the runtime, and return the ID it was registered under. + * @param {import("fbp-client").FbpClient} client Connected fbp-client instance + * @param {GraphPayload|import("fbp-graph").Graph} graph Graph to send, either a fbp-graph instance or graph JSON payload + * @param {boolean} [main=false] Whether this is the main graph of the runtime + * @returns {Promise} The graph ID assigned on the runtime + */ +export async function sendGraph(client, graph, main = false) { if (!graph) { - callback(new Error('Graph not defined')); - return; + throw new Error("Graph not defined"); } - let graphId = graph.name || graph.properties.id; - if (!graphId) { graphId = `fixture.${common.randomString(10)}`; } - const g = graph; - g.name = graphId; + let graphId = graph.name || graph.properties?.id; + if (!graphId) { + graphId = `fixture.${randomString(10)}`; + } + graph.name = graphId; if (!(graph instanceof fbpGraph.Graph)) { - // fbp-client operates on fbp-graph instances - fbpGraph.graph.loadJSON(graph, (err, loadedGraph) => { - if (err) { - callback(err); - return; - } - exports.sendGraph(client, loadedGraph, callback); - }); - return; + // fbp-client operates on fbp-graph instances; loadJSON always returns a Promise + const loadedGraph = await fbpGraph.graph.loadJSON(graph); + return sendGraph(client, loadedGraph, main); } - debug('sendgraph', graphId); - - Promise.resolve() - .then(() => client.protocol.graph.send(graph, main)) - .then(() => graphId) - .nodeify(callback); -}; + debug("sendgraph", graphId); + await client.protocol.graph.send(graph, main); + return graphId; +} -exports.startNetwork = function startNetwork(client, graphId, callback) { - debug('startnetwork', graphId); +/** + * Start the network for a graph on the runtime. + * @param {import("fbp-client").FbpClient} client Connected fbp-client instance + * @param {string} graphId ID of the graph to start + * @returns {Promise} Resolves when the network has started + */ +export async function startNetwork(client, graphId) { + debug("startnetwork", graphId); // TODO: Register start with recorder - Promise.resolve() - .then(() => client.protocol.network.start({ - graph: graphId, - })) - .nodeify(callback); -}; - -exports.stopNetwork = function stopNetwork(client, graphId, callback) { - debug('stopnetwork', graphId); + await client.protocol.network.start({ graph: graphId }); +} + +/** + * Stop the network for a graph on the runtime. + * @param {import("fbp-client").FbpClient} client Connected fbp-client instance + * @param {string} graphId ID of the graph to stop + * @returns {Promise} Resolves when the network has stopped + */ +export async function stopNetwork(client, graphId) { + debug("stopnetwork", graphId); // TODO: Register stop with recorder - Promise.resolve() - .then(() => client.protocol.network.stop({ - graph: graphId, - })) - .nodeify(callback); -}; - -exports.sendPackets = function sendPackets(client, graphId, packets, callback) { - debug('sendpackets', graphId, packets); - - Promise.all(Object.keys(packets).map((port) => client.protocol.runtime.packet({ - event: 'data', - port, - payload: packets[port], - graph: graphId, - }))) - .nodeify(callback); -}; - -exports.getComponents = function getComponents(client, callback) { - debug('get components'); - - Promise.resolve() - .then(() => client.protocol.component.list()) - .then((componentList) => { - const components = {}; - componentList.forEach((component) => { - components[component.name] = component; - }); - return components; - }) - .nodeify(callback); -}; - -exports.getCapabilities = function getCapabilities(client, callback) { + await client.protocol.network.stop({ graph: graphId }); +} + +/** + * Send input data packets to the input ports of a graph. + * @param {import("fbp-client").FbpClient} client Connected fbp-client instance + * @param {string} graphId ID of the graph to send to + * @param {Record} packets Data to send, keyed by port name + * @returns {Promise} Resolves when all packets have been sent + */ +export async function sendPackets(client, graphId, packets) { + debug("sendpackets", graphId, packets); + + await Promise.all( + Object.keys(packets).map((port) => + client.protocol.runtime.packet({ + event: "data", + port, + payload: packets[port], + graph: graphId, + }), + ), + ); +} + +/** + * Get the components available on the runtime. + * @param {import("fbp-client").FbpClient} client Connected fbp-client instance + * @returns {Promise>} Components keyed by name + */ +export async function getComponents(client) { + debug("get components"); + + const componentList = await client.protocol.component.list(); + /** @type {Record} */ + const components = {}; + for (const component of componentList) { + components[component.name] = component; + } + return components; +} + +/** + * Get the capabilities advertised by the runtime. + * @param {import("fbp-client").FbpClient} client Connected fbp-client instance + * @returns {Promise} Capability names + */ +export async function getCapabilities(client) { const def = client.definition; - if (def && def.capabilities && def.capabilities.length) { + if (def?.capabilities?.length) { // We already know the capabilities - callback(null, def.capabilities); - return; + return def.capabilities; } - Promise.resolve() - .then(() => client.protocol.runtime.getruntime()) - .then((definition) => definition.capabilities) - .nodeify(callback); -}; - -exports.getComponentTests = function getComponentTests(client, callback) { - debug('get component tests'); - - Promise.resolve() - .then(() => client.protocol.component.list()) - .then((components) => Promise.all(components.map((component) => client - .protocol.component.getsource({ - name: component.name, - }).then( + const definition = await client.protocol.runtime.getruntime(); + return definition.capabilities ?? []; +} + +/** + * Get the fbp-spec tests attached to components on the runtime. + * @param {import("fbp-client").FbpClient} client Connected fbp-client instance + * @returns {Promise>} YAML test sources, keyed by component name + */ +export async function getComponentTests(client) { + debug("get component tests"); + + const components = await client.protocol.component.list(); + const sources = await Promise.all( + components.map((component) => + client.protocol.component.getsource({ name: component.name }).then( (source) => source, - () => ({ - tests: null, - }), - )))) - .then((sources) => { - const tests = {}; - sources.forEach((source) => { - if (!source.tests) { return; } - const name = source.library ? `${source.library}/${source.name}` : source.name; - tests[name] = source.tests; - }); - return tests; - }) - .nodeify(callback); -}; + () => ({ tests: null }), + ), + ), + ); + /** @type {Record} */ + const tests = {}; + for (const source of sources) { + if (!source.tests) { + continue; + } + const name = source.library + ? `${source.library}/${source.name}` + : source.name; + tests[name] = source.tests; + } + return tests; +} diff --git a/lib/runner.js b/lib/runner.js index 2019959..b13e887 100644 --- a/lib/runner.js +++ b/lib/runner.js @@ -1,28 +1,71 @@ -const fbp = require('fbp'); -const fbpClient = require('fbp-client'); -const debug = require('debug')('fbp-spec:runner'); -const debugReceived = require('debug')('fbp-spec:runner:received'); -const Promise = require('bluebird'); -const { FlowtraceRecorder } = require('flowtrace'); -const common = require('./common'); -const protocol = require('./protocol'); -const testsuite = require('./testsuite'); -const expectation = require('./expectation'); - +/** + * Runs fbp-spec testsuites against an FBP runtime. + * @module lib/runner + */ + +import debugLib from "debug"; +import fbp from "fbp"; +import fbpClient from "fbp-client"; +import flowtrace from "flowtrace"; + +import { asyncSeries, isArray, retryUntil } from "./common.js"; +import { expect as checkExpectation } from "./expectation.js"; +import * as protocol from "./protocol.js"; +import * as testsuite from "./testsuite.js"; + +const debug = debugLib("fbp-spec:runner"); +const debugReceived = debugLib("fbp-spec:runner:received"); +const { FlowtraceRecorder } = flowtrace; + +/** + * Options for configuring a Runner. + * @typedef {object} RunnerOptions + * @property {number} [connectTimeout] Time to wait for the connection to come up, in milliseconds + * @property {number} [commandTimeout] Maximum time to wait for a command response, in milliseconds + * @property {((graphId: string, testcaseName: string, trace: object) => void)} [traceCallback] + * Called with a recorded Flowtrace for each test, when tracing is enabled + * @property {object} [parentElement] DOM element to attach iframe runtimes to (browser only) + */ + +/** + * One entry in a test sequence: inputs sent, expected outputs, and the actual outputs. + * @typedef {{ inputs: *, expect: *, actual?: * }} TestSequenceEntry + */ + +/** + * Anything that can run a testcase; satisfied by Runner. + * @typedef {{ runTest(testcase: object): Promise }} RunnerLike + */ + +/** + * Log all protocol messages received from a client. + * @param {import("fbp-client").FbpClient} client Client to listen on + * @returns {void} + */ function debugReceivedMessages(client) { - client.on('signal', ({ proto, command, payload }) => debugReceived(proto, command, payload)); + client.on("signal", (message) => { + const { proto, command, payload } = + /** @type {{ proto: string, command: string, payload: * }} */ (message); + debugReceived(proto, command, payload); + }); } -function synthesizeTopicFixture(topic, components, callback) { - debug('synthesizing fixture', topic); +/** + * Build a fixture graph that exposes all ports of the topic component. + * @param {string} topic Component name to use as fixture + * @param {Record} components Components available on the runtime + * @returns {import("./protocol.js").GraphPayload} A graph JSON payload + */ +function synthesizeTopicFixture(topic, components) { + debug("synthesizing fixture", topic); // export each of the ports for topic component // return a FBP graph? const component = components[topic]; if (!component) { - callback(new Error(`Could not find component for topic: ${topic}`)); - return; + throw new Error(`Could not find component for topic: ${topic}`); } + /** @type {import("./protocol.js").GraphPayload & { properties: Record, inports: Record, outports: Record, processes: Record, connections: *[] }} */ const graph = { properties: {}, inports: {}, @@ -31,385 +74,474 @@ function synthesizeTopicFixture(topic, components, callback) { connections: [], }; - const processName = 'testee'; + const processName = "testee"; graph.processes[processName] = { component: topic }; - component.inPorts.forEach((port) => { + for (const port of component.inPorts) { const portName = port.id; graph.inports[portName] = { process: processName, port: portName, }; - }); - component.outPorts.forEach((port) => { + } + for (const port of component.outPorts) { const portName = port.id; graph.outports[portName] = { process: processName, port: portName, }; - }); + } // Sanity checking if this is usable as a fixture if (Object.keys(graph.outports).length < 1) { - callback(new Error(`Component '${topic}' used as fixture has no outports`)); - return; + throw new Error(`Component '${topic}' used as fixture has no outports`); } if (Object.keys(graph.inports).length < 1) { - callback(new Error(`Component '${topic}' used as fixture has no inports`)); - return; + throw new Error(`Component '${topic}' used as fixture has no inports`); } - callback(null, graph); + return graph; } -// @context should have .components = {} property -function getFixtureGraph(context, suite, callback) { +/** + * Determine the fixture graph for a suite, loading components from the runtime if needed. + * @param {{ components: Record, client: import("fbp-client").FbpClient }} context + * Object holding the runtime context; the .components property is populated as needed + * @param {import("./testsuite.js").Suite} suite Testsuite to determine fixture for + * @returns {Promise} Graph JSON payload + */ +async function getFixtureGraph(context, suite) { // TODO: follow runtime for component changes - const hasComponents = (s) => (s.components != null) && (typeof s.components === 'object') && Object.keys(s.components).length; + const hasComponents = + /** @type {(s: { components?: Record }) => boolean} */ ( + (s) => + s.components != null && + typeof s.components === "object" && + Object.keys(s.components).length + ); - function updateComponents(cb) { - if (hasComponents(context)) { - cb(null); - return; - } - protocol.getComponents(context.client, (err, components) => { - if (err) { - cb(err); - return; - } - context.components = components; - cb(null); - }); + if (!hasComponents(context)) { + context.components = await protocol.getComponents(context.client); } - updateComponents((err) => { - if (err) { - callback(err); - return; - } + if (!suite.fixture) { + return synthesizeTopicFixture(suite.topic ?? "", context.components); + } - if (!suite.fixture) { - synthesizeTopicFixture(suite.topic, context.components, callback); - return; + if (suite.fixture.type === "json") { + let graph; + try { + graph = /** @type {import("./protocol.js").GraphPayload} */ ( + JSON.parse(suite.fixture.data) + ); + } catch (error) { + const err = /** @type {Error} */ (error); + throw new Error(`Could not parse JSON fixture: ${err.message}`); } - - if (suite.fixture.type === 'json') { - let graph; - try { - graph = JSON.parse(suite.fixture.data); - } catch (error) { - callback(error); - return; - } - callback(null, graph); - return; + return graph; + } + if (suite.fixture.type === "fbp") { + let graph; + try { + graph = /** @type {import("./protocol.js").GraphPayload} */ ( + fbp.parse(suite.fixture.data) + ); + } catch (error) { + const err = /** @type {Error} */ (error); + throw new Error(`Could not parse FBP fixture: ${err.message}`); } - if (suite.fixture.type === 'fbp') { - let graph; - try { - graph = fbp.parse(suite.fixture.data); - } catch (error) { - callback(error); - return; - } - if (!graph.properties) { - graph.properties = {}; - } - callback(null, graph); - return; + if (!graph.properties) { + graph.properties = {}; } - callback(new Error(`Unknown fixture type ${suite.fixture.type}`)); - }); + return graph; + } + throw new Error(`Unknown fixture type ${suite.fixture.type}`); } -function sendMessageAndWait(client, currentGraph, inputData, expectData, callback) { - const observer = client.observe((s) => (s.protocol === 'runtime') && (s.command === 'packet') && (s.payload.graph === currentGraph)); - +/** + * Send input data to the graph, and wait for the expected output packets to arrive. + * @param {import("fbp-client").FbpClient} client Connected client + * @param {string} currentGraph ID of the graph under test + * @param {Record} inputData Input data keyed by port name + * @param {Record} expectData Expected output data keyed by port name + * @returns {Promise>} Data received on the outports + */ +async function sendMessageAndWait(client, currentGraph, inputData, expectData) { + const observer = client.observe( + (s) => + s.protocol === "runtime" && + s.command === "packet" && + s.payload.graph === currentGraph, + ); + + /** + * Convert observed signals into received data, keyed by port. + * @param {{ payload: Record }[]} signals Observed signals + * @returns {Record} Data keyed by port name + */ function signalsToReceived(signals) { + /** @type {Record} */ const received = {}; - signals.forEach((signal) => { + for (const signal of signals) { received[signal.payload.port] = signal.payload.payload; - }); + } return received; } + /** + * Check whether a signal means we have received all the expected output. + * @param {{ protocol: string, command: string, payload: Record }} s Signal to check + * @returns {boolean} True when complete + */ function checkSuccess(s) { // We're only interested in response packets - if ((s.protocol !== 'runtime') || (s.command !== 'packet')) { return false; } + if (s.protocol !== "runtime" || s.command !== "packet") { + return false; + } // Check that is for the current graph under test - if (s.payload.graph !== currentGraph) { return false; } + if (s.payload.graph !== currentGraph) { + return false; + } // We're only interested in data IPs, not brackets - if (s.payload.event !== 'data') { return false; } + if (s.payload.event !== "data") { + return false; + } // Get signals received until this message - const receivedSignals = observer.signals.slice(0, observer.signals.indexOf(s) + 1); + const receivedSignals = observer.signals.slice( + 0, + observer.signals.indexOf(s) + 1, + ); const received = signalsToReceived(receivedSignals); - const result = (Object.keys(received).length === Object.keys(expectData).length); + const result = + Object.keys(received).length === Object.keys(expectData).length; return result; } + /** + * Check whether a signal means the test has failed. + * @param {{ protocol: string, command: string, payload: Record }} s Signal to check + * @returns {boolean} True when failed + */ function checkFailure(s) { - if ((s.protocol === 'network') && (s.command === 'error')) { + if (s.protocol === "network" && s.command === "error") { // network:error should imply failed test - if (s.payload.graph && (s.payload.graph !== currentGraph)) { return false; } + if (s.payload.graph && s.payload.graph !== currentGraph) { + return false; + } return true; } - if ((s.protocol === 'network') && (s.command === 'processerror')) { + if (s.protocol === "network" && s.command === "processerror") { // network:processerror should imply failed test - if (s.payload.graph && (s.payload.graph !== currentGraph)) { return false; } + if (s.payload.graph && s.payload.graph !== currentGraph) { + return false; + } return true; } - if ((s.protocol === 'runtime') && (s.command === 'packet')) { + if (s.protocol === "runtime" && s.command === "packet") { // Output packet, see if it is an unexpected error // Check that is for the current graph under test - if (s.payload.graph !== currentGraph) { return false; } + if (s.payload.graph !== currentGraph) { + return false; + } // We're only interested in data IPs, not brackets - if (s.payload.event !== 'data') { return false; } + if (s.payload.event !== "data") { + return false; + } // We only care for packets to error port - if (s.payload.port !== 'error') { return false; } + if (s.payload.port !== "error") { + return false; + } // We only care if spec isn't expecting errors - if (typeof expectData.error !== 'undefined') { return false; } + if (typeof expectData.error !== "undefined") { + return false; + } return true; } return false; } // send input packets - const sendPackets = Promise.promisify(protocol.sendPackets); - Promise.resolve() - .then(() => sendPackets(client, currentGraph, inputData)) - .then(() => observer.until(checkSuccess, checkFailure)) - .then((signals) => signalsToReceived(signals)) - .nodeify(callback); + await protocol.sendPackets(client, currentGraph, inputData); + const signals = await observer.until(checkSuccess, checkFailure); + return signalsToReceived(signals); } +/** + * Whether a suite has any testcases that are not skipped. + * @param {import("./testsuite.js").Suite} suite Testsuite + * @returns {boolean} True if the suite needs setup + */ function needsSetup(suite) { - const notSkipped = suite.cases.filter((c) => !c.skip); + const notSkipped = (suite.cases ?? []).filter((c) => !c.skip); return notSkipped.length > 0; } -class Runner { +/** + * Runs testsuites against a connected FBP runtime. + */ +export class Runner { + /** + * @param {object|import("fbp-client").FbpClient} client Either a runtime definition, + * or an already-connected fbp-client instance + * @param {RunnerOptions} [options] Runner options; timeouts are in milliseconds + */ constructor(client, options = {}) { - this.client = client; + /** @type {import("fbp-client").FbpClient} */ + this.client = /** @type {import("fbp-client").FbpClient} */ (client); this.currentGraphId = null; + /** @type {Record} */ this.components = {}; - this.parentElement = null; + this.parentElement = /** @type {object|null} */ (null); + /** @type {RunnerOptions} */ this.options = options; + /** @type {import("flowtrace").FlowtraceRecorder|null} */ this.recorder = null; - if ((this.options.connectTimeout == null)) { this.options.connectTimeout = 5 * 1000; } - if ((this.options.commandTimeout == null)) { this.options.commandTimeout = 3 * 1000; } + if (this.options.connectTimeout == null) { + this.options.connectTimeout = 5 * 1000; + } + if (this.options.commandTimeout == null) { + this.options.commandTimeout = 3 * 1000; + } } - prepareClient(callback) { - if ((this.client.protocol != null) && (this.client.address != null)) { + /** + * Ensure this.client is a connected fbp-client instance. + * @returns {Promise} Connected client + */ + async prepareClient() { + const isDefinition = "protocol" in this.client && "address" in this.client; + if (isDefinition) { // is a runtime definition - Promise.resolve() - .then(() => fbpClient(this.client, { commandTimeout: this.options.commandTimeout })) - .then((client) => { - this.client = client; - - if (this.parentElement && (client.definition.protocol === 'iframe')) { - // We need to set up the parent element in this case - client.transport.setParentElement(this.parentElement); - } - - return client; - }) - .nodeify(callback); - return; + const client = await fbpClient(this.client, { + commandTimeout: this.options.commandTimeout, + }); + + if (this.parentElement && client.definition.protocol === "iframe") { + // We need to set up the parent element in this case + client.transport.setParentElement(this.parentElement); + } + + this.client = client; + return client; } // This is a client instance - callback(null, this.client); + return this.client; } // TODO: check the runtime capabilities before continuing - connect(callback) { - debug('connect'); + /** + * Connect to the runtime, retrying until the connection is up. + * @returns {Promise} Resolves when connected and capabilities are checked + */ + async connect() { + debug("connect"); + + const client = await this.prepareClient(); + debugReceivedMessages(client); + client.on("network", (message) => { + const { command, payload } = + /** @type {{ command: string, payload: { message?: string } }} */ ( + message + ); + if (command === "output" && payload.message) { + console.log(payload.message); + } + }); - this.prepareClient((err) => { - if (err) { - callback(err); + const timeBetweenAttempts = 500; + const attempts = Math.floor( + (this.options.connectTimeout ?? 5000) / timeBetweenAttempts, + ); + const isOnline = async () => { + const connected = this.client.isConnected(); + if (connected) { return; } - - debugReceivedMessages(this.client); - this.client.on('network', ({ command, payload }) => { - if ((command === 'output') && payload.message) { - console.log(payload.message); - } - }); - - const timeBetweenAttempts = 500; - const attempts = Math.floor(this.options.connectTimeout / timeBetweenAttempts); - const isOnline = () => { - const connected = this.client.isConnected(); - if (connected) { - return Promise.resolve(); - } - return Promise.reject(new Error('Not connected to runtime')); - }; - const tryConnect = () => { - debug('trying to connect'); - return this.client.connect(); - }; - common.retryUntil(tryConnect, isOnline, timeBetweenAttempts, attempts) - .then(() => this.checkCapabilities(['protocol:graph', 'protocol:network', 'protocol:runtime'])) - .nodeify(callback); - }); + throw new Error("Not connected to runtime"); + }; + const tryConnect = async () => { + debug("trying to connect"); + return this.client.connect(); + }; + await retryUntil(tryConnect, isOnline, timeBetweenAttempts, attempts); + await this.checkCapabilities([ + "protocol:graph", + "protocol:network", + "protocol:runtime", + ]); } - disconnect(callback) { - debug('disconnect'); + /** + * Disconnect from the runtime, if connected. + * @returns {Promise} Resolves when disconnected + */ + async disconnect() { + debug("disconnect"); - if (!(this.client != null ? this.client.isConnected() : undefined)) { - callback(); + if (!this.client?.isConnected()) { return; } - Promise.resolve() - .then(() => this.client.disconnect()) - .nodeify(callback); + await this.client.disconnect(); } - checkCapabilities(capabilities) { + /** + * Check that the runtime provides the given capabilities. + * @param {string[]} capabilities Capability names required + * @returns {Promise} Rejects if a capability is missing + */ + async checkCapabilities(capabilities) { if (!this.client.isConnected()) { - return Promise.reject(new Error('Not connected to runtime')); + throw new Error("Not connected to runtime"); } - if (!this.client.definition - || !this.client.definition.capabilities - || !this.client.definition.capabilities.length) { - return Promise.reject(new Error('Runtime provides no capabilities')); + if (!this.client.definition?.capabilities?.length) { + throw new Error("Runtime provides no capabilities"); } - for (let i = 0; i < capabilities.length; i += 1) { - const capability = capabilities[i]; - if (this.client.definition.capabilities.indexOf(capability) === -1) { - return Promise.reject(new Error(`Runtime doesn't provide ${capability}`)); + const caps = this.client.definition?.capabilities ?? []; + for (const capability of capabilities) { + if (caps.indexOf(capability) === -1) { + throw new Error(`Runtime doesn't provide ${capability}`); } } - return Promise.resolve(); } - setupSuite(suite, callback) { + /** + * Set up the fixture graph for a suite, and start its network. + * @param {import("./testsuite.js").Suite} suite Testsuite to set up + * @returns {Promise} Resolves when the network is running + */ + async setupSuite(suite) { debug(`setup suite "${suite.name}"`); if (this.options.traceCallback && !this.recorder) { this.recorder = new FlowtraceRecorder(this.client); } if (!needsSetup(suite)) { - callback(null); return; } if (!this.client.isConnected()) { - callback(new Error('Disconnected from runtime')); - return; + throw new Error("Disconnected from runtime"); } - getFixtureGraph(this, suite, (err, graph) => { - if (err) { - callback(err); - return; - } - protocol.sendGraph(this.client, graph, (sendErr, graphId) => { - if (sendErr) { - callback(sendErr); - return; - } - this.currentGraphId = graphId; - if (this.recorder) { - this.recorder.start(graphId) - .then(() => { - protocol.startNetwork(this.client, graphId, callback); - }, callback); - return; - } - protocol.startNetwork(this.client, graphId, callback); - }); - }); + const graph = await getFixtureGraph(this, suite); + const graphId = await protocol.sendGraph(this.client, graph); + this.currentGraphId = graphId; + if (this.recorder) { + await this.recorder.start(graphId); + } + await protocol.startNetwork(this.client, graphId); } - teardownSuite(suite, callback) { + /** + * Stop the network for a suite, and stop any tracing. + * @param {import("./testsuite.js").Suite} suite Testsuite to tear down + * @returns {Promise} Resolves when the network is stopped + */ + async teardownSuite(suite) { debug(`teardown suite "${suite.name}"`); if (this.recorder && this.currentGraphId) { - // FIXME: This returns a Promise that we should wait for - this.recorder.stop(this.currentGraphId); + await this.recorder.stop(this.currentGraphId); } if (!needsSetup(suite)) { - callback(null); return; } if (!this.client.isConnected()) { - callback(new Error('Disconnected from runtime')); - return; + throw new Error("Disconnected from runtime"); } if (!this.currentGraphId) { // Graph was never successfully set up, so no need to stop - callback(); return; } // FIXME: also remove the graph. Ideally using a 'destroy' message in FBP protocol - protocol.stopNetwork(this.client, this.currentGraphId, callback); + await protocol.stopNetwork(this.client, this.currentGraphId); } - runTest(testcase, callback) { + /** + * Run a single testcase, recording the actual outputs in the sequence. + * @param {import("./testsuite.js").Testcase} testcase Testcase to run + * @returns {Promise} The input/expect sequence with actuals filled in + */ + async runTest(testcase) { debug(`runtest "${testcase.name}"`); if (!this.client.isConnected()) { - callback(new Error('Disconnected from runtime')); - return; + throw new Error("Disconnected from runtime"); + } + if (!this.currentGraphId) { + // Suite has not been set up + throw new Error("No graph is set up for the suite"); } // XXX: normalize and validate in testsuite.js instead? - const inputs = common.isArray(testcase.inputs) ? testcase.inputs : [testcase.inputs]; - const expects = common.isArray(testcase.expect) ? testcase.expect : [testcase.expect]; + const inputs = /** @type {Record|Record[]} */ ( + testcase.inputs + ); + const expects = /** @type {Record|Record[]} */ ( + testcase.expect + ); + const inputList = /** @type {Record[]} */ ( + isArray(inputs) ? inputs : [inputs] + ); + const expectList = /** @type {Record[]} */ ( + isArray(expects) ? expects : [expects] + ); + /** @type {TestSequenceEntry[]} */ const sequence = []; - for (let i = 0, end = inputs.length, asc = end >= 0; - asc ? i < end : i > end; - asc ? i += 1 : i -= 1) { + for (let i = 0; i < inputList.length; i += 1) { sequence.push({ - inputs: inputs[i], - expect: expects[i], + inputs: inputList[i], + expect: expectList[i], }); } - const sendWait = (data, cb) => sendMessageAndWait( - this.client, - this.currentGraphId, - data.inputs, - data.expect, - cb, - ); - common.asyncSeries(sequence, sendWait, (err, actuals) => { - if (this.options.traceCallback && this.currentGraphId && this.recorder) { - this.options.traceCallback( + const actuals = []; + for (const data of sequence) { + actuals.push( + await sendMessageAndWait( + // eslint-disable-line no-await-in-loop + this.client, this.currentGraphId, - testcase.name, - this.recorder.dump(this.currentGraphId), - ); - } - if (err) { - callback(err); - return; - } - actuals.forEach((r, idx) => { - sequence[idx].actual = r; - }); - callback(null, sequence); + data.inputs, + data.expect, + ), + ); + } + + if (this.options.traceCallback && this.currentGraphId && this.recorder) { + this.options.traceCallback( + this.currentGraphId, + testcase.name ?? "", + this.recorder.dump(this.currentGraphId), + ); + } + actuals.forEach((r, idx) => { + sequence[idx].actual = r; }); + return sequence; } } -// TODO: should this go into expectation? -function checkResults(results) { +/** + * Check actual test results against the expectations. + * @param {Array} results + * Input/expect/actual sequence as produced by Runner#runTest + * @returns {{ passed: boolean, error?: Error }} Result of the check + */ +export function checkResults(results) { + /** @type {{ passed: boolean, error?: Error }} */ let result; const actuals = results.filter((r) => r.actual != null); const expects = results.filter((r) => r.expect != null); if (actuals.length < expects.length) { result = { passed: false, - error: new Error(`Only got ${actuals.length} output messages out of ${expects.length}`), + error: new Error( + `Only got ${actuals.length} output messages out of ${expects.length}`, + ), }; return result; } @@ -417,10 +549,10 @@ function checkResults(results) { const mappedResults = results.map((res) => { res.error = null; try { - expectation.expect(res.expect, res.actual); + checkExpectation(res.expect, res.actual); } catch (e) { // FIXME: only catch actual AssertionErrors - res.error = e; + res.error = /** @type {Error} */ (e); } return res; }); @@ -429,168 +561,188 @@ function checkResults(results) { if (failures.length === 0) { result = { passed: true }; } else if (expects.length === 1) { - result = { error: failures[0].error }; - } else if ((expects.length > 1) && (failures.length === 1)) { + result = { + passed: false, + error: failures[0].error ?? new Error("unknown expectation failure"), + }; + } else if (expects.length > 1 && failures.length === 1) { const index = mappedResults.findIndex((r) => r.error); const failed = mappedResults[index]; - result = { error: new Error(`Expectation ${index} of sequence failed: ${failed.error.message}`) }; + result = { + passed: false, + error: new Error( + `Expectation ${index} of sequence failed: ${failed.error?.message}`, + ), + }; } else { - const errors = mappedResults.map((r) => (r.error != null ? r.error.message : undefined) || ''); - result = { error: new Error(`Multiple failures in sequence: ${errors}`) }; + const errors = mappedResults.map( + (r) => (r.error != null ? r.error.message : undefined) || "", + ); + result = { + passed: false, + error: new Error(`Multiple failures in sequence: ${errors}`), + }; } return result; } -function runTestAndCheck(runner, testcase, callback) { +/** + * Run a single testcase and check its results. + * @param {RunnerLike} runner Runner with a set up suite + * @param {import("./testsuite.js").Testcase} testcase Testcase to run + * @returns {Promise<{ passed: boolean, error?: Error }>} Test result + */ +export async function runTestAndCheck(runner, testcase) { if (testcase.skip) { // TODO: pass some skipped state? its indirectly in .skip though - callback(null, { passed: true }); - return; + return { passed: true }; } // XXX: normalize and validate in testsuite.js instead? - const inputs = common.isArray(testcase.inputs) ? testcase.inputs : [testcase.inputs]; - const expects = common.isArray(testcase.expect) ? testcase.expect : [testcase.expect]; - if (inputs.length !== expects.length) { - callback(null, { + const inputs = /** @type {Record|Record[]} */ ( + testcase.inputs + ); + const expects = /** @type {Record|Record[]} */ ( + testcase.expect + ); + const inputList = /** @type {Record[]} */ ( + isArray(inputs) ? inputs : [inputs] + ); + const expectList = /** @type {Record[]} */ ( + isArray(expects) ? expects : [expects] + ); + if (inputList.length !== expectList.length) { + return { passed: false, - error: new Error(`Test sequence length mismatch. Got ${inputs.length} inputs and ${expects.length} expectations`), - }); - return; + error: new Error( + `Test sequence length mismatch. Got ${inputList.length} inputs and ${expectList.length} expectations`, + ), + }; } - runner.runTest(testcase, (err, results) => { - let result; - if (err) { - // Map error to a test failure - result = { - passed: false, - error: err, - }; - callback(null, result); - return; - } - result = checkResults(results); - if (result.error) { - result.passed = false; - } - callback(null, result); - }); + /** @type {TestSequenceEntry[]} */ + let results; + try { + results = await runner.runTest(testcase); + } catch (err) { + // Map error to a test failure + return { + passed: false, + error: /** @type {Error} */ (err), + }; + } + + const result = checkResults(results); + return result; } -function runSuite(runner, suite, runTest, callback) { +/** + * Run all testcases in a suite. + * @param {Runner} runner Runner to run with + * @param {import("./testsuite.js").Suite} suite Testsuite to run + * @param {(testcase: import("./testsuite.js").Testcase) => Promise} runTest Async function running and checking one testcase + * @returns {Promise} The suite + */ +export async function runSuite(runner, suite, runTest) { if (suite.skip) { // TODO: pass some skipped state? its indirectly in .skip though - callback(null, suite); - return; + return suite; } - runner.setupSuite(suite, (err) => { - debug('setup suite', err); - if (err) { - callback(err, suite); - return; - } - - common.asyncSeries(suite.cases, runTest, (runErr) => { - debug('testrun complete', runErr); - if (runErr) { - runner.teardownSuite(suite, (teardownErr) => { - debug('teardown suite', teardownErr); - callback(runErr, suite); - }); - return; - } + try { + await runner.setupSuite(suite); + } catch (err) { + debug("setup suite", err); + throw err; + } - runner.teardownSuite(suite, (teardownErr) => { - debug('teardown suite', teardownErr); - callback(teardownErr, suite); - }); - }); - }); + try { + await asyncSeries(suite.cases ?? [], runTest); + } finally { + await runner + .teardownSuite(suite) + .catch((err) => debug("teardown suite", err)); + } + return suite; } -function loadComponentSuites(componentTests, callback) { - function loadYAML(source, cb) { - testsuite.loadYAML(source, (err, suite) => { - if (err) { - // ignore, could be non fbp-spec test - // TODO: include tests type in FBP protocol, so we know whether this is error or legit - cb(null, null); - return; - } - cb(null, suite); - }); - } +/** + * Load and parse the fbp-spec tests attached to components. + * Non-fbp-spec test sources are ignored. + * @param {Record} componentTests YAML test sources, keyed by component name + * @returns {Promise} Parsed testsuites + */ +export async function loadComponentSuites(componentTests) { const tests = Object.keys(componentTests).map((name) => componentTests[name]); - common.asyncSeries(tests, loadYAML, (err, suites) => { - if (err) { - callback(err); - return; + const suites = await asyncSeries(tests, (source) => { + try { + return Promise.resolve(testsuite.loadYAML(source)); + } catch { + // ignore, could be non fbp-spec test + // TODO: include tests type in FBP protocol, so we know whether this is error or legit + return Promise.resolve(null); } - callback(null, suites.filter((s) => s !== null).reduce((a, b) => a.concat(b), [])); }); + return suites.filter((s) => s !== null).reduce((a, b) => a.concat(b), []); } -exports.getComponentSuites = function getComponentSuites(runner, callback) { - protocol.getCapabilities(runner.client, (err, caps) => { - if (err) { - callback(err); - return; +/** + * Get the testsuites attached to components on the runtime. + * Requires the runtime to support the 'component:getsource' capability. + * @param {Runner} runner Runner connected to the runtime + * @returns {Promise} Testsuites + */ +export async function getComponentSuites(runner) { + const caps = await protocol.getCapabilities(runner.client); + if (caps.indexOf("component:getsource") === -1) { + return []; + } + + const tests = await protocol.getComponentTests(runner.client); + const suites = await loadComponentSuites(tests); + debug("get component suites", Object.keys(tests).length, suites.length); + return suites; +} + +/** + * Run all the given suites, updating the testcase objects in-place + * with .passed and .error states as tests are run. + * @param {Runner} runner Runner to run with + * @param {import("./testsuite.js").Suite[]} suites Suites to run + * @param {(suites: import("./testsuite.js").Suite[]) => void} [updateCallback] Called when test state is updated + * @returns {Promise} Resolves when all suites have run + */ +export async function runAll(runner, suites, updateCallback = () => {}) { + debug( + "running suites", + suites.map((s) => s.name), + ); + + /** + * Run and check a single testcase, storing the result on the testcase. + * @param {import("./testsuite.js").Testcase} testcase Testcase to run + * @returns {Promise} + */ + async function runTest(testcase) { + const results = await runTestAndCheck(runner, testcase); + // ignore error to not bail out early + const resultRecord = /** @type {Record} */ (results); + for (const key of Object.keys(resultRecord)) { + testcase[key] = resultRecord[key]; } - if (caps.indexOf('component:getsource') === -1) { - callback(null, []); - return; + if (testcase.error instanceof Error) { + testcase.error = testcase.error.message; } - - protocol.getComponentTests(runner.client, (testsErr, tests) => { - if (testsErr) { - callback(testsErr); - return; - } - loadComponentSuites(tests, (loadErr, suites) => { - if (loadErr) { - callback(loadErr); - return; - } - debug('get component suites', tests.length, suites.length); - callback(null, suites); - }); - }); - }); -}; - -// will update each of the testcases in @suites -// with .passed and .error states as tests are ran -function runAll(runner, suites, updateCallback, doneCallback) { - function runTest(testcase, callback) { - function done(error) { - updateCallback(suites); - callback(error); - } - - runTestAndCheck(runner, testcase, (err, results) => { - // ignore error to not bail out early - const tcase = testcase; - Object.keys(results).forEach((key) => { - const val = results[key]; - tcase[key] = val; - }); - if (testcase.error) { - tcase.error = testcase.error.message; - } - debug(`ran test "${testcase.name}"`, testcase.passed, err); - done(null); - }); + debug(`ran test "${testcase.name}"`, testcase.passed); + updateCallback(suites); } - const runOneSuite = (suite, cb) => runSuite(runner, suite, runTest, cb); - - debug('running suites', suites.map((s) => s.name)); - common.asyncSeries(suites, runOneSuite, (err) => doneCallback(err)); + for (const suite of suites) { + try { + await runSuite(runner, suite, runTest); // eslint-disable-line no-await-in-loop + } catch (err) { + debug("suite failed", err); + } + } } - -exports.Runner = Runner; -exports.runAll = runAll; -exports.runTestAndCheck = runTestAndCheck; diff --git a/lib/subprocess.js b/lib/subprocess.js index 43f5534..06b380f 100644 --- a/lib/subprocess.js +++ b/lib/subprocess.js @@ -1,63 +1,66 @@ -const debug = require('debug')('fbp-spec:subprocess'); - -exports.start = function start(command, options, callback) { - let childProcess; - try { - // eslint-disable-next-line global-require - childProcess = require('child_process'); - } catch (error) { - callback(error); - return null; - } - - if ((options.timeout == null)) { - const opts = options; - opts.timeout = 300; - } - - let started = false; - let stderr = ''; - - // FIXME: using sh to interpret command will Unix-specific - const prog = 'sh'; - const args = ['-c', command]; - const child = childProcess.spawn(prog, args); - - debug('spawned', `'${prog} ${args.join(' ')}'`); - debug('waiting for output'); - - child.on('error', (err) => callback(err)); - - child.stdout.on('data', (data) => { - const dataString = data.toString(); - debug('sub stdout', dataString); - if (!started) { - debug('got output, transitioning to started'); - started = true; - // give process some time to open port - setTimeout(callback, 100); - } - }); +/** + * Start runtimes-under-test as subprocesses. + * @module lib/subprocess + */ - child.stderr.on('data', (data) => { - const dataString = data.toString(); - stderr += dataString; - debug('sub stderr', dataString); - if (!started) { - debug('got stderr, failing'); - started = true; - callback(new Error(`Subprocess wrote on stderr: '${stderr}'`)); - } - }); +import debugLib from "debug"; + +const debug = debugLib("fbp-spec:subprocess"); + +/** + * Start a runtime as a subprocess, using sh to interpret the command. + * Considers the process started once it produces output, or after a timeout. + * Uses a dynamic import so that this module can also be bundled for browsers. + * @param {string} command Shell command to launch the runtime + * @param {{ timeout?: number }} [options] Options; timeout in ms before assuming startup succeeded + * @returns {Promise} The spawned child process + */ +export async function start(command, options = {}) { + const timeout = options.timeout ?? 300; + const childProcess = await import("node:child_process"); + + return new Promise((resolve, reject) => { + // FIXME: using sh to interpret command will Unix-specific + const prog = "sh"; + const args = ["-c", command]; + const child = childProcess.spawn(prog, args); + + debug("spawned", `'${prog} ${args.join(" ")}'`); + debug("waiting for output"); - setTimeout(() => { - if (!started) { - debug('timeout waiting for output, assuming started'); - started = true; - callback(null); - } - }, - options.timeout); - - return child; -}; + let started = false; + let stderr = ""; + + child.on("error", (err) => reject(err)); + + child.stdout.on("data", (data) => { + const dataString = data.toString(); + debug("sub stdout", dataString); + if (!started) { + debug("got output, transitioning to started"); + started = true; + // give process some time to open port + setTimeout(() => resolve(child), 100); + } + }); + + child.stderr.on("data", (data) => { + const dataString = data.toString(); + stderr += dataString; + debug("sub stderr", dataString); + if (!started) { + debug("got stderr, failing"); + started = true; + reject(new Error(`Subprocess wrote on stderr: '${stderr}'`)); + } + }); + + setTimeout(() => { + if (!started) { + debug("timeout waiting for output, assuming started"); + started = true; + resolve(child); + } + }, timeout); + }); +} diff --git a/lib/testsuite.js b/lib/testsuite.js index 33e95e7..ae08dfc 100644 --- a/lib/testsuite.js +++ b/lib/testsuite.js @@ -1,165 +1,159 @@ -/* eslint-env browser, node */ -const fetch = require('isomorphic-fetch'); -const path = require('path'); -const common = require('./common'); -// eslint-disable-next-line import/no-unresolved -const schemas = require('../schema'); - -exports.validate = function validate(obj) { - // eslint-disable-next-line global-require - const tv4 = (typeof window !== 'undefined' && window.tv4) ? window.tv4 : require('tv4'); - if (!tv4) { - throw new Error('TV4 not available, unable to validate'); - } - +/** + * Loading, normalizing and validating of fbp-spec testsuites. + * @module lib/testsuite + */ + +import { extname, join } from "node:path"; +import debugLib from "debug"; +import { loadAll } from "js-yaml"; +import tv4 from "tv4"; + +import schemas from "../schema/index.js"; +import { isArray, startsWith } from "./common.js"; + +const debug = debugLib("fbp-spec:testsuite"); + +/** + * A single test case within a testsuite. + * @typedef {object} TestcaseProps + * @property {string} [name] Name of this testcase + * @property {string} [assertion] Description of what this testcase asserts + * @property {Record|Array>} [inputs] Inputs sent to fixture inports, or a sequence of such + * @property {Record|Array>} [expect] Expected data on outports, or a sequence of such + * @property {string} [skip] Reason to skip this testcase + * @property {number} [timeout] Maximum time to run this testcase for, in milliseconds + * @property {boolean} [passed] Set to the test result by the runner + * @property {string|Error} [error] Set to the test error message, if the test failed + * @property {boolean} [shown] Internal: whether testcase result has been shown in CLI output + * @property {*} [actual] Set to the actual output data by the runner + */ + +/** + * A testcase, with free-form additional properties from the YAML source. + * @typedef {TestcaseProps & Record} Testcase + */ + +/** + * A fbp-spec testsuite: a fixture plus a set of testcases. + * @typedef {object} SuiteProps + * @property {string} [name] Name of the suite; defaults to the topic + * @property {string} [topic] Name of the component or graph under test + * @property {{ type: string, data: string }} [fixture] Fixture graph definition + * @property {Testcase[]} [cases] Testcases to run against the fixture + * @property {string} [skip] Reason to skip the whole suite + * @property {number} [timeout] Maximum time to run the suite for, in milliseconds + * @property {boolean} [titleshown] Internal: whether suite title has been shown in CLI output + */ + +/** + * A testsuite, with free-form additional properties from the YAML source. + * @typedef {SuiteProps & Record} Suite + */ + +/** + * JSON Schema definition for fbp-spec data. + * @typedef {{ id: string, title?: string, description?: string, [key: string]: * }} Schema + */ + +/** + * Validate an fbp-spec object against the fbp-spec schemas. + * @param {*} obj Object to validate, typically a testsuite or testsfile structure + * @returns {{ passed: boolean, errors: object[], missing: string[], valid: boolean }} Validation results + */ +export function validate(obj) { tv4.reset(); - Object.keys(schemas).forEach((name) => { + for (const name of Object.keys(schemas)) { const schema = schemas[name]; tv4.addSchema(schema.id, schema); - }); - - const results = tv4.validateMultiple(obj, 'testsfile.json'); - if (results.missing.length) { - results.passed = false; } + + const results = tv4.validateMultiple(obj, "testsfile.json"); + results.passed = results.valid && results.missing.length === 0; return results; -}; +} -function normalize(suite) { - // Default name to topic - const s = suite; +/** + * Apply defaults to a testsuite object. + * Defaults name to the topic, and cases to an empty list. + * @param {Suite} base Testsuite definition, as parsed from YAML + * @returns {Suite} Normalized testsuite + */ +export function create(base) { + const suite = base; if (!suite.name) { - s.name = suite.topic; + suite.name = suite.topic; } if (!suite.cases) { - s.cases = []; + suite.cases = []; } - return suite; } -exports.create = (base) => normalize(base); - -exports.loadYAML = function loadYAML(data, callback) { - const suites = []; - try { - // eslint-disable-next-line global-require - const yaml = (typeof window !== 'undefined' && window.jsyaml) ? window.jsyaml : require('js-yaml'); - yaml.loadAll(data, (doc) => suites.push(normalize(doc))); - } catch (e) { - callback(e); - return; - } - callback(null, suites); -}; - -exports.loadYAMLSync = function loadYAMLSync(data) { - // eslint-disable-next-line global-require - const yaml = (typeof window !== 'undefined' && window.jsyaml) ? window.jsyaml : require('js-yaml'); - const suites = []; - yaml.loadAll(data, (doc) => suites.push(normalize(doc))); - return suites; -}; - -function loadHTTP(url, callback) { - fetch(url) - .then((res) => res.text()) - .then((source) => new Promise((resolve, reject) => { - exports.loadYAML(source, (err, suites) => { - if (err) { - reject(err); - return; - } - resolve(suites); - }); - })) - .then((suites) => { - callback(null, suites); - }, callback); +/** + * Parse YAML containing one or more testsuites. + * @param {string} data YAML source + * @returns {Suite[]} Normalized testsuites + */ +export function loadYAML(data) { + const docs = /** @type {Suite[]} */ (loadAll(data)); + return docs.map((doc) => create(doc)); } -function getFileSync(test) { - // eslint-disable-next-line global-require - const fs = require('fs'); - - const stat = fs.statSync(test); - if (stat.isDirectory()) { - const files = fs.readdirSync(test); - const filePaths = files - .filter((f) => path.extname(f) === '.yaml') - .map((f) => path.join(test, f)); - return exports.getSuitesSync(filePaths); +/** + * Read testsuites from YAML files, directories of YAML files, or HTTP URLs. + * @param {string|string[]} tests File or directory path, or HTTP(S) URL, or a list of such + * @returns {Promise} Normalized testsuites + */ +export async function getSuites(tests) { + const list = /** @type {string[]} */ (isArray(tests) ? tests : [tests]); + const suitesList = []; + for (const test of list) { + suitesList.push(await loadLocation(test)); // eslint-disable-line no-await-in-loop } - const c = fs.readFileSync(test, 'utf-8'); - return exports.loadYAMLSync(c); + return suitesList.reduce((a, b) => a.concat(b), []); } -function getFile(test, callback) { - // eslint-disable-next-line global-require - const fs = require('fs'); - - fs.stat(test, (statErr, stat) => { - if (statErr) { - callback(statErr); - return; - } - if (stat.isDirectory()) { - fs.readdir(test, (dirErr, files) => { - if (dirErr) { - callback(dirErr); - return; - } - const filePaths = files - .filter((f) => path.extname(f) === '.yaml') - .map((f) => path.join(test, f)); - exports.getSuites(filePaths, callback); - }); - return; - } - fs.readFile(test, 'utf-8', (readErr, yaml) => { - if (readErr) { - callback(readErr); - return; - } - exports.loadYAML(yaml, callback); - }); - }); +/** + * Load suites from a single file path, directory, or URL. + * @param {string} test Location to load from + * @returns {Promise} Normalized testsuites + */ +async function loadLocation(test) { + if (startsWith(test, "http://") || startsWith(test, "https://")) { + return loadHTTP(test); + } + return getFile(test); } -// FIXME: get rid of this... -exports.getSuitesSync = function getSuitesSync(tests) { - if (!common.isArray(tests)) { - return exports.getSuitesSync([tests]); - } +/** + * Fetch a YAML testsfile over HTTP(S), and parse it. + * @param {string} url URL to fetch from + * @returns {Promise} Normalized testsuites + */ +async function loadHTTP(url) { + debug("loading suites from", url); + const res = await fetch(url); + const source = await res.text(); + return loadYAML(source); +} - let suites = []; - tests.forEach((test) => { - suites = suites.concat(getFileSync(test)); - }); - return suites; -}; - -exports.getSuites = function getSuites(tests, callback) { - if (!common.isArray(tests)) { - exports.getSuites([tests], callback); - return; +/** + * Read a testsfile from disk. + * Directories are scanned for `.yaml` files. + * Uses a dynamic import so that this module can also be bundled for browsers. + * @param {string} test File or directory path + * @returns {Promise} Normalized testsuites + */ +async function getFile(test) { + const fs = await import("node:fs"); + const info = await fs.promises.stat(test); + if (info.isDirectory()) { + const files = await fs.promises.readdir(test); + const filePaths = files + .filter((f) => extname(f) === ".yaml") + .map((f) => join(test, f)); + return getSuites(filePaths); } - - const loadTest = (test, cb) => { - if (common.startsWith(test, 'http://' || common.startsWith('https://'))) { - loadHTTP(test, cb); - return; - } - getFile(test, cb); - }; - - common.asyncSeries(tests, loadTest, (err, suitesList) => { - if (err) { - callback(err); - return; - } - // flatten list - const suites = suitesList.reduce((a, b) => a.concat(b), []); - callback(null, suites); - }); -}; + const yaml = await fs.promises.readFile(test, "utf-8"); + return loadYAML(yaml); +} diff --git a/lib/websocket.js b/lib/websocket.js deleted file mode 100644 index 3a1bf55..0000000 --- a/lib/websocket.js +++ /dev/null @@ -1,73 +0,0 @@ -// eslint-disable-next-line import/no-extraneous-dependencies -const WebSocketServer = require('websocket').server; - -// XXX: Copied from https://github.com/noflo/noflo-runtime-websocket/blob/master/runtime/network.js -// should probably reuse it as-is - -class WebSocketRuntime { - constructor(options = {}) { - this.options = options; - this.connections = []; - } - - send(protocol, topic, payload, context) { - if (!context.connection || !context.connection.connected) { - return; - } - if ((topic === 'error') && payload instanceof Error) { - this.send(protocol, topic, { - message: payload.message, - stack: payload.stack, - }, context); - return; - } - context.connection.sendUTF(JSON.stringify({ - protocol, - command: topic, - payload, - })); - } - - sendAll(protocol, topic, payload) { - this.connections.forEach((connection) => { - this.send(protocol, topic, payload, { - connection, - }); - }); - } -} - -module.exports = (httpServer, options) => { - const wsServer = new WebSocketServer({ httpServer }); - const runtime = new WebSocketRuntime(options); - - function handleMessage(message, connection) { - if (message.type === 'utf8') { - let contents; - try { - contents = JSON.parse(message.utf8Data); - } catch (e) { - if (e.stack) { - console.error(e.stack); - } else { - console.error(`Error: ${e.toString()}`); - } - return; - } - runtime.receive(contents.protocol, contents.command, contents.payload, { connection }); - } - } - - wsServer.on('request', (request) => { - const subProtocol = request.requestedProtocols.indexOf('noflo') !== -1 ? 'noflo' : null; - const connection = request.accept(subProtocol, request.origin); - runtime.connections.push(connection); - connection.on('message', (message) => handleMessage(message, connection)); - return connection.on('close', () => { - if (runtime.connections.indexOf(connection) === -1) { return; } - runtime.connections.splice(runtime.connections.indexOf(connection), 1); - }); - }); - - return runtime; -}; diff --git a/package.json b/package.json index 51500a2..b6e28de 100644 --- a/package.json +++ b/package.json @@ -13,64 +13,52 @@ "bugs": "http://github.com/flowbased/fbp-spec/issues", "homepage": "http://flowbased.org", "license": "MIT", + "type": "module", "main": "index.js", + "exports": { + ".": "./index.js", + "./schema": "./schema/index.js", + "./runner": "./lib/runner.js", + "./testsuite": "./lib/testsuite.js", + "./expectation": "./lib/expectation.js", + "./package.json": "./package.json" + }, + "bin": { + "fbp-spec": "./bin/fbp-spec" + }, "engines": { - "node": ">=6.0.0" + "node": ">=20.0.0" }, + "files": [ + "bin", + "lib", + "schema", + "index.js" + ], "scripts": { - "build": "grunt build", - "pretest": "eslint *.js lib/*.js spec/*.js", - "test": "grunt test --stack" - }, - "bin": { - "fbp-spec": "./bin/fbp-spec", - "fbp-spec-mocha": "./bin/fbp-spec-mocha" + "build": "node scripts/build-schemas.js", + "lint": "npx @biomejs/biome check --use-editorconfig=true lib/ schema/ scripts/ spec/ examples/ index.js bin/", + "format": "npx @biomejs/biome check --use-editorconfig=true --write lib/ schema/ scripts/ spec/ examples/ index.js bin/", + "typecheck": "tsc", + "test": "npm run lint && npm run typecheck && node --test spec/*.test.js" }, "dependencies": { - "bluebird": "^3.5.0", - "chai": "~4.2.0", - "commander": "~6.2.0", - "debug": "~4.3.1", - "fbp": "^1.7.0", - "fbp-client": "^0.4.1", + "debug": "^4.4.3", + "fbp": "^1.8.0", + "fbp-client": "^0.4.3", "fbp-graph": "^0.7.0", - "flowtrace": "^0.1.2", - "isomorphic-fetch": "^3.0.0", - "js-yaml": "~4.0.0", - "jsonpath": "^1.0.2", - "mocha": "~8.2.0", - "slug": "^4.0.2", - "tv4": "~1.3.0" + "flowtrace": "^0.1.10", + "js-yaml": "^5.4.3", + "jsonpath": "^1.3.0", + "tv4": "^1.3.0" }, "devDependencies": { - "@babel/core": "^7.11.5", - "@babel/preset-env": "^7.11.5", - "babel-loader": "^8.1.0", - "buffer": "^6.0.1", - "es6-shim": "^0.35.3", - "eslint": "^7.11.0", - "eslint-config-airbnb-base": "^14.2.0", - "eslint-plugin-chai": "0.0.1", - "eslint-plugin-import": "^2.22.1", - "eslint-plugin-mocha": "^8.0.0", - "events": "^3.2.0", - "grunt": "^1.0.1", - "grunt-contrib-copy": "^1.0.0", - "grunt-contrib-watch": "^1.0.0", - "grunt-exec": "^3.0.0", - "grunt-karma": "^4.0.0", - "grunt-mocha-test": "^0.13.2", - "grunt-webpack": "^4.0.2", - "grunt-yaml": "^0.4.2", - "grunt-yamllint": "https://github.com/gruntjs-updater/grunt-yamllint.git#8e7c04a", - "karma": "^5.2.3", - "karma-chai": "^0.1.0", - "karma-chrome-launcher": "^3.1.0", - "karma-mocha": "^2.0.1", - "karma-mocha-reporter": "^2.2.5", - "path-browserify": "^1.0.1", - "url": "^0.11.0", - "webpack": "^5.1.3", - "websocket": "^1.0.26" + "@biomejs/biome": "^2.5.15", + "@types/debug": "^4.1.13", + "@types/jsonpath": "^0.2.4", + "@types/node": "^26.6.5", + "@types/websocket": "^1.0.10", + "typescript": "^7.0.2", + "websocket": "^1.0.35" } } diff --git a/schema/base.json b/schema/base.json index bcefb98..619a911 100644 --- a/schema/base.json +++ b/schema/base.json @@ -10,4 +10,4 @@ "example": "products.book[2]" } } -} \ No newline at end of file +} diff --git a/schema/expectation.json b/schema/expectation.json index 01814eb..05e5bba 100644 --- a/schema/expectation.json +++ b/schema/expectation.json @@ -42,4 +42,4 @@ "description": "String or array contains substring/item" } } -} \ No newline at end of file +} diff --git a/schema/expectations.json b/schema/expectations.json index 3269564..26e5e8c 100644 --- a/schema/expectations.json +++ b/schema/expectations.json @@ -7,4 +7,4 @@ "items": { "$ref": "expectation.json" } -} \ No newline at end of file +} diff --git a/schema/index.js b/schema/index.js index 4984f82..1080bce 100644 --- a/schema/index.js +++ b/schema/index.js @@ -1,13 +1,34 @@ -/* eslint global-require: "off" */ -try { - module.exports = { - base: require('./base.json'), - testcase: require('./testcase.json'), - testsuite: require('./testsuite.json'), - testsuites: require('./testsuites.json'), - expectations: require('./expectations.json'), - testsfile: require('./testsfile.json'), - }; -} catch (e) { - console.log('fbp-spec: Failed to load schemas', e); +/** + * JSON Schema definitions for the fbp-spec data format. + * Built from the YAML sources in `schemata/` by `npm run build`. + * @module schema/index + */ + +/** + * JSON Schema definition for fbp-spec data. + * @typedef {{ id: string, title?: string, description?: string, [key: string]: * }} Schema + */ + +import schemas from "./schemas.js"; + +/** @type {Record} */ +const allSchemas = schemas; +export default allSchemas; +export const { + base, + testcase, + testsuite, + testsuites, + expectations, + testsfile, +} = allSchemas; + +/** + * Get a single schema by name. + * @param {string} name Schema name, with or without `.json` suffix + * @returns {Schema|undefined} The schema, if found + */ +export function getSchema(name) { + const schemaId = name.replace(".json", ""); + return allSchemas[schemaId]; } diff --git a/schema/outputdata.json b/schema/outputdata.json index caa2275..74ce837 100644 --- a/schema/outputdata.json +++ b/schema/outputdata.json @@ -14,4 +14,4 @@ } ] } -} \ No newline at end of file +} diff --git a/schema/schemas.js b/schema/schemas.js new file mode 100644 index 0000000..47697b4 --- /dev/null +++ b/schema/schemas.js @@ -0,0 +1,219 @@ +// Generated by scripts/build-schemas.js from schemata/*.yaml. Do not edit. +export default { + "base": { + "id": "base.json", + "$schema": "http://json-schema.org/draft-04/schema", + "title": "Base definitions", + "definitions": { + "JsonPath": { + "type": "string", + "description": "JsonPath, see http://goessner.net/articles/JsonPath/", + "format": "jsonpath", + "example": "products.book[2]" + } + } + }, + "expectation": { + "id": "expectation.json", + "$schema": "http://json-schema.org/draft-04/schema", + "title": "Expectation", + "description": "An expectation on data.", + "type": "object", + "properties": { + "path": { + "$ref": "base.json#/definitions/JsonPath", + "description": "JSONPath to run messages data through. Result is passed to assertion predicate." + }, + "equals": { + "description": "Strict equality" + }, + "above": { + "description": "Greater than (>)", + "type": "number" + }, + "below": { + "description": "Less than than (<)", + "type": "number" + }, + "type": { + "description": "JSON type", + "type": "string" + }, + "haveKeys": { + "description": "Object has only these keys", + "type": "array", + "items": { + "type": "string" + } + }, + "includeKeys": { + "description": "Object has at least these keys", + "type": "array", + "items": { + "type": "string" + } + }, + "contains": { + "description": "String or array contains substring/item" + } + } + }, + "expectations": { + "id": "testsuites.json", + "$schema": "http://json-schema.org/draft-04/schema", + "title": "Expectation list", + "description": "Array of Expectation", + "type": "array", + "items": { + "$ref": "expectation.json" + } + }, + "outputdata": { + "id": "outputdata.json", + "$schema": "http://json-schema.org/draft-04/schema", + "title": "Output data", + "description": "Data expected in a testcase.\nEach key refers to a port, and the value is a description of the expected message on that port.\nThere can be one expectation (check) on each message, or an array of them.\n", + "type": "object", + "items": { + "oneOf": [ + { + "$ref": "expectation.json" + }, + { + "$ref": "expectations.json" + } + ] + } + }, + "testcase": { + "id": "testcase.json", + "$schema": "http://json-schema.org/draft-04/schema", + "title": "Testcase", + "description": "A single test case", + "type": "object", + "properties": { + "name": { + "type": "string", + "description": "Name of this testcase/scenario" + }, + "assertion": { + "type": "string", + "description": "Description of what this testcase asserts" + }, + "inputs": { + "description": "The inputs send to the fixture input ports, or a sequence of such inputs", + "oneOf": [ + { + "type": "object" + }, + { + "type": "array", + "items": { + "type": "object" + } + } + ] + }, + "expect": { + "description": "Expected data on fixture output ports.\nCan either be an object, or a sequence of such objects.\n", + "oneOf": [ + { + "$ref": "outputdata.json" + }, + { + "type": "array", + "items": [ + { + "$ref": "outputdata.json" + } + ] + } + ] + }, + "timeout": { + "type": "number", + "description": "Timeout (in milliseconds). Overrides that set in the suite", + "default": 2000 + }, + "skip": { + "type": "string", + "description": "Flag to skip testcase.\nShould be a description of the reason for skipping.\n" + } + }, + "required": [ + "name", + "assertion", + "inputs", + "expect" + ] + }, + "testsfile": { + "id": "testsfile.json", + "$schema": "http://json-schema.org/draft-04/schema", + "title": "Tests file", + "description": "One or more suites", + "oneOf": [ + { + "$ref": "testsuites.json" + }, + { + "$ref": "testsuite.json" + } + ] + }, + "testsuite": { + "id": "testsuite.json", + "$schema": "http://json-schema.org/draft-04/schema", + "title": "Test suite", + "description": "A collection of test cases", + "type": [ + "object" + ], + "properties": { + "topic": { + "type": "string", + "description": "The component/graph under test", + "example": "MyComponent3" + }, + "name": { + "type": "string", + "description": "Name of this test suite. Topic is used if no name is provided.", + "default": "" + }, + "fixture": { + "type": "object", + "description": "The graph fixture to run for each test case" + }, + "cases": { + "type": "array", + "description": "The test cases", + "items": { + "$ref": "testcase.json" + }, + "default": [] + }, + "timeout": { + "type": "number", + "description": "Timeout (in milliseconds) for setup and default for each test case.", + "default": 2000 + }, + "skip": { + "type": "string", + "description": "Flag to skip test suite (and all containing test cases).\nShould be a description of the reason for skipping.\n" + } + }, + "required": [ + "topic" + ] + }, + "testsuites": { + "id": "testsuites.json", + "$schema": "http://json-schema.org/draft-04/schema", + "title": "TestSuite list", + "description": "Array of TestSuite", + "type": "array", + "items": { + "$ref": "testsuite.json" + } + } +}; diff --git a/schema/testcase.json b/schema/testcase.json index 9cf419f..5c7e8ce 100644 --- a/schema/testcase.json +++ b/schema/testcase.json @@ -59,4 +59,4 @@ "inputs", "expect" ] -} \ No newline at end of file +} diff --git a/schema/testsfile.json b/schema/testsfile.json index 93aa608..250020b 100644 --- a/schema/testsfile.json +++ b/schema/testsfile.json @@ -11,4 +11,4 @@ "$ref": "testsuite.json" } ] -} \ No newline at end of file +} diff --git a/schema/testsuite.json b/schema/testsuite.json index 2c6957b..809bd82 100644 --- a/schema/testsuite.json +++ b/schema/testsuite.json @@ -42,4 +42,4 @@ "required": [ "topic" ] -} \ No newline at end of file +} diff --git a/schema/testsuites.json b/schema/testsuites.json index 35d0291..63011a3 100644 --- a/schema/testsuites.json +++ b/schema/testsuites.json @@ -7,4 +7,4 @@ "items": { "$ref": "testsuite.json" } -} \ No newline at end of file +} diff --git a/scripts/build-schemas.js b/scripts/build-schemas.js new file mode 100644 index 0000000..547f904 --- /dev/null +++ b/scripts/build-schemas.js @@ -0,0 +1,39 @@ +/** + * Build JSON Schema artifacts from the YAML sources in schemata/. + * + * Outputs: + * - schema/.json: one JSON Schema document per source file + * - schema/schemas.js: an ES module exporting all schemas, for runtime loading + * without filesystem access (also usable in browsers) + * + * Replaces the old grunt-yaml build step. Run via `npm run build`. + */ + +import { readdir, readFile, writeFile } from "node:fs/promises"; +import { join } from "node:path"; +import { load } from "js-yaml"; + +const schemataDir = join(import.meta.dirname, "..", "schemata"); +const schemaDir = join(import.meta.dirname, "..", "schema"); + +const files = (await readdir(schemataDir)) + .filter((f) => f.endsWith(".yaml")) + .sort(); + +/** @type {Record} */ +const schemas = {}; +for (const file of files) { + const name = file.replace(/\.yaml$/, ""); + const source = await readFile(join(schemataDir, file), "utf-8"); + const doc = /** @type {object} */ (load(source)); + schemas[name] = doc; + + const jsonDest = join(schemaDir, `${name}.json`); + await writeFile(jsonDest, `${JSON.stringify(doc, null, 2)}\n`); + console.log(`built schema/${name}.json`); +} + +const jsDest = join(schemaDir, "schemas.js"); +const jsSource = `// Generated by scripts/build-schemas.js from schemata/*.yaml. Do not edit.\nexport default ${JSON.stringify(schemas, null, 2)};\n`; +await writeFile(jsDest, jsSource); +console.log("built schema/schemas.js"); diff --git a/spec/.eslintrc.json b/spec/.eslintrc.json deleted file mode 100644 index 6dfc3b1..0000000 --- a/spec/.eslintrc.json +++ /dev/null @@ -1,21 +0,0 @@ -{ - "extends": "airbnb-base", - "plugins": [ - "chai", - "mocha" - ], - "env": { - "browser": true, - "node": true, - "mocha": true - }, - "rules": { - "func-names": 0 - }, - "globals": { - "baseDir": false, - "chai": false, - "noflo": false, - "window": false - } -} diff --git a/spec/cli.js b/spec/cli.js deleted file mode 100644 index c75759c..0000000 --- a/spec/cli.js +++ /dev/null @@ -1,62 +0,0 @@ -const chai = require('chai'); -const childProcess = require('child_process'); -const path = require('path'); - -const fbpSpec = function (suite, callback) { - const prog = './bin/fbp-spec'; - const args = [ - suite, - '--command', 'python2 protocol-examples/python/runtime.py', - ]; - const options = {}; - return childProcess.execFile(prog, args, options, callback); -}; - -const example = (name) => path.join('examples', name); - -const countTestcases = function (str) { - let count = 0; - const passChars = ['✓', '✗']; - for (let i = 0; i < str.length; i += 1) { - const char = str[i]; - if (passChars.indexOf(char) !== -1) { - count += 1; - } - } - return count; -}; - -const pyTimeout = 3000; -describe('fbp-spec', () => { - describe('with failing testcases', () => it('should exit with non-zero code', function (done) { - this.timeout(pyTimeout); - fbpSpec(example('simple-failing.yaml'), (err) => { - chai.expect(err).to.be.an('error'); - chai.expect(err.code).to.not.equal(0); - chai.expect(err.message).to.contain('Command failed'); - done(); - }); - })); - - describe('with passing testcases', () => it('should exit with 0 code', function (done) { - this.timeout(pyTimeout); - fbpSpec(example('simple-passing.yaml'), (err) => { - if (err) { - done(err); - return; - } - done(); - }); - })); - - describe('with multiple suites and some failing cases', () => it('should run all testcases', function (done) { - this.timeout(pyTimeout); - fbpSpec(example('multisuite-failandpass.yaml'), (err, stdout) => { - chai.expect(err).to.be.an('error'); - chai.expect(countTestcases(stdout)).to.equal(4); - chai.expect(stdout).to.contain('sending a boolean with wrong expect'); - chai.expect(stdout).to.contain('should repeat the same'); - done(); - }); - })); -}); diff --git a/spec/cli.test.js b/spec/cli.test.js new file mode 100644 index 0000000..77a29fc --- /dev/null +++ b/spec/cli.test.js @@ -0,0 +1,207 @@ +import assert from "node:assert/strict"; +import { execFile } from "node:child_process"; +import { after, before, describe, it } from "node:test"; +import { fileURLToPath } from "node:url"; +import { hasErrors, parse, testStatusText } from "../lib/cli.js"; +import { startDummyRuntime } from "./fixtures/dummy-runtime.js"; + +const binPath = fileURLToPath(new URL("../bin/fbp-spec", import.meta.url)); +/** @param {string} name */ +const examplePath = (name) => + fileURLToPath(new URL(`../examples/${name}`, import.meta.url)); + +/** + * Run the fbp-spec CLI as a subprocess. + * @param {string[]} args CLI arguments + * @returns {Promise<{ code: number, stdout: string, stderr: string }>} Process result + */ +function runCli(args) { + return new Promise((resolve) => { + execFile( + process.execPath, + [binPath, ...args], + { encoding: "utf-8", timeout: 30000 }, + (err, stdout, stderr) => { + resolve({ + code: typeof err?.code === "number" ? err.code : 0, + stdout, + stderr, + }); + }, + ); + }); +} + +describe("CLI", () => { + /** @type {Awaited>} */ + let runtime; + + before(async () => { + runtime = await startDummyRuntime(); + }); + after(async () => { + await runtime.close(); + }); + + describe("parse", () => { + it("should parse options and positionals", () => { + const { options, help } = parse([ + "--address", + "ws://localhost:1234", + "--secret", + "s3cret", + "--start-timeout", + "2", + "--command-timeout", + "5", + "--component-tests", + "--trace", + "examples/simple-passing.yaml", + ]); + assert.equal(help, false); + assert.equal(options.address, "ws://localhost:1234"); + assert.equal(options.secret, "s3cret"); + assert.equal(options.startTimeout, 2); + assert.equal(options.commandTimeout, 5); + assert.equal(options.componentTests, true); + assert.equal(options.trace, true); + assert.deepEqual(options.suites, ["examples/simple-passing.yaml"]); + }); + it("should apply defaults", () => { + const { options } = parse(["suite.yaml"]); + assert.equal(options.address, "ws://localhost:3569"); + assert.equal(options.startTimeout, 10); + assert.equal(options.commandTimeout, 3); + assert.equal(options.componentTests, false); + assert.equal(options.trace, false); + assert.deepEqual(options.suites, ["suite.yaml"]); + }); + it("should support multiple suite files", () => { + const { options } = parse(["a.yaml", "b.yaml"]); + assert.deepEqual(options.suites, ["a.yaml", "b.yaml"]); + }); + it("should reject unknown options", () => { + assert.throws(() => parse(["--nope", "a.yaml"]), /Unknown option/); + }); + }); + + describe("hasErrors", () => { + it("should detect failing testcases", () => { + assert.equal(hasErrors([{ cases: [{ passed: true }, {}] }]), false); + assert.equal( + hasErrors([{ cases: [{ passed: true }, { error: "boom" }] }]), + true, + ); + }); + }); + + describe("testStatusText", () => { + it("should render testcase results", () => { + const lines = testStatusText([ + { + name: "My suite", + cases: [ + { name: "case a", assertion: "should work", passed: true }, + { + name: "case b", + assertion: "should fail", + passed: false, + error: "expected 1 to equal 2", + }, + { + name: "case c", + assertion: "should skip", + passed: true, + skip: "no runtime", + }, + ], + }, + ]); + assert.equal(lines.length, 4); + assert.match(lines[0], /My suite/); + assert.match(lines[1], /case a/); + assert.match(lines[1], /✓/); + assert.match(lines[2], /case b/); + assert.match(lines[2], /✗ Error: expected 1 to equal 2/); + }); + }); + + describe("against a runtime (end-to-end)", () => { + it("should exit 0 for passing suites", async () => { + const result = await runCli([ + examplePath("simple-passing.yaml"), + "--address", + runtime.address, + "--start-timeout", + "2", + ]); + assert.equal(result.code, 0, `stderr: ${result.stderr}`); + assert.match(result.stdout, /should repeat the same: ✓/); + }); + it("should exit non-zero for failing suites", async () => { + const result = await runCli([ + examplePath("simple-failing.yaml"), + "--address", + runtime.address, + "--start-timeout", + "2", + ]); + assert.equal(result.code, 2, `stderr: ${result.stderr}`); + assert.match( + result.stdout, + /should fail: ✗ Error: expected true to equal false/, + ); + }); + it("should run all testcases across multiple suites", async () => { + const result = await runCli([ + examplePath("multisuite-failandpass.yaml"), + "--address", + runtime.address, + "--start-timeout", + "2", + ]); + assert.equal(result.code, 2); + assert.match(result.stdout, /sending a boolean with wrong expect/); + assert.match(result.stdout, /should repeat the same/); + assert.equal((result.stdout.match(/✓/g) ?? []).length, 2); + assert.equal((result.stdout.match(/✗/g) ?? []).length, 2); + }); + it("should load component tests with --component-tests", async () => { + const result = await runCli([ + examplePath("simple-passing.yaml"), + "--address", + runtime.address, + "--start-timeout", + "2", + "--component-tests", + ]); + assert.equal(result.code, 0, `stderr: ${result.stderr}`); + assert.match(result.stdout, /DummyComponent tests/); + assert.match(result.stdout, /should repeat the same: ✓/); + }); + it("should start the runtime with --command", async () => { + const runtimePath = fileURLToPath( + new URL("./fixtures/dummy-runtime.js", import.meta.url), + ); + const result = await runCli([ + examplePath("simple-passing.yaml"), + "--command", + `node ${runtimePath} 33577`, + "--address", + "ws://127.0.0.1:33577", + "--start-timeout", + "10", + ]); + assert.equal( + result.code, + 0, + `stdout: ${result.stdout}\nstderr: ${result.stderr}`, + ); + }); + it("should print usage and exit non-zero without suites", async () => { + const result = await runCli(["--address", runtime.address]); + assert.equal(result.code, 2); + assert.match(result.stderr, /no suites specified/); + }); + }); +}); diff --git a/spec/common.test.js b/spec/common.test.js new file mode 100644 index 0000000..28e1422 --- /dev/null +++ b/spec/common.test.js @@ -0,0 +1,108 @@ +import assert from "node:assert/strict"; +import { describe, it } from "node:test"; + +import { + asyncSeries, + isArray, + isBrowser, + randomString, + retryUntil, + startsWith, +} from "../lib/common.js"; + +describe("common", () => { + describe("isBrowser", () => { + it("should detect Node.js", () => { + assert.equal(isBrowser(), false); + }); + }); + + describe("randomString", () => { + it("should generate string of requested length", () => { + assert.equal(randomString(10).length, 10); + assert.equal(randomString(0).length, 0); + }); + it("should only contain ASCII alphanumerics", () => { + assert.match(randomString(100), /^[A-Za-z0-9]+$/); + }); + it("should produce different strings", () => { + assert.notEqual(randomString(20), randomString(20)); + }); + }); + + describe("asyncSeries", () => { + it("should run items in order and collect results", async () => { + /** @type {number[]} */ + const order = []; + const results = await asyncSeries([1, 2, 3], async (item) => { + order.push(item); + return item * 2; + }); + assert.deepEqual(order, [1, 2, 3]); + assert.deepEqual(results, [2, 4, 6]); + }); + it("should reject on first error", async () => { + await assert.rejects( + asyncSeries([1, 2, 3], async (item) => { + if (item === 2) { + throw new Error("boom"); + } + return item; + }), + /boom/, + ); + }); + }); + + describe("isArray", () => { + it("should detect arrays", () => { + assert.equal(isArray([]), true); + assert.equal(isArray([1, 2]), true); + assert.equal(isArray({}), false); + assert.equal(isArray("string"), false); + assert.equal(isArray(null), false); + }); + }); + + describe("startsWith", () => { + it("should check string prefixes", () => { + assert.equal(startsWith("foobar", "foo"), true); + assert.equal(startsWith("foobar", "bar"), false); + assert.equal(startsWith("foobar", "foobar"), true); + assert.equal(startsWith("", "x"), false); + }); + }); + + describe("retryUntil", () => { + it("should return first passing result", async () => { + let attempts = 0; + const result = await retryUntil( + async () => { + attempts += 1; + return attempts; + }, + async (value) => { + if (value < 3) { + throw new Error("not yet"); + } + }, + 1, + 5, + ); + assert.equal(result, 3); + }); + it("should reject after max attempts", async () => { + await assert.rejects( + retryUntil( + async () => false, + async () => { + throw new Error("nope"); + }, + 1, + 2, + ), + /nope/, + ); + }); + }); +}); diff --git a/spec/examples.js b/spec/examples.js deleted file mode 100644 index 1ab93af..0000000 --- a/spec/examples.js +++ /dev/null @@ -1,148 +0,0 @@ -const startRuntime = function (fbpspec, runner, info, callback) { - let runtime = null; - if (info.command) { - runtime = fbpspec.subprocess.start(info.command, {}, callback); - } else if (info.protocol === 'iframe') { - const parent = document.createElement('div'); - document.body.appendChild(parent); - const r = runner; - r.parentElement = parent; - callback(null); - } else { - callback(null); - } - - return runtime; -}; - -const stopRuntime = function (runtime) { - if (runtime) { - runtime.kill(); - } -}; - -const setupAndRun = (fbpspec, runner, suite, testcase, callback) => { - runner.setupSuite(suite, (err) => { - if (err) { - callback(err); - return; - } - fbpspec.runner.runTestAndCheck(runner, testcase, (runErr, r) => { - if (runErr) { - callback(runErr); - return; - } - const results = r; - runner.teardownSuite(suite, (e) => callback(e, results)); - }); - }); -}; - -describe('Examples', () => { - const isBrowser = () => !((typeof process !== 'undefined' && process !== null) && process.execPath && process.execPath.match(/node|iojs/)); - // eslint-disable-next-line global-require - const chai = require('chai'); - // eslint-disable-next-line import/no-unresolved,global-require - const fbpspec = isBrowser() ? require('fbp-spec') : require('..'); - let examples; - let runtimeInfo; - if (isBrowser()) { - examples = window.fbpspec_examples; - runtimeInfo = { - protocol: 'iframe', - address: '/base/browser/spec/fixtures/everything.html?fbp_noload=true&fbp_protocol=iframe', - }; - } else { - // eslint-disable-next-line global-require,import/no-unresolved - examples = require('../examples/bundle.json'); - runtimeInfo = { - protocol: 'websocket', - address: 'ws://localhost:3335', - command: 'python2 protocol-examples/python/runtime.py --port 3335', - }; - } - - let runner = null; - let runtime = null; - before(function (done) { - this.timeout(6000); - runner = new fbpspec.runner.Runner(runtimeInfo); - runtime = startRuntime(fbpspec, runner, runtimeInfo, (err) => { - if (err) { - done(err); - return; - } - runner.connect(done); - }); - }); - after((done) => { - stopRuntime(runtime); - runner.disconnect(done); - }); - - Object.keys(examples).forEach((name) => { - let example = null; - describe(`${name}`, () => { - before((done) => { - try { - example = examples[name]; - done(); - } catch (e) { - done(e); - } - }); - - it('should validate against schema', function () { - if (isBrowser()) { - this.skip(); - return; - } - const results = fbpspec.testsuite.validate(example); - chai.expect(results.errors).to.eql([]); - chai.expect(results.missing).to.eql([]); - chai.expect(results.valid).to.equal(true); - }); - - describe('testcases', () => { - if (!example) { example = []; } - const suites = Array.isArray(example) ? example.slice(0) : [example]; - suites.forEach((suite) => suite.cases.forEach((testcase) => describe(`${testcase.name}`, () => { - let itOrSkip = testcase.skip ? it.skip : it; - if (isBrowser() && (suite.topic === 'DummyComponent')) { - // These tests only work with the Python runtime - itOrSkip = it.skip; - } - - if (testcase.assertion === 'should pass') { - itOrSkip('should pass', function (done) { - this.timeout(10000); - setupAndRun(fbpspec, runner, suite, testcase, (err, results) => { - if (err) { - done(err); - return; - } - chai.expect(results.error).to.be.a('null'); - chai.expect(results.passed).to.equal(true); - done(); - }); - }); - } if (testcase.assertion === 'should fail') { - itOrSkip('should fail', function (done) { - this.timeout(10000); - setupAndRun(fbpspec, runner, suite, testcase, (err, results) => { - if (err) { - done(err); - return; - } - chai.expect(results.error, 'missing error').be.an('error'); - chai.expect(results.error.message).to.contain('expect'); - chai.expect(results.passed).equal(false); - done(); - }); - }); - } - }))); - }); - }); - }); -}); diff --git a/spec/examples.test.js b/spec/examples.test.js new file mode 100644 index 0000000..84e3ea7 --- /dev/null +++ b/spec/examples.test.js @@ -0,0 +1,134 @@ +import assert from "node:assert/strict"; +import { readdirSync } from "node:fs"; +import { readFile } from "node:fs/promises"; +import { after, before, describe, it } from "node:test"; +import { + getComponentSuites, + Runner, + runAll, + runTestAndCheck, +} from "../lib/runner.js"; +import { getSuites, loadYAML, validate } from "../lib/testsuite.js"; +import { startDummyRuntime } from "./fixtures/dummy-runtime.js"; + +describe("Examples (end-to-end with dummy runtime)", () => { + /** @type {Awaited>} */ + let runtime; + /** @type {Runner} */ + let runner; + /** @typedef {import("../lib/testsuite.js").Suite} Suite */ + + before(async () => { + runtime = await startDummyRuntime(); + runner = new Runner( + { protocol: "websocket", address: runtime.address }, + { connectTimeout: 2000 }, + ); + await runner.connect(); + }); + after(async () => { + await runner.disconnect(); + await runtime.close(); + }); + + /** + * Set up a suite, run one testcase, and tear down. + * @param {object} suite Testsuite + * @param {object} testcase Testcase + * @returns {Promise<{ passed: boolean, error?: Error }>} Test result + */ + async function setupAndRun(suite, testcase) { + await runner.setupSuite(suite); + try { + return await runTestAndCheck(runner, testcase); + } finally { + await runner.teardownSuite(suite); + } + } + + const examplesDir = new URL("../examples/", import.meta.url); + const exampleFiles = readdirSync(examplesDir).filter((f) => + f.endsWith(".yaml"), + ); + + for (const file of exampleFiles) { + describe(file, () => { + /** @type {Suite[]} */ + let suites; + + before(async () => { + const source = await readFile(new URL(file, examplesDir), "utf-8"); + suites = loadYAML(source); + }); + + it("suites should validate against schema", () => { + assert.equal(suites.length > 0, true); + const results = validate(suites); + assert.deepEqual(results.errors, []); + assert.deepEqual(results.missing, []); + assert.equal(results.valid, true); + }); + + it("testcases should pass or fail as declared", async () => { + for (const suite of suites) { + for (const testcase of suite.cases ?? []) { + const result = await setupAndRun(suite, testcase); + const shouldFail = testcase.assertion === "should fail"; + if (shouldFail) { + assert.equal( + result.passed, + false, + `expected fail: ${suite.name} / ${testcase.name}`, + ); + assert.ok( + result.error instanceof Error, + "failing testcase should have an error", + ); + } else { + assert.deepEqual( + result, + { passed: true }, + `expected pass: ${suite.name} / ${testcase.name}`, + ); + } + } + } + }); + }); + } + + describe("runAll", () => { + it("should fill in results on all testcases", async () => { + const suites = await getSuites( + new URL("multisuite-failandpass.yaml", examplesDir).pathname, + ); + await runAll(runner, suites); + const cases = suites.flatMap((s) => s.cases ?? []); + assert.equal(cases.length, 4); + assert.deepEqual( + cases.map((c) => c.passed), + [false, false, true, true], + ); + for (const c of cases) { + if (!c.passed) { + assert.ok(typeof c.error === "string"); + } + } + }); + }); + + describe("component tests", () => { + it("should load and run tests attached to runtime components", async () => { + const suites = /** @type {Suite[]} */ (await getComponentSuites(runner)); + assert.equal(suites.length, 1); + const suite = suites[0]; + assert.ok(suite, "expected a suite"); + assert.equal(suite.name, "DummyComponent tests"); + await runAll(runner, suites); + assert.deepEqual( + (suite.cases ?? []).map((c) => c.passed), + [true], + ); + }); + }); +}); diff --git a/spec/expectation.test.js b/spec/expectation.test.js new file mode 100644 index 0000000..dbe337e --- /dev/null +++ b/spec/expectation.test.js @@ -0,0 +1,174 @@ +import assert from "node:assert/strict"; +import { describe, it } from "node:test"; + +import { expect, noError } from "../lib/expectation.js"; + +describe("expectation", () => { + describe("expect", () => { + it("should pass on matching expectations", () => { + expect({ out: { equals: 42 } }, { out: 42 }); + expect({ out: { equals: { a: 1 } } }, { out: { a: 1 } }); + expect({ out: { type: "number" } }, { out: 42 }); + }); + it("should throw on non-matching expectations", () => { + assert.throws( + () => expect({ out: { equals: 42 } }, { out: 41 }), + /expected 41 to equal 42/, + ); + }); + it("should throw when no data received on port", () => { + assert.throws( + () => expect({ out: { equals: 42 } }, {}), + /No data received on port out/, + ); + }); + it("should throw on unknown operator", () => { + assert.throws( + () => expect({ out: { unknownop: 42 } }, { out: 42 }), + /No operator matching/, + ); + }); + it("should support sequences of expectations", () => { + expect([{ out: { equals: 1 } }, { out: { equals: 1 } }], { out: 1 }); + }); + it("should support arrays of expectations on one port", () => { + expect( + { out: [{ type: "object" }, { haveKeys: ["a"] }] }, + { out: { a: 1 } }, + ); + assert.throws( + () => + expect( + { out: [{ type: "object" }, { haveKeys: ["b"] }] }, + { out: { a: 1 } }, + ), + /have keys/, + ); + }); + + describe("operators", () => { + it("type", () => { + expect({ out: { type: "object" } }, { out: {} }); + expect({ out: { type: "string" } }, { out: "hello" }); + assert.throws( + () => expect({ out: { type: "string" } }, { out: 42 }), + /to be of type string/, + ); + }); + it("above and below", () => { + expect({ out: { above: 41 } }, { out: 42 }); + expect({ out: { below: 43 } }, { out: 42 }); + assert.throws( + () => expect({ out: { above: 42 } }, { out: 42 }), + /to be above 42/, + ); + assert.throws( + () => expect({ out: { below: 42 } }, { out: 42 }), + /to be below 42/, + ); + }); + it("haveKeys", () => { + expect({ out: { haveKeys: ["a", "b"] } }, { out: { a: 1, b: 2 } }); + assert.throws( + () => expect({ out: { haveKeys: ["a"] } }, { out: { a: 1, b: 2 } }), + /have keys/, + ); + }); + it("includeKeys", () => { + expect({ out: { includeKeys: ["a"] } }, { out: { a: 1, b: 2 } }); + assert.throws( + () => expect({ out: { includeKeys: ["c"] } }, { out: { a: 1 } }), + /include key c/, + ); + }); + it("contains on strings", () => { + expect( + { out: { contains: "sub" } }, + { out: "a string with substring" }, + ); + assert.throws( + () => expect({ out: { contains: "nope" } }, { out: "string" }), + /to contain 'nope'/, + ); + }); + it("contains on arrays", () => { + expect({ out: { contains: { a: 1 } } }, { out: [{ a: 1 }, { b: 2 }] }); + assert.throws( + () => expect({ out: { contains: 3 } }, { out: [1, 2] }), + /to contain 3/, + ); + }); + it("contains on objects", () => { + expect({ out: { contains: { a: 1 } } }, { out: { a: 1, b: 2 } }); + assert.throws( + () => expect({ out: { contains: { c: 3 } } }, { out: { a: 1 } }), + /to contain/, + ); + }); + it("noterror", () => { + expect({ out: { noterror: null } }, { out: "value" }); + assert.throws( + () => + expect({ out: { noterror: null } }, { out: new Error("broken") }), + /broken/, + ); + }); + }); + + describe("JSONPath", () => { + it("should extract and match values", () => { + expect( + { out: { path: "$.outer.inner.foo", equals: "bar" } }, + { out: { outer: { inner: { foo: "bar" } } } }, + ); + }); + it("should match multiple values", () => { + expect( + { out: { path: "$.outer.*.foo", equals: "bar" } }, + { out: { outer: { first: { foo: "bar" }, second: { foo: "bar" } } } }, + ); + }); + it("should fail when path does not match", () => { + assert.throws( + () => + expect( + { out: { path: "$.outer.nonexist", equals: "bar" } }, + { out: { outer: { inner: { foo: "bar" } } } }, + ), + /to match data/, + ); + }); + it("should fail when a match violates the expectation", () => { + assert.throws( + () => + expect( + { out: { path: "$.outer.*.foo", equals: "bar" } }, + { + out: { + outer: { first: { foo: "bar" }, second: { foo: "bazz" } }, + }, + }, + ), + /expected "bazz" to equal "bar"/, + ); + }); + }); + }); + + describe("noError", () => { + it("should pass on nullish values", () => { + noError(null); + noError(undefined); + }); + it("should throw the error itself", () => { + const err = new Error("failure"); + assert.throws( + () => noError(err), + (e) => e === err, + ); + }); + it("should throw on other defined values", () => { + assert.throws(() => noError("value"), /value/); + }); + }); +}); diff --git a/spec/fixtures/.gitignore b/spec/fixtures/.gitignore deleted file mode 100644 index f638875..0000000 --- a/spec/fixtures/.gitignore +++ /dev/null @@ -1,2 +0,0 @@ -everything.* - diff --git a/spec/fixtures/dummy-runtime.js b/spec/fixtures/dummy-runtime.js new file mode 100644 index 0000000..3797dcb --- /dev/null +++ b/spec/fixtures/dummy-runtime.js @@ -0,0 +1,256 @@ +/** + * A minimal in-process FBP runtime, speaking the FBP runtime protocol over WebSocket. + * Mirrors the behaviour of protocol-examples/python/runtime.py (the 'python-example' + * runtime used by the fbp-spec examples): it echoes any input packet back on the + * 'out' port, and exposes a single DummyComponent. + * + * Used by the test suite so that end-to-end tests can run without Python. + * @module spec/fixtures/dummy-runtime + */ + +import http from "node:http"; +import { fileURLToPath } from "node:url"; +import { server as WebSocketServer } from "websocket"; + +/** YAML testsuite attached to DummyComponent, served via component:getsource. */ +export const DUMMY_COMPONENT_TESTS = ` +name: DummyComponent tests +topic: DummyComponent +cases: +- + name: 'sending a boolean' + assertion: 'should repeat the same' + inputs: + in: true + expect: + out: + equals: true +`; + +/** @type {Record} */ +const components = { + DummyComponent: { + name: "DummyComponent", + description: "An example component which does exactly nothing", + subgraph: false, + inPorts: [ + { + id: "portA", + type: "boolean", + description: "A boolean port", + addressable: false, + required: true, + }, + { id: "portB", type: "any" }, + ], + outPorts: [{ id: "out1", type: "string" }], + }, +}; + +/** + * Start a dummy FBP runtime listening on a port. + * @param {{ secret?: string, port?: number }} [options] Optional secret the runtime requires, and port (0 = ephemeral) + * @returns {Promise<{ address: string, port: number, close: () => Promise, httpServer: import('node:http').Server }>} Handle to the runtime + */ +export function startDummyRuntime(options = {}) { + const httpServer = http.createServer((_req, res) => { + res.writeHead(404); + res.end(); + }); + const wsServer = new WebSocketServer({ httpServer }); + + wsServer.on("request", (request) => { + const subProtocol = + request.requestedProtocols.indexOf("noflo") !== -1 ? "noflo" : null; + const connection = request.accept(subProtocol, request.origin); + + /** + * Send an FBP message to the client, stripping any runtime secret. + * @param {string} protocol FBP protocol name + * @param {string} command FBP command name + * @param {*} payload Message payload + * @returns {void} + */ + function send(protocol, command, payload) { + if (payload && typeof payload === "object" && "secret" in payload) { + const { secret, ...rest } = payload; // eslint-disable-line no-unused-vars + payload = rest; + } + connection.sendUTF(JSON.stringify({ protocol, command, payload })); + } + + /** + * Acknowledge a graph protocol message by echoing it back. + * @param {string} command FBP command name + * @param {object} payload Message payload + * @returns {void} + */ + /** + * @param {string} command FBP command name + * @param {Record} payload Message payload + * @returns {void} + */ + function handleGraph(command, payload) { + send("graph", command, payload); + } + + /** + * Handle a network protocol message. + * @param {string} command FBP command name + * @param {object} payload Message payload + * @returns {void} + */ + /** + * @param {string} command FBP command name + * @param {Record} payload Message payload + * @returns {void} + */ + function handleNetwork(command, payload) { + const graph = payload?.graph ?? null; + if (command === "getstatus") { + send("network", "status", { graph, started: false, running: false }); + return; + } + if (command === "start") { + send("network", "started", { graph, started: true, running: true }); + return; + } + if (command === "stop") { + send("network", "stopped", { graph, started: false, running: false }); + return; + } + console.warn(`Unknown network command ${command}`); + } + + /** + * Handle a runtime protocol message. + * @param {string} command FBP command name + * @param {object} payload Message payload + * @returns {void} + */ + /** + * @param {string} command FBP command name + * @param {Record} payload Message payload + * @returns {void} + */ + function handleRuntime(command, payload) { + if (command === "getruntime") { + if (options.secret && payload.secret !== options.secret) { + send("runtime", "error", { message: "Invalid secret" }); + return; + } + send("runtime", "runtime", { + type: "fbp-js-dummy", + version: "0.7", + capabilities: [ + "protocol:graph", + "protocol:component", + "protocol:network", + "protocol:runtime", + "component:getsource", + ], + }); + return; + } + if (command === "packet") { + // We don't actually run anything, just echo input back on 'out' + send("runtime", "packetsent", payload); + send("runtime", "packet", { ...payload, port: "out" }); + return; + } + console.warn(`Unknown runtime command ${command}`); + } + + /** + * Handle a component protocol message. + * @param {string} command FBP command name + * @param {object} payload Message payload + * @returns {void} + */ + /** + * @param {string} command FBP command name + * @param {Record} payload Message payload + * @returns {void} + */ + function handleComponent(command, payload) { + if (command === "list") { + for (const component of Object.values(components)) { + send("component", "component", component); + } + send("component", "componentsready", Object.keys(components).length); + return; + } + if (command === "getsource") { + const component = components[payload.name]; + if (!component) { + send("component", "error", { + message: `Component not found: ${payload.name}`, + }); + return; + } + send("component", "source", { + name: component.name, + code: "", + language: "whitespace", + tests: DUMMY_COMPONENT_TESTS, + }); + return; + } + console.warn(`Unknown component command ${command}`); + } + + connection.on("message", (message) => { + if (message.type !== "utf8") { + return; + } + let contents; + try { + contents = JSON.parse(message.utf8Data); + } catch (e) { + const err = /** @type {Error} */ (e); + console.error(`Error: ${err.toString()}`); + return; + } + switch (contents.protocol) { + case "runtime": + handleRuntime(contents.command, contents.payload); + break; + case "component": + handleComponent(contents.command, contents.payload); + break; + case "graph": + handleGraph(contents.command, contents.payload); + break; + case "network": + handleNetwork(contents.command, contents.payload); + break; + default: + console.warn(`Unknown protocol ${contents.protocol}`); + } + }); + }); + + return new Promise((resolve) => { + httpServer.listen(options.port ?? 0, "127.0.0.1", () => { + const addr = httpServer.address(); + const port = typeof addr === "object" && addr ? addr.port : 0; + resolve({ + address: `ws://127.0.0.1:${port}`, + port, + httpServer, + close: () => + new Promise((res) => { + wsServer.shutDown(); + httpServer.close(() => res()); + }), + }); + }); + }); +} + +// Allow running standalone, for use as a runtime-under-test via the CLI --command option +if (process.argv[1] && fileURLToPath(import.meta.url) === process.argv[1]) { + const port = Number(process.argv[2] ?? process.env.PORT ?? 3569); + await startDummyRuntime({ port }); + console.log(`INFO: Dummy runtime listening on port ${port}`); +} diff --git a/spec/fixtures/mochacases/bdd-nested-passing.js b/spec/fixtures/mochacases/bdd-nested-passing.js deleted file mode 100644 index 5091f1a..0000000 --- a/spec/fixtures/mochacases/bdd-nested-passing.js +++ /dev/null @@ -1,19 +0,0 @@ -/* - * decaffeinate suggestions: - * DS102: Remove unnecessary code created because of implicit returns - * Full docs: https://github.com/decaffeinate/decaffeinate/blob/master/docs/suggestions.md - */ - -const chai = require('chai'); - -describe('nested passing', () => describe('sub topic', () => { - it('should pass', (done) => { - chai.expect(42).to.equal(42); - return done(); - }); - - return describe('sub sub topic', () => it('should pass', (done) => { - chai.expect(42).to.equal(42); - return done(); - })); -})); diff --git a/spec/fixtures/mochacases/bdd-simple-failing.js b/spec/fixtures/mochacases/bdd-simple-failing.js deleted file mode 100644 index 580cf57..0000000 --- a/spec/fixtures/mochacases/bdd-simple-failing.js +++ /dev/null @@ -1,12 +0,0 @@ -/* - * decaffeinate suggestions: - * DS102: Remove unnecessary code created because of implicit returns - * Full docs: https://github.com/decaffeinate/decaffeinate/blob/master/docs/suggestions.md - */ - -const chai = require('chai'); - -describe('simple failing', () => it('should fail', (done) => { - chai.expect(42).to.equal(41); - return done(); -})); diff --git a/spec/fixtures/mochacases/bdd-simple-passing.js b/spec/fixtures/mochacases/bdd-simple-passing.js deleted file mode 100644 index 9ca678e..0000000 --- a/spec/fixtures/mochacases/bdd-simple-passing.js +++ /dev/null @@ -1,12 +0,0 @@ -/* - * decaffeinate suggestions: - * DS102: Remove unnecessary code created because of implicit returns - * Full docs: https://github.com/decaffeinate/decaffeinate/blob/master/docs/suggestions.md - */ - -const chai = require('chai'); - -describe('simple passing', () => it('should pass', (done) => { - chai.expect(42).to.equal(42); - return done(); -})); diff --git a/spec/import.js b/spec/import.js deleted file mode 100644 index 45118b2..0000000 --- a/spec/import.js +++ /dev/null @@ -1,11 +0,0 @@ -describe("require('fbp-spec')", () => { - const isBrowser = () => !((typeof process !== 'undefined' && process !== null) && process.execPath && process.execPath.match(/node|iojs/)); - // eslint-disable-next-line global-require - const chai = require('chai'); - - it('should not throw', () => { - // eslint-disable-next-line global-require,import/no-unresolved - const fbpspec = isBrowser() ? require('fbp-spec') : require('..'); - chai.expect(fbpspec).to.be.an('object'); - }); -}); diff --git a/spec/import.test.js b/spec/import.test.js new file mode 100644 index 0000000..c73694f --- /dev/null +++ b/spec/import.test.js @@ -0,0 +1,23 @@ +import assert from "node:assert/strict"; +import { describe, it } from "node:test"; + +import * as fbpSpec from "../index.js"; + +describe("require('fbp-spec')", () => { + it("should not throw", () => { + assert.equal(typeof fbpSpec, "object"); + }); + + it("should export the public API", () => { + assert.equal(typeof fbpSpec.runner.Runner, "function"); + assert.equal(typeof fbpSpec.runner.runAll, "function"); + assert.equal(typeof fbpSpec.runner.runTestAndCheck, "function"); + assert.equal(typeof fbpSpec.subprocess.start, "function"); + assert.equal(typeof fbpSpec.testsuite.loadYAML, "function"); + assert.equal(typeof fbpSpec.testsuite.validate, "function"); + assert.equal(typeof fbpSpec.schema.testcase, "object"); + const schema = fbpSpec.getSchema("testsfile.json"); + assert.ok(schema, "expected testsfile schema"); + assert.equal(schema.id, "testsfile.json"); + }); +}); diff --git a/spec/json-schema.json b/spec/json-schema.json index 85eb502..644bf82 100644 --- a/spec/json-schema.json +++ b/spec/json-schema.json @@ -1,150 +1,146 @@ { - "id": "http://json-schema.org/draft-04/schema#", - "$schema": "http://json-schema.org/draft-04/schema#", - "description": "Core schema meta-schema", + "id": "http://json-schema.org/draft-04/schema#", + "$schema": "http://json-schema.org/draft-04/schema#", + "description": "Core schema meta-schema", + "definitions": { + "schemaArray": { + "type": "array", + "minItems": 1, + "items": { "$ref": "#" } + }, + "positiveInteger": { + "type": "integer", + "minimum": 0 + }, + "positiveIntegerDefault0": { + "allOf": [{ "$ref": "#/definitions/positiveInteger" }, { "default": 0 }] + }, + "simpleTypes": { + "enum": [ + "array", + "boolean", + "integer", + "null", + "number", + "object", + "string" + ] + }, + "stringArray": { + "type": "array", + "items": { "type": "string" }, + "minItems": 1, + "uniqueItems": true + } + }, + "type": "object", + "properties": { + "id": { + "type": "string", + "format": "uri" + }, + "$schema": { + "type": "string", + "format": "uri" + }, + "title": { + "type": "string" + }, + "description": { + "type": "string" + }, + "default": {}, + "multipleOf": { + "type": "number", + "minimum": 0, + "exclusiveMinimum": true + }, + "maximum": { + "type": "number" + }, + "exclusiveMaximum": { + "type": "boolean", + "default": false + }, + "minimum": { + "type": "number" + }, + "exclusiveMinimum": { + "type": "boolean", + "default": false + }, + "maxLength": { "$ref": "#/definitions/positiveInteger" }, + "minLength": { "$ref": "#/definitions/positiveIntegerDefault0" }, + "pattern": { + "type": "string", + "format": "regex" + }, + "additionalItems": { + "anyOf": [{ "type": "boolean" }, { "$ref": "#" }], + "default": {} + }, + "items": { + "anyOf": [{ "$ref": "#" }, { "$ref": "#/definitions/schemaArray" }], + "default": {} + }, + "maxItems": { "$ref": "#/definitions/positiveInteger" }, + "minItems": { "$ref": "#/definitions/positiveIntegerDefault0" }, + "uniqueItems": { + "type": "boolean", + "default": false + }, + "maxProperties": { "$ref": "#/definitions/positiveInteger" }, + "minProperties": { "$ref": "#/definitions/positiveIntegerDefault0" }, + "required": { "$ref": "#/definitions/stringArray" }, + "additionalProperties": { + "anyOf": [{ "type": "boolean" }, { "$ref": "#" }], + "default": {} + }, "definitions": { - "schemaArray": { - "type": "array", - "minItems": 1, - "items": { "$ref": "#" } - }, - "positiveInteger": { - "type": "integer", - "minimum": 0 - }, - "positiveIntegerDefault0": { - "allOf": [ { "$ref": "#/definitions/positiveInteger" }, { "default": 0 } ] - }, - "simpleTypes": { - "enum": [ "array", "boolean", "integer", "null", "number", "object", "string" ] - }, - "stringArray": { - "type": "array", - "items": { "type": "string" }, - "minItems": 1, - "uniqueItems": true - } + "type": "object", + "additionalProperties": { "$ref": "#" }, + "default": {} }, - "type": "object", "properties": { - "id": { - "type": "string", - "format": "uri" - }, - "$schema": { - "type": "string", - "format": "uri" - }, - "title": { - "type": "string" - }, - "description": { - "type": "string" - }, - "default": {}, - "multipleOf": { - "type": "number", - "minimum": 0, - "exclusiveMinimum": true - }, - "maximum": { - "type": "number" - }, - "exclusiveMaximum": { - "type": "boolean", - "default": false - }, - "minimum": { - "type": "number" - }, - "exclusiveMinimum": { - "type": "boolean", - "default": false - }, - "maxLength": { "$ref": "#/definitions/positiveInteger" }, - "minLength": { "$ref": "#/definitions/positiveIntegerDefault0" }, - "pattern": { - "type": "string", - "format": "regex" - }, - "additionalItems": { - "anyOf": [ - { "type": "boolean" }, - { "$ref": "#" } - ], - "default": {} - }, - "items": { - "anyOf": [ - { "$ref": "#" }, - { "$ref": "#/definitions/schemaArray" } - ], - "default": {} - }, - "maxItems": { "$ref": "#/definitions/positiveInteger" }, - "minItems": { "$ref": "#/definitions/positiveIntegerDefault0" }, - "uniqueItems": { - "type": "boolean", - "default": false - }, - "maxProperties": { "$ref": "#/definitions/positiveInteger" }, - "minProperties": { "$ref": "#/definitions/positiveIntegerDefault0" }, - "required": { "$ref": "#/definitions/stringArray" }, - "additionalProperties": { - "anyOf": [ - { "type": "boolean" }, - { "$ref": "#" } - ], - "default": {} - }, - "definitions": { - "type": "object", - "additionalProperties": { "$ref": "#" }, - "default": {} - }, - "properties": { - "type": "object", - "additionalProperties": { "$ref": "#" }, - "default": {} - }, - "patternProperties": { - "type": "object", - "additionalProperties": { "$ref": "#" }, - "default": {} - }, - "dependencies": { - "type": "object", - "additionalProperties": { - "anyOf": [ - { "$ref": "#" }, - { "$ref": "#/definitions/stringArray" } - ] - } - }, - "enum": { - "type": "array", - "minItems": 1, - "uniqueItems": true - }, - "type": { - "anyOf": [ - { "$ref": "#/definitions/simpleTypes" }, - { - "type": "array", - "items": { "$ref": "#/definitions/simpleTypes" }, - "minItems": 1, - "uniqueItems": true - } - ] - }, - "allOf": { "$ref": "#/definitions/schemaArray" }, - "anyOf": { "$ref": "#/definitions/schemaArray" }, - "oneOf": { "$ref": "#/definitions/schemaArray" }, - "not": { "$ref": "#" } + "type": "object", + "additionalProperties": { "$ref": "#" }, + "default": {} + }, + "patternProperties": { + "type": "object", + "additionalProperties": { "$ref": "#" }, + "default": {} }, "dependencies": { - "exclusiveMaximum": [ "maximum" ], - "exclusiveMinimum": [ "minimum" ] + "type": "object", + "additionalProperties": { + "anyOf": [{ "$ref": "#" }, { "$ref": "#/definitions/stringArray" }] + } + }, + "enum": { + "type": "array", + "minItems": 1, + "uniqueItems": true + }, + "type": { + "anyOf": [ + { "$ref": "#/definitions/simpleTypes" }, + { + "type": "array", + "items": { "$ref": "#/definitions/simpleTypes" }, + "minItems": 1, + "uniqueItems": true + } + ] }, - "default": {} + "allOf": { "$ref": "#/definitions/schemaArray" }, + "anyOf": { "$ref": "#/definitions/schemaArray" }, + "oneOf": { "$ref": "#/definitions/schemaArray" }, + "not": { "$ref": "#" } + }, + "dependencies": { + "exclusiveMaximum": ["maximum"], + "exclusiveMinimum": ["minimum"] + }, + "default": {} } diff --git a/spec/mocha.js b/spec/mocha.js deleted file mode 100644 index 22bc598..0000000 --- a/spec/mocha.js +++ /dev/null @@ -1,10 +0,0 @@ -const fbpspec = require('..'); - -const rt = { - protocol: 'websocket', - address: 'ws://localhost:3569', - secret: 'py3k', // Optional. If needed to connect/authenticate to runtime - command: 'python2 protocol-examples/python/runtime.py', // Optional. Can be used to start runtime automatically -}; - -fbpspec.mocha.run(rt, './examples/simple-passing.yaml', { starttimeout: 1000 }); diff --git a/spec/mochacompat.js b/spec/mochacompat.js deleted file mode 100644 index 0aeae3c..0000000 --- a/spec/mochacompat.js +++ /dev/null @@ -1,192 +0,0 @@ -const chai = require('chai'); -const fbpClient = require('fbp-client'); -const mochacompat = require('../lib/mochacompat'); -const protocol = require('../lib/protocol'); -const testsuite = require('../lib/testsuite'); -const runner = require('../lib/runner'); - -const testPath = function (name) { - // eslint-disable-next-line global-require - const path = require('path'); - const test = path.join(__dirname, 'fixtures/mochacases', name); - return test; -}; - -const runtimeDefinition = function (options) { - const def = { - protocol: 'websocket', - address: `ws://localhost:${options.port || 3333}`, - }; - return def; -}; - -const setupAndConnect = function (options, callback) { - mochacompat.setup(options, (err, state, httpServer) => { - if (err) { - callback(err); - return; - } - const def = runtimeDefinition(options); - let client = null; - fbpClient(def) - .then((c) => { - client = c; - return client.connect(); - }) - .then((() => callback(null, client, def, state, httpServer)), callback); - }); -}; - -const runAllComponentTests = function (ru, callback) { - let state = null; - const onUpdate = (s) => { - state = s; - }; - ru.connect((err) => { - if (err) { - callback(err); - return; - } - runner.getComponentSuites(ru, (suiteErr, suites) => { - if (suiteErr) { - callback(suiteErr); - return; - } - runner.runAll(ru, suites, onUpdate, (runErr) => callback(runErr, state)); - }); - }); -}; - -describe('Mocha compatibility runner', () => { - let httpServer = null; - let definition = null; - let ru = null; - - afterEach((done) => { - if (httpServer) { - httpServer.close(); - httpServer = null; - } - if (ru) { - ru.disconnect((err) => { - ru = null; - done(err); - }); - return; - } - done(); - }); - - it('should implement the FBP runtime protocol', (done) => { - const options = { files: [testPath('bdd-nested-passing.js')] }; - setupAndConnect(options, (err, client, def, state, server) => { - httpServer = server; - definition = def; - if (err) { - done(err); - return; - } - chai.expect(def).to.include.keys(['protocol', 'type', 'version', 'capabilities']); - chai.expect(def.protocol).to.equal('websocket'); - chai.expect(def.type).to.equal('fbp-spec-mocha'); - chai.expect(def.version).to.equal('0.5'); - done(); - }); - }); - it('has the required FBP runtime capabilities', () => { - const c = definition.capabilities; - chai.expect(c).to.include('protocol:graph'); - chai.expect(c).to.include('protocol:component'); - chai.expect(c).to.include('protocol:network'); - chai.expect(c).to.include('component:getsource'); - }); - - describe('loading test file with nested describe()', () => it('should list each it() as separate fbp-spec testcase', (done) => { - const options = { files: [testPath('bdd-nested-passing.js')] }; - setupAndConnect(options, (err, client, def, state, server) => { - httpServer = server; - if (err) { - done(err); - return; - } - protocol.getComponentTests(client, (componentTestsErr, suites) => { - if (componentTestsErr) { - done(componentTestsErr); - return; - } - chai.expect(suites).to.be.a('object'); - const suiteNames = Object.keys(suites); - chai.expect(suiteNames).to.have.length(1); - const t = suites[suiteNames[0]]; - testsuite.loadYAML(t, (yamlErr, tests) => { - if (yamlErr) { - done(yamlErr); - return; - } - chai.expect(tests).to.have.length(1); - chai.expect(tests[0]).to.include.keys(['name', 'fixture', 'cases']); - chai.expect(tests[0].cases).to.have.length(2); - const [caseA, caseB] = tests[0].cases; - chai.expect(caseA.name).to.include('sub topic'); - chai.expect(caseB.name).to.include('sub sub topic'); - done(); - }); - }); - }); - })); - - describe('running a passing test', () => it('should recorded 1 passed test', (done) => { - const options = { files: [testPath('bdd-simple-passing.js')] }; - mochacompat.setup(options, (err, state, server) => { - httpServer = server; - - ru = new runner.Runner(runtimeDefinition(options)); - runAllComponentTests(ru, (runErr, updatedState) => { - if (runErr) { - done(runErr); - return; - } - chai.expect(updatedState).to.have.length(1); - const { - cases, - } = updatedState[0]; - chai.expect(cases).to.have.length(1); - chai.expect(cases[0].passed, 'testcase did not pass').to.equal(true); - done(); - }); - }); - })); - - describe('running a failing test', () => { - let testcase = null; - it('should recorded 1 failed test', (done) => { - const options = { files: [testPath('bdd-simple-failing.js')] }; - mochacompat.setup(options, (err, state, server) => { - httpServer = server; - ru = new runner.Runner(runtimeDefinition(options)); - runAllComponentTests(ru, (runErr, updatedState) => { - if (runErr) { - done(runErr); - return; - } - chai.expect(updatedState).to.have.length(1); - const { - cases, - } = updatedState[0]; - chai.expect(cases).to.have.length(1); - [testcase] = cases; - chai.expect(testcase.passed, 'failing testcase passed').to.equal(false); - done(); - }); - }); - }); - it('has error message of the Chai assertion', () => { - chai.expect(testcase != null ? testcase.error : undefined).to.contain('expected 42 to equal 41'); - }); - }); - - describe('suite with some skipped tests', () => { - it('skipped tests should be marked as such'); - it('non-skipped tests should be ran'); - }); -}); diff --git a/spec/runner.test.js b/spec/runner.test.js new file mode 100644 index 0000000..a19c877 --- /dev/null +++ b/spec/runner.test.js @@ -0,0 +1,164 @@ +import assert from "node:assert/strict"; +import { describe, it } from "node:test"; + +import { checkResults, Runner, runTestAndCheck } from "../lib/runner.js"; + +describe("runner", () => { + describe("checkResults", () => { + it("should pass when actual matches expect", () => { + const result = checkResults([ + { + inputs: { in: 1 }, + expect: { out: { equals: 1 } }, + actual: { out: 1 }, + }, + ]); + assert.deepEqual(result, { passed: true }); + }); + it("should fail when actual does not match expect", () => { + const result = checkResults([ + { + inputs: { in: 1 }, + expect: { out: { equals: 2 } }, + actual: { out: 1 }, + }, + ]); + assert.equal(result.passed, false); + assert.ok(result.error instanceof Error); + assert.match(result.error.message, /to equal/); + }); + it("should fail when not enough outputs received", () => { + const result = checkResults([ + { inputs: { in: 1 }, expect: { out: { equals: 1 } } }, + ]); + assert.ok(result.error instanceof Error); + assert.match(result.error.message, /Only got 0 output messages out of 1/); + }); + it("should report sequence position of failures", () => { + const result = checkResults([ + { + inputs: { in: 1 }, + expect: { out: { equals: 1 } }, + actual: { out: 1 }, + }, + { + inputs: { in: 2 }, + expect: { out: { equals: 2 } }, + actual: { out: 3 }, + }, + ]); + assert.ok(result.error instanceof Error); + assert.match(result.error.message, /Expectation 1 of sequence failed/); + }); + it("should report multiple failures", () => { + const result = checkResults([ + { + inputs: { in: 1 }, + expect: { out: { equals: 2 } }, + actual: { out: 1 }, + }, + { + inputs: { in: 2 }, + expect: { out: { equals: 3 } }, + actual: { out: 2 }, + }, + ]); + assert.ok(result.error instanceof Error); + assert.match(result.error.message, /Multiple failures in sequence/); + }); + it("should not fail when actual is null but expects are empty", () => { + const result = checkResults([ + { inputs: { in: 1 }, expect: {}, actual: { out: 1 } }, + ]); + assert.deepEqual(result, { passed: true }); + }); + }); + + describe("runTestAndCheck", () => { + /** + * A Runner stand-in with canned results. + * @param {import("../lib/runner.js").RunnerLike["runTest"]} runTest Implementation of runTest + * @returns {import("../lib/runner.js").RunnerLike} Fake runner + */ + function fakeRunner(runTest) { + return { runTest }; + } + + it("should pass skipped testcases without running them", async () => { + let ran = false; + const result = await runTestAndCheck( + fakeRunner(async () => { + ran = true; + return []; + }), + { skip: "not now" }, + ); + assert.deepEqual(result, { passed: true }); + assert.equal(ran, false); + }); + it("should fail on inputs/expect length mismatch", async () => { + const result = await runTestAndCheck( + fakeRunner(async () => []), + { + inputs: [{ in: 1 }], + expect: [{ out: { equals: 1 } }, { out: { equals: 2 } }], + }, + ); + assert.equal(result.passed, false); + assert.ok(result.error instanceof Error); + assert.match(result.error.message, /sequence length mismatch/); + }); + it("should map runner errors to test failures", async () => { + const result = await runTestAndCheck( + fakeRunner(async () => { + throw new Error("runtime went away"); + }), + { inputs: { in: 1 }, expect: { out: { equals: 1 } } }, + ); + assert.equal(result.passed, false); + assert.ok(result.error instanceof Error); + assert.match(result.error.message, /runtime went away/); + }); + it("should pass on matching results", async () => { + const result = await runTestAndCheck( + fakeRunner(async (testcase) => [ + { + inputs: /** @type {Record} */ (testcase).inputs, + expect: /** @type {Record} */ (testcase).expect, + actual: { out: 42 }, + }, + ]), + { inputs: { in: 1 }, expect: { out: { equals: 42 } } }, + ); + assert.deepEqual(result, { passed: true }); + }); + it("should stringify errors into testcase results", async () => { + const testcase = { inputs: { in: 1 }, expect: { out: { equals: 2 } } }; + const runner = fakeRunner(async () => [ + { + inputs: testcase.inputs, + expect: testcase.expect, + actual: { out: 1 }, + }, + ]); + const result = await runTestAndCheck(runner, testcase); + assert.equal(result.passed, false); + }); + }); + + describe("Runner", () => { + it("should default options", () => { + const runner = new Runner({}); + assert.equal(runner.options.connectTimeout, 5000); + assert.equal(runner.options.commandTimeout, 3000); + }); + + it("should fail to connect to an unavailable runtime", async () => { + const runner = new Runner( + { protocol: "websocket", address: "ws://127.0.0.1:1" }, + { connectTimeout: 1500 }, + ); + await assert.rejects(() => runner.connect(), Error); + }); + }); +}); diff --git a/spec/schema.test.js b/spec/schema.test.js new file mode 100644 index 0000000..204c9ce --- /dev/null +++ b/spec/schema.test.js @@ -0,0 +1,39 @@ +import assert from "node:assert/strict"; +import { readFileSync } from "node:fs"; +import { describe, it } from "node:test"; +import tv4 from "tv4"; + +import schemas from "../schema/index.js"; + +const metaSchema = JSON.parse( + readFileSync(new URL("./json-schema.json", import.meta.url), "utf-8"), +); + +describe("Schema meta validation", () => { + for (const name of Object.keys(schemas)) { + const schema = schemas[name]; + describe(`${name}.json (${schema.title || schema.description})`, () => { + it("should validate against JSON meta schema", () => { + tv4.reset(); + tv4.addSchema("http://json-schema.org/draft-04/schema", metaSchema); + tv4.addSchema(schema.id, schema); + const result = tv4.validateResult( + schema, + "http://json-schema.org/draft-04/schema", + ); + assert.equal(result.valid, true, JSON.stringify(result.errors)); + }); + }); + } +}); + +describe("Schema module", () => { + it("should provide getSchema lookup", async () => { + const { getSchema } = await import("../schema/index.js"); + const schema = getSchema("testcase.json"); + assert.ok(schema, "expected testcase schema"); + assert.equal(schema.id, "testcase.json"); + assert.equal(getSchema("testcase")?.id, "testcase.json"); + assert.equal(getSchema("nonexistent.json"), undefined); + }); +}); diff --git a/spec/testsuite.test.js b/spec/testsuite.test.js new file mode 100644 index 0000000..784198d --- /dev/null +++ b/spec/testsuite.test.js @@ -0,0 +1,159 @@ +import assert from "node:assert/strict"; +import { mkdtemp, writeFile } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { after, before, describe, it } from "node:test"; + +import { create, getSuites, loadYAML, validate } from "../lib/testsuite.js"; + +const MULTI_DOC_YAML = `--- +name: First suite +topic: DummyComponent +cases: +- + name: 'case one' + assertion: 'should pass' + inputs: + in: true + expect: + out: + equals: true +--- +name: Second suite +topic: core/Repeat +cases: [] +`; + +const PASSING_YAML = ` +name: Simple passing +topic: DummyComponent +cases: +- + name: 'sending a boolean' + assertion: 'should pass' + inputs: + in: true + expect: + out: + equals: true +`; + +describe("testsuite", () => { + describe("create", () => { + it("should default name to topic", () => { + const suite = create({ topic: "core/Repeat" }); + assert.equal(suite.name, "core/Repeat"); + assert.deepEqual(suite.cases, []); + }); + it("should keep existing name and cases", () => { + const suite = create({ + name: "Mine", + topic: "core/Repeat", + cases: [{ name: "x" }], + }); + assert.equal(suite.name, "Mine"); + assert.equal(suite.cases?.length, 1); + }); + }); + + describe("loadYAML", () => { + it("should parse single-document YAML", () => { + const suites = loadYAML(PASSING_YAML); + assert.equal(suites.length, 1); + assert.equal(suites[0].name, "Simple passing"); + assert.equal(suites[0].cases?.length, 1); + }); + it("should parse multi-document YAML", () => { + const suites = loadYAML(MULTI_DOC_YAML); + assert.equal(suites.length, 2); + assert.equal(suites[0].name, "First suite"); + assert.equal(suites[1].name, "Second suite"); + }); + it("should throw on invalid YAML", () => { + assert.throws(() => loadYAML("foo: [unclosed"), Error); + }); + }); + + describe("getSuites", () => { + /** @type {string} */ + let dir; + + before(async () => { + dir = await mkdtemp(join(tmpdir(), "fbp-spec-test-")); + await writeFile(join(dir, "passing.yaml"), PASSING_YAML); + await writeFile(join(dir, "other.yaml"), MULTI_DOC_YAML); + }); + + it("should load from a single file", async () => { + const suites = await getSuites(join(dir, "passing.yaml")); + assert.equal(suites.length, 1); + assert.equal(suites[0].name, "Simple passing"); + }); + it("should load from a list of files", async () => { + const suites = await getSuites([ + join(dir, "passing.yaml"), + join(dir, "other.yaml"), + ]); + assert.equal(suites.length, 3); + }); + it("should load from a directory", async () => { + const suites = await getSuites(dir); + assert.equal(suites.length, 3); + }); + it("should reject on missing files", async () => { + await assert.rejects(getSuites(join(dir, "missing.yaml")), Error); + }); + + describe("over HTTP", () => { + /** @type {import('node:http').Server} */ + let server; + /** @type {string} */ + let baseUrl; + + before(async () => { + const http = await import("node:http"); + server = http.createServer((_req, res) => { + res.writeHead(200, { "Content-Type": "text/yaml" }); + res.end(PASSING_YAML); + }); + /** @type {Promise} */ + const listening = new Promise((resolve) => { + server.listen({ port: 0, host: "127.0.0.1" }, resolve); + }); + await listening; + const addr = server.address(); + const port = typeof addr === "object" && addr ? addr.port : 0; + baseUrl = `http://127.0.0.1:${port}/tests.yaml`; + }); + after( + () => + new Promise((resolve) => { + server.close(resolve); + }), + ); + + it("should load suites from a URL", async () => { + const suites = await getSuites(baseUrl); + assert.equal(suites.length, 1); + assert.equal(suites[0].name, "Simple passing"); + }); + }); + }); + + describe("validate", () => { + it("should accept a valid testsfile", () => { + const results = validate(loadYAML(MULTI_DOC_YAML)); + assert.equal(results.valid, true); + assert.equal(results.passed, true); + assert.deepEqual(results.errors, []); + assert.deepEqual(results.missing, []); + }); + it("should reject invalid testcases", () => { + const results = validate([ + { name: "Broken suite", cases: [{ inputs: 42 }] }, + ]); + assert.equal(results.valid, false); + assert.ok(results.errors.length > 0); + }); + }); +}); diff --git a/spec/validate-schemas.js b/spec/validate-schemas.js deleted file mode 100644 index c62dd0f..0000000 --- a/spec/validate-schemas.js +++ /dev/null @@ -1,38 +0,0 @@ -const tv4 = require('tv4'); -const chai = require('chai'); -const path = require('path'); -const fs = require('fs'); - -const schemaPath = '../schema'; - -function loadSchema(filepath) { - const content = fs.readFileSync(filepath, { encoding: 'utf-8' }); - return JSON.parse(content); -} -function getSchema(name) { - const filepath = path.join(__dirname, schemaPath, name); - return loadSchema(filepath); -} - -describe('Schema meta validation', () => { - let schemas = fs.readdirSync(path.join(__dirname, schemaPath)); - schemas = schemas.filter((s) => s.indexOf('.json') !== -1); - - before(() => { - const metaSchema = loadSchema(path.join(__dirname, 'json-schema.json')); - tv4.addSchema('http://json-schema.org/draft-04/schema', metaSchema); - }); - after(() => { - tv4.reset(); - }); - schemas.forEach((schemaFile) => { - const schema = getSchema(schemaFile); - tv4.addSchema(schema.id, schema); - describe(`${schemaFile} (${schema.title || schema.description})`, () => { - it('should validate against JSON meta schema', () => { - const result = tv4.validateResult(schema, 'http://json-schema.org/draft-04/schema'); - chai.expect(result.valid).to.equal(true); - }); - }); - }); -}); diff --git a/tsconfig.json b/tsconfig.json new file mode 100644 index 0000000..b5a9b6f --- /dev/null +++ b/tsconfig.json @@ -0,0 +1,25 @@ +{ + "compilerOptions": { + "allowJs": true, + "checkJs": true, + "noEmit": true, + "target": "ES2022", + "lib": ["ES2022", "DOM", "DOM.Iterable"], + "module": "Node16", + "moduleResolution": "Node16", + "strict": true, + "noImplicitAny": true, + "noUnusedLocals": false, + "skipLibCheck": true, + "resolveJsonModule": true + }, + "include": [ + "index.js", + "lib/**/*.js", + "schema/**/*.js", + "scripts/**/*.js", + "spec/**/*.js", + "examples/**/*.js", + "types.d.ts" + ] +} diff --git a/types.d.ts b/types.d.ts new file mode 100644 index 0000000..8f57382 --- /dev/null +++ b/types.d.ts @@ -0,0 +1,146 @@ +/** + * Ambient type declarations for dependencies without their own type information. + * The shapes here are limited to what fbp-spec uses. + */ + +declare module "fbp" { + /** + * Parse FBP graph definition text into a graph JSON payload. + * @param source FBP graph language source + * @param options Parsing options + * @returns Graph JSON payload + */ + export function parse(source: string, options?: { caseSensitive?: boolean }): object; +} + +declare module "fbp-client" { + /** A connected FBP protocol client. */ + export interface FbpClient { + definition: { + protocol?: string; + address?: string; + capabilities?: string[]; + namespace?: string; + [key: string]: unknown; + }; + options: Record; + isConnected(): boolean; + connect(): Promise; + disconnect(): Promise; + observe(filter: (message: any) => boolean): { + signals: { payload: Record }[]; + until( + onSuccess: (message: any) => boolean, + onFailure?: (message: any) => boolean, + ): Promise<{ payload: Record }[]>; + }; + on(event: string, listener: (...args: any[]) => void): void; + transport: { + setParentElement(element: unknown): void; + [key: string]: unknown; + }; + protocol: { + graph: { + send(graph: object, main?: boolean): Promise; + clear(payload: object): Promise; + addnode(payload: object): Promise; + addedge(payload: object): Promise; + addinitial(payload: object): Promise; + addinport(payload: object): Promise; + addoutport(payload: object): Promise; + }; + network: { + start(payload: object): Promise; + stop(payload: object): Promise; + }; + runtime: { + getruntime(payload?: object): Promise<{ capabilities?: string[] }>; + packet(payload: object): Promise; + }; + component: { + list(): Promise<{ name: string; inPorts: { id: string }[]; outPorts: { id: string }[]; [key: string]: unknown }[]>; + getsource(payload: { name: string }): Promise<{ + name: string; + library?: string; + tests?: string | null; + [key: string]: unknown; + }>; + }; + }; + } + + /** + * Create a FBP protocol client for a runtime. + * @param definition Runtime definition + * @param options Client options + * @returns Client instance + */ + function fbpClient( + definition: object, + options?: { commandTimeout?: number; connectionTimeout?: number }, + ): Promise; + export default fbpClient; +} + +declare module "fbp-graph" { + /** A FBP graph instance. */ + export interface Graph { + name?: string; + properties: Record; + nodes: unknown[]; + edges: unknown[]; + initializers: unknown[]; + inports: Record; + outports: Record; + [key: string]: unknown; + } + const fbpGraph: { + Graph: new (name: string, options?: object) => Graph; + graph: { + loadJSON( + definition: object | string, + callback?: (err: Error | null, graph?: Graph) => void, + metadata?: object, + ): Promise; + loadFBP( + source: string, + callback?: (err: Error | null, graph?: Graph) => void, + metadata?: object, + caseSensitive?: boolean, + ): Promise; + }; + }; + export default fbpGraph; +} + +declare module "flowtrace" { + /** Records FBP protocol traffic as replayable Flowtraces. */ + export class FlowtraceRecorder { + constructor(client: object, options?: object); + start(graphId: string): Promise; + stop(graphId: string): Promise; + dump(graphId: string): object; + [key: string]: unknown; + } +} + +declare module "tv4" { + /** Result of a tv4 validation. */ + export interface Tv4Result { + valid: boolean; + errors: { message: string; [key: string]: unknown }[]; + missing: string[]; + passed: boolean; + } + const tv4: { + addSchema(id: string, schema: object): void; + reset(): void; + validateMultiple(data: unknown, schemaId: string): Tv4Result; + validateResult( + data: unknown, + schemaId: string, + ): { valid: boolean; errors: { message: string }[]; missing: string[] }; + [key: string]: unknown; + }; + export default tv4; +} diff --git a/webpack.config.js b/webpack.config.js deleted file mode 100644 index 7987dd3..0000000 --- a/webpack.config.js +++ /dev/null @@ -1,34 +0,0 @@ -module.exports = { - entry: './webpack.entry.js', - output: { - path: __dirname, - filename: 'browser/fbp-spec.js', - }, - module: { - rules: [ - { - test: /\.js$/, - use: [ - { - loader: 'babel-loader', - options: { - presets: ['@babel/preset-env'], - }, - }, - ], - }, - ], - }, - resolve: { - extensions: ['.js'], - fallback: { - buffer: require.resolve('buffer/'), - events: require.resolve('events/'), - fs: false, - path: require.resolve('path-browserify'), - url: require.resolve('url/'), - }, - }, - externals: { - }, -}; diff --git a/webpack.entry.js b/webpack.entry.js deleted file mode 100644 index 0799b3a..0000000 --- a/webpack.entry.js +++ /dev/null @@ -1,18 +0,0 @@ -/* eslint-env browser */ -const exported = { - // eslint-disable-next-line global-require - 'fbp-spec': require('./index'), - // eslint-disable-next-line global-require - chai: require('chai'), - // eslint-disable-next-line global-require - 'js-yaml': require('js-yaml'), -}; - -if (window) { - window.require = function (moduleName) { - if (exported[moduleName]) { - return exported[moduleName]; - } - throw new Error(`Module ${moduleName} not available`); - }; -}