feat: implement new result output format for validation

packages/ruleset/src/ibm-oas.js

@@ -12,11 +12,13 @@ const ibmRules = require('./rules');
 // However, we want to enable this rule also for OpenAPI 3.1.x documents,
 // so we'll just tweak Spectral's rule definition here.
 oas.rules['no-$ref-siblings'].formats = [oas3];
+oas.documentationUrl =
+  'https://meta.stoplight.io/docs/spectral/4dec24461f3af-open-api-rules';
 
 module.exports = {
   extends: oas,
   documentationUrl:
-    'https://github.com/IBM/openapi-validator/blob/main/docs/ibm-cloud-rules.md',
+    'https://ibm.github.io/openapi-validator/docs/ibm-cloud-rules.html',
   formats: [oas3],
   rules: {
     // Original list created from Spectral with:

packages/validator/src/cli-validator/utils/print-results.js

@@ -38,6 +38,7 @@ module.exports = function print(context, results) {
       console.log(chalk[color](`  Message :   ${result.message}`));
       console.log(chalk[color](`  Rule    :   ${result.rule}`));
       console.log(chalk[color](`  Path    :   ${result.path.join('.')}`));
+      console.log(chalk[color](`  DocLink :   ${result.docLink}`));
       console.log(chalk[color](`  Line    :   ${result.line}`));
       console.log('');
     });

packages/validator/src/markdown-report/report.js

@@ -34,8 +34,14 @@ inherently weighted by the scoring algorithm, so that security violations are 5
 as usability violations, evolution 3 times, and robustness 2 times.
 
 ## Scoring information
+<details>
+
+<summary>Information about how the scoring gets calculated</summary>
+
 ${scoringData(results)}
 
+</details>
+
 ## Error summary
 ${errorSummary(results)}
 

packages/validator/src/markdown-report/tables/rule-violation-details.js

@@ -5,24 +5,102 @@
 
 const MarkdownTable = require('../markdown-table');
 
-function getTable({ error, warning }) {
-  const table = new MarkdownTable(
-    'Rule',
-    'Message',
-    'Path',
-    'Line',
-    'Severity'
-  );
-
-  error.results.forEach(({ message, path, rule, line }) => {
-    table.addRow(rule, message, path.join('.'), line, 'error');
+function splitMessage(message) {
+  const colonIdx = message.indexOf(':');
+  if (colonIdx === -1) {
+    return { base: message, detail: null };
+  }
+  return {
+    base: message.slice(0, colonIdx).trim(),
+    detail: message.slice(colonIdx + 1).trim() || null,
+  };
+}
+
+function groupResultsByRule(results) {
+  const groupedByRule = {};
+
+  results.forEach(result => {
+    const { base, detail } = splitMessage(result.message);
+
+    if (!groupedByRule[result.rule]) {
+      groupedByRule[result.rule] = {
+        base,
+        docLink: result.docLink,
+        violations: [],
+      };
+    }
+
+    const pathStr = result.path.join('.');
+
+    if (
+      !groupedByRule[result.rule].violations.some(
+        v => v.line === result.line && v.path === pathStr
+      )
+    ) {
+      groupedByRule[result.rule].violations.push({
+        line: result.line,
+        path: pathStr,
+        detail,
+      });
+    }
   });
 
-  warning.results.forEach(({ message, path, rule, line }) => {
-    table.addRow(rule, message, path.join('.'), line, 'warning');
+  return groupedByRule;
+}
+
+function renderRuleGroups(groupedByRule) {
+  const sections = [];
+
+  Object.entries(groupedByRule).forEach(([rule, data]) => {
+    // Create heading with link
+    sections.push(`### [${rule}](${data.docLink})`);
+    sections.push('');
+
+    // Add the base message as italic description
+    sections.push(`_${data.base}_`);
+    sections.push('');
+
+    // Use a 3-col table when any violation carries extra detail
+    const hasDetail = data.violations.some(v => v.detail);
+    const table = hasDetail
+      ? new MarkdownTable('Line', 'Path', 'Detail')
+      : new MarkdownTable('Line', 'Path');
+
+    data.violations.forEach(({ line, path, detail }) => {
+      if (hasDetail) {
+        table.addRow(line, path, detail);
+      } else {
+        table.addRow(line, path);
+      }
+    });
+
+    sections.push(table.render());
+    sections.push('');
   });
 
-  return table.render();
+  return sections.join('\n');
+}
+
+function getTable({ error, warning }) {
+  const sections = [];
+
+  // Process errors
+  if (error.results.length > 0) {
+    sections.push('### Errors');
+    sections.push('');
+    const errorGroups = groupResultsByRule(error.results);
+    sections.push(renderRuleGroups(errorGroups));
+  }
+
+  // Process warnings
+  if (warning.results.length > 0) {
+    sections.push('### Warnings');
+    sections.push('');
+    const warningGroups = groupResultsByRule(warning.results);
+    sections.push(renderRuleGroups(warningGroups));
+  }
+
+  return sections.join('\n').trim();
 }
 
 module.exports = getTable;

packages/validator/src/schemas/results-object.yaml

@@ -125,6 +125,7 @@ $defs:
       - message
       - path
       - rule
+      - docLink
       - line
     properties:
       message:
@@ -139,6 +140,9 @@ $defs:
       rule:
         type: string
         description: The rule identifier in the Spectral ruleset
+      docLink:
+        type: string
+        description: The documentation link to the problem discovered in the API
       line:
         type: integer
         description: The line number in the original file that the problem
@@ -169,3 +173,4 @@ $defs:
     type: number
     description: A number describing the demerit quality a rule has on an API
     minimum: 0.01
+

packages/validator/src/spectral/index.js

@@ -47,10 +47,10 @@ const runSpectral = async function ({ originalFile, validFile }, context) {
   // Save the resolved API definition for use in the scoring tool logic.
   context.apiDefinition = doc.data;
 
-  return convertResults(spectralResults, context);
+  return convertResults(spectralResults, context, spectral.ruleset.rules);
 };
 
-function convertResults(spectralResults, { config, logger }) {
+function convertResults(spectralResults, { config, logger }, rules) {
   const { errorsOnly } = config;
 
   // This structure must match the JSON Schema defined for JSON output
@@ -89,6 +89,7 @@ function convertResults(spectralResults, { config, logger }) {
       message: r.message,
       path: r.path,
       rule: r.code,
+      docLink: rules[r.code]?.documentationUrl,
       line: r.range.start.line + 1,
     });
 

packages/validator/test/cli-validator/tests/expected-output.test.js

@@ -87,39 +87,39 @@ describe('Expected output tests', function () {
 
         // errors
         const errorStart = 3;
-        expect(capturedText[errorStart + 5].match(/\S+/g)[2]).toEqual('52');
-        expect(capturedText[errorStart + 10].match(/\S+/g)[2]).toEqual('96');
-        expect(capturedText[errorStart + 15].match(/\S+/g)[2]).toEqual('103');
+        expect(capturedText[errorStart + 6].match(/\S+/g)[2]).toEqual('52');
+        expect(capturedText[errorStart + 12].match(/\S+/g)[2]).toEqual('96');
+        expect(capturedText[errorStart + 18].match(/\S+/g)[2]).toEqual('103');
 
         // Specifically verify that the no-$ref-siblings error occurred.
         // We do this because this rule is inherited from Spectral's oas ruleset,
         // but we modify the rule definition in ibmoas.js so that it is run
         // against both OpenAPI 3.0 and 3.1 documents.
-        expect(capturedText[errorStart + 17].split(':')[1].trim()).toEqual(
+        expect(capturedText[errorStart + 20].split(':')[1].trim()).toEqual(
           '$ref must not be placed next to any other properties'
         );
-        expect(capturedText[errorStart + 18].split(':')[1].trim()).toEqual(
+        expect(capturedText[errorStart + 21].split(':')[1].trim()).toEqual(
           'no-$ref-siblings'
         );
-        expect(capturedText[errorStart + 19].split(':')[1].trim()).toEqual(
+        expect(capturedText[errorStart + 22].split(':')[1].trim()).toEqual(
           'components.schemas.Pet.properties.category.description'
         );
-        expect(capturedText[errorStart + 20].match(/\S+/g)[2]).toEqual('184');
+        expect(capturedText[errorStart + 24].match(/\S+/g)[2]).toEqual('184');
 
-        // warnings
-        const warningStart = 25;
+        // warnings - each warning block is now 6 lines (5 content + 1 blank)
+        const warningStart = 30;
         expect(capturedText[warningStart + 5].match(/\S+/g)[2]).toEqual('22');
-        expect(capturedText[warningStart + 10].match(/\S+/g)[2]).toEqual('24');
-        expect(capturedText[warningStart + 15].match(/\S+/g)[2]).toEqual('40');
-        expect(capturedText[warningStart + 20].match(/\S+/g)[2]).toEqual('41');
-        expect(capturedText[warningStart + 25].match(/\S+/g)[2]).toEqual('52');
-        expect(capturedText[warningStart + 30].match(/\S+/g)[2]).toEqual('56');
-        expect(capturedText[warningStart + 35].match(/\S+/g)[2]).toEqual('57');
-        expect(capturedText[warningStart + 40].match(/\S+/g)[2]).toEqual('59');
-        expect(capturedText[warningStart + 45].match(/\S+/g)[2]).toEqual('61');
-        expect(capturedText[warningStart + 50].match(/\S+/g)[2]).toEqual('96');
-        // Skip a few, then verify the last one.
-        expect(capturedText[warningStart + 145].match(/\S+/g)[2]).toEqual(
+        expect(capturedText[warningStart + 11].match(/\S+/g)[2]).toEqual('24');
+        expect(capturedText[warningStart + 17].match(/\S+/g)[2]).toEqual('40');
+        expect(capturedText[warningStart + 23].match(/\S+/g)[2]).toEqual('41');
+        expect(capturedText[warningStart + 29].match(/\S+/g)[2]).toEqual('52');
+        expect(capturedText[warningStart + 35].match(/\S+/g)[2]).toEqual('56');
+        expect(capturedText[warningStart + 41].match(/\S+/g)[2]).toEqual('57');
+        expect(capturedText[warningStart + 47].match(/\S+/g)[2]).toEqual('59');
+        expect(capturedText[warningStart + 53].match(/\S+/g)[2]).toEqual('61');
+        expect(capturedText[warningStart + 59].match(/\S+/g)[2]).toEqual('96');
+        // Skip a few, then verify the last one (29 warnings × 6 lines = 174 lines)
+        expect(capturedText[warningStart + 173].match(/\S+/g)[2]).toEqual(
           '210'
         );
       }
@@ -214,7 +214,10 @@ describe('Expected output tests', function () {
           []
         );
 
-        allMessages.forEach(msg => expect(msg).toHaveProperty('rule'));
+        allMessages.forEach(msg => {
+          expect(msg).toHaveProperty('rule');
+          expect(msg).toHaveProperty('docLink');
+        });
       }
     );
 
@@ -244,6 +247,7 @@ describe('Expected output tests', function () {
         const warningToCheck = jsonOutput.warning.results[0];
 
         expect(warningToCheck.rule).toEqual('ibm-prefer-token-pagination');
+        expect(warningToCheck.docLink).toContain('ibm-prefer-token-pagination');
         expect(warningToCheck.path.join('.')).toBe('paths./letters.get');
         expect(warningToCheck.line).toEqual(20);
       }

packages/validator/test/markdown-report/report.test.js

@@ -18,7 +18,7 @@ describe('getReport tests', function () {
     // Check all subtitle-level headers.
     const headers = report
       .split('\n')
-      .filter(l => l.startsWith('##'))
+      .filter(l => /^## /.test(l))
       .map(l => l.slice(3));
     expect(headers).toEqual([
       'Quick view',