lib: implement bench/reporters

Signed-off-by: James M Snell <jasnell@gmail.com>
Assisted-by: Opencode
PR-URL: #65606
Reviewed-By: Filip Skokan <panva.ip@gmail.com>
Reviewed-By: Robert Nagy <ronagy@icloud.com>
Reviewed-By: Vinícius Lourenço Claro Cardoso <contact@viniciusl.com.br>
Reviewed-By: Gürgün Dayıoğlu <hey@gurgun.day>

Author: jasnell

doc/api/bench.md

@@ -65,6 +65,66 @@ system load can all affect results. Keep raw samples when comparing results and
 investigate noisy or skewed distributions rather than treating a confidence
 interval as a pass/fail threshold.
 
+## Benchmark reporters
+
+The built-in reporters are available from the scheme-only
+`node:bench/reporters` module:
+
+```mjs
+import { json, spec } from 'node:bench/reporters';
+```
+
+```cjs
+const { json, spec } = require('node:bench/reporters');
+```
+
+Reporter values can be passed directly to `stream.compose()`:
+
+```mjs
+import { bench, run } from 'node:bench';
+import { spec } from 'node:bench/reporters';
+import process from 'node:process';
+
+bench('example', (b) => {
+  b.start();
+  doWork();
+  b.end(1);
+});
+
+run().compose(spec).pipe(process.stdout);
+```
+
+The `spec` reporter buffers results and outputs a concise table containing the
+sample count, mean rate, 95% confidence interval for the mean, median rate, and
+warnings. A coefficient of variation above 5% is reported as `noisy`, and an
+absolute skewness above 1 is reported as `skewed`. The exact human-readable
+format is subject to change.
+
+The `json` reporter emits every lifecycle record as newline-delimited JSON.
+BigInt values, including `duration_ns`, are encoded as decimal strings. Errors
+are represented using their `name`, `message`, `stack`, `code`, `cause`, and
+`errors` properties. As required by JSON, non-finite numbers are encoded as
+`null`.
+
+Custom reporters use the same composition contract. They can be transforms or
+functions accepted by `stream.compose()`. The composed readable can be piped to
+any writable destination:
+
+```mjs
+import { run } from 'node:bench';
+import process from 'node:process';
+
+async function* names(source) {
+  for await (const { type, data } of source) {
+    if (type === 'bench:complete') {
+      yield `${data.name}\n`;
+    }
+  }
+}
+
+run().compose(names).pipe(process.stdout);
+```
+
 ## `bench([name][, options], fn)`
 
 <!-- YAML

lib/bench/reporters.js

@@ -0,0 +1,34 @@
+'use strict';
+
+const {
+  ObjectDefineProperties,
+  ReflectConstruct,
+} = primordials;
+const { emitExperimentalWarning } = require('internal/util');
+
+let json;
+let specFn;
+
+emitExperimentalWarning('Benchmarks');
+
+ObjectDefineProperties(module.exports, {
+  __proto__: null,
+  json: {
+    __proto__: null,
+    configurable: true,
+    enumerable: true,
+    get() {
+      json ??= require('internal/bench_runner/reporter/json');
+      return json;
+    },
+  },
+  spec: {
+    __proto__: null,
+    configurable: true,
+    enumerable: true,
+    value: function spec() {
+      specFn ??= require('internal/bench_runner/reporter/spec');
+      return ReflectConstruct(specFn, arguments);
+    },
+  },
+});

lib/internal/bench_runner/reporter/json.js

@@ -0,0 +1,64 @@
+'use strict';
+
+const {
+  ArrayIsArray,
+  ArrayPrototypePush,
+  JSONStringify,
+  ObjectKeys,
+  SafeWeakSet,
+  String,
+} = primordials;
+const { isError } = require('internal/util');
+
+function toJSONValue(value, seen) {
+  if (typeof value === 'bigint') return String(value);
+  if (value === null || typeof value !== 'object') return value;
+  if (seen.has(value)) return '[Circular]';
+  seen.add(value);
+
+  if (isError(value)) {
+    const result = {
+      __proto__: null,
+      name: value.name,
+      message: value.message,
+      stack: value.stack,
+      code: value.code,
+    };
+    if (value.cause !== undefined) {
+      result.cause = toJSONValue(value.cause, seen);
+    }
+    if (value.errors !== undefined) {
+      result.errors = toJSONValue(value.errors, seen);
+    }
+    return result;
+  }
+
+  if (ArrayIsArray(value)) {
+    const result = [];
+    for (let i = 0; i < value.length; i++) {
+      ArrayPrototypePush(result, toJSONValue(value[i], seen));
+    }
+    return result;
+  }
+
+  const result = { __proto__: null };
+  const keys = ObjectKeys(value);
+  for (let i = 0; i < keys.length; i++) {
+    const key = keys[i];
+    result[key] = toJSONValue(value[key], seen);
+  }
+  return result;
+}
+
+function stringify(record) {
+  return JSONStringify(record === undefined ? null :
+    toJSONValue(record, new SafeWeakSet()));
+}
+
+async function* jsonReporter(source) {
+  for await (const record of source) {
+    yield `${stringify(record)}\n`;
+  }
+}
+
+module.exports = jsonReporter;

lib/internal/bench_runner/reporter/spec.js

@@ -0,0 +1,149 @@
+'use strict';
+
+const {
+  ArrayPrototypeJoin,
+  ArrayPrototypePush,
+  JSONStringify,
+  MathAbs,
+  NumberIsFinite,
+  NumberPrototypeToFixed,
+  NumberPrototypeToPrecision,
+  ObjectEntries,
+  String,
+  StringPrototypeReplaceAll,
+} = primordials;
+const Transform = require('internal/streams/transform');
+
+const kHeader =
+  'benchmark | samples | mean rate | 95% CI | median rate | warning\n';
+
+function escapeCell(value) {
+  let result = String(value);
+  result = StringPrototypeReplaceAll(result, '\r', '\\r');
+  result = StringPrototypeReplaceAll(result, '\n', '\\n');
+  return StringPrototypeReplaceAll(result, '|', '\\|');
+}
+
+function formatName(data) {
+  const name = escapeCell(data.name);
+  const entries = ObjectEntries(data.params);
+  if (entries.length === 0) return name;
+
+  const params = [];
+  for (let i = 0; i < entries.length; i++) {
+    const { 0: key, 1: value } = entries[i];
+    ArrayPrototypePush(
+      params, `${escapeCell(key)}=${escapeCell(JSONStringify(value))}`);
+  }
+  return `${name} [${ArrayPrototypeJoin(params, ', ')}]`;
+}
+
+function formatRate(rate) {
+  if (!NumberIsFinite(rate)) return '-';
+
+  const absolute = MathAbs(rate);
+  let divisor = 1;
+  let prefix = '';
+  if (absolute >= 1e9) {
+    divisor = 1e9;
+    prefix = 'G';
+  } else if (absolute >= 1e6) {
+    divisor = 1e6;
+    prefix = 'M';
+  } else if (absolute >= 1e3) {
+    divisor = 1e3;
+    prefix = 'k';
+  }
+
+  const scaled = rate / divisor;
+  const formatted = MathAbs(scaled) > 0 && MathAbs(scaled) < 1 ?
+    NumberPrototypeToPrecision(scaled, 3) :
+    NumberPrototypeToFixed(scaled, 2);
+  return `${formatted}${prefix} ops/s`;
+}
+
+function formatWarning(summary) {
+  const warnings = [];
+  if (NumberIsFinite(summary.coefficientOfVariation) &&
+      summary.coefficientOfVariation > 0.05) {
+    ArrayPrototypePush(warnings, 'noisy');
+  }
+  if (NumberIsFinite(summary.skewness) && MathAbs(summary.skewness) > 1) {
+    ArrayPrototypePush(warnings, 'skewed');
+  }
+  return ArrayPrototypeJoin(warnings, ', ');
+}
+
+function formatResult(data) {
+  const name = formatName(data);
+  if (data.skip !== undefined) {
+    const reason = typeof data.skip === 'string' && data.skip.length > 0 ?
+      `: ${escapeCell(data.skip)}` : '';
+    return `${name} | 0 | - | - | - | skipped${reason}\n`;
+  }
+  if (data.error !== undefined) {
+    const message = data.error?.message ?? data.error;
+    return `${name} | ${data.samples.length} | - | - | - | ` +
+      `error: ${escapeCell(message)}\n`;
+  }
+
+  const { confidenceInterval, median } = data.summary;
+  return `${name} | ${data.samples.length} | ` +
+    `${formatRate(data.summary.mean)} | ` +
+    `[${formatRate(confidenceInterval.lower)}, ` +
+    `${formatRate(confidenceInterval.upper)}] | ` +
+    `${formatRate(median)} | ${formatWarning(data.summary)}\n`;
+}
+
+class SpecReporter extends Transform {
+  #diagnostics = [];
+  #reported = false;
+  #results = [];
+
+  constructor() {
+    super({ __proto__: null, writableObjectMode: true });
+  }
+
+  #format(summary = undefined) {
+    let output = kHeader;
+    for (let i = 0; i < this.#results.length; i++) {
+      output += formatResult(this.#results[i]);
+    }
+    for (let i = 0; i < this.#diagnostics.length; i++) {
+      output += `diagnostic: ${escapeCell(this.#diagnostics[i].message)}\n`;
+    }
+    if (summary !== undefined) {
+      const { completed, failed, skipped } = summary.counts;
+      output += `\n${completed} completed, ${failed} failed, ` +
+        `${skipped} skipped\n`;
+    }
+    return output;
+  }
+
+  _transform({ type, data }, _encoding, callback) {
+    switch (type) {
+      case 'bench:complete':
+        ArrayPrototypePush(this.#results, data);
+        break;
+      case 'bench:diagnostic':
+        ArrayPrototypePush(this.#diagnostics, data);
+        break;
+      case 'bench:summary':
+        this.#reported = true;
+        callback(null, this.#format(data));
+        return;
+    }
+    callback();
+  }
+
+  _flush(callback) {
+    if (this.#reported ||
+        (this.#results.length === 0 && this.#diagnostics.length === 0)) {
+      callback();
+      return;
+    }
+    callback(null, this.#format());
+  }
+}
+
+module.exports = SpecReporter;

lib/internal/bootstrap/realm.js

@@ -125,6 +125,7 @@ const legacyWrapperList = new SafeSet([
 // Modules that can only be imported via the node: scheme.
 const schemelessBlockList = new SafeSet([
   'bench',
+  'bench/reporters',
   'dtls',
   'ffi',
   'sea',

test/module-hooks/test-module-hooks-builtin-require.js

@@ -12,6 +12,7 @@ const { registerHooks } = require('module');
 
 const schemelessBlockList = new Set([
   'bench',
+  'bench/reporters',
   'sea',
   'test',
   'test/reporters',

test/module-hooks/test-module-hooks-load-builtin-require.js

@@ -36,6 +36,7 @@ hook.deregister();
 // stripped for internal lookups should not get passed into the hooks.
 const schemelessBlockList = new Set([
   'bench',
+  'bench/reporters',
   'sea',
   'test',
   'test/reporters',

test/parallel/test-bench-custom-reporter.js

@@ -0,0 +1,37 @@
+// Flags: --no-warnings
+'use strict';
+
+const common = require('../common');
+const assert = require('assert');
+const { Writable } = require('stream');
+const { finished } = require('stream/promises');
+const { bench, run } = require('node:bench');
+
+bench('completed', { samples: 1 }, (b) => {
+  b.start();
+  process.hrtime.bigint();
+  b.end(1);
+});
+bench.skip('skipped', { samples: 1 }, common.mustNotCall());
+
+async function* customReporter(source) {
+  for await (const { type, data } of source) {
+    if (type !== 'bench:complete') continue;
+    yield `${data.name}:${data.skip === undefined ? 'completed' : 'skipped'}\n`;
+  }
+}
+
+let output = '';
+const destination = new Writable({
+  write(chunk, _encoding, callback) {
+    output += chunk;
+    callback();
+  },
+});
+
+run().compose(customReporter).pipe(destination);
+
+(async () => {
+  await finished(destination);
+  assert.strictEqual(output, 'completed:completed\nskipped:skipped\n');
+})().then(common.mustCall());