From 523ff0e7edfc47afde3604c3ba12fd53613a97d4 Mon Sep 17 00:00:00 2001 From: RafaelGSS Date: Thu, 26 Oct 2023 00:06:34 -0300 Subject: [PATCH 1/3] lib: add experimental benchmark module MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Vinícius Lourenço --- doc/api/benchmark.md | 293 ++++++++++++++++++ doc/api/errors.md | 7 + doc/api/index.md | 1 + lib/benchmark.js | 10 + lib/internal/benchmark/clock.js | 195 ++++++++++++ lib/internal/benchmark/histogram.js | 207 +++++++++++++ lib/internal/benchmark/lifecycle.js | 95 ++++++ lib/internal/benchmark/report.js | 42 +++ lib/internal/benchmark/runner.js | 122 ++++++++ lib/internal/bootstrap/realm.js | 1 + lib/internal/errors.js | 1 + .../test-benchmarkmodule-experimental.js | 12 + .../test-benchmarkmodule-managed-benchmark.js | 284 +++++++++++++++++ tools/doc/type-parser.mjs | 2 + 14 files changed, 1272 insertions(+) create mode 100644 doc/api/benchmark.md create mode 100644 lib/benchmark.js create mode 100644 lib/internal/benchmark/clock.js create mode 100644 lib/internal/benchmark/histogram.js create mode 100644 lib/internal/benchmark/lifecycle.js create mode 100644 lib/internal/benchmark/report.js create mode 100644 lib/internal/benchmark/runner.js create mode 100644 test/parallel/test-benchmarkmodule-experimental.js create mode 100644 test/parallel/test-benchmarkmodule-managed-benchmark.js diff --git a/doc/api/benchmark.md b/doc/api/benchmark.md new file mode 100644 index 00000000000000..544ebe7d62eb4d --- /dev/null +++ b/doc/api/benchmark.md @@ -0,0 +1,293 @@ +# Benchmark + + + +> Stability: 1.1 - Active Development + + + +The `node:benchmark` module gives the ability to measure +performance of JavaScript code. To access it: + +```mjs +import benchmark from 'node:benchmark'; +``` + +```cjs +const benchmark = require('node:benchmark'); +``` + +This module is only available under the `node:` scheme. The following will not +work: + +```mjs +import benchmark from 'benchmark'; +``` + +```cjs +const benchmark = require('benchmark'); +``` + +The following example illustrates how benchmarks are written using the +`benchmark` module. + +```mjs +import { Suite } from 'node:benchmark'; + +const suite = new Suite(); + +suite.add('Using delete to remove property from object', function() { + const data = { x: 1, y: 2, z: 3 }; + delete data.y; + + data.x; + data.y; + data.z; +}); + +suite.run(); +``` + +```cjs +const { Suite } = require('node:benchmark'); + +const suite = new Suite(); + +suite.add('Using delete to remove property from object', function() { + const data = { x: 1, y: 2, z: 3 }; + delete data.y; + + data.x; + data.y; + data.z; +}); + +suite.run(); +``` + +```console +$ node my-benchmark.js +(node:14165) ExperimentalWarning: The benchmark module is an experimental feature and might change at any time +(Use `node --trace-warnings ...` to show where the warning was created) +Using delete property x 5,853,505 ops/sec ± 0.01% (10 runs sampled) min..max=(169ns ... 171ns) p75=170ns p99=171ns +``` + +## Class: `Suite` + +> Stability: 1.1 Active Development + + + +An `Suite` is responsible for managing and executing +benchmark functions. It provides two methods: `add()` and `run()`. + +### `new Suite([options])` + + + +* `options` {Object} Configuration options for the suite. The following + properties are supported: + * `reporter` {Function} Callback function with results to be called after + benchmark is concluded. The callback function should receive two arguments: + `suite` - A {Suite} object and + `result` - A object containing three properties: + `opsSec` {string}, `iterations {Number}`, `histogram` {Histogram} instance. + +If no `reporter` is provided, the results will printed to the console. + +```mjs +import { Suite } from 'node:benchmark'; +const suite = new Suite(); +``` + +```cjs +const { Suite } = require('node:benchmark'); +const suite = new Suite(); +``` + +### `suite.add(name[, options], fn)` + + + +* `name` {string} The name of the benchmark, which is displayed when reporting + benchmark results. +* `options` {Object} Configuration options for the benchmark. The following + properties are supported: + * `minTime` {number} The minimum time a benchmark can run. + **Default:** `0.05` seconds. + * `maxTime` {number} The maximum time a benchmark can run. + **Default:** `0.5` seconds. +* `fn` {Function|AsyncFunction} +* Returns: {Suite} + +This method stores the benchmark of a given function (`fn`). +The `fn` parameter can be either an asynchronous (`async function () {}`) or +a synchronous (`function () {}`) function. + +```console +$ node my-benchmark.js +(node:14165) ExperimentalWarning: The benchmark module is an experimental feature and might change at any time +(Use `node --trace-warnings ...` to show where the warning was created) +Using delete property x 5,853,505 ops/sec ± 0.01% (10 runs sampled) min..max=(169ns ... 171ns) p75=170ns p99=171ns +``` + +### `suite.run()` + +* Returns: `{Promise>}` + * `opsSec` {number} The amount of operations per second + * `iterations` {number} The amount executions of `fn` + * `histogram` {Histogram} Histogram object used to record benchmark iterations + + + +The purpose of the run method is to run all the benchmarks that have been +added to the suite using the [`suite.add()`][] function. +By calling the run method, you can easily trigger the execution of all +the stored benchmarks and obtain the corresponding results. + +### Using custom reporter + +You can customize the data reporting by passing an function to the `reporter` argument while creating your `Suite`: + +```mjs +import { Suite } from 'node:benchmark'; + +function reporter(bench, result) { + console.log(`Benchmark: ${bench.name} - ${result.opsSec} ops/sec`); +} + +const suite = new Suite({ reporter }); + +suite.add('Using delete to remove property from object', () => { + const data = { x: 1, y: 2, z: 3 }; + delete data.y; + + data.x; + data.y; + data.z; +}); + +suite.run(); +``` + +```cjs +const { Suite } = require('node:benchmark'); + +function reporter(bench, result) { + console.log(`Benchmark: ${bench.name} - ${result.opsSec} ops/sec`); +} + +const suite = new Suite({ reporter }); + +suite.add('Using delete to remove property from object', () => { + const data = { x: 1, y: 2, z: 3 }; + delete data.y; + + data.x; + data.y; + data.z; +}); + +suite.run(); +``` + +```console +$ node my-benchmark.js +Benchmark: Using delete to remove property from object - 6032212 ops/sec +``` + +### Setup and Teardown + +The benchmark function has a special handling when you pass an argument, +for example: + +```cjs +const { Suite } = require('node:benchmark'); +const { readFileSync, writeFileSync, rmSync } = require('node:fs'); + +const suite = new Suite(); + +suite.add('readFileSync', (timer) => { + const randomFile = Date.now(); + const filePath = `./${randomFile}.txt`; + writeFileSync(filePath, Math.random().toString()); + + timer.start(); + readFileSync(filePath, 'utf8'); + timer.end(); + + rmSync(filePath); +}).run(); +``` + +In this way, you can control when the `timer` will start +and also when the `timer` will stop. + +In the timer, we also give you a property `count` +that will tell you how much iterations +you should run your function to achieve the `benchmark.minTime`, +see the following example: + +```mjs +import { Suite } from 'node:benchmark'; +import { readFileSync, writeFileSync, rmSync } from 'node:fs'; + +const suite = new Suite(); + +suite.add('readFileSync', (timer) => { + const randomFile = Date.now(); + const filePath = `./${randomFile}.txt`; + writeFileSync(filePath, Math.random().toString()); + + timer.start(); + for (let i = 0; i < timer.count; i++) + readFileSync(filePath, 'utf8'); + // You must send to the `.end` function the amount of + // times you executed the function, by default, + // the end will be called with value 1. + timer.end(timer.count); + + rmSync(filePath); +}); + +suite.run(); +``` + +```cjs +const { Suite } = require('node:benchmark'); +const { readFileSync, writeFileSync, rmSync } = require('node:fs'); + +const suite = new Suite(); + +suite.add('readFileSync', (timer) => { + const randomFile = Date.now(); + const filePath = `./${randomFile}.txt`; + writeFileSync(filePath, Math.random().toString()); + + timer.start(); + for (let i = 0; i < timer.count; i++) + readFileSync(filePath, 'utf8'); + // You must send to the `.end` function the amount of + // times you executed the function, by default, + // the end will be called with value 1. + timer.end(timer.count); + + rmSync(filePath); +}); + +suite.run(); +``` + +Once your function has at least one argument, +you must call `.start` and `.end`, if you didn't, +it will throw the error [ERR\_BENCHMARK\_MISSING\_OPERATION](./errors.md#err_benchmark_missing_operation). + +[`suite.add()`]: #suiteaddname-options-fn diff --git a/doc/api/errors.md b/doc/api/errors.md index 50e9f658fcbf3a..ede8a6845f316b 100644 --- a/doc/api/errors.md +++ b/doc/api/errors.md @@ -705,6 +705,13 @@ An attempt was made to register something that is not a function as an The type of an asynchronous resource was invalid. Users are also able to define their own types if using the public embedder API. + + +### `ERR_BENCHMARK_MISSING_OPERATION` + +The user forgot to call .start or .end during the execution of +the benchmark. + ### `ERR_BROTLI_COMPRESSION_FAILED` diff --git a/doc/api/index.md b/doc/api/index.md index 81ef77491b1f1b..aa5f08b60d1376 100644 --- a/doc/api/index.md +++ b/doc/api/index.md @@ -13,6 +13,7 @@ * [Assertion testing](assert.md) * [Asynchronous context tracking](async_context.md) * [Async hooks](async_hooks.md) +* [Benchmark](benchmark.md) * [Buffer](buffer.md) * [C++ addons](addons.md) * [C/C++ addons with Node-API](n-api.md) diff --git a/lib/benchmark.js b/lib/benchmark.js new file mode 100644 index 00000000000000..d1490c974e589c --- /dev/null +++ b/lib/benchmark.js @@ -0,0 +1,10 @@ +'use strict'; +const { ObjectAssign } = primordials; +const { Suite } = require('internal/benchmark/runner'); +const { emitExperimentalWarning } = require('internal/util'); + +emitExperimentalWarning('The benchmark module'); + +ObjectAssign(module.exports, { + Suite, +}); diff --git a/lib/internal/benchmark/clock.js b/lib/internal/benchmark/clock.js new file mode 100644 index 00000000000000..bd69ca646ceeb5 --- /dev/null +++ b/lib/internal/benchmark/clock.js @@ -0,0 +1,195 @@ +'use strict'; + +const { + FunctionPrototypeBind, + MathMax, + NumberPrototypeToFixed, + Symbol, + Number, + globalThis, +} = primordials; + +const { + codes: { ERR_BENCHMARK_MISSING_OPERATION }, +} = require('internal/errors'); +const { validateInteger, validateNumber } = require('internal/validators'); + +let debugBench = require('internal/util/debuglog').debuglog('benchmark', (fn) => { + debugBench = fn; +}); + +const kUnmanagedTimerResult = Symbol('kUnmanagedTimerResult'); + +// If the smallest time measurement is 1ns +// the minimum resolution of this timer is 0.5 +const MIN_RESOLUTION = 0.5; + +class Timer { + constructor() { + this.now = process.hrtime.bigint; + } + + get scale() { + return 1e9; + } + + get resolution() { + return 1 / 1e9; + } + + /** + * @param {number} timeInNs + * @returns {string} + */ + format(timeInNs) { + validateNumber(timeInNs, 'timeInNs', 0); + + if (timeInNs > 1e9) { + return `${NumberPrototypeToFixed(timeInNs / 1e9, 2)}s`; + } + + if (timeInNs > 1e6) { + return `${NumberPrototypeToFixed(timeInNs / 1e6, 2)}ms`; + } + + if (timeInNs > 1e3) { + return `${NumberPrototypeToFixed(timeInNs / 1e3, 2)}us`; + } + + if (timeInNs > 1e2) { + return `${NumberPrototypeToFixed(timeInNs, 0)}ns`; + } + + return `${NumberPrototypeToFixed(timeInNs, 2)}ns`; + } +} + +const timer = new Timer(); + +class ManagedTimer { + /** + * @type {bigint} + */ + #start; + /** + * @type {bigint} + */ + #end; + /** + * @type {number} + */ + #iterations; + /** + * @type {number} + */ + #recommendedCount; + + /** + * @param {number} recommendedCount + */ + constructor(recommendedCount) { + this.#recommendedCount = recommendedCount; + } + + /** + * Returns the recommended value to be used to benchmark your code + * @returns {number} + */ + get count() { + return this.#recommendedCount; + } + + /** + * Starts the timer + */ + start() { + this.#start = timer.now(); + } + + /** + * Stops the timer + * @param {number} [iterations=1] The amount of iterations that run + */ + end(iterations = 1) { + this.#end = timer.now(); + + validateInteger(iterations, 'iterations', 1); + + this.#iterations = iterations; + } + + [kUnmanagedTimerResult]() { + if (this.#start === undefined) + throw new ERR_BENCHMARK_MISSING_OPERATION('You forgot to call .start()'); + + if (this.#end === undefined) + throw new ERR_BENCHMARK_MISSING_OPERATION('You forgot to call .end(count)'); + + return [Number(this.#end - this.#start), this.#iterations]; + } +} + +function createRunUnmanagedBenchmark(awaitOrEmpty) { + return ` +const startedAt = timer.now(); + +for (let i = 0; i < count; i++) + ${awaitOrEmpty}bench.fn(); + +const duration = Number(timer.now() - startedAt); +return [duration, count]; +`; +} + +function createRunManagedBenchmark(awaitOrEmpty) { + return ` +${awaitOrEmpty}bench.fn(timer); + +return timer[kUnmanagedTimerResult](); + `; +} + +const AsyncFunction = async function() { }.constructor; +const SyncFunction = function() { }.constructor; + +function createRunner(bench, recommendedCount) { + const isAsync = bench.fn.constructor === AsyncFunction; + const hasArg = bench.fn.length >= 1; + + if (bench.fn.length > 1) { + process.emitWarning(`The benchmark "${bench.name}" function should not have more than 1 argument.`); + } + + const compiledFnStringFactory = hasArg ? createRunManagedBenchmark : createRunUnmanagedBenchmark; + const compiledFnString = compiledFnStringFactory(isAsync ? 'await ' : ''); + const createFnPrototype = isAsync ? AsyncFunction : SyncFunction; + const compiledFn = createFnPrototype('bench', 'timer', 'count', 'kUnmanagedTimerResult', compiledFnString); + + const selectedTimer = hasArg ? new ManagedTimer(recommendedCount) : timer; + + const runner = FunctionPrototypeBind( + compiledFn, globalThis, bench, selectedTimer, recommendedCount, kUnmanagedTimerResult); + + debugBench(`Created compiled benchmark, hasArg=${hasArg}, isAsync=${isAsync}, recommendedCount=${recommendedCount}`); + + return runner; +} + +async function clockBenchmark(bench, recommendedCount) { + const runner = createRunner(bench, recommendedCount); + const result = await runner(); + + // Just to avoid issues with empty fn + result[0] = MathMax(MIN_RESOLUTION, result[0]); + + debugBench(`Took ${timer.format(result[0])} to execute ${result[1]} iterations`); + + return result; +} + +module.exports = { + clockBenchmark, + debugBench, + timer, + MIN_RESOLUTION, +}; diff --git a/lib/internal/benchmark/histogram.js b/lib/internal/benchmark/histogram.js new file mode 100644 index 00000000000000..86774c6ad841c6 --- /dev/null +++ b/lib/internal/benchmark/histogram.js @@ -0,0 +1,207 @@ +'use strict'; + +const { validateNumber } = require('internal/validators'); +const { codes: { ERR_OUT_OF_RANGE } } = require('internal/errors'); +const { debugBench } = require('internal/benchmark/clock'); +const { + MathMin, + MathMax, + MathCeil, + MathSqrt, + MathPow, + MathFloor, + NumberMAX_SAFE_INTEGER, + ArrayPrototypeSort, + ArrayPrototypePush, + ArrayPrototypeFilter, + ArrayPrototypeReduce, + ArrayPrototypeSlice, + NumberIsNaN, + Symbol, +} = primordials; + +const kStatisticalHistogramRecord = Symbol('kStatisticalHistogramRecord'); +const kStatisticalHistogramFinish = Symbol('kStatisticalHistogramFinish'); + +class StatisticalHistogram { + /** + * @type {number[]} + * @default [] + */ + #all = []; + /** + * @type {number} + */ + #min = undefined; + /** + * @type {number} + */ + #max = undefined; + /** + * @type {number} + */ + #mean = undefined; + /** + * @type {number} + */ + #cv = undefined; + + /** + * @returns {number[]} + */ + get samples() { + return ArrayPrototypeSlice(this.#all); + } + + /** + * @returns {number} + */ + get min() { + return this.#min; + } + + /** + * @returns {number} + */ + get max() { + return this.#max; + } + + /** + * @returns {number} + */ + get mean() { + return this.#mean; + } + + /** + * @returns {number} + */ + get cv() { + return this.#cv; + } + + /** + * @param {number} percentile + * @returns {number} + */ + percentile(percentile) { + validateNumber(percentile, 'percentile'); + + if (NumberIsNaN(percentile) || percentile < 0 || percentile > 100) + throw new ERR_OUT_OF_RANGE('percentile', '>= 0 && <= 100', percentile); + + if (this.#all.length === 0) + return 0; + + if (percentile === 0) + return this.#all[0]; + + return this.#all[MathCeil(this.#all.length * (percentile / 100)) - 1]; + } + + /** + * @param {number} value + */ + [kStatisticalHistogramRecord](value) { + validateNumber(value, 'value', 0); + + ArrayPrototypePush(this.#all, value); + } + + [kStatisticalHistogramFinish]() { + this.#removeOutliers(); + + this.#calculateMinMax(); + this.#calculateMean(); + this.#calculateCv(); + } + + /** + * References: + * - https://gist.github.com/rmeissn/f5b42fb3e1386a46f60304a57b6d215a + * - https://en.wikipedia.org/wiki/Interquartile_range + */ + #removeOutliers() { + ArrayPrototypeSort(this.#all, (a, b) => a - b); + + const size = this.#all.length; + + if (size < 4) + return; + + let q1, q3; + + if ((size - 1) / 4 % 1 === 0 || size / 4 % 1 === 0) { + q1 = 1 / 2 * (this.#all[MathFloor(size / 4) - 1] + this.#all[MathFloor(size / 4)]); + q3 = 1 / 2 * (this.#all[MathCeil(size * 3 / 4) - 1] + this.#all[MathCeil(size * 3 / 4)]); + } else { + q1 = this.#all[MathFloor(size / 4)]; + q3 = this.#all[MathFloor(size * 3 / 4)]; + } + + const iqr = q3 - q1; + const minValue = q1 - iqr * 1.5; + const maxValue = q3 + iqr * 1.5; + + this.#all = ArrayPrototypeFilter( + this.#all, + (value) => (value <= maxValue) && (value >= minValue), + ); + } + + #calculateMinMax() { + this.#min = Infinity; + this.#max = -Infinity; + + for (let i = 0; i < this.#all.length; i++) { + this.#min = MathMin(this.#all[i], this.#min); + this.#max = MathMax(this.#all[i], this.#max); + } + } + + #calculateMean() { + if (this.#all.length === 0) { + this.#mean = 0; + return; + } + + if (this.#all.length === 1) { + this.#mean = this.#all[0]; + return; + } + + this.#mean = ArrayPrototypeReduce( + this.#all, + (acc, value) => MathMin(NumberMAX_SAFE_INTEGER, acc + value), + 0, + ) / this.#all.length; + } + + /** + * References: + * - https://en.wikipedia.org/wiki/Coefficient_of_variation + * - https://github.com/google/benchmark/blob/159eb2d0ffb85b86e00ec1f983d72e72009ec387/src/statistics.cc#L81-L88 + */ + #calculateCv() { + if (this.#all.length < 2) { + this.#cv = 0; + return; + } + + const avgSquares = ArrayPrototypeReduce( + this.#all, + (acc, value) => MathMin(NumberMAX_SAFE_INTEGER, MathPow(value, 2) + acc), 0, + ) * (1 / this.#all.length); + + const stddev = MathSqrt(MathMax(0, this.#all.length / (this.#all.length - 1) * (avgSquares - (this.#mean * this.#mean)))); + + this.#cv = stddev / this.#mean; + } +} + +module.exports = { + StatisticalHistogram, + kStatisticalHistogramRecord, + kStatisticalHistogramFinish, +}; diff --git a/lib/internal/benchmark/lifecycle.js b/lib/internal/benchmark/lifecycle.js new file mode 100644 index 00000000000000..15c3e973b662a0 --- /dev/null +++ b/lib/internal/benchmark/lifecycle.js @@ -0,0 +1,95 @@ +'use strict'; + +const { + MathMin, + MathMax, + MathRound, + Number, + NumberMAX_SAFE_INTEGER, +} = primordials; +const { clockBenchmark, debugBench, timer, MIN_RESOLUTION } = require('internal/benchmark/clock'); +const { + StatisticalHistogram, + kStatisticalHistogramRecord, + kStatisticalHistogramFinish, +} = require('internal/benchmark/histogram'); + +// 0.05 - Arbitrary number used in some benchmark tools +const minTime = 0.05; + +// 0.5s - Arbitrary number used in some benchmark tools +const maxTime = 0.5; + +/** + * @param {number} durationPerOp The amount of time each operation takes + * @param {number} targetTime The amount of time we want the benchmark to execute + */ +function getItersForOpDuration(durationPerOp, targetTime) { + const totalOpsForMinTime = targetTime / (durationPerOp / timer.scale); + + return MathMin(NumberMAX_SAFE_INTEGER, MathMax(1, MathRound(totalOpsForMinTime))); +} + +async function getInitialIterations(bench) { + const { 0: duration, 1: realIterations } = await clockBenchmark(bench, 30); + + // Just to avoid issues with empty fn + const durationPerOp = MathMax(MIN_RESOLUTION, duration / realIterations); + debugBench(`Duration per operation on initial count: ${timer.format(durationPerOp)}`); + + if ((durationPerOp / timer.scale) >= bench.minTime) + process.emitWarning(`The benchmark "${bench.name}" has a duration per operation greater than the minTime.`); + + return getItersForOpDuration(durationPerOp, bench.minTime); +} + +async function runBenchmark(bench, initialIterations) { + const histogram = new StatisticalHistogram(); + + let startClock; + let benchTimeSpent = 0; + const maxDuration = bench.maxTime * timer.scale; + + let iterations = 0; + let timeSpent = 0; + + while (benchTimeSpent < maxDuration) { + startClock = timer.now(); + const { 0: duration, 1: realIterations } = await clockBenchmark(bench, initialIterations); + benchTimeSpent += Number(timer.now() - startClock); + + iterations += realIterations; + iterations = MathMin(NumberMAX_SAFE_INTEGER, iterations); + + timeSpent += duration; + + // Just to avoid issues with empty fn + const durationPerOp = MathMax(MIN_RESOLUTION, duration / realIterations); + + histogram[kStatisticalHistogramRecord](durationPerOp); + + if (benchTimeSpent >= maxDuration) + break; + + const minWindowTime = MathMax(0, MathMin((maxDuration - benchTimeSpent) / timer.scale, bench.minTime)); + initialIterations = getItersForOpDuration(durationPerOp, minWindowTime); + } + + histogram[kStatisticalHistogramFinish](); + + const opsSec = iterations / (timeSpent / timer.scale); + + return { + __proto__: null, + opsSec, + iterations, + histogram, + }; +} + +module.exports = { + minTime, + maxTime, + runBenchmark, + getInitialIterations, +}; diff --git a/lib/internal/benchmark/report.js b/lib/internal/benchmark/report.js new file mode 100644 index 00000000000000..e280399b53434f --- /dev/null +++ b/lib/internal/benchmark/report.js @@ -0,0 +1,42 @@ +'use strict'; + +const { + NumberPrototypeToFixed, + globalThis, +} = primordials; +const { Intl } = globalThis; + +const { timer } = require('internal/benchmark/clock'); + +const formatter = Intl.NumberFormat(undefined, { + notation: 'standard', + maximumFractionDigits: 2, +}); + +function reportConsoleBench(bench, result) { + const opsSecReported = result.opsSec < 100 ? + NumberPrototypeToFixed(result.opsSec, 2) : + NumberPrototypeToFixed(result.opsSec, 0); + + process.stdout.write(bench.name); + process.stdout.write(' x '); + process.stdout.write(`${formatter.format(opsSecReported)} ops/sec +/- `); + process.stdout.write(formatter.format( + NumberPrototypeToFixed(result.histogram.cv, 2)), + ); + process.stdout.write(`% (${result.histogram.samples.length} runs sampled)\t`); + + process.stdout.write('min..max=('); + process.stdout.write(timer.format(result.histogram.min)); + process.stdout.write(' ... '); + process.stdout.write(timer.format(result.histogram.max)); + process.stdout.write(') p75='); + process.stdout.write(timer.format(result.histogram.percentile(75))); + process.stdout.write(' p99='); + process.stdout.write(timer.format(result.histogram.percentile(99))); + process.stdout.write('\n'); +} + +module.exports = { + reportConsoleBench, +}; diff --git a/lib/internal/benchmark/runner.js b/lib/internal/benchmark/runner.js new file mode 100644 index 00000000000000..f5ec7684e89a1f --- /dev/null +++ b/lib/internal/benchmark/runner.js @@ -0,0 +1,122 @@ +'use strict'; + +const { ArrayPrototypePush } = primordials; +const { debugBench, timer } = require('internal/benchmark/clock'); +const { reportConsoleBench } = require('internal/benchmark/report'); +const { maxTime, minTime, runBenchmark, getInitialIterations } = require('internal/benchmark/lifecycle'); +const { validateNumber, validateString, validateFunction, validateObject } = require('internal/validators'); + +class Benchmark { + #name; + #fn; + #minTime; + #maxTime; + + constructor(name, fn, minTime, maxTime) { + this.#name = name; + this.#fn = fn; + this.#minTime = minTime; + this.#maxTime = maxTime; + } + + get name() { + return this.#name; + } + + get fn() { + return this.#fn; + } + + get minTime() { + return this.#minTime; + } + + get maxTime() { + return this.#maxTime; + } +} + +class Suite { + #benchmarks; + #reporter; + + constructor(options) { + this.#benchmarks = []; + + if (typeof options !== 'undefined') { + validateObject(options, 'options'); + + options.reporter ??= reportConsoleBench; + + validateFunction(options.reporter, 'options.reporter'); + + this.#reporter = options.reportConsoleBench; + } else { + this.#reporter = reportConsoleBench; + } + } + + add(name, options, fn) { + validateString(name, 'name'); + + let benchOptions; + let benchFn; + + if (typeof options === 'object') { + benchOptions = options; + benchFn = fn; + } else if (typeof options === 'function') { + benchFn = options; + benchOptions = fn; + } + + validateFunction(benchFn, 'fn'); + + if (typeof benchOptions !== 'undefined') { + validateObject(benchOptions, 'options'); + } + + const benchMinTime = benchOptions?.minTime ?? minTime; + const benchMaxTime = benchOptions?.maxTime ?? maxTime; + + validateNumber(benchMinTime, 'options.minTime', timer.resolution * 1e3); + validateNumber(benchMaxTime, 'options.maxTime', benchMinTime); + + const benchmark = new Benchmark(name, benchFn, benchMinTime, benchMaxTime); + + ArrayPrototypePush(this.#benchmarks, benchmark); + + return this; + } + + async run() { + const results = new Array(this.#benchmarks.length); + const initialIterations = new Array(this.#benchmarks.length); + + // We need to calculate the initial iterations first + // because this helps to reduce noise/bias on the results, + // when changing the order of the benchmarks also changes + // the amount of ops/sec + for (let i = 0; i < this.#benchmarks.length; i++) { + initialIterations[i] = await getInitialIterations(this.#benchmarks[i]); + } + + for (let i = 0; i < this.#benchmarks.length; i++) { + const benchmark = this.#benchmarks[i]; + const initialIteration = initialIterations[i]; + + debugBench(`Starting ${benchmark.name} with minTime=${benchmark.minTime}, maxTime=${benchmark.maxTime}`); + + const result = await runBenchmark(benchmark, initialIteration); + results[i] = result; + + this.#reporter(benchmark, result); + } + + return results; + } +} + +module.exports = { + Suite, +}; diff --git a/lib/internal/bootstrap/realm.js b/lib/internal/bootstrap/realm.js index 6034af9a36003c..fc9a60b877e83e 100644 --- a/lib/internal/bootstrap/realm.js +++ b/lib/internal/bootstrap/realm.js @@ -136,6 +136,7 @@ const legacyWrapperList = new SafeSet([ // beginning with "internal/". // Modules that can only be imported via the node: scheme. const schemelessBlockList = new SafeSet([ + 'benchmark', 'test', 'test/reporters', ]); diff --git a/lib/internal/errors.js b/lib/internal/errors.js index 39e512762a470c..31776b94b4702b 100644 --- a/lib/internal/errors.js +++ b/lib/internal/errors.js @@ -1091,6 +1091,7 @@ E('ERR_ARG_NOT_ITERABLE', '%s must be iterable', TypeError); E('ERR_ASSERTION', '%s', Error); E('ERR_ASYNC_CALLBACK', '%s must be a function', TypeError); E('ERR_ASYNC_TYPE', 'Invalid name for async "type": %s', TypeError); +E('ERR_BENCHMARK_MISSING_OPERATION', '%s', Error); E('ERR_BROTLI_INVALID_PARAM', '%s is not a valid Brotli parameter', RangeError); E('ERR_BUFFER_OUT_OF_BOUNDS', // Using a default argument here is important so the argument is not counted diff --git a/test/parallel/test-benchmarkmodule-experimental.js b/test/parallel/test-benchmarkmodule-experimental.js new file mode 100644 index 00000000000000..1d0bbcdaa51191 --- /dev/null +++ b/test/parallel/test-benchmarkmodule-experimental.js @@ -0,0 +1,12 @@ +'use strict'; + +const common = require('../common'); +const assert = require('assert'); + +require('node:benchmark'); + +// This test ensures that the experimental message is emitted +// when using permission system +process.on('warning', common.mustCall((warning) => { + assert.match(warning.message, /The benchmark module is an experimental feature/); +}, 1)); diff --git a/test/parallel/test-benchmarkmodule-managed-benchmark.js b/test/parallel/test-benchmarkmodule-managed-benchmark.js new file mode 100644 index 00000000000000..0e9fcb8631454f --- /dev/null +++ b/test/parallel/test-benchmarkmodule-managed-benchmark.js @@ -0,0 +1,284 @@ +'use strict'; + +const common = require('../common'); +const assert = require('assert'); +const { spawnSync } = require('child_process'); +const { Suite } = require('node:benchmark'); +const { setTimeout } = require('timers/promises'); + +(async () => { + { + const suite = new Suite(); + const results = await suite.run(); + + assert.ok(Array.isArray(results)); + assert.strictEqual(results.length, 0); + } + + { + const suite = new Suite(); + + suite.add('wait 10ms', async () => { + await setTimeout(10); + }); + + const results = await suite.run(); + + assert.ok(Array.isArray(results)); + assert.strictEqual(results.length, 1); + + const firstResult = results[0]; + + assert.strictEqual(typeof firstResult.opsSec, 'number'); + assert.strictEqual(typeof firstResult.iterations, 'number'); + assert.strictEqual(typeof firstResult.histogram, 'object'); + assert.strictEqual(typeof firstResult.histogram.cv, 'number'); + assert.strictEqual(typeof firstResult.histogram.min, 'number'); + assert.strictEqual(typeof firstResult.histogram.max, 'number'); + assert.strictEqual(typeof firstResult.histogram.mean, 'number'); + assert.strictEqual(typeof firstResult.histogram.percentile, 'function'); + assert.strictEqual(typeof firstResult.histogram.percentile(1), 'number'); + assert.strictEqual(firstResult.histogram.percentile(1), firstResult.histogram.samples[0]); + assert.strictEqual( + firstResult.histogram.percentile(100), + firstResult.histogram.samples[firstResult.histogram.samples.length - 1], + ); + assert.ok(Array.isArray(firstResult.histogram.samples)); + + assert.ok(firstResult.histogram.samples.length >= 1, `received ${firstResult.histogram.samples.length}`); + assert.ok(firstResult.opsSec >= 1, `received ${firstResult.opsSec}`); + } + + { + const suite = new Suite(); + + suite.add('async empty', async () => { }); + suite.add('sync empty', () => { }); + + const results = await suite.run(); + + assert.strictEqual(results.length, 2); + assert.ok(results[0].opsSec > 0); + assert.ok(results[1].opsSec > 0); + } + + { + const suite = new Suite(); + + suite.add('async empty', async (timer) => { + timer.start(); + timer.end(); + }); + suite.add('sync empty', (timer) => { + timer.start(); + timer.end(); + }); + + const results = await suite.run(); + + assert.strictEqual(results.length, 2); + assert.ok(results[0].opsSec > 0); + assert.ok(results[1].opsSec > 0); + } + + { + const suite = new Suite(); + + suite.add('empty', (timer) => { }); + + await assert.rejects(async () => { + return await suite.run(); + }, common.expectsError({ + code: 'ERR_BENCHMARK_MISSING_OPERATION', + message: 'You forgot to call .start()', + })); + } + + { + const suite = new Suite(); + + suite.add('async empty', async (timer) => { + timer.start(); + }); + + await assert.rejects(async () => { + return await suite.run(); + }, common.expectsError({ + code: 'ERR_BENCHMARK_MISSING_OPERATION', + message: 'You forgot to call .end(count)', + })); + } + + { + const suite = new Suite(); + + suite.add('empty', (timer) => { + timer.end(); + }); + + await assert.rejects(async () => { + return await suite.run(); + }, common.expectsError({ + code: 'ERR_BENCHMARK_MISSING_OPERATION', + message: 'You forgot to call .start()', + })); + } + + { + const suite = new Suite(); + + suite.add('empty', (timer) => { + timer.end('error'); + }); + + await assert.rejects(async () => { + return await suite.run(); + }, common.expectsError({ + code: 'ERR_INVALID_ARG_TYPE', + message: 'The "iterations" argument must be of type number. Received type string (\'error\')', + })); + } + + { + const suite = new Suite(); + + suite.add('empty', (timer) => { + timer.end(-1); + }); + + await assert.rejects(async () => { + return await suite.run(); + }, common.expectsError({ + code: 'ERR_OUT_OF_RANGE', + message: 'The value of "iterations" is out of range. It must be >= 1 && <= 9007199254740991. Received -1', + })); + } + + { + const { status, stdout, stderr } = spawnSync( + process.execPath, + [ + '-e', ` + const { Suite } = require('node:benchmark'); + const { setTimeout } = require('timers/promises'); + + (async () => { + const suite = new Suite(); + + suite.add('wait 10ms', async () => { + await setTimeout(10); + }); + + const results = await suite.run(); + })()`, + ], + ); + + assert.strictEqual(status, 0, stderr.toString()); + + const stdoutString = stdout.toString(); + assert.ok(stdoutString.includes('wait 10ms'), stdoutString); + assert.ok(stdoutString.includes('ops/sec'), stdoutString); + assert.ok(stdoutString.includes('+/-'), stdoutString); + assert.ok(stdoutString.includes('min..max'), stdoutString); + assert.ok(stdoutString.includes('runs sampled'), stdoutString); + assert.ok(stdoutString.includes('p75'), stdoutString); + assert.ok(stdoutString.includes('p99'), stdoutString); + } +})().then(common.mustCall()); + +{ + const suite = new Suite(); + + assert.throws(() => { + suite.add(); + }, common.expectsError({ + code: 'ERR_INVALID_ARG_TYPE', + message: 'The "name" argument must be of type string. Received undefined', + })); + + assert.throws(() => { + suite.add('test'); + }, common.expectsError({ + code: 'ERR_INVALID_ARG_TYPE', + message: 'The "fn" argument must be of type function. Received undefined', + })); + + assert.throws(() => { + suite.add('test', () => { }, 123); + }, common.expectsError({ + code: 'ERR_INVALID_ARG_TYPE', + message: 'The "options" argument must be of type object. Received type number (123)', + })); + + assert.throws(() => { + suite.add('empty fn', () => { }, { + minTime: 10, + maxTime: 1, + }); + }, common.expectsError({ + code: 'ERR_OUT_OF_RANGE', + message: 'The value of "options.maxTime" is out of range. It must be >= 10. Received 1', + })); + + assert.throws(() => { + suite.add('empty fn', { + minTime: 10, + maxTime: 1, + }, () => { }); + }, common.expectsError({ + code: 'ERR_OUT_OF_RANGE', + message: 'The value of "options.maxTime" is out of range. It must be >= 10. Received 1', + })); + + assert.throws(() => { + suite.add('empty fn', { + minTime: 1e9, + }, () => { }); + }, common.expectsError({ + code: 'ERR_OUT_OF_RANGE', + message: 'The value of "options.maxTime" is out of range. It must be >= 1000000000. Received 0.5', + })); + + assert.throws(() => { + suite.add('empty fn', { + minTime: '1e9', + }, () => { }); + }, common.expectsError({ + code: 'ERR_INVALID_ARG_TYPE', + message: 'The "options.minTime" property must be of type number. Received type string (\'1e9\')', + })); + + assert.throws(() => { + suite.add('empty fn', { + maxTime: '1e9', + }, () => { }); + }, common.expectsError({ + code: 'ERR_INVALID_ARG_TYPE', + message: 'The "options.maxTime" property must be of type number. Received type string (\'1e9\')', + })); +} + +{ + assert.throws( + () => new Suite(1), + common.expectsError({ + code: 'ERR_INVALID_ARG_TYPE', + message: 'The "options" argument must be of type object. Received type number (1)' + }), + ); + assert.throws( + () => new Suite({ reporter: 'non-function' }), + common.expectsError({ + code: 'ERR_INVALID_ARG_TYPE', + message: 'The "options.reporter" property must be of type function. Received type string (\'non-function\')', + }), + ); + assert.throws( + () => new Suite({ reporter: {} }), + common.expectsError({ + code: 'ERR_INVALID_ARG_TYPE', + message: 'The "options.reporter" property must be of type function. Received an instance of Object', + }), + ); +} diff --git a/tools/doc/type-parser.mjs b/tools/doc/type-parser.mjs index d4d75f3d7482d8..4c84bf552530aa 100644 --- a/tools/doc/type-parser.mjs +++ b/tools/doc/type-parser.mjs @@ -207,6 +207,8 @@ const customTypesMap = { 'stream.Writable': 'stream.html#class-streamwritable', 'Writable': 'stream.html#class-streamwritable', + 'Suite': 'benchmark.html#class-suite', + 'Immediate': 'timers.html#class-immediate', 'Timeout': 'timers.html#class-timeout', 'Timer': 'timers.html#timers', From ddde62e0fbd8952cc54900303b0f4ae4051de25a Mon Sep 17 00:00:00 2001 From: RafaelGSS Date: Sat, 18 Nov 2023 09:15:12 -0500 Subject: [PATCH 2/3] fixup! use ERR_INVALID_STATE instead of new error --- doc/api/errors.md | 7 ------- lib/internal/benchmark/clock.js | 6 +++--- lib/internal/errors.js | 1 - .../test-benchmarkmodule-managed-benchmark.js | 12 ++++++------ 4 files changed, 9 insertions(+), 17 deletions(-) diff --git a/doc/api/errors.md b/doc/api/errors.md index ede8a6845f316b..50e9f658fcbf3a 100644 --- a/doc/api/errors.md +++ b/doc/api/errors.md @@ -705,13 +705,6 @@ An attempt was made to register something that is not a function as an The type of an asynchronous resource was invalid. Users are also able to define their own types if using the public embedder API. - - -### `ERR_BENCHMARK_MISSING_OPERATION` - -The user forgot to call .start or .end during the execution of -the benchmark. - ### `ERR_BROTLI_COMPRESSION_FAILED` diff --git a/lib/internal/benchmark/clock.js b/lib/internal/benchmark/clock.js index bd69ca646ceeb5..58a045a110d385 100644 --- a/lib/internal/benchmark/clock.js +++ b/lib/internal/benchmark/clock.js @@ -10,7 +10,7 @@ const { } = primordials; const { - codes: { ERR_BENCHMARK_MISSING_OPERATION }, + codes: { ERR_INVALID_STATE }, } = require('internal/errors'); const { validateInteger, validateNumber } = require('internal/validators'); @@ -120,10 +120,10 @@ class ManagedTimer { [kUnmanagedTimerResult]() { if (this.#start === undefined) - throw new ERR_BENCHMARK_MISSING_OPERATION('You forgot to call .start()'); + throw new ERR_INVALID_STATE('You forgot to call .start()'); if (this.#end === undefined) - throw new ERR_BENCHMARK_MISSING_OPERATION('You forgot to call .end(count)'); + throw new ERR_INVALID_STATE('You forgot to call .end(count)'); return [Number(this.#end - this.#start), this.#iterations]; } diff --git a/lib/internal/errors.js b/lib/internal/errors.js index 31776b94b4702b..39e512762a470c 100644 --- a/lib/internal/errors.js +++ b/lib/internal/errors.js @@ -1091,7 +1091,6 @@ E('ERR_ARG_NOT_ITERABLE', '%s must be iterable', TypeError); E('ERR_ASSERTION', '%s', Error); E('ERR_ASYNC_CALLBACK', '%s must be a function', TypeError); E('ERR_ASYNC_TYPE', 'Invalid name for async "type": %s', TypeError); -E('ERR_BENCHMARK_MISSING_OPERATION', '%s', Error); E('ERR_BROTLI_INVALID_PARAM', '%s is not a valid Brotli parameter', RangeError); E('ERR_BUFFER_OUT_OF_BOUNDS', // Using a default argument here is important so the argument is not counted diff --git a/test/parallel/test-benchmarkmodule-managed-benchmark.js b/test/parallel/test-benchmarkmodule-managed-benchmark.js index 0e9fcb8631454f..aa097a8e0f0ad0 100644 --- a/test/parallel/test-benchmarkmodule-managed-benchmark.js +++ b/test/parallel/test-benchmarkmodule-managed-benchmark.js @@ -89,8 +89,8 @@ const { setTimeout } = require('timers/promises'); await assert.rejects(async () => { return await suite.run(); }, common.expectsError({ - code: 'ERR_BENCHMARK_MISSING_OPERATION', - message: 'You forgot to call .start()', + code: 'ERR_INVALID_STATE', + message: 'Invalid state: You forgot to call .start()', })); } @@ -104,8 +104,8 @@ const { setTimeout } = require('timers/promises'); await assert.rejects(async () => { return await suite.run(); }, common.expectsError({ - code: 'ERR_BENCHMARK_MISSING_OPERATION', - message: 'You forgot to call .end(count)', + code: 'ERR_INVALID_STATE', + message: 'Invalid state: You forgot to call .end(count)', })); } @@ -119,8 +119,8 @@ const { setTimeout } = require('timers/promises'); await assert.rejects(async () => { return await suite.run(); }, common.expectsError({ - code: 'ERR_BENCHMARK_MISSING_OPERATION', - message: 'You forgot to call .start()', + code: 'ERR_INVALID_STATE', + message: 'Invalid state: You forgot to call .start()', })); } From fde16c7c4c010a4aadd1c13c4827363483a6ab50 Mon Sep 17 00:00:00 2001 From: RafaelGSS Date: Sat, 18 Nov 2023 09:17:55 -0500 Subject: [PATCH 3/3] fixup! remove explicit undefined assignment --- lib/internal/benchmark/histogram.js | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/lib/internal/benchmark/histogram.js b/lib/internal/benchmark/histogram.js index 86774c6ad841c6..6ea2e46e74f193 100644 --- a/lib/internal/benchmark/histogram.js +++ b/lib/internal/benchmark/histogram.js @@ -2,7 +2,6 @@ const { validateNumber } = require('internal/validators'); const { codes: { ERR_OUT_OF_RANGE } } = require('internal/errors'); -const { debugBench } = require('internal/benchmark/clock'); const { MathMin, MathMax, @@ -32,19 +31,19 @@ class StatisticalHistogram { /** * @type {number} */ - #min = undefined; + #min; /** * @type {number} */ - #max = undefined; + #max; /** * @type {number} */ - #mean = undefined; + #mean; /** * @type {number} */ - #cv = undefined; + #cv; /** * @returns {number[]}