diff --git a/.ci/schedule-daily.groovy b/.ci/schedule-daily.groovy index 01d8f1add5..00d3cf0e44 100644 --- a/.ci/schedule-daily.groovy +++ b/.ci/schedule-daily.groovy @@ -41,6 +41,7 @@ pipeline { job: 'apm-agent-nodejs/apm-agent-nodejs-mbp/master', parameters: [ booleanParam(name: 'Run_As_Master_Branch', value: true), + booleanParam(name: 'bench_ci', value: false), booleanParam(name: 'doc_ci', value: true), booleanParam(name: 'tav_ci', value: true), booleanParam(name: 'test_edge_ci', value: true) diff --git a/.ci/scripts/prepare-benchmarks-env.sh b/.ci/scripts/prepare-benchmarks-env.sh new file mode 100755 index 0000000000..84ab44bf63 --- /dev/null +++ b/.ci/scripts/prepare-benchmarks-env.sh @@ -0,0 +1,21 @@ +#!/usr/bin/env bash +set -eo pipefail + +# This particular configuration is required to be installed in the baremetal +curl -o- https://raw.githubusercontent.com/nvm-sh/nvm/v0.34.0/install.sh | bash +export NVM_DIR="$HOME/.nvm" +[ -s "$NVM_DIR/nvm.sh" ] && \. "$NVM_DIR/nvm.sh" +command -v nvm + +## If NODE_VERSION env variable exists then use it otherwise use node as default +if [ -z "${NODE_VERSION}" ] ; then + NODE_VERSION="node" +fi +nvm install ${NODE_VERSION} + +set +x +npm config list +npm install + +node --version +npm --version diff --git a/.ci/scripts/run-benchmarks.sh b/.ci/scripts/run-benchmarks.sh new file mode 100755 index 0000000000..1dced5e7db --- /dev/null +++ b/.ci/scripts/run-benchmarks.sh @@ -0,0 +1,9 @@ +#!/usr/bin/env bash +set -ueo pipefail + +SCRIPTPATH=$(dirname "$0") +source ./${SCRIPTPATH}/prepare-benchmarks-env.sh + +RESULT_FILE=${1:-apm-agent-benchmark-results.json} + +npm run bench:ci ${RESULT_FILE} diff --git a/.gitignore b/.gitignore index 18715098e1..bf09eabe3e 100644 --- a/.gitignore +++ b/.gitignore @@ -10,3 +10,4 @@ test/types/transpile/index.js build coverage node_modules +test/benchmarks/.tmp diff --git a/Jenkinsfile b/Jenkinsfile index d16f662f96..a9c129c76d 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -28,6 +28,7 @@ pipeline { } parameters { booleanParam(name: 'Run_As_Master_Branch', defaultValue: false, description: 'Allow to run any steps on a PR, some steps normally only run on master branch.') + booleanParam(name: 'bench_ci', defaultValue: true, description: 'Enable benchmarks.') booleanParam(name: 'tav_ci', defaultValue: true, description: 'Enable TAV tests.') booleanParam(name: 'tests_ci', defaultValue: true, description: 'Enable tests.') booleanParam(name: 'test_edge_ci', defaultValue: true, description: 'Enable tests for edge versions of nodejs.') @@ -269,6 +270,52 @@ pipeline { githubNotify(context: "${env.GITHUB_CHECK_ITS_NAME}", description: "${env.GITHUB_CHECK_ITS_NAME} ...", status: 'PENDING', targetUrl: "${env.JENKINS_URL}search/?q=${env.ITS_PIPELINE.replaceAll('/','+')}") } } + /** + Run the benchmarks and store the results on ES. + The result JSON files are also archive into Jenkins. + */ + stage('Benchmarks') { + agent { label 'metal' } + options { skipDefaultCheckout() } + environment { + HOME = "${env.WORKSPACE}" + RESULT_FILE = 'apm-agent-benchmark-results.json' + NODE_VERSION = '12' + } + when { + beforeAgent true + allOf { + anyOf { + branch 'master' + tag pattern: 'v\\d+\\.\\d+\\.\\d+.*', comparator: 'REGEXP' + expression { return params.Run_As_Master_Branch } + } + expression { return params.bench_ci } + } + } + steps { + withGithubNotify(context: 'Benchmarks', tab: 'artifacts') { + dir(env.BUILD_NUMBER) { + deleteDir() + unstash 'source' + dir(BASE_DIR){ + sh '.ci/scripts/run-benchmarks.sh "${RESULT_FILE}"' + } + } + } + } + post { + always { + catchError(message: 'sendBenchmarks failed', buildResult: 'FAILURE') { + sendBenchmarks(file: "${BUILD_NUMBER}/${BASE_DIR}/${RESULT_FILE}", + index: 'benchmark-nodejs', archive: true) + } + catchError(message: 'deleteDir failed', buildResult: 'SUCCESS', stageResult: 'UNSTABLE') { + deleteDir() + } + } + } + } } post { cleanup { diff --git a/TESTING.md b/TESTING.md index 1bce59b045..37212f00ac 100644 --- a/TESTING.md +++ b/TESTING.md @@ -124,3 +124,9 @@ Clean up Docker containers and volumes: ``` npm run docker:clean ``` + +Run the benchmarks: + +``` +npm run bench +``` diff --git a/package.json b/package.json index 2fa424744e..9097264d14 100644 --- a/package.json +++ b/package.json @@ -21,6 +21,8 @@ "test:types": "tsc --project test/types/tsconfig.json && tsc --project test/types/transpile/tsconfig.json && node test/types/transpile/index.js", "test:babel": "babel test/babel/src.js --out-file test/babel/out.js && node test/babel/out.js", "test:esm": "node --experimental-modules test/esm", + "bench": "./test/benchmarks/scripts/run-benchmarks.sh", + "bench:ci": "./test/benchmarks/scripts/run-benchmarks-ci.sh", "local:start": "./test/script/local-deps-start.sh", "local:stop": "./test/script/local-deps-stop.sh", "docker:start": "docker-compose -f ./test/docker-compose.yml up -d", @@ -113,8 +115,10 @@ "@types/node": "^12.0.8", "apollo-server-express": "^2.6.3", "aws-sdk": "^2.477.0", + "benchmark": "^2.1.4", "bluebird": "^3.5.2", "cassandra-driver": "^4.0.0", + "columnify": "^1.5.4", "commitlint-config-squash-pr": "^1.0.0", "connect": "^3.7.0", "container-info": "^1.0.1", @@ -148,6 +152,7 @@ "mysql": "^2.16.0", "mysql2": "^1.6.3", "ndjson": "^1.5.0", + "numeral": "^2.0.6", "nyc": "^14.1.1", "once": "^1.4.0", "p-finally": "^1.0.0", diff --git a/test/benchmarks/001-transaction-and-span-no-stack-trace.js b/test/benchmarks/001-transaction-and-span-no-stack-trace.js new file mode 100644 index 0000000000..879d1ec673 --- /dev/null +++ b/test/benchmarks/001-transaction-and-span-no-stack-trace.js @@ -0,0 +1,29 @@ +'use strict' + +/* eslint-disable no-unused-vars, no-undef */ + +const bench = require('./utils/bench') + +bench('transaction-and-span-no-stack-trace', { + agentConf: { + captureSpanStackTraces: false + }, + setup () { + var agent = this.benchmark.agent + }, + fn (deferred) { + if (agent) agent.startTransaction() + setImmediate(() => { + const span = agent && agent.startSpan() + setImmediate(() => { + if (agent) { + span.end() + agent.endTransaction() + } + setImmediate(() => { + deferred.resolve() + }) + }) + }) + } +}) diff --git a/test/benchmarks/002-transaction-and-span-overhead-realistic-size.js b/test/benchmarks/002-transaction-and-span-overhead-realistic-size.js new file mode 100644 index 0000000000..ebd845a5f9 --- /dev/null +++ b/test/benchmarks/002-transaction-and-span-overhead-realistic-size.js @@ -0,0 +1,43 @@ +'use strict' + +/* eslint-disable no-unused-vars, no-undef */ + +const bench = require('./utils/bench') + +bench('transaction-and-span-overhead-realistic-size', { + agentConf: { + captureSpanStackTraces: true + }, + setup () { + var agent = this.benchmark.agent + var callstack = this.benchmark.callstack + + // To avoid randomness, but still generate what appears to be natural random + // call stacks, number of spans etc, use a pre-defined set of numbers + var numbers = [2, 5, 10, 1, 2, 21, 2, 5, 6, 9, 1, 11, 9, 8, 12] + var numbersSpanIndex = 5 + var numbersStackLevelIndex = 0 + + function addSpan (amount, cb) { + setImmediate(() => { + const span = agent && agent.startSpan() + setImmediate(() => { + if (agent) span.end() + if (--amount === 0) cb() + else addSpan(amount, cb) + }) + }) + } + }, + fn (deferred) { + if (agent) agent.startTransaction() + const amount = numbers[numbersStackLevelIndex++ % numbers.length] + callstack(amount, () => { + const amount = numbers[numbersSpanIndex++ % numbers.length] + addSpan(amount, () => { + if (agent) agent.endTransaction() + deferred.resolve() + }) + }) + } +}) diff --git a/test/benchmarks/003-transaction-and-span-with-stack-trace.js b/test/benchmarks/003-transaction-and-span-with-stack-trace.js new file mode 100644 index 0000000000..2353d758ac --- /dev/null +++ b/test/benchmarks/003-transaction-and-span-with-stack-trace.js @@ -0,0 +1,27 @@ +'use strict' + +/* eslint-disable no-unused-vars, no-undef */ + +const bench = require('./utils/bench') + +bench('transaction-and-span-with-stack-trace', { + agentConf: { + captureSpanStackTraces: false + }, + setup () { + var agent = this.benchmark.agent + }, + fn (deferred) { + if (agent) agent.startTransaction() + setImmediate(() => { + const span = agent && agent.startSpan() + setImmediate(() => { + if (agent) { + span.end() + agent.endTransaction() + } + deferred.resolve() + }) + }) + } +}) diff --git a/test/benchmarks/004-transaction.js b/test/benchmarks/004-transaction.js new file mode 100644 index 0000000000..c6b3d4a232 --- /dev/null +++ b/test/benchmarks/004-transaction.js @@ -0,0 +1,20 @@ +'use strict' + +/* eslint-disable no-unused-vars, no-undef */ + +const bench = require('./utils/bench') + +bench('transaction', { + setup () { + var agent = this.benchmark.agent + }, + fn (deferred) { + if (agent) agent.startTransaction() + setImmediate(() => { + if (agent) agent.endTransaction() + setImmediate(() => { + deferred.resolve() + }) + }) + } +}) diff --git a/test/benchmarks/005-transaction-reading-file.js b/test/benchmarks/005-transaction-reading-file.js new file mode 100644 index 0000000000..4d710b737d --- /dev/null +++ b/test/benchmarks/005-transaction-reading-file.js @@ -0,0 +1,21 @@ +'use strict' + +/* eslint-disable no-unused-vars, no-undef */ + +const bench = require('./utils/bench') + +bench('transaction-reading-file', { + setup () { + var agent = this.benchmark.agent + var fs = this.benchmark.fs + var filename = this.benchmark.testFile + }, + fn (deferred) { + if (agent) agent.startTransaction() + fs.readFile(filename, err => { + if (err) throw err + if (agent) agent.endTransaction() + deferred.resolve() + }) + } +}) diff --git a/test/benchmarks/scripts/run-benchmarks-ci.sh b/test/benchmarks/scripts/run-benchmarks-ci.sh new file mode 100755 index 0000000000..4c40d9c9d0 --- /dev/null +++ b/test/benchmarks/scripts/run-benchmarks-ci.sh @@ -0,0 +1,106 @@ +#!/usr/bin/env bash + +set -exo pipefail + +SCRIPTPATH=$(dirname "$0") +RESULT_FILE=${1} + +if [ -z "$1" ] +then + echo "Usage:" + echo " run-benchmarks-ci.sh " + echo + echo "Examples:" + echo " run-benchmarks-ci.sh out.ndjson - Run benchmark + store result in out.ndjson" + echo + exit +fi + +echo $(pwd) + +function setUp() { + echo "Setting CPU frequency to base frequency" + + CPU_MODEL=$(lscpu | grep "Model name" | awk '{for(i=3;i<=NF;i++){printf "%s ", $i}; printf "\n"}') + if [ "${CPU_MODEL}" == "Intel(R) Xeon(R) CPU E3-1246 v3 @ 3.50GHz " ] + then + # could also use `nproc` + CORE_INDEX=7 + BASE_FREQ="3.5GHz" + elif [ "${CPU_MODEL}" == "Intel(R) Core(TM) i7-6700 CPU @ 3.40GHz " ] + then + CORE_INDEX=7 + BASE_FREQ="3.4GHz" + elif [ "${CPU_MODEL}" == "Intel(R) Core(TM) i7-7700 CPU @ 3.60GHz " ] + then + CORE_INDEX=7 + BASE_FREQ="3.6GHz" + elif [ "${CPU_MODEL}" == "Intel(R) Core(TM) i9-8950HK CPU @ 2.90GHz " ] + then + CORE_INDEX=9 + BASE_FREQ="2.90GHz" + else + >&2 echo "Cannot determine base frequency for CPU model [${CPU_MODEL}]. Please adjust the build script." + exit 1 + fi + MIN_FREQ=$(cpufreq-info -l -c 0 | awk '{print $1}') + # This is the frequency including Turbo Boost. See also http://ark.intel.com/products/80916/Intel-Xeon-Processor-E3-1246-v3-8M-Cache-3_50-GHz + MAX_FREQ=$(cpufreq-info -l -c 0 | awk '{print $2}') + + # set all CPUs to the base frequency + for (( cpu=0; cpu<=${CORE_INDEX}; cpu++ )) + do + sudo -n cpufreq-set -c ${cpu} --min ${BASE_FREQ} --max ${BASE_FREQ} + done + + # Build cgroups to isolate microbenchmarks and JVM threads + echo "Creating groups for OS and microbenchmarks" + # Isolate the OS to the first core + sudo -n cset set --set=/os --cpu=0-1 + sudo -n cset proc --move --fromset=/ --toset=/os + + # Isolate the microbenchmarks to all cores except the first two (first physical core) + # On a 4 core CPU with hyper threading, this would be 6 cores (3 physical cores) + sudo -n cset set --set=/benchmark --cpu=2-${CORE_INDEX} +} + +function escape_quotes() { + echo $1 | sed -e 's/"/\\"/g' +} + +# Escapes double quites in environment variables so that they are exported correctly +function safe_env_export() { + echo "export $1=\"$(escape_quotes "${2}")\"" +} + +function benchmark() { + echo "export GIT_BUILD_CAUSE='${GIT_BUILD_CAUSE}'" > env_vars.sh + echo "export GIT_BASE_COMMIT='${GIT_BASE_COMMIT}'" >> env_vars.sh + echo "export GIT_COMMIT='${GIT_COMMIT}'" >> env_vars.sh + echo "export BRANCH_NAME='${BRANCH_NAME}'" >> env_vars.sh + echo "export CHANGE_ID='${CHANGE_ID}'" >> env_vars.sh + safe_env_export "CHANGE_TITLE" "${CHANGE_TITLE}" >> env_vars.sh + echo "export CHANGE_TARGET='${CHANGE_TARGET}'" >> env_vars.sh + echo "export CHANGE_URL='${CHANGE_URL}'" >> env_vars.sh + sudo -n cset proc --exec /benchmark -- ./"${SCRIPTPATH}"/run-benchmarks.sh all "${RESULT_FILE}" +} + +function tearDown() { + echo "Destroying cgroups" + sudo -n cset set --destroy /os + sudo -n cset set --destroy /benchmark + + echo "Setting normal frequency range" + for (( cpu=0; cpu<=${CORE_INDEX}; cpu++ )) + do + sudo -n cpufreq-set -c ${cpu} --min ${MIN_FREQ} --max ${MAX_FREQ} + done + + echo "Delete env_vars.sh" + rm env_vars.sh || true +} + +trap "tearDown" TERM EXIT + +setUp +benchmark diff --git a/test/benchmarks/scripts/run-benchmarks.sh b/test/benchmarks/scripts/run-benchmarks.sh new file mode 100755 index 0000000000..02edc830ac --- /dev/null +++ b/test/benchmarks/scripts/run-benchmarks.sh @@ -0,0 +1,125 @@ +#!/usr/bin/env bash + +set -e + +if [ "$1" == "--help" ] +then + echo "Usage:" + echo " run-benchmarks.sh [benchmark-file|all] [output-file] [node_version]" + echo + echo "If no benchmark file is provided, the default is to run all the benchmarks" + echo "(can also be specified using the \"all\" keyword)" + echo + echo "node_version is required when running within the CI context for sudo purposes" + echo + echo "Examples:" + echo " run-benchmarks.sh - Run all benchmarks" + echo " run-benchmarks.sh all - Run all benchmarks" + echo " run-benchmarks.sh all out.ndjson - Run all benchmarks + store result in out.ndjson" + echo " run-benchmarks.sh foo.js - Run foo.js benchmark" + echo " run-benchmarks.sh foo.js out.ndjson - Run foo.js benchmark + store result in out.ndjson" + echo + exit +fi + +function log() { + msg=$1 + if [ ! -z "$DEBUG" ] + then + echo $msg + fi +} + +function teardown() { + log "teardown..." + if [ -n "$pid" ] ; then + shutdownAPMServer + fi + if [ -n "${SUDO_COMMAND}" ] ; then + if [ -d "${outputdir}" ] ; then + chmod -R 777 "${outputdir}" + ls -l "${outputdir}" + fi + if [ -f "${result_file}" ] ; then + chmod 777 "${result_file}" + fi + HOME_CONFIG="${HOME}/.config" + if [ -d "${HOME_CONFIG}" ] ; then + chmod -R 777 "${HOME_CONFIG}" + fi + if [ -d "${NVM_DIR}" ] ; then + chmod -R 777 "${NVM_DIR}" + fi + # A bit dramatic but as it runs under the sudo folder let's get it fix once + chmod -R 777 "${HOME}" + fi +} + +function startAPMServer() { + log "Starting mock APM Server..." + node $utils/apm-server.js & + pid=$! + log "Mock APM Server running as pid $pid" + + sleep 1 +} + +function shutdownAPMServer() { + log "Shutting down mock APM Server (pid: $pid)..." + kill -SIGUSR2 $pid + unset pid +} + +function runBenchmark() { + benchmark=$1 + + sleep 1 + + log "Running benchmark $benchmark without agent..." + node $benchmark > $appout_no_agent + + startAPMServer + + log "Running benchmark $benchmark with agent..." + AGENT=1 node $benchmark > $appout_agent + + shutdownAPMServer + + log "Analyzing results..." + node $utils/analyzer.js $appout_agent $appout_no_agent $result_file +} + +trap teardown TERM EXIT + +basedir=$(dirname $0)/.. +utils=$basedir/utils +outputdir=$basedir/.tmp +appout_no_agent=$outputdir/app-no-agent.json +appout_agent=$outputdir/app-agent.json +result_file=$2 + +rm -f $result_file # in case it's left over from an old run somehow +rm -fr $outputdir +mkdir -p $outputdir + +# If running as sudo then prepare the environment +if [ -n "${SUDO_COMMAND}" ] ; then + log "Running benchmark as sudo, let's reload the node environment..." + export NODE_VERSION=$3 + source ./${basedir}/../../.ci/scripts/prepare-benchmarks-env.sh + [ -e env_vars.sh ] && source env_vars.sh || echo "env_vars.sh not found" +fi + +if [ "$1" == "all" ] +then + benchmarks=($basedir/0*.js) +elif [ ! -z "$1" ] +then + benchmarks=($1) +else + benchmarks=($basedir/0*.js) +fi + +for benchmark in "${benchmarks[@]}"; do + runBenchmark "$benchmark" +done diff --git a/test/benchmarks/utils/analyzer.js b/test/benchmarks/utils/analyzer.js new file mode 100644 index 0000000000..b39278ea2a --- /dev/null +++ b/test/benchmarks/utils/analyzer.js @@ -0,0 +1,78 @@ +'use strict' + +const { appendFile, readFileSync } = require('fs') +const os = require('os') +const { resolve } = require('path') + +const logResult = require('./result-logger') + +const input = process.argv.slice(2) +const outputFile = input.length > 2 ? resolve(input.pop()) : null +const [bench, control] = input + .map(file => readFileSync(file)) + .map(buf => JSON.parse(buf)) + +calculateDelta(bench, control) +logResult(bench, control) + +if (outputFile) storeResult() + +function storeResult () { + bench.controlStats = control.stats + + const result = { + '@timestamp': bench.times.timeStamp, + ci: { + build_cause: process.env.GIT_BUILD_CAUSE + }, + git: { + branch: process.env.BRANCH_NAME, + commit: process.env.GIT_BASE_COMMIT || process.env.GIT_COMMIT + }, + pr: { + id: Number(process.env.CHANGE_ID) || null, + title: process.env.CHANGE_TITLE, + target: process.env.CHANGE_TARGET, + url: process.env.CHANGE_URL + }, + os: { + arch: os.arch(), + cpus: os.cpus(), + freemem: os.freemem(), + homedir: os.homedir(), + hostname: os.hostname(), + loadavg: os.loadavg(), + platform: os.platform(), + release: os.release(), + totalmem: os.totalmem(), + type: os.type(), + uptime: os.uptime() + }, + process: { + arch: process.arch, + config: process.config, + env: process.env, + platform: process.platform, + release: process.release, + version: process.version, + versions: process.versions + }, + bench + } + + const data = `{"index":{"_index":"benchmark-nodejs","_type":"_doc"}}\n${JSON.stringify(result)}\n` + + appendFile(outputFile, data, function (err) { + if (err) throw err + }) +} + +function calculateDelta (bench, control) { + // moe: The margin of error + // rme: The relative margin of error (expressed as a percentage of the mean) + // sem: The standard error of the mean + // deviation: The sample standard deviation + // mean: The sample arithmetic mean (secs) + // variance: The sample variance + bench.overhead = bench.stats.mean - control.stats.mean +} diff --git a/test/benchmarks/utils/apm-server.js b/test/benchmarks/utils/apm-server.js new file mode 100644 index 0000000000..9bd7eaa644 --- /dev/null +++ b/test/benchmarks/utils/apm-server.js @@ -0,0 +1,16 @@ +'use strict' + +process.on('SIGUSR2', function () { + process.exit() +}) + +const http = require('http') + +const server = http.createServer(function (req, res) { + req.on('end', function () { + res.end() + }) + req.resume() +}) + +server.listen(8200) diff --git a/test/benchmarks/utils/bench.js b/test/benchmarks/utils/bench.js new file mode 100644 index 0000000000..ebab1bd05e --- /dev/null +++ b/test/benchmarks/utils/bench.js @@ -0,0 +1,47 @@ +'use strict' + +const withAgent = !!process.env.AGENT + +module.exports = function (name, { agentConf, ...benchConf }) { + let agent + + if (withAgent) { + agent = require('../../..').start(Object.assign({ + serviceName: 'test', + centralConfig: false, + captureExceptions: false, + metricsInterval: 0 + }, agentConf)) + } + + const Benchmark = require('benchmark') + + new Benchmark(Object.assign({ + name: `${name} (${withAgent ? 'With Agent' : 'Without Agent'})`, + initCount: 10000, + maxTime: 60, + defer: true, + onStart () { + // Prepare properties that need to be accessible inside the benchmark + // test code. These can be accessed via `this.benchmark.*` from within + // either the `setup` function or the `fn` function (as the regular + // Node.js globals like `require` and `__filename` are not available to + // those functions). + this.fs = require('fs') + this.callstack = require('./callstack') + this.testFile = __filename + this.agent = agent + }, + onComplete (result) { + process.stdout.write(JSON.stringify({ + name: name, + count: result.target.count, + cycles: result.target.cycles, + hz: result.target.hz, + stats: result.target.stats, + times: result.target.times + })) + process.exit() + } + }, benchConf)).run() +} diff --git a/test/benchmarks/utils/callstack.js b/test/benchmarks/utils/callstack.js new file mode 100644 index 0000000000..e6d7d61720 --- /dev/null +++ b/test/benchmarks/utils/callstack.js @@ -0,0 +1,18 @@ +'use strict' + +const stack = [] + +for (let depth = 0; depth < 50; depth++) { + /* eslint-disable no-eval */ + eval(` + stack[${depth}] = function level${depth} (depth, cb) { + if (--depth === 0) return cb() + else stack[depth](depth, cb) + } + `) + /* eslint-endable no-eval */ +} + +module.exports = function deep (depth, cb) { + stack[depth](depth, cb) +} diff --git a/test/benchmarks/utils/result-logger.js b/test/benchmarks/utils/result-logger.js new file mode 100644 index 0000000000..639bdcb2c8 --- /dev/null +++ b/test/benchmarks/utils/result-logger.js @@ -0,0 +1,121 @@ +'use strict' + +// To be able to properly display the results in microseconds the stats are +// being re-calculated. The code to calculate the stats inside the `calc` +// function is lifted from the benchmarkjs library. So is the `tTable` data. +// +// The benchmarkjs library is under the MIT license: +// https://github.com/bestiejs/benchmark.js/blob/42f3b732bac3640eddb3ae5f50e445f3141016fd/LICENSE + +const columnify = require('columnify') +const numeral = require('numeral') + +module.exports = logResult + +/** + * T-Distribution two-tailed critical values for 95% confidence. + * For more info see http://www.itl.nist.gov/div898/handbook/eda/section3/eda3672.htm. + */ +const tTable = { + '1': 12.706, + '2': 4.303, + '3': 3.182, + '4': 2.776, + '5': 2.571, + '6': 2.447, + '7': 2.365, + '8': 2.306, + '9': 2.262, + '10': 2.228, + '11': 2.201, + '12': 2.179, + '13': 2.16, + '14': 2.145, + '15': 2.131, + '16': 2.12, + '17': 2.11, + '18': 2.101, + '19': 2.093, + '20': 2.086, + '21': 2.08, + '22': 2.074, + '23': 2.069, + '24': 2.064, + '25': 2.06, + '26': 2.056, + '27': 2.052, + '28': 2.048, + '29': 2.045, + '30': 2.042, + 'infinity': 1.96 +} + +function logResult (bench, control) { + const a = calc(bench.stats.sample, 1e6) + const b = calc(control.stats.sample, 1e6) + + const data = [ + { name: 'ops/sec', bench: format(bench.hz, 0), control: format(control.hz, 0) }, + { name: 'sample size', bench: bench.stats.sample.length, control: control.stats.sample.length }, + { name: 'sample arithmetic mean', bench: format(a.mean), bu: 'μs', control: format(b.mean), cu: 'μs' }, + { name: 'sample variance', bench: format(a.variance), control: format(b.variance) }, + { name: 'sample standard deviation', bench: format(a.deviation), bu: 'μs', control: format(b.deviation), cu: 'μs' }, + { name: 'standard error of the mean', bench: format(a.sem), bu: 'μs', control: format(b.sem), cu: 'μs' }, + { name: 'margin of error', bench: `±${format(a.moe)}`, bu: 'μs', control: `±${format(b.moe)}`, cu: 'μs' }, + { name: 'relative margin of error', bench: `±${format(a.rme)}`, bu: '%', control: `±${format(b.rme)}`, cu: '%' }, + { name: 'overhead', bench: format(bench.overhead * 1e6), bu: 'μs' } + ] + + const options = { + columns: ['name', 'bench', 'bu', 'control', 'cu'], + config: { + name: { showHeaders: false }, + bench: { align: 'right' }, + bu: { showHeaders: false }, + control: { align: 'right' }, + cu: { showHeaders: false } + } + } + + console.error(`${bench.name}:`) + console.error(columnify(data, options)) + console.error() +} + +function calc (sample, scale = 1) { + sample = sample.map(sample => sample * scale) + + const size = sample.length + // Compute the sample mean (estimate of the population mean). + const mean = (sample.reduce((sum, x) => sum + x) / sample.length) || 0 + // Compute the sample variance (estimate of the population variance). + const variance = sample.reduce((sum, x) => sum + (x - mean) ** 2, 0) / (size - 1) || 0 + // Compute the sample standard deviation (estimate of the population standard deviation). + const sd = Math.sqrt(variance) + // Compute the standard error of the mean (a.k.a. the standard deviation of the sampling distribution of the sample mean). + const sem = sd / Math.sqrt(size) + // Compute the degrees of freedom. + const df = size - 1 + // Compute the critical value. + const critical = tTable[Math.round(df) || 1] || tTable.infinity + // Compute the margin of error. + const moe = sem * critical + // Compute the relative margin of error. + const rme = (moe / mean) * 100 || 0 + + return { + deviation: sd, + mean, + moe, + rme, + sem, + variance + } +} + +function format (n, decimals = 3) { + if (decimals === 0) { + return numeral(n).format('0,0') + } + return numeral(n).format(`0,0.${Array(decimals + 1).join(0)}`) +}