Add benchmarks

.ci/scripts/prepare-benchmarks-env.sh

@@ -0,0 +1,21 @@
+#!/usr/bin/env bash
+set -eo pipefail
+
+# This particular configuration is required to be installed in the baremetal
+curl -o- https://raw.githubusercontent.com/nvm-sh/nvm/v0.34.0/install.sh | bash
+export NVM_DIR="$HOME/.nvm"
+[ -s "$NVM_DIR/nvm.sh" ] && \. "$NVM_DIR/nvm.sh"
+command -v nvm
+
+## If NODE_VERSION env variable exists then use it otherwise use node as default
+if [ -z "${NODE_VERSION}" ] ; then
+  NODE_VERSION="node"
+fi
+nvm install ${NODE_VERSION}
+
+set +x
+npm config list
+npm install
+
+node --version
+npm --version

.ci/scripts/run-benchmarks.sh

@@ -0,0 +1,9 @@
+#!/usr/bin/env bash
+set -ueo pipefail
+
+SCRIPTPATH=$(dirname "$0")
+source ./${SCRIPTPATH}/prepare-benchmarks-env.sh
+
+RESULT_FILE=${1:-apm-agent-benchmark-results.json}
+
+npm run bench:ci ${RESULT_FILE}

.gitignore

@@ -10,3 +10,4 @@ test/types/transpile/index.js
 build
 coverage
 node_modules
+test/benchmarks/.tmp

Jenkinsfile

@@ -28,6 +28,7 @@ pipeline {
   }
   parameters {
     booleanParam(name: 'Run_As_Master_Branch', defaultValue: false, description: 'Allow to run any steps on a PR, some steps normally only run on master branch.')
+    booleanParam(name: 'bench_ci', defaultValue: true, description: 'Enable benchmarks.')
     booleanParam(name: 'tav_ci', defaultValue: true, description: 'Enable TAV tests.')
     booleanParam(name: 'tests_ci', defaultValue: true, description: 'Enable tests.')
     booleanParam(name: 'test_edge_ci', defaultValue: true, description: 'Enable tests for edge versions of nodejs.')
@@ -271,6 +272,51 @@ pipeline {
         githubNotify(context: "${env.GITHUB_CHECK_ITS_NAME}", description: "${env.GITHUB_CHECK_ITS_NAME} ...", status: 'PENDING', targetUrl: "${env.JENKINS_URL}search/?q=${env.ITS_PIPELINE.replaceAll('/','+')}")
       }
     }
+    /**
+      Run the benchmarks and store the results on ES.
+      The result JSON files are also archive into Jenkins.
+    */
+    stage('Benchmarks') {
+      agent { label 'metal' }
+      options { skipDefaultCheckout() }
+      environment {
+        HOME = "${env.WORKSPACE}"
+        RESULT_FILE = 'apm-agent-benchmark-results.json'
+        NODE_VERSION = '12'
+      }
+      when {
+        beforeAgent true
+        allOf {
+          anyOf {
+            branch 'master'
+            tag pattern: 'v\\d+\\.\\d+\\.\\d+.*', comparator: 'REGEXP'
+            expression { return params.Run_As_Master_Branch }
+          }
+          expression { return params.bench_ci }
+        }
+      }
+      steps {
+        withGithubNotify(context: 'Benchmarks', tab: 'artifacts') {
+          dir(env.BUILD_NUMBER) {
+            deleteDir()
+            unstash 'source'
+            dir(BASE_DIR){
+              sh '[ -e "${HOME}/.nvm" ] && rm -rf ${HOME}/.nvm || true' // to remove a broken env
+              sh '.ci/scripts/run-benchmarks.sh "${RESULT_FILE}"'
+            }
+          }
+        }
+      }
+      post {
+        always {
+          sendBenchmarks(file: "${BUILD_NUMBER}/${BASE_DIR}/${RESULT_FILE}",
+            index: "benchmark-nodejs", archive: true)
+          dir(env.BUILD_NUMBER) {
+            deleteDir()
+          }
+        }
+      }
+    }
   }
   post {
     cleanup {

TESTING.md

@@ -124,3 +124,9 @@ Clean up Docker containers and volumes:
 ```
 npm run docker:clean
 ```
+
+Run the benchmarks:
+
+```
+npm run bench
+```

package.json

@@ -21,6 +21,8 @@
     "test:types": "tsc --project test/types/tsconfig.json && tsc --project test/types/transpile/tsconfig.json && node test/types/transpile/index.js",
     "test:babel": "babel test/babel/src.js --out-file test/babel/out.js && node test/babel/out.js",
     "test:esm": "node --experimental-modules test/esm",
+    "bench": "./test/benchmarks/scripts/run-benchmarks.sh",
+    "bench:ci": "./test/benchmarks/scripts/run-benchmarks-ci.sh",
     "local:start": "./test/script/local-deps-start.sh",
     "local:stop": "./test/script/local-deps-stop.sh",
     "docker:start": "docker-compose -f ./test/docker-compose.yml up -d",
@@ -113,8 +115,10 @@
     "@types/node": "^12.0.8",
     "apollo-server-express": "^2.6.3",
     "aws-sdk": "^2.477.0",
+    "benchmark": "^2.1.4",
     "bluebird": "^3.5.2",
     "cassandra-driver": "^4.0.0",
+    "columnify": "^1.5.4",
     "commitlint-config-squash-pr": "^1.0.0",
     "connect": "^3.7.0",
     "container-info": "^1.0.1",
@@ -148,6 +152,7 @@
     "mysql": "^2.16.0",
     "mysql2": "^1.6.3",
     "ndjson": "^1.5.0",
+    "numeral": "^2.0.6",
     "nyc": "^14.1.1",
     "once": "^1.4.0",
     "p-finally": "^1.0.0",

test/benchmarks/001-transaction-and-span-no-stack-trace.js

@@ -0,0 +1,29 @@
+'use strict'
+
+/* eslint-disable no-unused-vars, no-undef */
+
+const bench = require('./utils/bench')
+
+bench('transaction-and-span-no-stack-trace', {
+  agentConf: {
+    captureSpanStackTraces: false
+  },
+  setup () {
+    var agent = this.benchmark.agent
+  },
+  fn (deferred) {
+    if (agent) agent.startTransaction()
+    setImmediate(() => {
+      const span = agent && agent.startSpan()
+      setImmediate(() => {
+        if (agent) {
+          span.end()
+          agent.endTransaction()
+        }
+        setImmediate(() => {
+          deferred.resolve()
+        })
+      })
+    })
+  }
+})

test/benchmarks/002-transaction-and-span-overhead-realistic-size.js

@@ -0,0 +1,43 @@
+'use strict'
+
+/* eslint-disable no-unused-vars, no-undef */
+
+const bench = require('./utils/bench')
+
+bench('transaction-and-span-overhead-realistic-size', {
+  agentConf: {
+    captureSpanStackTraces: true
+  },
+  setup () {
+    var agent = this.benchmark.agent
+    var callstack = this.benchmark.callstack
+
+    // To avoid randomness, but still generate what appears to be natural random
+    // call stacks, number of spans etc, use a pre-defined set of numbers
+    var numbers = [2, 5, 10, 1, 2, 21, 2, 5, 6, 9, 1, 11, 9, 8, 12]
+    var numbersSpanIndex = 5
+    var numbersStackLevelIndex = 0
+
+    function addSpan (amount, cb) {
+      setImmediate(() => {
+        const span = agent && agent.startSpan()
+        setImmediate(() => {
+          if (agent) span.end()
+          if (--amount === 0) cb()
+          else addSpan(amount, cb)
+        })
+      })
+    }
+  },
+  fn (deferred) {
+    if (agent) agent.startTransaction()
+    const amount = numbers[numbersStackLevelIndex++ % numbers.length]
+    callstack(amount, () => {
+      const amount = numbers[numbersSpanIndex++ % numbers.length]
+      addSpan(amount, () => {
+        if (agent) agent.endTransaction()
+        deferred.resolve()
+      })
+    })
+  }
+})

test/benchmarks/003-transaction-and-span-with-stack-trace.js

@@ -0,0 +1,27 @@
+'use strict'
+
+/* eslint-disable no-unused-vars, no-undef */
+
+const bench = require('./utils/bench')
+
+bench('transaction-and-span-with-stack-trace', {
+  agentConf: {
+    captureSpanStackTraces: false
+  },
+  setup () {
+    var agent = this.benchmark.agent
+  },
+  fn (deferred) {
+    if (agent) agent.startTransaction()
+    setImmediate(() => {
+      const span = agent && agent.startSpan()
+      setImmediate(() => {
+        if (agent) {
+          span.end()
+          agent.endTransaction()
+        }
+        deferred.resolve()
+      })
+    })
+  }
+})

test/benchmarks/004-transaction.js

@@ -0,0 +1,20 @@
+'use strict'
+
+/* eslint-disable no-unused-vars, no-undef */
+
+const bench = require('./utils/bench')
+
+bench('transaction', {
+  setup () {
+    var agent = this.benchmark.agent
+  },
+  fn (deferred) {
+    if (agent) agent.startTransaction()
+    setImmediate(() => {
+      if (agent) agent.endTransaction()
+      setImmediate(() => {
+        deferred.resolve()
+      })
+    })
+  }
+})

test/benchmarks/005-transaction-reading-file.js

@@ -0,0 +1,21 @@
+'use strict'
+
+/* eslint-disable no-unused-vars, no-undef */
+
+const bench = require('./utils/bench')
+
+bench('transaction-reading-file', {
+  setup () {
+    var agent = this.benchmark.agent
+    var fs = this.benchmark.fs
+    var filename = this.benchmark.testFile
+  },
+  fn (deferred) {
+    if (agent) agent.startTransaction()
+    fs.readFile(filename, err => {
+      if (err) throw err
+      if (agent) agent.endTransaction()
+      deferred.resolve()
+    })
+  }
+})

test/benchmarks/scripts/run-benchmarks-ci.sh

@@ -0,0 +1,86 @@
+#!/usr/bin/env bash
+
+set -exo pipefail
+
+SCRIPTPATH=$(dirname "$0")
+RESULT_FILE=${1}
+
+if [ -z "$1" ]
+then
+  echo "Usage:"
+  echo "  run-benchmarks-ci.sh <output-file>"
+  echo
+  echo "Examples:"
+  echo "  run-benchmarks-ci.sh out.ndjson  - Run benchmark + store result in out.ndjson"
+  echo
+  exit
+fi
+
+echo $(pwd)
+
+function setUp() {
+    echo "Setting CPU frequency to base frequency"
+
+    CPU_MODEL=$(lscpu | grep "Model name" | awk '{for(i=3;i<=NF;i++){printf "%s ", $i}; printf "\n"}')
+    if [ "${CPU_MODEL}" == "Intel(R) Xeon(R) CPU E3-1246 v3 @ 3.50GHz " ]
+    then
+        # could also use `nproc`
+        CORE_INDEX=7
+        BASE_FREQ="3.5GHz"
+    elif [ "${CPU_MODEL}" == "Intel(R) Core(TM) i7-6700 CPU @ 3.40GHz " ]
+    then
+        CORE_INDEX=7
+        BASE_FREQ="3.4GHz"
+    elif [ "${CPU_MODEL}" == "Intel(R) Core(TM) i7-7700 CPU @ 3.60GHz " ]
+    then
+        CORE_INDEX=7
+        BASE_FREQ="3.6GHz"
+    elif [ "${CPU_MODEL}" == "Intel(R) Core(TM) i9-8950HK CPU @ 2.90GHz " ]
+    then
+        CORE_INDEX=9
+        BASE_FREQ="2.90GHz"
+    else
+        >&2 echo "Cannot determine base frequency for CPU model [${CPU_MODEL}]. Please adjust the build script."
+        exit 1
+    fi
+    MIN_FREQ=$(cpufreq-info -l -c 0 | awk '{print $1}')
+    # This is the frequency including Turbo Boost. See also http://ark.intel.com/products/80916/Intel-Xeon-Processor-E3-1246-v3-8M-Cache-3_50-GHz
+    MAX_FREQ=$(cpufreq-info -l -c 0 | awk '{print $2}')
+
+    # set all CPUs to the base frequency
+    for (( cpu=0; cpu<=${CORE_INDEX}; cpu++ ))
+    do
+        sudo -n cpufreq-set -c ${cpu} --min ${BASE_FREQ} --max ${BASE_FREQ}
+    done
+
+    # Build cgroups to isolate microbenchmarks and JVM threads
+    echo "Creating groups for OS and microbenchmarks"
+    # Isolate the OS to the first core
+    sudo -n cset set --set=/os --cpu=0-1
+    sudo -n cset proc --move --fromset=/ --toset=/os
+
+    # Isolate the microbenchmarks to all cores except the first two (first physical core)
+    # On a 4 core CPU with hyper threading, this would be 6 cores (3 physical cores)
+    sudo -n cset set --set=/benchmark --cpu=2-${CORE_INDEX}
+}
+
+function benchmark() {
+    sudo -n cset proc --exec /benchmark -- ./${SCRIPTPATH}/run-benchmarks.sh all ${RESULT_FILE}
+}
+
+function tearDown() {
+    echo "Destroying cgroups"
+    sudo -n cset set --destroy /os
+    sudo -n cset set --destroy /benchmark
+
+    echo "Setting normal frequency range"
+    for (( cpu=0; cpu<=${CORE_INDEX}; cpu++ ))
+    do
+        sudo -n cpufreq-set -c ${cpu} --min ${MIN_FREQ} --max ${MAX_FREQ}
+    done
+}
+
+trap "tearDown" EXIT
+
+setUp
+benchmark