Add benchmarks#1145
Merged
Merged
Conversation
Member
|
I managed to have something in place to be used, although it's now failing during the runtime: Further details: here |
Contributor
Author
|
@v1v great to hear! Yeah, I've been using the |
Contributor
Author
|
The structure of the result JSON object is: {
"git" : {...},
"os" : {...},
"process" : {...},
"results" : [...]
}The Each result is an object with the following structure (I left out most of the samples in the two {
"controlStats" : {
"deviation" : 3.76651188587459e-07,
"mean" : 1.63528747289572e-05,
"moe" : 8.20262588479355e-08,
"rme" : 0.501601462785535,
"sample" : [
1.58977693672383e-05,
1.66014547604967e-05,
...
],
"sem" : 4.18501320652732e-08,
"variance" : 1.41866117864345e-13
},
"count" : 31,
"cycles" : 8,
"hz" : 525.053939322554,
"name" : "transaction-and-span-overhead-realistic-size",
"overhead" : 0.00188821335191481,
"stats" : {
"deviation" : 0.000208476484359058,
"mean" : 0.00190456622664376,
"moe" : 4.68712299409083e-05,
"rme" : 2.46099239213671,
"sample" : [
0.00258644335,
0.00256365236363636,
...
],
"sem" : 2.3913892826994e-05,
"variance" : 4.34624445307127e-08
},
"times" : {
"cycle" : 0.0590415530259567,
"elapsed" : 6.313,
"period" : 0.00190456622664376,
"timeStamp" : 1567588648954
}
}The Example of full JSON result document{
"git" : {
"branch" : "benchmark",
"message" : "feat: add debug feature, 'DEBUG=1 npm run bench'",
"rev" : "4b5295db"
},
"os" : {
"arch" : "x64",
"cpus" : [
{
"model" : "Intel(R) Core(TM) i7-8569U CPU @ 2.80GHz",
"speed" : 2800,
"times" : {
"idle" : 256542410,
"irq" : 0,
"nice" : 0,
"sys" : 25006600,
"user" : 33869010
}
},
{
"model" : "Intel(R) Core(TM) i7-8569U CPU @ 2.80GHz",
"speed" : 2800,
"times" : {
"idle" : 309978340,
"irq" : 0,
"nice" : 0,
"sys" : 2797450,
"user" : 2625410
}
},
{
"model" : "Intel(R) Core(TM) i7-8569U CPU @ 2.80GHz",
"speed" : 2800,
"times" : {
"idle" : 265770990,
"irq" : 0,
"nice" : 0,
"sys" : 20144830,
"user" : 29485880
}
},
{
"model" : "Intel(R) Core(TM) i7-8569U CPU @ 2.80GHz",
"speed" : 2800,
"times" : {
"idle" : 310681520,
"irq" : 0,
"nice" : 0,
"sys" : 2313350,
"user" : 2406310
}
},
{
"model" : "Intel(R) Core(TM) i7-8569U CPU @ 2.80GHz",
"speed" : 2800,
"times" : {
"idle" : 274798900,
"irq" : 0,
"nice" : 0,
"sys" : 16778990,
"user" : 23823790
}
},
{
"model" : "Intel(R) Core(TM) i7-8569U CPU @ 2.80GHz",
"speed" : 2800,
"times" : {
"idle" : 311155340,
"irq" : 0,
"nice" : 0,
"sys" : 1980600,
"user" : 2265220
}
},
{
"model" : "Intel(R) Core(TM) i7-8569U CPU @ 2.80GHz",
"speed" : 2800,
"times" : {
"idle" : 284847830,
"irq" : 0,
"nice" : 0,
"sys" : 11858520,
"user" : 18695310
}
},
{
"model" : "Intel(R) Core(TM) i7-8569U CPU @ 2.80GHz",
"speed" : 2800,
"times" : {
"idle" : 311447070,
"irq" : 0,
"nice" : 0,
"sys" : 1735330,
"user" : 2218750
}
}
],
"freemem" : 636014592,
"homedir" : "/Users/watson",
"hostname" : "watson",
"loadavg" : [
1.31884765625,
1.50537109375,
1.65625
],
"platform" : "darwin",
"release" : "18.7.0",
"totalmem" : 17179869184,
"type" : "Darwin",
"uptime" : 534192
},
"process" : {
"arch" : "x64",
"config" : {
"target_defaults" : {
"cflags" : [],
"default_configuration" : "Release",
"defines" : [],
"include_dirs" : [],
"libraries" : []
},
"variables" : {
"asan" : 0,
"build_v8_with_gn" : false,
"coverage" : false,
"debug_nghttp2" : false,
"enable_lto" : false,
"enable_pgo_generate" : false,
"enable_pgo_use" : false,
"force_dynamic_crt" : 0,
"host_arch" : "x64",
"icu_data_in" : "../../deps/icu-small/source/data/in/icudt64l.dat",
"icu_endianness" : "l",
"icu_gyp_path" : "tools/icu/icu-generic.gyp",
"icu_locales" : "en,root",
"icu_path" : "deps/icu-small",
"icu_small" : true,
"icu_ver_major" : "64",
"is_debug" : 0,
"llvm_version" : "0",
"napi_build_version" : "4",
"node_byteorder" : "little",
"node_code_cache" : "yes",
"node_debug_lib" : false,
"node_enable_d8" : false,
"node_install_npm" : true,
"node_module_version" : 72,
"node_no_browser_globals" : false,
"node_prefix" : "/",
"node_release_urlbase" : "https://nodejs.org/download/release/",
"node_report" : true,
"node_shared" : false,
"node_shared_cares" : false,
"node_shared_http_parser" : false,
"node_shared_libuv" : false,
"node_shared_nghttp2" : false,
"node_shared_openssl" : false,
"node_shared_zlib" : false,
"node_tag" : "",
"node_target_type" : "executable",
"node_use_bundled_v8" : true,
"node_use_dtrace" : true,
"node_use_etw" : false,
"node_use_large_pages" : false,
"node_use_large_pages_script_lld" : false,
"node_use_node_snapshot" : true,
"node_use_openssl" : true,
"node_use_v8_platform" : true,
"node_with_ltcg" : false,
"node_without_node_options" : false,
"openssl_fips" : "",
"openssl_is_fips" : false,
"shlib_suffix" : "72.dylib",
"target_arch" : "x64",
"v8_enable_gdbjit" : 0,
"v8_enable_i18n_support" : 1,
"v8_enable_inspector" : 1,
"v8_no_strict_aliasing" : 1,
"v8_optimized_debug" : 1,
"v8_promise_internal_field_count" : 1,
"v8_random_seed" : 0,
"v8_trace_maps" : 0,
"v8_use_siphash" : 1,
"v8_use_snapshot" : 1,
"want_separate_host_toolset" : 0,
"xcode_version" : "8.0"
}
},
"env" : {
"Apple_PubSub_Socket_Render" : "/private/tmp/com.apple.launchd.SQAZ6o4HI9/Render",
"GITHUB_USERNAME" : "watson",
"GOPATH" : "/Users/watson/code/go",
"GOROOT" : "/Users/watson/.gvm/versions/go1.12.7.darwin.amd64",
"GPG_TTY" : "/dev/ttys002",
"HISTFILE" : "/Users/watson/.history",
"HISTSIZE" : "10000",
"HOME" : "/Users/watson",
"LANG" : "en_US.UTF-8",
"LC_ALL" : "",
"LC_COLLATE" : "en_US.UTF-8",
"LC_CTYPE" : "en_US.UTF-8",
"LC_MESSAGES" : "en_US.UTF-8",
"LC_MONETARY" : "en_US.UTF-8",
"LC_NUMERIC" : "en_US.UTF-8",
"LC_TIME" : "en_US.UTF-8",
"LESS" : "-R",
"LOGNAME" : "watson",
"LSCOLORS" : "Gxfxcxdxbxegedabagacad",
"NVM_BIN" : "/Users/watson/.nvm/versions/node/v12.9.1/bin",
"NVM_CD_FLAGS" : "-q",
"NVM_DIR" : "/Users/watson/.nvm",
"PAGER" : "less",
"PATH" : "/Users/watson/.cargo/bin:/Users/watson/.gvm/versions/go1.12.7.darwin.amd64/bin:/Users/watson/code/go/bin:/usr/local/opt/python/libexec/bin:/usr/local/opt/mysql@5.7/bin:/Users/watson/.nvm/versions/node/v12.9.1/bin:/usr/local/bin:/usr/bin:/bin:/usr/sbin:/sbin:/Users/watson/.cargo/bin:/Users/watson/.gvm/versions/go1.12.7.darwin.amd64/bin:/Users/watson/code/go/bin:/usr/local/opt/python/libexec/bin:/usr/local/opt/mysql@5.7/bin:/Users/watson/.nvm/versions/node/v12.7.0/bin:/usr/local/sbin:/usr/local/sbin",
"PWD" : "/Users/watson/code/node_modules/elastic-apm-node",
"SAVEHIST" : "10000",
"SECURITYSESSIONID" : "186ab",
"SHELL" : "/bin/zsh",
"SHLVL" : "3",
"SSH_AUTH_SOCK" : "/private/tmp/com.apple.launchd.9AXpqYBk4o/Listeners",
"TERM" : "screen-256color",
"TERM_PROGRAM" : "Apple_Terminal",
"TERM_PROGRAM_VERSION" : "421.2",
"TERM_SESSION_ID" : "8D2CDF1A-3F0D-425D-8A26-01117E3858A3",
"TMPDIR" : "/var/folders/tn/9gmgdzbx0xsbp6_1ykklgh8c0000gn/T/",
"TMUX" : "/tmp//tmux-501/default,2490,0",
"TMUX_PANE" : "%12",
"USER" : "watson",
"VAULT_ADDR" : "https://secrets.elastic.co:8200",
"XPC_FLAGS" : "0x0",
"XPC_SERVICE_NAME" : "0",
"ZSH" : "/Users/watson/.oh-my-zsh",
"_" : "/Users/watson/.nvm/versions/node/v12.9.1/bin/node",
"__CF_USER_TEXT_ENCODING" : "0x1F5:0x0:0x0"
},
"platform" : "darwin",
"release" : {
"headersUrl" : "https://nodejs.org/download/release/v12.9.1/node-v12.9.1-headers.tar.gz",
"name" : "node",
"sourceUrl" : "https://nodejs.org/download/release/v12.9.1/node-v12.9.1.tar.gz"
},
"version" : "v12.9.1",
"versions" : {
"ares" : "1.15.0",
"brotli" : "1.0.7",
"cldr" : "35.1",
"http_parser" : "2.8.0",
"icu" : "64.2",
"llhttp" : "1.1.4",
"modules" : "72",
"napi" : "4",
"nghttp2" : "1.39.2",
"node" : "12.9.1",
"openssl" : "1.1.1c",
"tz" : "2019a",
"unicode" : "12.1",
"uv" : "1.31.0",
"v8" : "7.6.303.29-node.15",
"zlib" : "1.2.11"
}
},
"results" : [
{
"controlStats" : {
"deviation" : 6.94785045212906e-08,
"mean" : 3.53943624237264e-06,
"moe" : 1.513087431797e-08,
"rme" : 0.427493908121002,
"sample" : [
3.41185549169859e-06,
3.60359176245211e-06,
3.52106040868455e-06,
3.50845427841635e-06,
3.54549514687101e-06,
3.54900166028097e-06,
3.36568767560664e-06,
3.54585619412516e-06,
3.68100849297573e-06,
3.58006264367816e-06,
3.50323237547893e-06,
3.55211979565773e-06,
3.51824770114943e-06,
3.56654195402299e-06,
3.50776877394636e-06,
3.53530779054917e-06,
3.52234993614304e-06,
3.52332388250319e-06,
3.59003569604087e-06,
3.56684278416347e-06,
3.53160229885057e-06,
3.53349738186462e-06,
3.53180446998723e-06,
3.54071743295019e-06,
3.59577713920817e-06,
3.82444508301405e-06,
3.6089469348659e-06,
3.82217924648787e-06,
3.58759054916986e-06,
3.49688352490421e-06,
3.54877656449553e-06,
3.5438288633461e-06,
3.53970197956577e-06,
3.53938077905492e-06,
3.55496896551724e-06,
3.55108346104725e-06,
3.52088965517241e-06,
3.46443920817369e-06,
3.56283512132822e-06,
3.55794578544061e-06,
3.70148122605364e-06,
3.55767222222222e-06,
3.57879731800766e-06,
3.50745287356322e-06,
3.46326845466156e-06,
3.58441385696041e-06,
3.52596251596424e-06,
3.51970836526181e-06,
3.52309195402299e-06,
3.55188876117497e-06,
3.49301877394636e-06,
3.56430791826309e-06,
3.52756813537676e-06,
3.52774182630907e-06,
3.50199610472542e-06,
3.5084462962963e-06,
3.53086590038314e-06,
3.51728729246488e-06,
3.50510491698595e-06,
3.49523218390805e-06,
3.53597247765006e-06,
3.51727464878672e-06,
3.55786449553001e-06,
3.37928001277139e-06,
3.49822809706258e-06,
3.52024655172414e-06,
3.51594099616858e-06,
3.57051021711367e-06,
3.52591040868455e-06,
3.68097343550447e-06,
3.5187217752235e-06,
3.50937560664112e-06,
3.53220293742018e-06,
3.44019540229885e-06,
3.50404750957854e-06,
3.53567139208174e-06,
3.46396851851852e-06,
3.49515261813538e-06,
3.4738588761175e-06,
3.572812899106e-06,
3.5056809706258e-06
],
"sem" : 7.71983383569896e-09,
"variance" : 4.827262590515e-15
},
"count" : 610,
"cycles" : 7,
"hz" : 10055.6259946895,
"name" : "transaction-and-span-no-stack-trace",
"overhead" : 9.59073809451512e-05,
"stats" : {
"deviation" : 1.4412186595689e-05,
"mean" : 9.94468171875239e-05,
"moe" : 3.1781354458808e-06,
"rme" : 3.19581414042431,
"sample" : [
0.00020242596812749,
0.000163442530674847,
0.000103402183712121,
0.000103588712121212,
0.000100262117424242,
0.000104108356060606,
9.99283200757576e-05,
0.000102081685606061,
9.86714699646643e-05,
0.000100834607773852,
9.60791042402827e-05,
9.74089045936396e-05,
9.18224098939929e-05,
0.000106893136042403,
9.46360424028269e-05,
9.17760070671378e-05,
0.000110479408127208,
9.65583374558304e-05,
9.43232102473498e-05,
9.52836201413428e-05,
9.80880406360424e-05,
9.79460123674912e-05,
0.000104794637809187,
9.64507791519435e-05,
9.26716519434629e-05,
9.92226148409894e-05,
9.4553796819788e-05,
9.42350424028268e-05,
9.72568922261484e-05,
9.82031360424028e-05,
0.000102325823321555,
0.000101651243816254,
9.52946643109541e-05,
9.82240088339223e-05,
9.7014480565371e-05,
9.12581130742049e-05,
9.4956148409894e-05,
9.84897385159011e-05,
0.000102610883392226,
9.75126501766784e-05,
9.26578833922262e-05,
9.33979310954064e-05,
9.65010282685512e-05,
9.62074558303887e-05,
9.64409010600707e-05,
9.08452084805654e-05,
9.49569858657244e-05,
0.000103497750883392,
9.77818462897526e-05,
9.93511872791519e-05,
9.95190141342756e-05,
9.73150265017668e-05,
9.58905123674912e-05,
9.67795989399293e-05,
9.73144187279152e-05,
9.9912609540636e-05,
9.43973727915194e-05,
9.27015106007067e-05,
9.59850512367491e-05,
9.70724434628975e-05,
9.69551130742049e-05,
9.60585830388693e-05,
9.17911395759717e-05,
0.000104153310954064,
9.17473692579505e-05,
9.56618639344262e-05,
9.84105180327869e-05,
9.34306508196721e-05,
9.6050362295082e-05,
9.6487737704918e-05,
0.000102321447540984,
9.43655737704918e-05,
9.23474737704918e-05,
9.48532032786885e-05,
9.59943278688525e-05,
9.54879278688525e-05,
9.55495278688525e-05,
9.55112163934426e-05,
9.38289836065574e-05
],
"sem" : 1.6214976764698e-06,
"variance" : 2.07711122468959e-10
},
"times" : {
"cycle" : 0.0606625584843896,
"elapsed" : 6.584,
"period" : 9.94468171875239e-05,
"timeStamp" : 1567599253925
}
},
{
"controlStats" : {
"deviation" : 4.057117049322e-07,
"mean" : 1.67705343135937e-05,
"moe" : 8.78145909352977e-08,
"rme" : 0.523624288250123,
"sample" : [
1.66588719790986e-05,
1.72610764206401e-05,
1.72036913781842e-05,
1.6853451992162e-05,
1.68902563683867e-05,
1.68011832135859e-05,
1.8393079480684e-05,
1.66455921469284e-05,
1.65773134895503e-05,
1.64245202659911e-05,
1.66779680177327e-05,
1.74084835338822e-05,
1.65927729575681e-05,
1.68374021532616e-05,
1.81911529449018e-05,
1.64945531982267e-05,
1.71484965167828e-05,
1.6616832488917e-05,
1.65068603546548e-05,
1.69408885370488e-05,
1.66665918302723e-05,
1.65635335655478e-05,
1.67592552248258e-05,
1.67405668144395e-05,
1.65915896136795e-05,
1.68946830272324e-05,
1.6611642178594e-05,
1.75708378720709e-05,
1.62420316656111e-05,
1.64933831538949e-05,
1.58383679544015e-05,
1.64682903736542e-05,
1.66987507916403e-05,
1.75397792906903e-05,
1.66448112729576e-05,
1.66503755541482e-05,
1.68661377454085e-05,
1.69568682710576e-05,
1.68628343888537e-05,
1.64836155794807e-05,
1.66555310322989e-05,
1.65003993033566e-05,
1.69507539582014e-05,
1.65145614312856e-05,
1.64486621279291e-05,
1.6507687460418e-05,
1.65289616846105e-05,
1.6533269791007e-05,
1.65083049398353e-05,
1.66977362254592e-05,
1.6621190310323e-05,
1.6448116529449e-05,
1.71507777074098e-05,
1.66834832172261e-05,
1.67530924635845e-05,
1.65035946801773e-05,
1.67645075997467e-05,
1.65505842305257e-05,
1.65260066497783e-05,
1.66093936035465e-05,
1.66482248258391e-05,
1.63287504749842e-05,
1.67295310322989e-05,
1.67124783159228e-05,
1.67108420386198e-05,
1.64045121874011e-05,
1.79854586894587e-05,
1.67254700854701e-05,
1.66583244697689e-05,
1.66864463437797e-05,
1.66480259575815e-05,
1.7017611269389e-05,
1.66094472934473e-05,
1.68259100981323e-05,
1.62890481164926e-05,
1.66868455207344e-05,
1.63538236783792e-05,
1.68394358974359e-05,
1.72899721430833e-05,
1.7325804368471e-05,
1.73971722063944e-05,
1.65856701487813e-05
],
"sem" : 4.48033627220906e-08,
"variance" : 1.64601987518992e-13
},
"count" : 28,
"cycles" : 6,
"hz" : 449.381670895515,
"name" : "transaction-and-span-overhead-realistic-size",
"overhead" : 0.00220850936641585,
"stats" : {
"deviation" : 0.000312155234182256,
"mean" : 0.00222527990072944,
"moe" : 7.01810556888301e-05,
"rme" : 3.15380800706576,
"sample" : [
0.00363913321428571,
0.00300412394444444,
0.002841155,
0.0029162595,
0.00292815716666667,
0.002810822,
0.00271975239130435,
0.00283783843478261,
0.00261374673913043,
0.00256444434782609,
0.00238873117391304,
0.00266457534782609,
0.00254485673913044,
0.00237881091304348,
0.002443787,
0.00254500186956522,
0.0022577272173913,
0.00214932229166667,
0.00224789870833333,
0.002098898125,
0.00229055175,
0.00210602720833333,
0.00214851941666667,
0.002200001875,
0.00219898520833333,
0.00226000979166667,
0.00204321124,
0.00214197972,
0.00214040184,
0.00212717148,
0.0021269448,
0.00215937132,
0.00230662616,
0.00209744504,
0.00217064372,
0.00217769036,
0.00216761756,
0.00209716172,
0.0020833392,
0.00208808228,
0.00196853665384615,
0.00206463792307692,
0.00201145792307692,
0.00201568338461538,
0.00205128269230769,
0.00200555784615385,
0.00206920719230769,
0.00197816546153846,
0.00196935761538462,
0.002035474,
0.00205042180769231,
0.00209476342307692,
0.00202709492307692,
0.00203788396296296,
0.00206773622222222,
0.00210193474074074,
0.00208499144444444,
0.0020320332962963,
0.00210367418518519,
0.00189096311111111,
0.00201018685185185,
0.00195714851851852,
0.00211144392592593,
0.00194697159259259,
0.00188156811111111,
0.00198783622222222,
0.00198179911111111,
0.00202057796296296,
0.00219770146428571,
0.00206694664285714,
0.00206222771428571,
0.00206032746428571,
0.00213319553571429,
0.00211415382142857,
0.00217657560714286,
0.00202493028571429
],
"sem" : 3.58066610657296e-05,
"variance" : 9.7440890227379e-08
},
"times" : {
"cycle" : 0.0623078372204243,
"elapsed" : 6.168,
"period" : 0.00222527990072944,
"timeStamp" : 1567599269564
}
},
{
"controlStats" : {
"deviation" : 7.53998453902881e-08,
"mean" : 2.39232293221665e-06,
"moe" : 1.63199791848237e-08,
"rme" : 0.68218127933515,
"sample" : [
2.4546809081709e-06,
2.43714009768117e-06,
2.46755722268667e-06,
2.45910507326088e-06,
2.50126290315484e-06,
2.52347150965812e-06,
2.33108082896995e-06,
2.25423839486074e-06,
2.52540682888195e-06,
2.42168240418885e-06,
2.41288309059709e-06,
2.46866568398821e-06,
2.35492704712457e-06,
2.44876547718573e-06,
2.47732542790514e-06,
2.45193949927399e-06,
2.23624759097109e-06,
2.46954688256259e-06,
2.49811765741189e-06,
2.44899273991288e-06,
2.45399542394509e-06,
2.49711088133058e-06,
2.46600378404541e-06,
2.47995863950367e-06,
2.50002266027192e-06,
2.48905720068641e-06,
2.49715105381265e-06,
2.35589945879351e-06,
2.40132173186078e-06,
2.47069146829762e-06,
2.39113785365424e-06,
2.42856580278963e-06,
2.42429951159414e-06,
2.45757689092269e-06,
2.48479412152946e-06,
2.46501421217055e-06,
2.4571324415893e-06,
2.39997817573811e-06,
2.4502235666828e-06,
2.42434734016808e-06,
2.50397144365732e-06,
2.45319439433273e-06,
2.36431240374885e-06,
2.52174748096977e-06,
2.39416957803494e-06,
2.31594240330884e-06,
2.34353693844326e-06,
2.35353522242267e-06,
2.31794671536058e-06,
2.29239446473358e-06,
2.32593219518634e-06,
2.31575680908171e-06,
2.37049267391209e-06,
2.37916412196946e-06,
2.33446306155674e-06,
2.33725089101069e-06,
2.32805046860562e-06,
2.36030413164958e-06,
2.35669908038896e-06,
2.38748365380385e-06,
2.30526057112685e-06,
2.31025150701808e-06,
2.31785396224755e-06,
2.20828877546531e-06,
2.32848440181282e-06,
2.32227553130638e-06,
2.32353095437145e-06,
2.33232560390725e-06,
2.35668082016984e-06,
2.44191411096933e-06,
2.31988045056541e-06,
2.35382153385841e-06,
2.32279922559071e-06,
2.32475236502838e-06,
2.32329735556827e-06,
2.32667171206054e-06,
2.28492027104325e-06,
2.33455660667928e-06,
2.32862014344172e-06,
2.32543604523254e-06,
2.33257781493378e-06,
2.35261305935671e-06
],
"sem" : 8.326519992257e-09,
"variance" : 5.68513668487936e-15
},
"count" : 615,
"cycles" : 11,
"hz" : 10748.3262085998,
"name" : "transaction-and-span-with-stack-trace",
"overhead" : 9.06454190000941e-05,
"stats" : {
"deviation" : 7.58950259587224e-06,
"mean" : 9.30377419323107e-05,
"moe" : 1.65282500976773e-06,
"rme" : 1.77651023707157,
"sample" : [
0.000148704867977528,
0.000109474045548654,
0.000100364715384615,
0.000100068640384615,
0.000101751360344828,
8.85214310344828e-05,
9.19315206896552e-05,
9.07328189655172e-05,
9.18298086206897e-05,
9.18091327586207e-05,
8.82977586206896e-05,
9.20188034482759e-05,
8.97940827586207e-05,
0.000110702146551724,
9.46660103448276e-05,
8.94898879310345e-05,
9.15739189655172e-05,
9.11885448275862e-05,
9.55968862068965e-05,
9.61191344827586e-05,
9.28575568965517e-05,
9.28431810344828e-05,
9.89653068965517e-05,
9.10590517241379e-05,
9.15624086206897e-05,
9.0354375862069e-05,
9.14725189655172e-05,
9.08584310344828e-05,
9.19326844827586e-05,
8.75212327586207e-05,
9.13903810344828e-05,
9.92983568965517e-05,
8.85866051724138e-05,
9.22368931034483e-05,
9.06030448275862e-05,
9.51601155172414e-05,
9.129395e-05,
9.03949344827586e-05,
9.14830224137931e-05,
9.12699534482759e-05,
9.90467775862069e-05,
9.07468896551724e-05,
9.11801172413793e-05,
9.01589465517241e-05,
8.91063534482759e-05,
9.16068810344828e-05,
8.98933706896552e-05,
8.96599068965517e-05,
9.02037068965517e-05,
9.88202344827586e-05,
8.97504310344828e-05,
9.12489051724138e-05,
8.96460017241379e-05,
9.07664103448276e-05,
8.99121068965517e-05,
9.12486189655172e-05,
9.02825948275862e-05,
9.70127448275862e-05,
8.99815137931034e-05,
8.63179568965517e-05,
9.01257310344828e-05,
8.971245e-05,
9.31966362068966e-05,
9.05925189655172e-05,
9.07124362068966e-05,
9.20757482758621e-05,
9.22144620689655e-05,
9.7469974137931e-05,
9.14288775862069e-05,
9.00318931034483e-05,
8.92950482758621e-05,
8.93550551724138e-05,
9.16161155172414e-05,
8.98266396551724e-05,
9.09023853658537e-05,
9.53422032520325e-05,
8.85528487804878e-05,
8.8170430894309e-05,
8.60041414634146e-05,
9.56816162601626e-05,
9.13798943089431e-05
],
"sem" : 8.43278066208027e-07,
"variance" : 5.76005496527515e-11
},
"times" : {
"cycle" : 0.0572182112883711,
"elapsed" : 6.689,
"period" : 9.30377419323107e-05,
"timeStamp" : 1567599284228
}
},
{
"controlStats" : {
"deviation" : 5.55208115118853e-08,
"mean" : 2.35713045604708e-06,
"moe" : 1.19446335476732e-08,
"rme" : 0.506744695315015,
"sample" : [
2.33339918960458e-06,
2.3958560849518e-06,
2.40386260537469e-06,
2.34403898281403e-06,
2.39212188533371e-06,
2.35463536863677e-06,
2.34892073028736e-06,
2.37108634902892e-06,
2.39703488426249e-06,
2.37178091379069e-06,
2.38808970238927e-06,
2.38762875506497e-06,
2.35912854547995e-06,
2.41516659680499e-06,
2.34361231428438e-06,
2.37588146802664e-06,
2.35955572632854e-06,
2.33549680965023e-06,
2.38997061152252e-06,
2.41581598435099e-06,
2.35641497834288e-06,
2.35829979972987e-06,
2.35785091518793e-06,
2.34886661077733e-06,
2.34935927530157e-06,
2.42784500023287e-06,
2.39661194168879e-06,
2.45454287178054e-06,
2.4324325834847e-06,
2.33759047086768e-06,
2.38612854547995e-06,
2.3840878394113e-06,
2.37457389036375e-06,
2.35395379814634e-06,
2.38585501373946e-06,
2.41273592287271e-06,
2.37107787247916e-06,
2.35710432676634e-06,
2.47796399795072e-06,
2.36089422942574e-06,
2.39842997531554e-06,
2.36647971682735e-06,
2.38597713194541e-06,
2.52632793069722e-06,
2.35719831400494e-06,
2.35315411485259e-06,
2.33517749522612e-06,
2.39869204974151e-06,
2.38803958828187e-06,
2.34171976153882e-06,
2.36488752270504e-06,
2.54817297750454e-06,
2.24233083493845e-06,
2.3360171677414e-06,
2.3045150741144e-06,
2.31890846662759e-06,
2.35673896658571e-06,
2.29969726153588e-06,
2.31680956368813e-06,
2.30961644753371e-06,
2.31724554057449e-06,
2.27561657315133e-06,
2.33972531613768e-06,
2.23147324344695e-06,
2.31339908717863e-06,
2.22915182983e-06,
2.30702709153337e-06,
2.47846704631103e-06,
2.31206557239762e-06,
2.31540084582531e-06,
2.31340557742233e-06,
2.28519533539905e-06,
2.31803986265807e-06,
2.3107118331798e-06,
2.30778255589984e-06,
2.34180315718952e-06,
2.25767226362951e-06,
2.32755384808642e-06,
2.3091294280211e-06,
2.37754170504983e-06,
2.32025730675823e-06,
2.30708634117746e-06,
2.32991273762666e-06
],
"sem" : 6.09420078962917e-09,
"variance" : 3.0825605109383e-15
},
"count" : 976,
"cycles" : 9,
"hz" : 17873.1842078389,
"name" : "transaction",
"overhead" : 5.35926090179866e-05,
"stats" : {
"deviation" : 4.32783668578196e-06,
"mean" : 5.59497394740336e-05,
"moe" : 9.42506656014739e-07,
"rme" : 1.68455950800657,
"sample" : [
8.47341086956522e-05,
6.49896762246117e-05,
7.02907407407407e-05,
6.08277395459976e-05,
5.49357489270386e-05,
5.43452060085837e-05,
5.68371301229508e-05,
5.46848422131148e-05,
5.22453924180328e-05,
6.14137889344262e-05,
5.4139768442623e-05,
5.38693637295082e-05,
5.47839528688525e-05,
5.79995768442623e-05,
5.18178616803279e-05,
5.55920819672131e-05,
6.07027489754098e-05,
5.6535756147541e-05,
5.53660399590164e-05,
6.06592090163934e-05,
5.40411178278689e-05,
5.50440081967213e-05,
5.4859643442623e-05,
5.49877930327869e-05,
5.49719354508197e-05,
5.46854067622951e-05,
5.26168452868853e-05,
5.34803155737705e-05,
5.7610943647541e-05,
5.54543145491803e-05,
5.40808391393443e-05,
5.58615645491803e-05,
5.41459795081967e-05,
5.50509918032787e-05,
5.54828790983607e-05,
5.5273237704918e-05,
5.47078452868852e-05,
5.5145675204918e-05,
5.4946481557377e-05,
6.27716711065574e-05,
5.71546782786885e-05,
5.42312694672131e-05,
5.46963975409836e-05,
5.56395512295082e-05,
5.26942776639344e-05,
5.39073165983607e-05,
5.49943514344262e-05,
5.63473319672131e-05,
5.56257038934426e-05,
5.89858831967213e-05,
5.49028206967213e-05,
5.50953801229508e-05,
5.37248954918033e-05,
5.44239856557377e-05,
5.42556219262295e-05,
5.57182581967213e-05,
5.53510706967213e-05,
5.42790604508197e-05,
5.2454824795082e-05,
6.01219405737705e-05,
5.32335676229508e-05,
5.41202090163934e-05,
5.57700696721312e-05,
5.4294925204918e-05,
5.38264764344262e-05,
5.36293114754098e-05,
5.37632510245902e-05,
5.41327213114754e-05,
5.51102366803279e-05,
5.46368288934426e-05,
5.7439643442623e-05,
5.25322715163934e-05,
5.61746260245902e-05,
5.30297991803279e-05,
5.49115911885246e-05,
5.42047510245902e-05,
5.38467079918033e-05,
5.41687079918033e-05,
5.46651424180328e-05,
5.55945020491803e-05,
5.62487151639344e-05
],
"sem" : 4.80870742864663e-07,
"variance" : 1.87301703788002e-11
},
"times" : {
"cycle" : 0.0546069457266568,
"elapsed" : 6.479,
"period" : 5.59497394740336e-05,
"timeStamp" : 1567599299281
}
},
{
"controlStats" : {
"deviation" : 3.84362207510179e-06,
"mean" : 7.13019513969182e-05,
"moe" : 8.53000820699514e-07,
"rme" : 1.19632184531822,
"sample" : [
8.9467610619469e-05,
7.92358525280899e-05,
7.07228539325843e-05,
7.12308286516854e-05,
7.6174220362622e-05,
7.01412635983264e-05,
7.44909184210526e-05,
6.96807736842105e-05,
7.94834684210526e-05,
7.7113622368421e-05,
7.42937592105263e-05,
6.77773315789474e-05,
7.31050947368421e-05,
7.97250065789474e-05,
7.19376710526316e-05,
7.55329171052632e-05,
7.15950657894737e-05,
6.65810592105263e-05,
7.32631842105263e-05,
7.09807197368421e-05,
7.20694802631579e-05,
6.8946922368421e-05,
6.90292315789474e-05,
6.84408407894737e-05,
6.73915171052632e-05,
7.54589355263158e-05,
6.6383682642487e-05,
7.16808419689119e-05,
6.79283393782383e-05,
7.28812422279793e-05,
7.729703626943e-05,
7.03618160621762e-05,
6.97280220207254e-05,
7.1929146373057e-05,
6.92700569948186e-05,
7.04967525906736e-05,
7.01371178756477e-05,
7.18034494818653e-05,
7.43569054404145e-05,
6.68274935233161e-05,
7.1401103626943e-05,
6.99948652849741e-05,
7.36954158031088e-05,
6.99562979274611e-05,
6.84271178756477e-05,
7.32673393782383e-05,
6.89491113989637e-05,
7.35335025906736e-05,
6.73927849740933e-05,
6.9679835492228e-05,
7.02316593264249e-05,
7.01199313471503e-05,
7.03870621761658e-05,
7.00071722797927e-05,
7.22401062176166e-05,
7.73759611398964e-05,
7.50053329253366e-05,
7.40840514075887e-05,
6.83958066095471e-05,
6.98032717258262e-05,
6.6894123623011e-05,
7.40384541003672e-05,
7.19469265605875e-05,
6.4077605875153e-05,
6.78425520195838e-05,
6.97835973072215e-05,
6.87861248470012e-05,
6.88507062423501e-05,
6.84731664626683e-05,
6.63288714810281e-05,
6.76532386780906e-05,
6.91461138310893e-05,
7.08003317013464e-05,
6.94337845777234e-05,
6.93795336597307e-05,
7.00701089351285e-05,
7.0844029375765e-05,
6.83031578947368e-05
],
"sem" : 4.35204500356895e-07,
"variance" : 1.47734306562098e-11
},
"count" : 382,
"cycles" : 5,
"hz" : 6759.22025482594,
"name" : "transaction-reading-file",
"overhead" : 7.6644107808065e-05,
"stats" : {
"deviation" : 2.09264731254705e-05,
"mean" : 0.000147946059204983,
"moe" : 4.70484494561435e-06,
"rme" : 3.18010832522119,
"sample" : [
0.000237168341880342,
0.000224169547008547,
0.000214306260683761,
0.000207921043650794,
0.000199132637873754,
0.000175003564784053,
0.000168901119601329,
0.000168468315614618,
0.000176233073089701,
0.000161183521341463,
0.000149754219444444,
0.000164938181081081,
0.000140593783783784,
0.000135671827027027,
0.000146332445945946,
0.000145199372972973,
0.000147305527027027,
0.000140627802702703,
0.000146240424324324,
0.000155564308108108,
0.000139555705405405,
0.000137849327027027,
0.000139637927027027,
0.000142137667567568,
0.000142163935135135,
0.0001363072,
0.000139957683783784,
0.000137854994594595,
0.000141221297297297,
0.000140260767567568,
0.000135797348648649,
0.000149838805405405,
0.000142816637837838,
0.000143071227027027,
0.000141663924324324,
0.000134339468586387,
0.000149182623036649,
0.000143772908376963,
0.000137501518324607,
0.000138625623036649,
0.000136291842931937,
0.000136439630890052,
0.000140697916230366,
0.000139785981675393,
0.000140886761780105,
0.000132861314136126,
0.000133324845549738,
0.00013802242408377,
0.000141579458115183,
0.000140815748691099,
0.000170995497382199,
0.000139992743455497,
0.000141736316753927,
0.000137776146596859,
0.000134903256544503,
0.000140727910994764,
0.000136617685863874,
0.0001391175,
0.000137762083769633,
0.000135072968586387,
0.000136886431937173,
0.000140397013089005,
0.000136441031413613,
0.000144621476439791,
0.000132282013089005,
0.00013986682460733,
0.000139950170157068,
0.000141572719895288,
0.000138399222513089,
0.000135641819371728,
0.000139427654450262,
0.000141343947643979,
0.000138501384816754,
0.000149132463350785,
0.000140657764397906,
0.000135098620418848
],
"sem" : 2.4004310947012e-06,
"variance" : 4.37917277471037e-10
},
"times" : {
"cycle" : 0.0565153946163036,
"elapsed" : 6.023,
"period" : 0.000147946059204983,
"timeStamp" : 1567599314262
}
}
]
} |
Member
|
For the time being, we could potentially add some |
Contributor
Author
|
AFAIK it's better to use double quotes as those can easier be escaped (at least in my shell). So something like this would probably work + retain the quotes right? |
Contributor
Author
|
... so something like this if we put it all together |
Contributor
Author
|
Oh, even more clever: In zsh, you can do: In bash 4.4+, you can do: |
Member
|
So what about: Test casesBased on the below snippet
|
Contributor
Author
|
@v1v I tried to run this on my machine, but couldn't get it to work (maybe I have a too old version of bash). But have you tried it with double quotes as well? The 3 test cases you posted doesn't use those. (nit: maybe call the function |
Member
|
Or even the below snippet might be a bit simple if you like: Test caseswith the below snippet
|
Member
|
Contributor
Author
|
The |
Qard
reviewed
Sep 11, 2019
Qard
left a comment
Qard
left a comment
Contributor
There was a problem hiding this comment.
If I understand correctly, the intent is to only run (and save the result) on master? Has there been any thought about if we can run some of this on PRs to compare the delta of a PR against the data from master, before landing it? Or would that be too difficult?
| captureSpanStackTraces: false | ||
| }, | ||
| setup () { | ||
| var agent = this.benchmark.agent |
Contributor
There was a problem hiding this comment.
I guess this gets into the fn(...) scope below by benchmark.js generating code from the contents of the input functions and eval'ing that? 🤔
Seems a bit confusing and looks like dead code at first glance. Also, I feel like the linter would likely complain about this sort of thing.
Contributor
Author
There was a problem hiding this comment.
Correct. Which is also why I have to use var instead of const as those are not hoisted in the generated code where it happens to wrap the setup in a try-catch. To get the linter to behave, I've added a eslint-disable comment at the top of the file.
Check out this example to see how the code compilation works: https://benchmarkjs.com/docs#prototype_setup
Contributor
Author
|
@Qard I'd like to have this run for PR's as well. I don't yet know how to best make a Kibana dashboard to visualize this against Optimally I'd like a bot to comment on the PR if the difference to master crosses a threshold, but that's probably a project in itself. @v1v Can we make sure this automatically runs for PR's as well if it doesn't already? |
Member
If so, there will be a bottleneck for this repo and some other ones since there is only one baremetal worker in the CI. Could it be tackled in an incremental this new requirement? Then we can work out whether to ask for another baremetal or reconsider to run in a kind of async mode this particular stage to avoid adding any delay in providing any feedback regarding the pipeline WDYT? |
Contributor
Author
|
@v1v makes sense. I actually thought it was supposed to be running for PRs, but understand the current limitations. I think it's soon going to be critical for us to run benchmarks for PRs so we don't accidentally degrade the performance when merging a PR. |
Member
|
I love this swift left. I've just raised an issue with the infra team to get more |
Contributor
Author
|
@Qard When using function anonymous (window, t) {
var global = window,
clearTimeout = global.clearTimeout,
setTimeout = global.setTimeout;
var d = this,
deferred = d,
m = d.benchmark._original,
fn = m.fn,
su = m.setup,
td = m.teardown;
if (!d.cycles) {
d.fn = function () {
var deferred = d;
if (typeof f == "function") {
try {
// test function body
} catch (e) {
f(d)
}
} else {
// test function body
}
};
d.teardown = function () {
d.cycles = 0;
if (typeof td == "function") {
try{
// teardown function body
} catch (e) {
td()
}
} else {
// teardown function body
}
};
if (typeof su == "function") {
try {
// setup function body
} catch (e) {
su()
}
} else {
// setup function body
};
t.start(d);
}
d.fn();
return { uid: "..." }
}When instead using function anonymous (window, t) {
var global = window,
clearTimeout = global.clearTimeout,
setTimeout = global.setTimeout;
var r,
s,
m = this,
f = m.fn,
i = m.count,
n = t.ns;
// setup function body
s = n();
while (i--) {
// test function body
}
r = n(s);
r = r[0] + (r[1] / 1e9);
// teardown function body
return { elapsed: r, uid: "..." }
} |
Qard
approved these changes
Sep 11, 2019
Qard
left a comment
Qard
left a comment
Contributor
There was a problem hiding this comment.
LGTM other than some commit titles being too long, but we can squash that away. 👍
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment
3 participants
Add this suggestion to a batch that can be applied as a single commit.This suggestion is invalid because no changes were made to the code.Suggestions cannot be applied while the pull request is closed.Suggestions cannot be applied while viewing a subset of changes.Only one suggestion per line can be applied in a batch.Add this suggestion to a batch that can be applied as a single commit.Applying suggestions on deleted lines is not supported.You must change the existing code in this line in order to create a valid suggestion.Outdated suggestions cannot be applied.This suggestion has been applied or marked resolved.Suggestions cannot be applied from pending reviews.Suggestions cannot be applied on multi-line comments.Suggestions cannot be applied while the pull request is queued to merge.Suggestion cannot be applied right now. Please check back later.
This runs 5 different benchmarks, defined in each their own file:
test/benchmarks/001-transaction-and-span-no-stack-trace.js- Measures the overhead of starting and ending a transaction and a span without a stack tracetest/benchmarks/002-transaction-and-span-overhead-realistic-size.js- Measures the overhead of starting and ending a transaction and a span with a realistic size stack tracetest/benchmarks/003-transaction-and-span-with-stack-trace.js- Measures the overhead of starting and ending a transaction and a span with a simple uniform stack tracetest/benchmarks/004-transaction.js- Measures the overhead of starting and ending a transaction onlytest/benchmarks/005-transaction-reading-file.js- Measures the overhead of starting and ending a transaction only while reading a fileThe benchmarks are using the benchmark.js benchmarking tool. For each of the 5 benchmarks, I wanted to get the relative margin of error down below 1%, but benchmark.js doesn't support running the benchmark until a given relative margin of error threshold. So for now I just configured the benchmarks to run for 60 seconds, which in all but one case gets them below the magic 1%. I've opened a PR to add a feture to benchmark.js to allow for this type of setting: bestiejs/benchmark.js#223
Each of the 5 benchmarks generates a new document in Elasticsearch when running on Jenkins. Each of these documents looks like this:
Click to expand example benchmark document
{ "_index": "benchmark-nodejs", "_type": "_doc", "_id": "GawHGm0By9Yf0rV8w84W", "_version": 1, "_score": null, "_source": { "@timestamp": 1568099076042, "ci": { "build_cause": "pr" }, "git": { "branch": "PR-1145", "commit": "6a4303d71753081024daa0c10c74e42b0a9beb3a" }, "pr": { "id": "1145", "title": "Add", "target": "master", "url": "https://github.com/elastic/apm-agent-nodejs/pull/1145" }, "os": { "arch": "x64", "cpus": [ { "model": "Intel(R) Core(TM) i7-7700 CPU @ 3.60GHz", "speed": 3600, "times": { "user": 4084229000, "nice": 19632000, "sys": 1080955300, "idle": 420119126300, "irq": 0 } }, { "model": "Intel(R) Core(TM) i7-7700 CPU @ 3.60GHz", "speed": 3600, "times": { "user": 3817286900, "nice": 18511800, "sys": 1029496900, "idle": 420691376700, "irq": 0 } }, { "model": "Intel(R) Core(TM) i7-7700 CPU @ 3.60GHz", "speed": 3600, "times": { "user": 3995793400, "nice": 17760900, "sys": 989468700, "idle": 420362602400, "irq": 0 } }, { "model": "Intel(R) Core(TM) i7-7700 CPU @ 3.60GHz", "speed": 3600, "times": { "user": 4398593200, "nice": 17829400, "sys": 1021458100, "idle": 420189053900, "irq": 0 } }, { "model": "Intel(R) Core(TM) i7-7700 CPU @ 3.60GHz", "speed": 3600, "times": { "user": 4017084200, "nice": 17794900, "sys": 1007035400, "idle": 420455171200, "irq": 0 } }, { "model": "Intel(R) Core(TM) i7-7700 CPU @ 3.60GHz", "speed": 3600, "times": { "user": 3979874600, "nice": 17592200, "sys": 1004401000, "idle": 420552976900, "irq": 0 } }, { "model": "Intel(R) Core(TM) i7-7700 CPU @ 3.60GHz", "speed": 3600, "times": { "user": 4152233800, "nice": 18154400, "sys": 1024004100, "idle": 420378694200, "irq": 0 } }, { "model": "Intel(R) Core(TM) i7-7700 CPU @ 3.60GHz", "speed": 3600, "times": { "user": 4029285200, "nice": 17654800, "sys": 1016009900, "idle": 420556260800, "irq": 0 } } ], "freemem": 9343033344, "homedir": "/var/lib/jenkins/workspace/ejs_apm-agent-nodejs-mbp_PR-1145@2", "hostname": "worker-854309", "loadavg": [ 1.19189453125, 0.52099609375, 0.224609375 ], "platform": "linux", "release": "4.13.0-36-generic", "totalmem": 33504399360, "type": "Linux", "uptime": 42631607 }, "process": { "arch": "x64", "config": { "target_defaults": { "cflags": [], "default_configuration": "Release", "defines": [], "include_dirs": [], "libraries": [] }, "variables": { "asan": 0, "build_v8_with_gn": false, "coverage": false, "debug_nghttp2": false, "enable_lto": false, "enable_pgo_generate": false, "enable_pgo_use": false, "force_dynamic_crt": 0, "gas_version": "2.27", "host_arch": "x64", "icu_data_in": "../../deps/icu-small/source/data/in/icudt64l.dat", "icu_endianness": "l", "icu_gyp_path": "tools/icu/icu-generic.gyp", "icu_locales": "en,root", "icu_path": "deps/icu-small", "icu_small": true, "icu_ver_major": "64", "is_debug": 0, "llvm_version": 0, "napi_build_version": "4", "node_byteorder": "little", "node_code_cache": "yes", "node_debug_lib": false, "node_enable_d8": false, "node_install_npm": true, "node_module_version": 72, "node_no_browser_globals": false, "node_prefix": "/", "node_release_urlbase": "https://nodejs.org/download/release/", "node_report": true, "node_shared": false, "node_shared_cares": false, "node_shared_http_parser": false, "node_shared_libuv": false, "node_shared_nghttp2": false, "node_shared_openssl": false, "node_shared_zlib": false, "node_tag": "", "node_target_type": "executable", "node_use_bundled_v8": true, "node_use_dtrace": false, "node_use_etw": false, "node_use_large_pages": false, "node_use_large_pages_script_lld": false, "node_use_node_snapshot": true, "node_use_openssl": true, "node_use_v8_platform": true, "node_with_ltcg": false, "node_without_node_options": false, "openssl_fips": "", "openssl_is_fips": false, "shlib_suffix": "so.72", "target_arch": "x64", "v8_enable_gdbjit": 0, "v8_enable_i18n_support": 1, "v8_enable_inspector": 1, "v8_no_strict_aliasing": 1, "v8_optimized_debug": 1, "v8_promise_internal_field_count": 1, "v8_random_seed": 0, "v8_trace_maps": 0, "v8_use_siphash": 1, "v8_use_snapshot": 1, "want_separate_host_toolset": 0 } }, "env": { "NODE_VERSION": "node", "NVM_CD_FLAGS": "", "SHELL": "/bin/bash", "TERM": "xterm", "CHANGE_URL": "https://github.com/elastic/apm-agent-nodejs/pull/1145", "BRANCH_NAME": "PR-1145", "NVM_DIR": "/var/lib/jenkins/workspace/ejs_apm-agent-nodejs-mbp_PR-1145@2/.nvm", "USER": "root", "CHANGE_TITLE": "Add", "SUDO_USER": "jenkins", "SUDO_UID": "1106", "GIT_BUILD_CAUSE": "pr", "CHANGE_TARGET": "master", "USERNAME": "root", "GIT_COMMIT": "0198525593137c5c8f48505eaf0d201a1115a8c5", "MAIL": "/var/mail/root", "PATH": "/var/lib/jenkins/workspace/ejs_apm-agent-nodejs-mbp_PR-1145@2/.nvm/versions/node/v12.10.0/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/snap/bin", "CHANGE_ID": "1145", "PWD": "/var/lib/jenkins/workspace/ejs_apm-agent-nodejs-mbp_PR-1145@2/127/src/github.com/elastic/apm-agent-nodejs", "LANG": "en_US.UTF-8", "SHLVL": "1", "SUDO_COMMAND": "/usr/bin/cset proc --exec /benchmark -- ././test/benchmarks/scripts/run-benchmarks.sh all apm-agent-benchmark-results.json", "HOME": "/var/lib/jenkins/workspace/ejs_apm-agent-nodejs-mbp_PR-1145@2", "LOGNAME": "root", "NVM_BIN": "/var/lib/jenkins/workspace/ejs_apm-agent-nodejs-mbp_PR-1145@2/.nvm/versions/node/v12.10.0/bin", "SUDO_GID": "1106", "GIT_BASE_COMMIT": "6a4303d71753081024daa0c10c74e42b0a9beb3a", "_": "/var/lib/jenkins/workspace/ejs_apm-agent-nodejs-mbp_PR-1145@2/.nvm/versions/node/v12.10.0/bin/node" }, "platform": "linux", "release": { "name": "node", "sourceUrl": "https://nodejs.org/download/release/v12.10.0/node-v12.10.0.tar.gz", "headersUrl": "https://nodejs.org/download/release/v12.10.0/node-v12.10.0-headers.tar.gz" }, "version": "v12.10.0", "versions": { "node": "12.10.0", "v8": "7.6.303.29-node.16", "uv": "1.31.0", "zlib": "1.2.11", "brotli": "1.0.7", "ares": "1.15.0", "modules": "72", "nghttp2": "1.39.2", "napi": "4", "llhttp": "1.1.4", "http_parser": "2.8.0", "openssl": "1.1.1c", "cldr": "35.1", "icu": "64.2", "tz": "2019a", "unicode": "12.1" } }, "bench": { "name": "transaction-and-span-no-stack-trace", "count": 10000, "cycles": 1, "hz": 10965.37345916329, "stats": { "moe": 9.575091322752012e-7, "rme": 1.0499445225956963, "sem": 4.885250674873475e-7, "deviation": 0.000004087293959265252, "mean": 0.00009119616433714286, "sample": [ 0.0001233536633, 0.0000971044264, 0.0000938828668, ... ], "variance": 1.6705971909446222e-11 }, "times": { "cycle": 0.9119616433714286, "elapsed": 64.79, "period": 0.00009119616433714286, "timeStamp": 1568099076042 }, "overhead": 0.00008845952489404925, "controlStats": { "moe": 3.3894080613459174e-9, "rme": 0.1238529273521828, "sem": 1.7292898272173049e-9, "deviation": 5.39970575481139e-8, "mean": 0.0000027366394430936173, "sample": [ 0.000002516306237244263, 0.0000027454286923191797, 0.0000027217545422868236, ... ], "variance": 2.9156822238543244e-15 } } }, "fields": { "@timestamp": [ "2019-09-10T07:04:36.042Z" ] }, "sort": [ 1568099076042 ] }To add a new benchmark, simply add a new file to the
test/benchmarksdirectory.To get more info about how to run the benchmarks locally, run:
Closes #293
Closes #306
Tasks
Count CPU ticks for generating a transaction or spanUpdate: This is too complicated to do in this iteration