From 3142af584724178c61b74217f65a65c028386a04 Mon Sep 17 00:00:00 2001 From: Ruben Bridgewater Date: Mon, 26 Oct 2015 19:34:37 +0100 Subject: [PATCH] Update benchmark and diff output --- benchmarks/diff_multi_bench_output.js | 101 +++++++++++++------------- benchmarks/multi_bench.js | 99 +++++++++++++++---------- 2 files changed, 111 insertions(+), 89 deletions(-) diff --git a/benchmarks/diff_multi_bench_output.js b/benchmarks/diff_multi_bench_output.js index fb2a4a66ce..6ba8c6900a 100755 --- a/benchmarks/diff_multi_bench_output.js +++ b/benchmarks/diff_multi_bench_output.js @@ -1,50 +1,55 @@ -#!/usr/bin/env node - 'use strict'; -/* jshint -W079: Ignore redefinitions (before & after) */ +var fs = require('fs'); +var metrics = require('metrics'); + // `node diff_multi_bench_output.js beforeBench.txt afterBench.txt` +var file1 = process.argv[2]; +var file2 = process.argv[3]; -var fs = require('fs'), - metrics = require('metrics'), - - // `node diff_multi_bench_output.js before.txt after.txt` - before = process.argv[2], - after = process.argv[3]; - -if (!before || !after) { +if (!file1 || !file2) { console.log('Please supply two file arguments:'); var n = __filename; n = n.substring(n.lastIndexOf('/', n.length)); - console.log(' ./' + n + ' multiBenchBefore.txt multiBenchAfter.txt'); - console.log('To generate multiBenchBefore.txt, run'); - console.log(' node multi_bench.js > multiBenchBefore.txt'); + console.log(' node .' + n + ' benchBefore.txt benchAfter.txt\n'); + console.log('To generate the benchmark files, run'); + console.log(' npm run benchmark > benchBefore.txt\n'); console.log('Thank you for benchmarking responsibly.'); return; } -var before_lines = fs.readFileSync(before, 'utf8').split('\n'), - after_lines = fs.readFileSync(after, 'utf8').split('\n'); - -console.log('Comparing before,', before.green, '(', before_lines.length, - 'lines)', 'to after,', after.green, '(', after_lines.length, 'lines)'); - +var before_lines = fs.readFileSync(file1, 'utf8').split('\n'); +var after_lines = fs.readFileSync(file2, 'utf8').split('\n'); var total_ops = new metrics.Histogram.createUniformHistogram(); +console.log('Comparing before,', file1, '(', before_lines.length, 'lines)', 'to after,', file2, '(', after_lines.length, 'lines)'); + function is_whitespace(s) { return !!s.trim(); } -function parseInt10(s) { - return parseInt(s, 10); +function pad(input, len, chr, right) { + var str = input.toString(); + chr = chr || ' '; + + if (right) { + while (str.length < len) { + str += chr; + } + } else { + while (str.length < len) { + str = chr + str; + } + } + return str; } // green if greater than 0, red otherwise -function humanize_diff(num, unit) { +function humanize_diff(num, unit, toFixed) { unit = unit || ''; if (num > 0) { - return ('+' + num + unit).green; + return ' +' + pad(num.toFixed(toFixed || 0) + unit, 7); } - return ('' + num + unit).red; + return ' -' + pad(Math.abs(num).toFixed(toFixed || 0) + unit, 7); } function command_name(words) { @@ -58,35 +63,33 @@ before_lines.forEach(function(b, i) { // console.log('#ignored#', '>'+a+'<', '>'+b+'<'); return; } - var b_words = b.split(' ').filter(is_whitespace); var a_words = a.split(' ').filter(is_whitespace); - var ops = - [b_words, a_words] - .map(function(words) { - // console.log(words); - return parseInt10(words.slice(-2, -1)); - }).filter(function(num) { - var isNaN = !num && num !== 0; - return !isNaN; - }); - if (ops.length !== 2) return; - + var ops = [b_words, a_words].map(function(words) { + // console.log(words); + return words.slice(-2, -1) | 0; + }).filter(function(num) { + var isNaN = !num && num !== 0; + return !isNaN; + }); + if (ops.length !== 2) { + return; + } var delta = ops[1] - ops[0]; - var pct = ((delta / ops[0]) * 100).toPrecision(3); - + var pct = +((delta / ops[0]) * 100); + ops[0] = pad(ops[0], 6); + ops[1] = pad(ops[1], 6); total_ops.update(delta); - delta = humanize_diff(delta); - pct = humanize_diff(pct, '%'); - console.log( - // name of test - command_name(a_words) === command_name(b_words) ? - command_name(a_words) + ':' : - '404:', - // results of test - ops.join(' -> '), 'ops/sec (∆', delta, pct, ')'); + var small_delta = pct < 3 && pct > -3; + // Let's mark differences above 20% bold + var big_delta = pct > 20 || pct < -20 ? ';1' : ''; + pct = humanize_diff(pct, '', 2) + '%'; + var str = pad((command_name(a_words) === command_name(b_words) ? command_name(a_words) + ':' : '404:'), 14, false, true) + + (pad(ops.join(' -> '), 15) + ' ops/sec (∆' + delta + pct + ')'); + str = (small_delta ? '' : (/-[^>]/.test(str) ? '\x1b[31' : '\x1b[32') + big_delta + 'm') + str + '\x1b[0m'; + console.log(str); }); -console.log('Mean difference in ops/sec:', humanize_diff(total_ops.mean().toPrecision(6))); +console.log('Mean difference in ops/sec:', humanize_diff(total_ops.mean(), '', 1)); diff --git a/benchmarks/multi_bench.js b/benchmarks/multi_bench.js index 62b79668ef..43831414de 100644 --- a/benchmarks/multi_bench.js +++ b/benchmarks/multi_bench.js @@ -7,24 +7,33 @@ var client_nr = 0; var redis = require('../index'); var totalTime = 0; var metrics = require('metrics'); -var num_clients = parseInt(process.argv[2], 10) || 5; -var num_requests = 50000; var tests = []; // var bluebird = require('bluebird'); // bluebird.promisifyAll(redis.RedisClient.prototype); // bluebird.promisifyAll(redis.Multi.prototype); + +function returnArg (name, def) { + var matches = process.argv.filter(function(entry) { + return entry.indexOf(name + '=') === 0; + }); + if (matches.length) { + return matches[0].substr(name.length + 1); + } + return def; +} +var num_clients = returnArg('clients', 1); +var run_time = returnArg('time', 2500); // ms var versions_logged = false; var client_options = { return_buffers: false, max_attempts: 4, - parser: process.argv.indexOf('parser=javascript') === -1 ? 'hiredis' : 'javascript' + parser: returnArg('parser', 'hiredis') }; var small_str, large_str, small_buf, large_buf, very_large_str, very_large_buf; function lpad(input, len, chr) { var str = input.toString(); chr = chr || ' '; - while (str.length < len) { str = chr + str; } @@ -33,23 +42,19 @@ function lpad(input, len, chr) { metrics.Histogram.prototype.print_line = function () { var obj = this.printObj(); - return lpad(obj.min, 4) + '/' + lpad(obj.max, 4) + '/' + lpad(obj.mean.toFixed(2), 7) + '/' + lpad(obj.p95.toFixed(2), 7); }; function Test(args) { this.args = args; - this.callback = null; this.clients = []; this.clients_ready = 0; this.commands_sent = 0; this.commands_completed = 0; - this.max_pipeline = this.args.pipeline || num_requests; + this.max_pipeline = this.args.pipeline || 50; this.batch_pipeline = this.args.batch || 0; this.client_options = args.client_options || client_options; - this.num_requests = args.reqs || num_requests; - this.connect_latency = new metrics.Histogram(); this.ready_latency = new metrics.Histogram(); this.command_latency = new metrics.Histogram(); @@ -57,9 +62,7 @@ function Test(args) { Test.prototype.run = function (callback) { var i; - this.callback = callback; - for (i = 0; i < num_clients ; i++) { this.new_client(i); } @@ -112,27 +115,39 @@ Test.prototype.new_client = function (id) { Test.prototype.on_clients_ready = function () { process.stdout.write(lpad(this.args.descr, 13) + ', ' + (this.args.batch ? lpad('batch ' + this.args.batch, 9) : lpad(this.args.pipeline, 9)) + '/' + this.clients_ready + ' '); this.test_start = Date.now(); - this.fill_pipeline(); }; Test.prototype.fill_pipeline = function () { var pipeline = this.commands_sent - this.commands_completed; - if (this.batch_pipeline && this.commands_sent < this.num_requests) { + if (this.test_start < Date.now() - run_time) { + if (this.ended) { + return; + } + this.ended = true; + this.print_stats(); + this.stop_clients(); + return; + } + + if (this.clients[0].should_buffer) { + var self = this; + setTimeout(function() { + self.fill_pipeline(); + }, 1); + return; + } + + if (this.batch_pipeline) { this.batch(); } else { - while (this.commands_sent < this.num_requests && pipeline < this.max_pipeline) { + while (pipeline < this.max_pipeline) { this.commands_sent++; pipeline++; this.send_next(); } } - - if (this.commands_completed === this.num_requests) { - this.print_stats(); - this.stop_clients(); - } }; Test.prototype.batch = function () { @@ -191,7 +206,7 @@ Test.prototype.print_stats = function () { totalTime += duration; console.log('min/max/avg/p95: ' + this.command_latency.print_line() + ' ' + lpad(duration, 6) + 'ms total, ' + - lpad((this.num_requests / (duration / 1000)).toFixed(2), 9) + ' ops/sec'); + lpad((this.commands_completed / (duration / 1000)).toFixed(2), 9) + ' ops/sec'); }; small_str = '1234'; @@ -203,19 +218,19 @@ very_large_buf = new Buffer(very_large_str); tests.push(new Test({descr: 'PING', command: 'ping', args: [], pipeline: 1})); tests.push(new Test({descr: 'PING', command: 'ping', args: [], pipeline: 50})); -tests.push(new Test({descr: 'PING', command: 'ping', args: [], batch: 50, reqs: num_requests * 2})); +tests.push(new Test({descr: 'PING', command: 'ping', args: [], batch: 50})); tests.push(new Test({descr: 'SET 4B str', command: 'set', args: ['foo_rand000000000000', small_str], pipeline: 1})); tests.push(new Test({descr: 'SET 4B str', command: 'set', args: ['foo_rand000000000000', small_str], pipeline: 50})); -tests.push(new Test({descr: 'SET 4B str', command: 'set', args: ['foo_rand000000000000', small_str], batch: 50, reqs: num_requests * 2})); +tests.push(new Test({descr: 'SET 4B str', command: 'set', args: ['foo_rand000000000000', small_str], batch: 50})); tests.push(new Test({descr: 'SET 4B buf', command: 'set', args: ['foo_rand000000000000', small_buf], pipeline: 1})); tests.push(new Test({descr: 'SET 4B buf', command: 'set', args: ['foo_rand000000000000', small_buf], pipeline: 50})); -tests.push(new Test({descr: 'SET 4B buf', command: 'set', args: ['foo_rand000000000000', small_buf], batch: 50, reqs: num_requests * 2})); +tests.push(new Test({descr: 'SET 4B buf', command: 'set', args: ['foo_rand000000000000', small_buf], batch: 50})); tests.push(new Test({descr: 'GET 4B str', command: 'get', args: ['foo_rand000000000000'], pipeline: 1})); tests.push(new Test({descr: 'GET 4B str', command: 'get', args: ['foo_rand000000000000'], pipeline: 50})); -tests.push(new Test({descr: 'GET 4B str', command: 'get', args: ['foo_rand000000000000'], batch: 50, reqs: num_requests * 2})); +tests.push(new Test({descr: 'GET 4B str', command: 'get', args: ['foo_rand000000000000'], batch: 50})); tests.push(new Test({descr: 'GET 4B buf', command: 'get', args: ['foo_rand000000000000'], pipeline: 1, client_opts: { return_buffers: true} })); tests.push(new Test({descr: 'GET 4B buf', command: 'get', args: ['foo_rand000000000000'], pipeline: 50, client_opts: { return_buffers: true} })); @@ -223,15 +238,15 @@ tests.push(new Test({descr: 'GET 4B buf', command: 'get', args: ['foo_rand000000 tests.push(new Test({descr: 'SET 4KiB str', command: 'set', args: ['foo_rand000000000001', large_str], pipeline: 1})); tests.push(new Test({descr: 'SET 4KiB str', command: 'set', args: ['foo_rand000000000001', large_str], pipeline: 50})); -tests.push(new Test({descr: 'SET 4KiB str', command: 'set', args: ['foo_rand000000000001', large_str], batch: 50, reqs: num_requests * 2})); +tests.push(new Test({descr: 'SET 4KiB str', command: 'set', args: ['foo_rand000000000001', large_str], batch: 50})); tests.push(new Test({descr: 'SET 4KiB buf', command: 'set', args: ['foo_rand000000000001', large_buf], pipeline: 1})); tests.push(new Test({descr: 'SET 4KiB buf', command: 'set', args: ['foo_rand000000000001', large_buf], pipeline: 50})); -tests.push(new Test({descr: 'SET 4KiB buf', command: 'set', args: ['foo_rand000000000001', large_buf], batch: 50, reqs: num_requests * 2})); +tests.push(new Test({descr: 'SET 4KiB buf', command: 'set', args: ['foo_rand000000000001', large_buf], batch: 50})); tests.push(new Test({descr: 'GET 4KiB str', command: 'get', args: ['foo_rand000000000001'], pipeline: 1})); tests.push(new Test({descr: 'GET 4KiB str', command: 'get', args: ['foo_rand000000000001'], pipeline: 50})); -tests.push(new Test({descr: 'GET 4KiB str', command: 'get', args: ['foo_rand000000000001'], batch: 50, reqs: num_requests * 2})); +tests.push(new Test({descr: 'GET 4KiB str', command: 'get', args: ['foo_rand000000000001'], batch: 50})); tests.push(new Test({descr: 'GET 4KiB buf', command: 'get', args: ['foo_rand000000000001'], pipeline: 1, client_opts: { return_buffers: true} })); tests.push(new Test({descr: 'GET 4KiB buf', command: 'get', args: ['foo_rand000000000001'], pipeline: 50, client_opts: { return_buffers: true} })); @@ -239,31 +254,35 @@ tests.push(new Test({descr: 'GET 4KiB buf', command: 'get', args: ['foo_rand0000 tests.push(new Test({descr: 'INCR', command: 'incr', args: ['counter_rand000000000000'], pipeline: 1})); tests.push(new Test({descr: 'INCR', command: 'incr', args: ['counter_rand000000000000'], pipeline: 50})); -tests.push(new Test({descr: 'INCR', command: 'incr', args: ['counter_rand000000000000'], batch: 50, reqs: num_requests * 2})); +tests.push(new Test({descr: 'INCR', command: 'incr', args: ['counter_rand000000000000'], batch: 50})); tests.push(new Test({descr: 'LPUSH', command: 'lpush', args: ['mylist', small_str], pipeline: 1})); tests.push(new Test({descr: 'LPUSH', command: 'lpush', args: ['mylist', small_str], pipeline: 50})); -tests.push(new Test({descr: 'LPUSH', command: 'lpush', args: ['mylist', small_str], batch: 50, reqs: num_requests * 2})); +tests.push(new Test({descr: 'LPUSH', command: 'lpush', args: ['mylist', small_str], batch: 50})); tests.push(new Test({descr: 'LRANGE 10', command: 'lrange', args: ['mylist', '0', '9'], pipeline: 1})); tests.push(new Test({descr: 'LRANGE 10', command: 'lrange', args: ['mylist', '0', '9'], pipeline: 50})); -tests.push(new Test({descr: 'LRANGE 10', command: 'lrange', args: ['mylist', '0', '9'], batch: 50, reqs: num_requests * 2})); +tests.push(new Test({descr: 'LRANGE 10', command: 'lrange', args: ['mylist', '0', '9'], batch: 50})); tests.push(new Test({descr: 'LRANGE 100', command: 'lrange', args: ['mylist', '0', '99'], pipeline: 1})); tests.push(new Test({descr: 'LRANGE 100', command: 'lrange', args: ['mylist', '0', '99'], pipeline: 50})); -tests.push(new Test({descr: 'LRANGE 100', command: 'lrange', args: ['mylist', '0', '99'], batch: 50, reqs: num_requests * 2})); +tests.push(new Test({descr: 'LRANGE 100', command: 'lrange', args: ['mylist', '0', '99'], batch: 50})); -tests.push(new Test({descr: 'SET 4MiB buf', command: 'set', args: ['foo_rand000000000002', very_large_buf], pipeline: 1, reqs: 500})); -tests.push(new Test({descr: 'SET 4MiB buf', command: 'set', args: ['foo_rand000000000002', very_large_buf], pipeline: 20, reqs: 500})); -tests.push(new Test({descr: 'SET 4MiB buf', command: 'set', args: ['foo_rand000000000002', very_large_buf], batch: 20, reqs: 500})); +tests.push(new Test({descr: 'SET 4MiB buf', command: 'set', args: ['foo_rand000000000002', very_large_buf], pipeline: 1})); +tests.push(new Test({descr: 'SET 4MiB buf', command: 'set', args: ['foo_rand000000000002', very_large_buf], pipeline: 20})); +tests.push(new Test({descr: 'SET 4MiB buf', command: 'set', args: ['foo_rand000000000002', very_large_buf], batch: 20})); -tests.push(new Test({descr: 'GET 4MiB str', command: 'get', args: ['foo_rand000000000002'], pipeline: 1, reqs: 100})); -tests.push(new Test({descr: 'GET 4MiB str', command: 'get', args: ['foo_rand000000000002'], pipeline: 20, reqs: 100})); -tests.push(new Test({descr: 'GET 4MiB str', command: 'get', args: ['foo_rand000000000002'], batch: 20, reqs: 100})); +tests.push(new Test({descr: 'SET 4MiB str', command: 'set', args: ['foo_rand000000000002', very_large_str], pipeline: 1})); +tests.push(new Test({descr: 'SET 4MiB str', command: 'set', args: ['foo_rand000000000002', very_large_str], pipeline: 20})); +tests.push(new Test({descr: 'SET 4MiB str', command: 'set', args: ['foo_rand000000000002', very_large_str], batch: 20})); -tests.push(new Test({descr: 'GET 4MiB buf', command: 'get', args: ['foo_rand000000000002'], pipeline: 1, reqs: 100, client_opts: { return_buffers: true} })); -tests.push(new Test({descr: 'GET 4MiB buf', command: 'get', args: ['foo_rand000000000002'], pipeline: 20, reqs: 100, client_opts: { return_buffers: true} })); -tests.push(new Test({descr: 'GET 4MiB buf', command: 'get', args: ['foo_rand000000000002'], batch: 20, reqs: 100, client_opts: { return_buffers: true} })); +tests.push(new Test({descr: 'GET 4MiB str', command: 'get', args: ['foo_rand000000000002'], pipeline: 1})); +tests.push(new Test({descr: 'GET 4MiB str', command: 'get', args: ['foo_rand000000000002'], pipeline: 20})); +tests.push(new Test({descr: 'GET 4MiB str', command: 'get', args: ['foo_rand000000000002'], batch: 20})); + +tests.push(new Test({descr: 'GET 4MiB buf', command: 'get', args: ['foo_rand000000000002'], pipeline: 1, client_opts: { return_buffers: true} })); +tests.push(new Test({descr: 'GET 4MiB buf', command: 'get', args: ['foo_rand000000000002'], pipeline: 20, client_opts: { return_buffers: true} })); +tests.push(new Test({descr: 'GET 4MiB buf', command: 'get', args: ['foo_rand000000000002'], batch: 20, client_opts: { return_buffers: true} })); function next() { var test = tests.shift();