Update benchmark and diff output

benchmarks/diff_multi_bench_output.js

@@ -1,50 +1,55 @@
-#!/usr/bin/env node
-
 'use strict';
 
-/* jshint -W079: Ignore redefinitions (before & after) */
+var fs = require('fs');
+var metrics = require('metrics');
+    // `node diff_multi_bench_output.js beforeBench.txt afterBench.txt`
+var file1 = process.argv[2];
+var file2 = process.argv[3];
 
-var fs = require('fs'),
+if (!file1 || !file2) {
-    metrics = require('metrics'),
-
-    // `node diff_multi_bench_output.js before.txt after.txt`
-    before = process.argv[2],
-    after = process.argv[3];
-
-if (!before || !after) {
     console.log('Please supply two file arguments:');
     var n = __filename;
     n = n.substring(n.lastIndexOf('/', n.length));
-    console.log('    ./' + n + ' multiBenchBefore.txt multiBenchAfter.txt');
+    console.log('    node .' + n + ' benchBefore.txt benchAfter.txt\n');
-    console.log('To generate multiBenchBefore.txt, run');
+    console.log('To generate the benchmark files, run');
-    console.log('    node multi_bench.js > multiBenchBefore.txt');
+    console.log('    npm run benchmark > benchBefore.txt\n');
     console.log('Thank you for benchmarking responsibly.');
     return;
 }
 
-var before_lines = fs.readFileSync(before, 'utf8').split('\n'),
+var before_lines = fs.readFileSync(file1, 'utf8').split('\n');
-    after_lines = fs.readFileSync(after, 'utf8').split('\n');
+var after_lines = fs.readFileSync(file2, 'utf8').split('\n');
-
-console.log('Comparing before,', before.green, '(', before_lines.length,
-    'lines)', 'to after,', after.green, '(', after_lines.length, 'lines)');
-
 var total_ops = new metrics.Histogram.createUniformHistogram();
 
+console.log('Comparing before,', file1, '(', before_lines.length, 'lines)', 'to after,', file2, '(', after_lines.length, 'lines)');
+
 function is_whitespace(s) {
     return !!s.trim();
 }
 
-function parseInt10(s) {
+function pad(input, len, chr, right) {
-    return parseInt(s, 10);
+    var str = input.toString();
+    chr = chr || ' ';
+
+    if (right) {
+        while (str.length < len) {
+            str += chr;
+        }
+    } else {
+        while (str.length < len) {
+            str = chr + str;
+        }
+    }
+    return str;
 }
 
 // green if greater than 0, red otherwise
-function humanize_diff(num, unit) {
+function humanize_diff(num, unit, toFixed) {
     unit = unit || '';
     if (num > 0) {
-        return ('+' + num + unit).green;
+        return ' +' + pad(num.toFixed(toFixed || 0) + unit, 7);
     }
-    return ('' + num + unit).red;
+    return ' -' + pad(Math.abs(num).toFixed(toFixed || 0) + unit, 7);
 }
 
 function command_name(words) {
@@ -58,35 +63,33 @@ before_lines.forEach(function(b, i) {
         // console.log('#ignored#', '>'+a+'<', '>'+b+'<');
         return;
     }
-
     var b_words = b.split(' ').filter(is_whitespace);
     var a_words = a.split(' ').filter(is_whitespace);
 
-    var ops =
+    var ops = [b_words, a_words].map(function(words) {
-        [b_words, a_words]
+        // console.log(words);
-        .map(function(words) {
+        return words.slice(-2, -1) | 0;
-            // console.log(words);
+    }).filter(function(num) {
-            return parseInt10(words.slice(-2, -1));
+        var isNaN = !num && num !== 0;
-        }).filter(function(num) {
+        return !isNaN;
-            var isNaN = !num && num !== 0;
+    });
-            return !isNaN;
+    if (ops.length !== 2) {
-        });
+        return;
-    if (ops.length !== 2) return;
+    }
-
     var delta = ops[1] - ops[0];
-    var pct = ((delta / ops[0]) * 100).toPrecision(3);
+    var pct = +((delta / ops[0]) * 100);
-
+    ops[0] = pad(ops[0], 6);
+    ops[1] = pad(ops[1], 6);
     total_ops.update(delta);
-
     delta = humanize_diff(delta);
-    pct = humanize_diff(pct, '%');
+    var small_delta = pct < 3 && pct > -3;
-    console.log(
+    // Let's mark differences above 20% bold
-        // name of test
+    var big_delta = pct > 20 || pct < -20 ? ';1' : '';
-        command_name(a_words) === command_name(b_words) ?
+    pct = humanize_diff(pct, '', 2) + '%';
-            command_name(a_words) + ':' :
+    var str = pad((command_name(a_words) === command_name(b_words) ? command_name(a_words) + ':' : '404:'), 14, false, true) +
-            '404:',
+        (pad(ops.join(' -> '), 15) + ' ops/sec (∆' + delta + pct + ')');
-        // results of test
+    str = (small_delta ? '' : (/-[^>]/.test(str) ? '\x1b[31' : '\x1b[32') + big_delta + 'm') + str + '\x1b[0m';
-        ops.join(' -> '), 'ops/sec (∆', delta, pct, ')');
+    console.log(str);
 });
 
-console.log('Mean difference in ops/sec:', humanize_diff(total_ops.mean().toPrecision(6)));
+console.log('Mean difference in ops/sec:', humanize_diff(total_ops.mean(), '', 1));

benchmarks/multi_bench.js

@@ -7,24 +7,33 @@ var client_nr = 0;
 var redis = require('../index');
 var totalTime = 0;
 var metrics = require('metrics');
-var num_clients = parseInt(process.argv[2], 10) || 5;
-var num_requests = 50000;
 var tests = [];
 // var bluebird = require('bluebird');
 // bluebird.promisifyAll(redis.RedisClient.prototype);
 // bluebird.promisifyAll(redis.Multi.prototype);
+
+function returnArg (name, def) {
+    var matches = process.argv.filter(function(entry) {
+        return entry.indexOf(name + '=') === 0;
+    });
+    if (matches.length) {
+        return matches[0].substr(name.length + 1);
+    }
+    return def;
+}
+var num_clients = returnArg('clients', 1);
+var run_time = returnArg('time', 2500); // ms
 var versions_logged = false;
 var client_options = {
         return_buffers: false,
         max_attempts: 4,
-        parser: process.argv.indexOf('parser=javascript') === -1 ? 'hiredis' : 'javascript'
+        parser: returnArg('parser', 'hiredis')
     };
 var small_str, large_str, small_buf, large_buf, very_large_str, very_large_buf;
 
 function lpad(input, len, chr) {
     var str = input.toString();
     chr = chr || ' ';
-
     while (str.length < len) {
         str = chr + str;
     }
@@ -33,23 +42,19 @@ function lpad(input, len, chr) {
 
 metrics.Histogram.prototype.print_line = function () {
     var obj = this.printObj();
-
     return lpad(obj.min, 4) + '/' + lpad(obj.max, 4) + '/' + lpad(obj.mean.toFixed(2), 7) + '/' + lpad(obj.p95.toFixed(2), 7);
 };
 
 function Test(args) {
     this.args = args;
-
     this.callback = null;
     this.clients = [];
     this.clients_ready = 0;
     this.commands_sent = 0;
     this.commands_completed = 0;
-    this.max_pipeline = this.args.pipeline || num_requests;
+    this.max_pipeline = this.args.pipeline || 50;
     this.batch_pipeline = this.args.batch || 0;
     this.client_options = args.client_options || client_options;
-    this.num_requests = args.reqs || num_requests;
-
     this.connect_latency = new metrics.Histogram();
     this.ready_latency = new metrics.Histogram();
     this.command_latency = new metrics.Histogram();
@@ -57,9 +62,7 @@ function Test(args) {
 
 Test.prototype.run = function (callback) {
     var i;
-
     this.callback = callback;
-
     for (i = 0; i < num_clients ; i++) {
         this.new_client(i);
     }
@@ -112,27 +115,39 @@ Test.prototype.new_client = function (id) {
 Test.prototype.on_clients_ready = function () {
     process.stdout.write(lpad(this.args.descr, 13) + ', ' + (this.args.batch ? lpad('batch ' + this.args.batch, 9) : lpad(this.args.pipeline, 9)) + '/' + this.clients_ready + ' ');
     this.test_start = Date.now();
-
     this.fill_pipeline();
 };
 
 Test.prototype.fill_pipeline = function () {
     var pipeline = this.commands_sent - this.commands_completed;
 
-    if (this.batch_pipeline && this.commands_sent < this.num_requests) {
+    if (this.test_start < Date.now() - run_time) {
+        if (this.ended) {
+            return;
+        }
+        this.ended = true;
+        this.print_stats();
+        this.stop_clients();
+        return;
+    }
+
+    if (this.clients[0].should_buffer) {
+        var self = this;
+        setTimeout(function() {
+            self.fill_pipeline();
+        }, 1);
+        return;
+    }
+
+    if (this.batch_pipeline) {
         this.batch();
     } else {
-        while (this.commands_sent < this.num_requests && pipeline < this.max_pipeline) {
+        while (pipeline < this.max_pipeline) {
             this.commands_sent++;
             pipeline++;
             this.send_next();
         }
     }
-
-    if (this.commands_completed === this.num_requests) {
-        this.print_stats();
-        this.stop_clients();
-    }
 };
 
 Test.prototype.batch = function () {
@@ -191,7 +206,7 @@ Test.prototype.print_stats = function () {
     totalTime += duration;
 
     console.log('min/max/avg/p95: ' + this.command_latency.print_line() + ' ' + lpad(duration, 6) + 'ms total, ' +
-        lpad((this.num_requests / (duration / 1000)).toFixed(2), 9) + ' ops/sec');
+        lpad((this.commands_completed / (duration / 1000)).toFixed(2), 9) + ' ops/sec');
 };
 
 small_str = '1234';
@@ -203,19 +218,19 @@ very_large_buf = new Buffer(very_large_str);
 
 tests.push(new Test({descr: 'PING', command: 'ping', args: [], pipeline: 1}));
 tests.push(new Test({descr: 'PING', command: 'ping', args: [], pipeline: 50}));
-tests.push(new Test({descr: 'PING', command: 'ping', args: [], batch: 50, reqs: num_requests * 2}));
+tests.push(new Test({descr: 'PING', command: 'ping', args: [], batch: 50}));
 
 tests.push(new Test({descr: 'SET 4B str', command: 'set', args: ['foo_rand000000000000', small_str], pipeline: 1}));
 tests.push(new Test({descr: 'SET 4B str', command: 'set', args: ['foo_rand000000000000', small_str], pipeline: 50}));
-tests.push(new Test({descr: 'SET 4B str', command: 'set', args: ['foo_rand000000000000', small_str], batch: 50, reqs: num_requests * 2}));
+tests.push(new Test({descr: 'SET 4B str', command: 'set', args: ['foo_rand000000000000', small_str], batch: 50}));
 
 tests.push(new Test({descr: 'SET 4B buf', command: 'set', args: ['foo_rand000000000000', small_buf], pipeline: 1}));
 tests.push(new Test({descr: 'SET 4B buf', command: 'set', args: ['foo_rand000000000000', small_buf], pipeline: 50}));
-tests.push(new Test({descr: 'SET 4B buf', command: 'set', args: ['foo_rand000000000000', small_buf], batch: 50, reqs: num_requests * 2}));
+tests.push(new Test({descr: 'SET 4B buf', command: 'set', args: ['foo_rand000000000000', small_buf], batch: 50}));
 
 tests.push(new Test({descr: 'GET 4B str', command: 'get', args: ['foo_rand000000000000'], pipeline: 1}));
 tests.push(new Test({descr: 'GET 4B str', command: 'get', args: ['foo_rand000000000000'], pipeline: 50}));
-tests.push(new Test({descr: 'GET 4B str', command: 'get', args: ['foo_rand000000000000'], batch: 50, reqs: num_requests * 2}));
+tests.push(new Test({descr: 'GET 4B str', command: 'get', args: ['foo_rand000000000000'], batch: 50}));
 
 tests.push(new Test({descr: 'GET 4B buf', command: 'get', args: ['foo_rand000000000000'], pipeline: 1, client_opts: { return_buffers: true} }));
 tests.push(new Test({descr: 'GET 4B buf', command: 'get', args: ['foo_rand000000000000'], pipeline: 50, client_opts: { return_buffers: true} }));
@@ -223,15 +238,15 @@ tests.push(new Test({descr: 'GET 4B buf', command: 'get', args: ['foo_rand000000
 
 tests.push(new Test({descr: 'SET 4KiB str', command: 'set', args: ['foo_rand000000000001', large_str], pipeline: 1}));
 tests.push(new Test({descr: 'SET 4KiB str', command: 'set', args: ['foo_rand000000000001', large_str], pipeline: 50}));
-tests.push(new Test({descr: 'SET 4KiB str', command: 'set', args: ['foo_rand000000000001', large_str], batch: 50, reqs: num_requests * 2}));
+tests.push(new Test({descr: 'SET 4KiB str', command: 'set', args: ['foo_rand000000000001', large_str], batch: 50}));
 
 tests.push(new Test({descr: 'SET 4KiB buf', command: 'set', args: ['foo_rand000000000001', large_buf], pipeline: 1}));
 tests.push(new Test({descr: 'SET 4KiB buf', command: 'set', args: ['foo_rand000000000001', large_buf], pipeline: 50}));
-tests.push(new Test({descr: 'SET 4KiB buf', command: 'set', args: ['foo_rand000000000001', large_buf], batch: 50, reqs: num_requests * 2}));
+tests.push(new Test({descr: 'SET 4KiB buf', command: 'set', args: ['foo_rand000000000001', large_buf], batch: 50}));
 
 tests.push(new Test({descr: 'GET 4KiB str', command: 'get', args: ['foo_rand000000000001'], pipeline: 1}));
 tests.push(new Test({descr: 'GET 4KiB str', command: 'get', args: ['foo_rand000000000001'], pipeline: 50}));
-tests.push(new Test({descr: 'GET 4KiB str', command: 'get', args: ['foo_rand000000000001'], batch: 50, reqs: num_requests * 2}));
+tests.push(new Test({descr: 'GET 4KiB str', command: 'get', args: ['foo_rand000000000001'], batch: 50}));
 
 tests.push(new Test({descr: 'GET 4KiB buf', command: 'get', args: ['foo_rand000000000001'], pipeline: 1, client_opts: { return_buffers: true} }));
 tests.push(new Test({descr: 'GET 4KiB buf', command: 'get', args: ['foo_rand000000000001'], pipeline: 50, client_opts: { return_buffers: true} }));
@@ -239,31 +254,35 @@ tests.push(new Test({descr: 'GET 4KiB buf', command: 'get', args: ['foo_rand0000
 
 tests.push(new Test({descr: 'INCR', command: 'incr', args: ['counter_rand000000000000'], pipeline: 1}));
 tests.push(new Test({descr: 'INCR', command: 'incr', args: ['counter_rand000000000000'], pipeline: 50}));
-tests.push(new Test({descr: 'INCR', command: 'incr', args: ['counter_rand000000000000'], batch: 50, reqs: num_requests * 2}));
+tests.push(new Test({descr: 'INCR', command: 'incr', args: ['counter_rand000000000000'], batch: 50}));
 
 tests.push(new Test({descr: 'LPUSH', command: 'lpush', args: ['mylist', small_str], pipeline: 1}));
 tests.push(new Test({descr: 'LPUSH', command: 'lpush', args: ['mylist', small_str], pipeline: 50}));
-tests.push(new Test({descr: 'LPUSH', command: 'lpush', args: ['mylist', small_str], batch: 50, reqs: num_requests * 2}));
+tests.push(new Test({descr: 'LPUSH', command: 'lpush', args: ['mylist', small_str], batch: 50}));
 
 tests.push(new Test({descr: 'LRANGE 10', command: 'lrange', args: ['mylist', '0', '9'], pipeline: 1}));
 tests.push(new Test({descr: 'LRANGE 10', command: 'lrange', args: ['mylist', '0', '9'], pipeline: 50}));
-tests.push(new Test({descr: 'LRANGE 10', command: 'lrange', args: ['mylist', '0', '9'], batch: 50, reqs: num_requests * 2}));
+tests.push(new Test({descr: 'LRANGE 10', command: 'lrange', args: ['mylist', '0', '9'], batch: 50}));
 
 tests.push(new Test({descr: 'LRANGE 100', command: 'lrange', args: ['mylist', '0', '99'], pipeline: 1}));
 tests.push(new Test({descr: 'LRANGE 100', command: 'lrange', args: ['mylist', '0', '99'], pipeline: 50}));
-tests.push(new Test({descr: 'LRANGE 100', command: 'lrange', args: ['mylist', '0', '99'], batch: 50, reqs: num_requests * 2}));
+tests.push(new Test({descr: 'LRANGE 100', command: 'lrange', args: ['mylist', '0', '99'], batch: 50}));
 
-tests.push(new Test({descr: 'SET 4MiB buf', command: 'set', args: ['foo_rand000000000002', very_large_buf], pipeline: 1, reqs: 500}));
+tests.push(new Test({descr: 'SET 4MiB buf', command: 'set', args: ['foo_rand000000000002', very_large_buf], pipeline: 1}));
-tests.push(new Test({descr: 'SET 4MiB buf', command: 'set', args: ['foo_rand000000000002', very_large_buf], pipeline: 20, reqs: 500}));
+tests.push(new Test({descr: 'SET 4MiB buf', command: 'set', args: ['foo_rand000000000002', very_large_buf], pipeline: 20}));
-tests.push(new Test({descr: 'SET 4MiB buf', command: 'set', args: ['foo_rand000000000002', very_large_buf], batch: 20, reqs: 500}));
+tests.push(new Test({descr: 'SET 4MiB buf', command: 'set', args: ['foo_rand000000000002', very_large_buf], batch: 20}));
 
-tests.push(new Test({descr: 'GET 4MiB str', command: 'get', args: ['foo_rand000000000002'], pipeline: 1, reqs: 100}));
+tests.push(new Test({descr: 'SET 4MiB str', command: 'set', args: ['foo_rand000000000002', very_large_str], pipeline: 1}));
-tests.push(new Test({descr: 'GET 4MiB str', command: 'get', args: ['foo_rand000000000002'], pipeline: 20, reqs: 100}));
+tests.push(new Test({descr: 'SET 4MiB str', command: 'set', args: ['foo_rand000000000002', very_large_str], pipeline: 20}));
-tests.push(new Test({descr: 'GET 4MiB str', command: 'get', args: ['foo_rand000000000002'], batch: 20, reqs: 100}));
+tests.push(new Test({descr: 'SET 4MiB str', command: 'set', args: ['foo_rand000000000002', very_large_str], batch: 20}));
 
-tests.push(new Test({descr: 'GET 4MiB buf', command: 'get', args: ['foo_rand000000000002'], pipeline: 1, reqs: 100, client_opts: { return_buffers: true} }));
+tests.push(new Test({descr: 'GET 4MiB str', command: 'get', args: ['foo_rand000000000002'], pipeline: 1}));
-tests.push(new Test({descr: 'GET 4MiB buf', command: 'get', args: ['foo_rand000000000002'], pipeline: 20, reqs: 100, client_opts: { return_buffers: true} }));
+tests.push(new Test({descr: 'GET 4MiB str', command: 'get', args: ['foo_rand000000000002'], pipeline: 20}));
-tests.push(new Test({descr: 'GET 4MiB buf', command: 'get', args: ['foo_rand000000000002'], batch: 20, reqs: 100, client_opts: { return_buffers: true} }));
+tests.push(new Test({descr: 'GET 4MiB str', command: 'get', args: ['foo_rand000000000002'], batch: 20}));
+
+tests.push(new Test({descr: 'GET 4MiB buf', command: 'get', args: ['foo_rand000000000002'], pipeline: 1, client_opts: { return_buffers: true} }));
+tests.push(new Test({descr: 'GET 4MiB buf', command: 'get', args: ['foo_rand000000000002'], pipeline: 20, client_opts: { return_buffers: true} }));
+tests.push(new Test({descr: 'GET 4MiB buf', command: 'get', args: ['foo_rand000000000002'], batch: 20, client_opts: { return_buffers: true} }));
 
 function next() {
     var test = tests.shift();