diff --git a/README.md b/README.md index cd728c8f02..df7519c143 100644 --- a/README.md +++ b/README.md @@ -6,7 +6,7 @@ redis - a node.js redis client [![Windows Tests](https://img.shields.io/appveyor/ci/BridgeAR/node-redis/master.svg?label=Windows%20Tests)](https://ci.appveyor.com/project/BridgeAR/node-redis/branch/master) [![Gitter](https://badges.gitter.im/Join%20Chat.svg)](https://gitter.im/NodeRedis/node_redis?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge) -This is a complete and feature rich Redis client for node.js. It supports all Redis commands and focuses on performance. +This is a complete and feature rich Redis client for node.js. It supports all Redis commands and focuses on high performance. Install with: @@ -646,40 +646,40 @@ hiredis parser (Lenovo T450s i7-5600U): ``` Client count: 1, node version: 4.2.2, server version: 3.0.3, parser: hiredis - PING, 1/1 min/max/avg/p95: 0/ 3/ 0.02/ 0.00 2501ms total, 39862.85 ops/sec - PING, batch 50/1 min/max/avg/p95: 0/ 2/ 0.10/ 1.00 2501ms total, 491223.51 ops/sec - SET 4B str, 1/1 min/max/avg/p95: 0/ 3/ 0.03/ 0.00 2501ms total, 36387.45 ops/sec - SET 4B str, batch 50/1 min/max/avg/p95: 0/ 3/ 0.14/ 1.00 2501ms total, 346381.45 ops/sec - SET 4B buf, 1/1 min/max/avg/p95: 0/ 2/ 0.04/ 0.00 2501ms total, 24395.84 ops/sec - SET 4B buf, batch 50/1 min/max/avg/p95: 0/ 2/ 0.32/ 1.00 2501ms total, 156457.42 ops/sec - GET 4B str, 1/1 min/max/avg/p95: 0/ 3/ 0.03/ 0.00 2501ms total, 36906.44 ops/sec - GET 4B str, batch 50/1 min/max/avg/p95: 0/ 3/ 0.12/ 1.00 2501ms total, 425729.71 ops/sec - GET 4B buf, 1/1 min/max/avg/p95: 0/ 2/ 0.03/ 0.00 2501ms total, 36221.91 ops/sec - GET 4B buf, batch 50/1 min/max/avg/p95: 0/ 2/ 0.11/ 1.00 2501ms total, 430407.84 ops/sec - SET 4KiB str, 1/1 min/max/avg/p95: 0/ 3/ 0.03/ 0.00 2501ms total, 30951.22 ops/sec - SET 4KiB str, batch 50/1 min/max/avg/p95: 0/ 2/ 0.33/ 1.00 2501ms total, 150299.88 ops/sec - SET 4KiB buf, 1/1 min/max/avg/p95: 0/ 2/ 0.04/ 1.00 2501ms total, 23919.63 ops/sec - SET 4KiB buf, batch 50/1 min/max/avg/p95: 0/ 2/ 0.36/ 1.00 2501ms total, 139204.32 ops/sec - GET 4KiB str, 1/1 min/max/avg/p95: 0/ 2/ 0.03/ 0.00 2501ms total, 32739.30 ops/sec - GET 4KiB str, batch 50/1 min/max/avg/p95: 0/ 2/ 0.32/ 1.00 2501ms total, 154158.34 ops/sec - GET 4KiB buf, 1/1 min/max/avg/p95: 0/ 2/ 0.03/ 0.00 2501ms total, 34654.94 ops/sec - GET 4KiB buf, batch 50/1 min/max/avg/p95: 0/ 2/ 0.32/ 1.00 2501ms total, 153758.50 ops/sec - INCR, 1/1 min/max/avg/p95: 0/ 2/ 0.03/ 0.00 2501ms total, 37530.19 ops/sec - INCR, batch 50/1 min/max/avg/p95: 0/ 3/ 0.12/ 1.00 2501ms total, 415993.60 ops/sec - LPUSH, 1/1 min/max/avg/p95: 0/ 1/ 0.03/ 0.00 2501ms total, 37409.04 ops/sec - LPUSH, batch 50/1 min/max/avg/p95: 0/ 2/ 0.14/ 1.00 2501ms total, 354778.09 ops/sec - LRANGE 10, 1/1 min/max/avg/p95: 0/ 3/ 0.03/ 0.00 2501ms total, 31768.49 ops/sec - LRANGE 10, batch 50/1 min/max/avg/p95: 0/ 3/ 0.33/ 1.00 2501ms total, 151379.45 ops/sec - LRANGE 100, 1/1 min/max/avg/p95: 0/ 2/ 0.06/ 1.00 2501ms total, 16801.68 ops/sec - LRANGE 100, batch 50/1 min/max/avg/p95: 2/ 4/ 2.07/ 3.00 2501ms total, 24150.34 ops/sec - SET 4MiB str, 1/1 min/max/avg/p95: 1/ 5/ 1.96/ 2.00 2501ms total, 510.20 ops/sec - SET 4MiB str, batch 20/1 min/max/avg/p95: 83/ 108/ 94.44/ 106.40 2550ms total, 211.76 ops/sec - SET 4MiB buf, 1/1 min/max/avg/p95: 1/ 7/ 2.06/ 3.00 2501ms total, 484.21 ops/sec - SET 4MiB buf, batch 20/1 min/max/avg/p95: 38/ 48/ 40.90/ 46.00 2536ms total, 488.96 ops/sec - GET 4MiB str, 1/1 min/max/avg/p95: 3/ 13/ 5.20/ 9.00 2503ms total, 192.17 ops/sec - GET 4MiB str, batch 20/1 min/max/avg/p95: 74/ 105/ 87.24/ 104.00 2530ms total, 229.25 ops/sec - GET 4MiB buf, 1/1 min/max/avg/p95: 3/ 11/ 5.01/ 9.00 2501ms total, 199.12 ops/sec - GET 4MiB buf, batch 20/1 min/max/avg/p95: 78/ 93/ 84.23/ 91.90 2528ms total, 237.34 ops/sec + PING, 1/1 min/max/avg/p95: 0/ 2/ 0.02/ 0.00 2501ms total, 47503.80 ops/sec + PING, batch 50/1 min/max/avg/p95: 0/ 2/ 0.09/ 1.00 2501ms total, 529668.13 ops/sec + SET 4B str, 1/1 min/max/avg/p95: 0/ 2/ 0.02/ 0.00 2501ms total, 41900.04 ops/sec + SET 4B str, batch 50/1 min/max/avg/p95: 0/ 2/ 0.14/ 1.00 2501ms total, 354658.14 ops/sec + SET 4B buf, 1/1 min/max/avg/p95: 0/ 4/ 0.04/ 0.00 2501ms total, 23499.00 ops/sec + SET 4B buf, batch 50/1 min/max/avg/p95: 0/ 2/ 0.31/ 1.00 2501ms total, 159836.07 ops/sec + GET 4B str, 1/1 min/max/avg/p95: 0/ 4/ 0.02/ 0.00 2501ms total, 43489.80 ops/sec + GET 4B str, batch 50/1 min/max/avg/p95: 0/ 2/ 0.11/ 1.00 2501ms total, 444202.32 ops/sec + GET 4B buf, 1/1 min/max/avg/p95: 0/ 3/ 0.02/ 0.00 2501ms total, 38561.38 ops/sec + GET 4B buf, batch 50/1 min/max/avg/p95: 0/ 2/ 0.11/ 1.00 2501ms total, 452139.14 ops/sec + SET 4KiB str, 1/1 min/max/avg/p95: 0/ 2/ 0.03/ 0.00 2501ms total, 32990.80 ops/sec + SET 4KiB str, batch 50/1 min/max/avg/p95: 0/ 2/ 0.34/ 1.00 2501ms total, 146161.54 ops/sec + SET 4KiB buf, 1/1 min/max/avg/p95: 0/ 1/ 0.04/ 0.00 2501ms total, 23294.28 ops/sec + SET 4KiB buf, batch 50/1 min/max/avg/p95: 0/ 2/ 0.36/ 1.00 2501ms total, 137584.97 ops/sec + GET 4KiB str, 1/1 min/max/avg/p95: 0/ 2/ 0.03/ 0.00 2501ms total, 36350.66 ops/sec + GET 4KiB str, batch 50/1 min/max/avg/p95: 0/ 2/ 0.32/ 1.00 2501ms total, 155157.94 ops/sec + GET 4KiB buf, 1/1 min/max/avg/p95: 0/ 4/ 0.02/ 0.00 2501ms total, 39776.49 ops/sec + GET 4KiB buf, batch 50/1 min/max/avg/p95: 0/ 2/ 0.32/ 1.00 2501ms total, 155457.82 ops/sec + INCR, 1/1 min/max/avg/p95: 0/ 3/ 0.02/ 0.00 2501ms total, 43972.41 ops/sec + INCR, batch 50/1 min/max/avg/p95: 0/ 1/ 0.12/ 1.00 2501ms total, 425809.68 ops/sec + LPUSH, 1/1 min/max/avg/p95: 0/ 2/ 0.02/ 0.00 2501ms total, 38998.40 ops/sec + LPUSH, batch 50/1 min/max/avg/p95: 0/ 4/ 0.14/ 1.00 2501ms total, 365013.99 ops/sec + LRANGE 10, 1/1 min/max/avg/p95: 0/ 2/ 0.03/ 0.00 2501ms total, 31879.25 ops/sec + LRANGE 10, batch 50/1 min/max/avg/p95: 0/ 1/ 0.32/ 1.00 2501ms total, 153698.52 ops/sec + LRANGE 100, 1/1 min/max/avg/p95: 0/ 4/ 0.06/ 0.00 2501ms total, 16676.13 ops/sec + LRANGE 100, batch 50/1 min/max/avg/p95: 1/ 6/ 2.03/ 2.00 2502ms total, 24520.38 ops/sec + SET 4MiB str, 1/1 min/max/avg/p95: 1/ 6/ 2.11/ 3.00 2502ms total, 472.82 ops/sec + SET 4MiB str, batch 20/1 min/max/avg/p95: 85/ 112/ 94.93/ 109.60 2563ms total, 210.69 ops/sec + SET 4MiB buf, 1/1 min/max/avg/p95: 1/ 8/ 2.02/ 3.00 2502ms total, 490.01 ops/sec + SET 4MiB buf, batch 20/1 min/max/avg/p95: 37/ 52/ 39.48/ 46.75 2528ms total, 506.33 ops/sec + GET 4MiB str, 1/1 min/max/avg/p95: 3/ 13/ 5.26/ 9.00 2504ms total, 190.10 ops/sec + GET 4MiB str, batch 20/1 min/max/avg/p95: 70/ 106/ 89.36/ 103.75 2503ms total, 223.73 ops/sec + GET 4MiB buf, 1/1 min/max/avg/p95: 3/ 11/ 5.04/ 8.15 2502ms total, 198.24 ops/sec + GET 4MiB buf, batch 20/1 min/max/avg/p95: 70/ 105/ 88.07/ 103.00 2554ms total, 227.09 ops/sec ``` The hiredis and js parser should most of the time be on the same level. But if you use Redis for big SUNION/SINTER/LRANGE/ZRANGE hiredis is significantly faster. diff --git a/index.js b/index.js index 422bc41965..2212f1b143 100644 --- a/index.js +++ b/index.js @@ -294,14 +294,10 @@ RedisClient.prototype.init_parser = function () { // Important: Only send results / errors async. // That way the result / error won't stay in a try catch block and catch user things this.reply_parser.send_error = function (data) { - process.nextTick(function() { - self.return_error(data); - }); + self.return_error(data); }; this.reply_parser.send_reply = function (data) { - process.nextTick(function() { - self.return_reply(data); - }); + self.return_reply(data); }; }; diff --git a/lib/parsers/javascript.js b/lib/parsers/javascript.js index 7decd0447b..bdb4a9f8bb 100644 --- a/lib/parsers/javascript.js +++ b/lib/parsers/javascript.js @@ -9,6 +9,8 @@ function JavascriptReplyParser() { this._big_offset = 0; this._chunks_size = 0; this._buffers = []; + this._type = 0; + this._protocol_error = false; } function IncompleteReadBuffer(message) { @@ -21,87 +23,73 @@ JavascriptReplyParser.prototype._parseResult = function (type) { var start = 0, end = 0, offset = 0, - packetHeader = 0; + packetHeader = 0, + res, + reply; if (type === 43 || type === 58 || type === 45) { // + or : or - - // up to the delimiter + // Up to the delimiter end = this._packetEndOffset(); start = this._offset; - - // include the delimiter + // Include the delimiter this._offset = end + 2; if (type === 43) { return this._buffer.slice(start, end); } else if (type === 58) { - // return the coerced numeric value + // Return the coerced numeric value return +this._buffer.toString('ascii', start, end); } return new Error(this._buffer.toString('utf-8', start, end)); } else if (type === 36) { // $ packetHeader = this.parseHeader(); - // packets with a size of -1 are considered null + // Packets with a size of -1 are considered null if (packetHeader === -1) { return null; } - end = this._offset + packetHeader; start = this._offset; - if (end > this._buffer.length) { + if (end + 2 > this._buffer.length) { this._chunks_size = this._buffer.length - this._offset - 2; this._big_offset = packetHeader; throw new IncompleteReadBuffer('Wait for more data.'); } - - // set the offset to after the delimiter + // Set the offset to after the delimiter this._offset = end + 2; return this._buffer.slice(start, end); } else if (type === 42) { // * - // set a rewind point, as the packet is larger than the buffer in memory + // Set a rewind point, as the packet is larger than the buffer in memory offset = this._offset; packetHeader = this.parseHeader(); if (packetHeader === -1) { return null; } - - if (packetHeader > this._buffer.length - this._offset) { - this._offset = offset - 1; - throw new IncompleteReadBuffer('Wait for more data.'); - } - - var reply = []; - var ntype, i, res; - + reply = []; offset = this._offset - 1; - for (i = 0; i < packetHeader; i++) { - ntype = this._buffer[this._offset++]; - - if (this._offset > this._buffer.length) { + for (var i = 0; i < packetHeader; i++) { + if (this._offset >= this._buffer.length) { throw new IncompleteReadBuffer('Wait for more data.'); } - res = this._parseResult(ntype); + res = this._parseResult(this._buffer[this._offset++]); reply.push(res); } - return reply; } else { - return null; + return void 0; } }; JavascriptReplyParser.prototype.execute = function (buffer) { - if (this._chunks_size !== 0 && this._big_offset > this._chunks_size + buffer.length) { this._buffers.push(buffer); this._chunks_size += buffer.length; return; } - if (this._buffers.length !== 0) { this._buffers.unshift(this._offset === 0 ? this._buffer : this._buffer.slice(this._offset)); this._buffers.push(buffer); @@ -115,44 +103,41 @@ JavascriptReplyParser.prototype.execute = function (buffer) { this._buffer = Buffer.concat([this._buffer.slice(this._offset), buffer]); } this._offset = 0; + this._protocol_error = true; this.run(); }; +JavascriptReplyParser.prototype.try_parsing = function () { + // Set a rewind point. If a failure occurs, wait for the next execute()/append() and try again + var offset = this._offset - 1; + try { + return this._parseResult(this._type); + } catch (err) { + // Catch the error (not enough data), rewind if it's an array, + // and wait for the next packet to appear + this._offset = offset; + this._protocol_error = false; + return void 0; + } +}; + JavascriptReplyParser.prototype.run = function (buffer) { - var type, offset = this._offset; + this._type = this._buffer[this._offset++]; + var reply = this.try_parsing(); - while (true) { - offset = this._offset; - // at least 4 bytes: :1\r\n - if (this._buffer.length - this._offset < 4) { - break; - } - - try { - type = this._buffer[this._offset++]; - - if (type === 43 || type === 58 || type === 36) { // Strings + // Integers : // Bulk strings $ - this.send_reply(this._parseResult(type)); - } else if (type === 45) { // Errors - - this.send_error(this._parseResult(type)); - } else if (type === 42) { // Arrays * - // set a rewind point. if a failure occurs, - // wait for the next execute()/append() and try again - offset = this._offset - 1; - - this.send_reply(this._parseResult(type)); - } else if (type !== 10 && type !== 13) { - // Reset the buffer so the parser can handle following commands properly - this._buffer = new Buffer(0); - var err = new Error('Protocol error, got "' + String.fromCharCode(type) + '" as reply type byte'); - this.send_error(err); - } - } catch (err) { - // catch the error (not enough data), rewind, and wait - // for the next packet to appear - this._offset = offset; - break; + while (reply !== undefined) { + if (this._type === 45) { // Errors - + this.send_error(reply); + } else { + this.send_reply(reply); // Strings + // Integers : // Bulk strings $ // Arrays * } + this._type = this._buffer[this._offset++]; + reply = this.try_parsing(); + } + if (this._type !== undefined && this._protocol_error === true) { + // Reset the buffer so the parser can handle following commands properly + this._buffer = new Buffer(0); + this.send_error(new Error('Protocol error, got "' + String.fromCharCode(this._type) + '" as reply type byte')); } }; @@ -161,21 +146,20 @@ JavascriptReplyParser.prototype.parseHeader = function () { value = this._buffer.toString('ascii', this._offset, end) | 0; this._offset = end + 2; - return value; }; JavascriptReplyParser.prototype._packetEndOffset = function () { - var offset = this._offset; + var offset = this._offset, + len = this._buffer.length - 1; while (this._buffer[offset] !== 0x0d && this._buffer[offset + 1] !== 0x0a) { offset++; - if (offset >= this._buffer.length) { + if (offset >= len) { throw new IncompleteReadBuffer('Did not see LF after NL reading multi bulk count (' + offset + ' => ' + this._buffer.length + ', ' + this._offset + ')'); } } - return offset; }; diff --git a/test/parser.spec.js b/test/parser.spec.js index c1444186b7..85f92f50de 100644 --- a/test/parser.spec.js +++ b/test/parser.spec.js @@ -16,7 +16,7 @@ describe('parsers', function () { describe(Parser.name, function () { - it('handles multi-bulk reply', function (done) { + it('handles multi-bulk reply', function () { var parser = new Parser(); var reply_count = 0; function check_reply(reply) { @@ -27,33 +27,53 @@ describe('parsers', function () { parser.send_reply = check_reply; parser.execute(new Buffer('*1\r\n*1\r\n$1\r\na\r\n')); + assert.strictEqual(reply_count, 1); parser.execute(new Buffer('*1\r\n*1\r')); parser.execute(new Buffer('\n$1\r\na\r\n')); + assert.strictEqual(reply_count, 2); parser.execute(new Buffer('*1\r\n*1\r\n')); parser.execute(new Buffer('$1\r\na\r\n')); assert.equal(reply_count, 3, "check reply should have been called three times"); - return done(); }); - it('parser error', function (done) { + it('parser error', function () { var parser = new Parser(); var reply_count = 0; - function check_reply(reply) { + function check_reply (reply) { assert.strictEqual(reply.message, 'Protocol error, got "a" as reply type byte'); reply_count++; } parser.send_error = check_reply; parser.execute(new Buffer('a*1\r*1\r$1`zasd\r\na')); - - assert.equal(reply_count, 1, "check reply should have been called one time"); - return done(); + assert.equal(reply_count, 1); }); - it('line breaks in the beginning of the last chunk', function (done) { + it('parser error v2', function () { + var parser = new Parser(); + var reply_count = 0; + var err_count = 0; + function check_reply (reply) { + reply = utils.reply_to_strings(reply); + assert.strictEqual(reply[0], 'OK'); + reply_count++; + } + function check_error (err) { + assert.strictEqual(err.message, 'Protocol error, got "b" as reply type byte'); + err_count++; + } + parser.send_error = check_error; + parser.send_reply = check_reply; + + parser.execute(new Buffer('*1\r\n+OK\r\nb$1`zasd\r\na')); + assert.strictEqual(reply_count, 1); + assert.strictEqual(err_count, 1); + }); + + it('line breaks in the beginning of the last chunk', function () { var parser = new Parser(); var reply_count = 0; function check_reply(reply) { @@ -64,15 +84,16 @@ describe('parsers', function () { parser.send_reply = check_reply; parser.execute(new Buffer('*1\r\n*1\r\n$1\r\na')); + assert.equal(reply_count, 0); parser.execute(new Buffer('\r\n*1\r\n*1\r')); + assert.equal(reply_count, 1); parser.execute(new Buffer('\n$1\r\na\r\n*1\r\n*1\r\n$1\r\na\r\n')); assert.equal(reply_count, 3, "check reply should have been called three times"); - return done(); }); - it('multiple chunks in a bulk string', function (done) { + it('multiple chunks in a bulk string', function () { var parser = new Parser(); var reply_count = 0; function check_reply(reply) { @@ -107,21 +128,27 @@ describe('parsers', function () { parser.execute(new Buffer('\n')); assert.equal(reply_count, 4, "check reply should have been called three times"); - return done(); }); - it('multiple chunks with arrays different types', function (done) { + it('multiple chunks with arrays different types', function () { var parser = new Parser(); var reply_count = 0; + var predefined_data = [ + 'abcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghij', + 'test', + 100, + new Error('Error message'), + ['The force awakens'] + ]; function check_reply(reply) { reply = utils.reply_to_strings(reply); - assert.deepStrictEqual(reply, [ - 'abcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghij', - 'test', - 100, - new Error('Error message'), - ['The force awakens'] - ]); + for (var i = 0; i < reply.length; i++) { + if (i < 3) { + assert.strictEqual(reply[i], predefined_data[i]); + } else { + assert.deepEqual(reply[i], predefined_data[i]); + } + } reply_count++; } parser.send_reply = check_reply; @@ -137,10 +164,9 @@ describe('parsers', function () { assert.strictEqual(reply_count, 0); parser.execute(new Buffer(' awakens\r\n$5')); assert.strictEqual(reply_count, 1); - return done(); }); - it('return normal errors', function (done) { + it('return normal errors', function () { var parser = new Parser(); var reply_count = 0; function check_reply(reply) { @@ -154,10 +180,9 @@ describe('parsers', function () { assert.strictEqual(reply_count, 1); parser.execute(new Buffer(' awakens\r\n$5')); assert.strictEqual(reply_count, 1); - return done(); }); - it('return null for empty arrays and empty bulk strings', function (done) { + it('return null for empty arrays and empty bulk strings', function () { var parser = new Parser(); var reply_count = 0; function check_reply(reply) { @@ -172,7 +197,44 @@ describe('parsers', function () { assert.strictEqual(reply_count, 1); parser.execute(new Buffer('\r\n$-')); assert.strictEqual(reply_count, 2); - return done(); + }); + + it('return value even if all chunks are only 1 character long', function () { + var parser = new Parser(); + var reply_count = 0; + function check_reply(reply) { + assert.equal(reply, 1); + reply_count++; + } + parser.send_reply = check_reply; + + parser.execute(new Buffer(':')); + assert.strictEqual(reply_count, 0); + parser.execute(new Buffer('1')); + assert.strictEqual(reply_count, 0); + parser.execute(new Buffer('\r')); + assert.strictEqual(reply_count, 0); + parser.execute(new Buffer('\n')); + assert.strictEqual(reply_count, 1); + }); + + it('do not return before \\r\\n', function () { + var parser = new Parser(); + var reply_count = 0; + function check_reply(reply) { + assert.equal(reply, 1); + reply_count++; + } + parser.send_reply = check_reply; + + parser.execute(new Buffer(':1\r\n:')); + assert.strictEqual(reply_count, 1); + parser.execute(new Buffer('1')); + assert.strictEqual(reply_count, 1); + parser.execute(new Buffer('\r')); + assert.strictEqual(reply_count, 1); + parser.execute(new Buffer('\n')); + assert.strictEqual(reply_count, 2); }); }); });