redis/node-redis
1
0
Fork 0
mirror of https://github.com/redis/node-redis.git synced 2025-08-06 02:15:48 +03:00
Code Activity

Refactor js parser

Browse Source 
Fix tests to work with Node.js 0.10
Improve average use case speed by up to 20%
Fix some small js parser issues
This commit is contained in:
Ruben Bridgewater
2015-11-23 11:17:05 +01:00
parent b6a81a4297
commit ac9ff9a3b1
4 changed files with 172 additions and 130 deletions
Download Patch File Download Diff File Expand all files Collapse all files

70
README.md
View File

@@ -6,7 +6,7 @@ redis - a node.js redis client
[![Windows Tests](https://img.shields.io/appveyor/ci/BridgeAR/node-redis/master.svg?label=Windows%20Tests)](https://ci.appveyor.com/project/BridgeAR/node-redis/branch/master)
[![Gitter](https://badges.gitter.im/Join%20Chat.svg)](https://gitter.im/NodeRedis/node_redis?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge)
This is a complete and feature rich Redis client for node.js. It supports all Redis commands and focuses on performance.
This is a complete and feature rich Redis client for node.js. It supports all Redis commands and focuses on high performance.
Install with:
@@ -646,40 +646,40 @@ hiredis parser (Lenovo T450s i7-5600U):
```
Client count: 1, node version: 4.2.2, server version: 3.0.3, parser: hiredis
PING, 1/1 min/max/avg/p95: 0/ 3/ 0.02/ 0.00 2501ms total, 39862.85 ops/sec
PING, batch 50/1 min/max/avg/p95: 0/ 2/ 0.10/ 1.00 2501ms total, 491223.51 ops/sec
SET 4B str, 1/1 min/max/avg/p95: 0/ 3/ 0.03/ 0.00 2501ms total, 36387.45 ops/sec
SET 4B str, batch 50/1 min/max/avg/p95: 0/ 3/ 0.14/ 1.00 2501ms total, 346381.45 ops/sec
SET 4B buf, 1/1 min/max/avg/p95: 0/ 2/ 0.04/ 0.00 2501ms total, 24395.84 ops/sec
SET 4B buf, batch 50/1 min/max/avg/p95: 0/ 2/ 0.32/ 1.00 2501ms total, 156457.42 ops/sec
GET 4B str, 1/1 min/max/avg/p95: 0/ 3/ 0.03/ 0.00 2501ms total, 36906.44 ops/sec
GET 4B str, batch 50/1 min/max/avg/p95: 0/ 3/ 0.12/ 1.00 2501ms total, 425729.71 ops/sec
GET 4B buf, 1/1 min/max/avg/p95: 0/ 2/ 0.03/ 0.00 2501ms total, 36221.91 ops/sec
GET 4B buf, batch 50/1 min/max/avg/p95: 0/ 2/ 0.11/ 1.00 2501ms total, 430407.84 ops/sec
SET 4KiB str, 1/1 min/max/avg/p95: 0/ 3/ 0.03/ 0.00 2501ms total, 30951.22 ops/sec
SET 4KiB str, batch 50/1 min/max/avg/p95: 0/ 2/ 0.33/ 1.00 2501ms total, 150299.88 ops/sec
SET 4KiB buf, 1/1 min/max/avg/p95: 0/ 2/ 0.04/ 1.00 2501ms total, 23919.63 ops/sec
SET 4KiB buf, batch 50/1 min/max/avg/p95: 0/ 2/ 0.36/ 1.00 2501ms total, 139204.32 ops/sec
GET 4KiB str, 1/1 min/max/avg/p95: 0/ 2/ 0.03/ 0.00 2501ms total, 32739.30 ops/sec
GET 4KiB str, batch 50/1 min/max/avg/p95: 0/ 2/ 0.32/ 1.00 2501ms total, 154158.34 ops/sec
GET 4KiB buf, 1/1 min/max/avg/p95: 0/ 2/ 0.03/ 0.00 2501ms total, 34654.94 ops/sec
GET 4KiB buf, batch 50/1 min/max/avg/p95: 0/ 2/ 0.32/ 1.00 2501ms total, 153758.50 ops/sec
INCR, 1/1 min/max/avg/p95: 0/ 2/ 0.03/ 0.00 2501ms total, 37530.19 ops/sec
INCR, batch 50/1 min/max/avg/p95: 0/ 3/ 0.12/ 1.00 2501ms total, 415993.60 ops/sec
LPUSH, 1/1 min/max/avg/p95: 0/ 1/ 0.03/ 0.00 2501ms total, 37409.04 ops/sec
LPUSH, batch 50/1 min/max/avg/p95: 0/ 2/ 0.14/ 1.00 2501ms total, 354778.09 ops/sec
LRANGE 10, 1/1 min/max/avg/p95: 0/ 3/ 0.03/ 0.00 2501ms total, 31768.49 ops/sec
LRANGE 10, batch 50/1 min/max/avg/p95: 0/ 3/ 0.33/ 1.00 2501ms total, 151379.45 ops/sec
LRANGE 100, 1/1 min/max/avg/p95: 0/ 2/ 0.06/ 1.00 2501ms total, 16801.68 ops/sec
LRANGE 100, batch 50/1 min/max/avg/p95: 2/ 4/ 2.07/ 3.00 2501ms total, 24150.34 ops/sec
SET 4MiB str, 1/1 min/max/avg/p95: 1/ 5/ 1.96/ 2.00 2501ms total, 510.20 ops/sec
SET 4MiB str, batch 20/1 min/max/avg/p95: 83/ 108/ 94.44/ 106.40 2550ms total, 211.76 ops/sec
SET 4MiB buf, 1/1 min/max/avg/p95: 1/ 7/ 2.06/ 3.00 2501ms total, 484.21 ops/sec
SET 4MiB buf, batch 20/1 min/max/avg/p95: 38/ 48/ 40.90/ 46.00 2536ms total, 488.96 ops/sec
GET 4MiB str, 1/1 min/max/avg/p95: 3/ 13/ 5.20/ 9.00 2503ms total, 192.17 ops/sec
GET 4MiB str, batch 20/1 min/max/avg/p95: 74/ 105/ 87.24/ 104.00 2530ms total, 229.25 ops/sec
GET 4MiB buf, 1/1 min/max/avg/p95: 3/ 11/ 5.01/ 9.00 2501ms total, 199.12 ops/sec
GET 4MiB buf, batch 20/1 min/max/avg/p95: 78/ 93/ 84.23/ 91.90 2528ms total, 237.34 ops/sec
PING, 1/1 min/max/avg/p95: 0/ 2/ 0.02/ 0.00 2501ms total, 47503.80 ops/sec
PING, batch 50/1 min/max/avg/p95: 0/ 2/ 0.09/ 1.00 2501ms total, 529668.13 ops/sec
SET 4B str, 1/1 min/max/avg/p95: 0/ 2/ 0.02/ 0.00 2501ms total, 41900.04 ops/sec
SET 4B str, batch 50/1 min/max/avg/p95: 0/ 2/ 0.14/ 1.00 2501ms total, 354658.14 ops/sec
SET 4B buf, 1/1 min/max/avg/p95: 0/ 4/ 0.04/ 0.00 2501ms total, 23499.00 ops/sec
SET 4B buf, batch 50/1 min/max/avg/p95: 0/ 2/ 0.31/ 1.00 2501ms total, 159836.07 ops/sec
GET 4B str, 1/1 min/max/avg/p95: 0/ 4/ 0.02/ 0.00 2501ms total, 43489.80 ops/sec
GET 4B str, batch 50/1 min/max/avg/p95: 0/ 2/ 0.11/ 1.00 2501ms total, 444202.32 ops/sec
GET 4B buf, 1/1 min/max/avg/p95: 0/ 3/ 0.02/ 0.00 2501ms total, 38561.38 ops/sec
GET 4B buf, batch 50/1 min/max/avg/p95: 0/ 2/ 0.11/ 1.00 2501ms total, 452139.14 ops/sec
SET 4KiB str, 1/1 min/max/avg/p95: 0/ 2/ 0.03/ 0.00 2501ms total, 32990.80 ops/sec
SET 4KiB str, batch 50/1 min/max/avg/p95: 0/ 2/ 0.34/ 1.00 2501ms total, 146161.54 ops/sec
SET 4KiB buf, 1/1 min/max/avg/p95: 0/ 1/ 0.04/ 0.00 2501ms total, 23294.28 ops/sec
SET 4KiB buf, batch 50/1 min/max/avg/p95: 0/ 2/ 0.36/ 1.00 2501ms total, 137584.97 ops/sec
GET 4KiB str, 1/1 min/max/avg/p95: 0/ 2/ 0.03/ 0.00 2501ms total, 36350.66 ops/sec
GET 4KiB str, batch 50/1 min/max/avg/p95: 0/ 2/ 0.32/ 1.00 2501ms total, 155157.94 ops/sec
GET 4KiB buf, 1/1 min/max/avg/p95: 0/ 4/ 0.02/ 0.00 2501ms total, 39776.49 ops/sec
GET 4KiB buf, batch 50/1 min/max/avg/p95: 0/ 2/ 0.32/ 1.00 2501ms total, 155457.82 ops/sec
INCR, 1/1 min/max/avg/p95: 0/ 3/ 0.02/ 0.00 2501ms total, 43972.41 ops/sec
INCR, batch 50/1 min/max/avg/p95: 0/ 1/ 0.12/ 1.00 2501ms total, 425809.68 ops/sec
LPUSH, 1/1 min/max/avg/p95: 0/ 2/ 0.02/ 0.00 2501ms total, 38998.40 ops/sec
LPUSH, batch 50/1 min/max/avg/p95: 0/ 4/ 0.14/ 1.00 2501ms total, 365013.99 ops/sec
LRANGE 10, 1/1 min/max/avg/p95: 0/ 2/ 0.03/ 0.00 2501ms total, 31879.25 ops/sec
LRANGE 10, batch 50/1 min/max/avg/p95: 0/ 1/ 0.32/ 1.00 2501ms total, 153698.52 ops/sec
LRANGE 100, 1/1 min/max/avg/p95: 0/ 4/ 0.06/ 0.00 2501ms total, 16676.13 ops/sec
LRANGE 100, batch 50/1 min/max/avg/p95: 1/ 6/ 2.03/ 2.00 2502ms total, 24520.38 ops/sec
SET 4MiB str, 1/1 min/max/avg/p95: 1/ 6/ 2.11/ 3.00 2502ms total, 472.82 ops/sec
SET 4MiB str, batch 20/1 min/max/avg/p95: 85/ 112/ 94.93/ 109.60 2563ms total, 210.69 ops/sec
SET 4MiB buf, 1/1 min/max/avg/p95: 1/ 8/ 2.02/ 3.00 2502ms total, 490.01 ops/sec
SET 4MiB buf, batch 20/1 min/max/avg/p95: 37/ 52/ 39.48/ 46.75 2528ms total, 506.33 ops/sec
GET 4MiB str, 1/1 min/max/avg/p95: 3/ 13/ 5.26/ 9.00 2504ms total, 190.10 ops/sec
GET 4MiB str, batch 20/1 min/max/avg/p95: 70/ 106/ 89.36/ 103.75 2503ms total, 223.73 ops/sec
GET 4MiB buf, 1/1 min/max/avg/p95: 3/ 11/ 5.04/ 8.15 2502ms total, 198.24 ops/sec
GET 4MiB buf, batch 20/1 min/max/avg/p95: 70/ 105/ 88.07/ 103.00 2554ms total, 227.09 ops/sec
```
The hiredis and js parser should most of the time be on the same level. But if you use Redis for big SUNION/SINTER/LRANGE/ZRANGE hiredis is significantly faster.

4
index.js
View File

@@ -294,14 +294,10 @@ RedisClient.prototype.init_parser = function () {
// Important: Only send results / errors async.
// That way the result / error won't stay in a try catch block and catch user things
this.reply_parser.send_error = function (data) {
process.nextTick(function() {
self.return_error(data);
});
};
this.reply_parser.send_reply = function (data) {
process.nextTick(function() {
self.return_reply(data);
});
};
};

110
lib/parsers/javascript.js
View File

@@ -9,6 +9,8 @@ function JavascriptReplyParser() {
this._big_offset = 0;
this._chunks_size = 0;
this._buffers = [];
this._type = 0;
this._protocol_error = false;
}
function IncompleteReadBuffer(message) {
@@ -21,87 +23,73 @@ JavascriptReplyParser.prototype._parseResult = function (type) {
var start = 0,
end = 0,
offset = 0,
packetHeader = 0;
packetHeader = 0,
res,
reply;
if (type === 43 || type === 58 || type === 45) { // + or : or -
// up to the delimiter
// Up to the delimiter
end = this._packetEndOffset();
start = this._offset;
// include the delimiter
// Include the delimiter
this._offset = end + 2;
if (type === 43) {
return this._buffer.slice(start, end);
} else if (type === 58) {
// return the coerced numeric value
// Return the coerced numeric value
return +this._buffer.toString('ascii', start, end);
}
return new Error(this._buffer.toString('utf-8', start, end));
} else if (type === 36) { // $
packetHeader = this.parseHeader();
// packets with a size of -1 are considered null
// Packets with a size of -1 are considered null
if (packetHeader === -1) {
return null;
}
end = this._offset + packetHeader;
start = this._offset;
if (end > this._buffer.length) {
if (end + 2 > this._buffer.length) {
this._chunks_size = this._buffer.length - this._offset - 2;
this._big_offset = packetHeader;
throw new IncompleteReadBuffer('Wait for more data.');
}
// set the offset to after the delimiter
// Set the offset to after the delimiter
this._offset = end + 2;
return this._buffer.slice(start, end);
} else if (type === 42) { // *
// set a rewind point, as the packet is larger than the buffer in memory
// Set a rewind point, as the packet is larger than the buffer in memory
offset = this._offset;
packetHeader = this.parseHeader();
if (packetHeader === -1) {
return null;
}
if (packetHeader > this._buffer.length - this._offset) {
this._offset = offset - 1;
throw new IncompleteReadBuffer('Wait for more data.');
}
var reply = [];
var ntype, i, res;
reply = [];
offset = this._offset - 1;
for (i = 0; i < packetHeader; i++) {
ntype = this._buffer[this._offset++];
if (this._offset > this._buffer.length) {
for (var i = 0; i < packetHeader; i++) {
if (this._offset >= this._buffer.length) {
throw new IncompleteReadBuffer('Wait for more data.');
}
res = this._parseResult(ntype);
res = this._parseResult(this._buffer[this._offset++]);
reply.push(res);
}
return reply;
} else {
return null;
return void 0;
}
};
JavascriptReplyParser.prototype.execute = function (buffer) {
if (this._chunks_size !== 0 && this._big_offset > this._chunks_size + buffer.length) {
this._buffers.push(buffer);
this._chunks_size += buffer.length;
return;
}
if (this._buffers.length !== 0) {
this._buffers.unshift(this._offset === 0 ? this._buffer : this._buffer.slice(this._offset));
this._buffers.push(buffer);
@@ -115,44 +103,41 @@ JavascriptReplyParser.prototype.execute = function (buffer) {
this._buffer = Buffer.concat([this._buffer.slice(this._offset), buffer]);
}
this._offset = 0;
this._protocol_error = true;
this.run();
};
JavascriptReplyParser.prototype.run = function (buffer) {
var type, offset = this._offset;
while (true) {
offset = this._offset;
// at least 4 bytes: :1\r\n
if (this._buffer.length - this._offset < 4) {
break;
}
JavascriptReplyParser.prototype.try_parsing = function () {
// Set a rewind point. If a failure occurs, wait for the next execute()/append() and try again
var offset = this._offset - 1;
try {
type = this._buffer[this._offset++];
return this._parseResult(this._type);
} catch (err) {
// Catch the error (not enough data), rewind if it's an array,
// and wait for the next packet to appear
this._offset = offset;
this._protocol_error = false;
return void 0;
}
};
if (type === 43 || type === 58 || type === 36) { // Strings + // Integers : // Bulk strings $
this.send_reply(this._parseResult(type));
} else if (type === 45) { // Errors -
this.send_error(this._parseResult(type));
} else if (type === 42) { // Arrays *
// set a rewind point. if a failure occurs,
// wait for the next execute()/append() and try again
offset = this._offset - 1;
JavascriptReplyParser.prototype.run = function (buffer) {
this._type = this._buffer[this._offset++];
var reply = this.try_parsing();
this.send_reply(this._parseResult(type));
} else if (type !== 10 && type !== 13) {
while (reply !== undefined) {
if (this._type === 45) { // Errors -
this.send_error(reply);
} else {
this.send_reply(reply); // Strings + // Integers : // Bulk strings $ // Arrays *
}
this._type = this._buffer[this._offset++];
reply = this.try_parsing();
}
if (this._type !== undefined && this._protocol_error === true) {
// Reset the buffer so the parser can handle following commands properly
this._buffer = new Buffer(0);
var err = new Error('Protocol error, got "' + String.fromCharCode(type) + '" as reply type byte');
this.send_error(err);
}
} catch (err) {
// catch the error (not enough data), rewind, and wait
// for the next packet to appear
this._offset = offset;
break;
}
this.send_error(new Error('Protocol error, got "' + String.fromCharCode(this._type) + '" as reply type byte'));
}
};
@@ -161,21 +146,20 @@ JavascriptReplyParser.prototype.parseHeader = function () {
value = this._buffer.toString('ascii', this._offset, end) | 0;
this._offset = end + 2;
return value;
};
JavascriptReplyParser.prototype._packetEndOffset = function () {
var offset = this._offset;
var offset = this._offset,
len = this._buffer.length - 1;
while (this._buffer[offset] !== 0x0d && this._buffer[offset + 1] !== 0x0a) {
offset++;
if (offset >= this._buffer.length) {
if (offset >= len) {
throw new IncompleteReadBuffer('Did not see LF after NL reading multi bulk count (' + offset + ' => ' + this._buffer.length + ', ' + this._offset + ')');
}
}
return offset;
};

102
test/parser.spec.js
View File

@@ -16,7 +16,7 @@ describe('parsers', function () {
describe(Parser.name, function () {
it('handles multi-bulk reply', function (done) {
it('handles multi-bulk reply', function () {
var parser = new Parser();
var reply_count = 0;
function check_reply(reply) {
@@ -27,18 +27,19 @@ describe('parsers', function () {
parser.send_reply = check_reply;
parser.execute(new Buffer('*1\r\n*1\r\n$1\r\na\r\n'));
assert.strictEqual(reply_count, 1);
parser.execute(new Buffer('*1\r\n*1\r'));
parser.execute(new Buffer('\n$1\r\na\r\n'));
assert.strictEqual(reply_count, 2);
parser.execute(new Buffer('*1\r\n*1\r\n'));
parser.execute(new Buffer('$1\r\na\r\n'));
assert.equal(reply_count, 3, "check reply should have been called three times");
return done();
});
it('parser error', function (done) {
it('parser error', function () {
var parser = new Parser();
var reply_count = 0;
function check_reply (reply) {
@@ -48,12 +49,31 @@ describe('parsers', function () {
parser.send_error = check_reply;
parser.execute(new Buffer('a*1\r*1\r$1`zasd\r\na'));
assert.equal(reply_count, 1, "check reply should have been called one time");
return done();
assert.equal(reply_count, 1);
});
it('line breaks in the beginning of the last chunk', function (done) {
it('parser error v2', function () {
var parser = new Parser();
var reply_count = 0;
var err_count = 0;
function check_reply (reply) {
reply = utils.reply_to_strings(reply);
assert.strictEqual(reply[0], 'OK');
reply_count++;
}
function check_error (err) {
assert.strictEqual(err.message, 'Protocol error, got "b" as reply type byte');
err_count++;
}
parser.send_error = check_error;
parser.send_reply = check_reply;
parser.execute(new Buffer('*1\r\n+OK\r\nb$1`zasd\r\na'));
assert.strictEqual(reply_count, 1);
assert.strictEqual(err_count, 1);
});
it('line breaks in the beginning of the last chunk', function () {
var parser = new Parser();
var reply_count = 0;
function check_reply(reply) {
@@ -64,15 +84,16 @@ describe('parsers', function () {
parser.send_reply = check_reply;
parser.execute(new Buffer('*1\r\n*1\r\n$1\r\na'));
assert.equal(reply_count, 0);
parser.execute(new Buffer('\r\n*1\r\n*1\r'));
assert.equal(reply_count, 1);
parser.execute(new Buffer('\n$1\r\na\r\n*1\r\n*1\r\n$1\r\na\r\n'));
assert.equal(reply_count, 3, "check reply should have been called three times");
return done();
});
it('multiple chunks in a bulk string', function (done) {
it('multiple chunks in a bulk string', function () {
var parser = new Parser();
var reply_count = 0;
function check_reply(reply) {
@@ -107,21 +128,27 @@ describe('parsers', function () {
parser.execute(new Buffer('\n'));
assert.equal(reply_count, 4, "check reply should have been called three times");
return done();
});
it('multiple chunks with arrays different types', function (done) {
it('multiple chunks with arrays different types', function () {
var parser = new Parser();
var reply_count = 0;
function check_reply(reply) {
reply = utils.reply_to_strings(reply);
assert.deepStrictEqual(reply, [
var predefined_data = [
'abcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghij',
'test',
100,
new Error('Error message'),
['The force awakens']
]);
];
function check_reply(reply) {
reply = utils.reply_to_strings(reply);
for (var i = 0; i < reply.length; i++) {
if (i < 3) {
assert.strictEqual(reply[i], predefined_data[i]);
} else {
assert.deepEqual(reply[i], predefined_data[i]);
}
}
reply_count++;
}
parser.send_reply = check_reply;
@@ -137,10 +164,9 @@ describe('parsers', function () {
assert.strictEqual(reply_count, 0);
parser.execute(new Buffer(' awakens\r\n$5'));
assert.strictEqual(reply_count, 1);
return done();
});
it('return normal errors', function (done) {
it('return normal errors', function () {
var parser = new Parser();
var reply_count = 0;
function check_reply(reply) {
@@ -154,10 +180,9 @@ describe('parsers', function () {
assert.strictEqual(reply_count, 1);
parser.execute(new Buffer(' awakens\r\n$5'));
assert.strictEqual(reply_count, 1);
return done();
});
it('return null for empty arrays and empty bulk strings', function (done) {
it('return null for empty arrays and empty bulk strings', function () {
var parser = new Parser();
var reply_count = 0;
function check_reply(reply) {
@@ -172,7 +197,44 @@ describe('parsers', function () {
assert.strictEqual(reply_count, 1);
parser.execute(new Buffer('\r\n$-'));
assert.strictEqual(reply_count, 2);
return done();
});
it('return value even if all chunks are only 1 character long', function () {
var parser = new Parser();
var reply_count = 0;
function check_reply(reply) {
assert.equal(reply, 1);
reply_count++;
}
parser.send_reply = check_reply;
parser.execute(new Buffer(':'));
assert.strictEqual(reply_count, 0);
parser.execute(new Buffer('1'));
assert.strictEqual(reply_count, 0);
parser.execute(new Buffer('\r'));
assert.strictEqual(reply_count, 0);
parser.execute(new Buffer('\n'));
assert.strictEqual(reply_count, 1);
});
it('do not return before \\r\\n', function () {
var parser = new Parser();
var reply_count = 0;
function check_reply(reply) {
assert.equal(reply, 1);
reply_count++;
}
parser.send_reply = check_reply;
parser.execute(new Buffer(':1\r\n:'));
assert.strictEqual(reply_count, 1);
parser.execute(new Buffer('1'));
assert.strictEqual(reply_count, 1);
parser.execute(new Buffer('\r'));
assert.strictEqual(reply_count, 1);
parser.execute(new Buffer('\n'));
assert.strictEqual(reply_count, 2);
});
});
});