redis/node-redis
1
0
Fork 0
mirror of https://github.com/redis/node-redis.git synced 2025-08-06 02:15:48 +03:00
Code Activity

Refactor pipelining

Browse Source
This commit is contained in:
Ruben Bridgewater
2016-04-14 00:57:34 +02:00
parent 5d12659583
commit 683815de9d
4 changed files with 47 additions and 54 deletions
Download Patch File Download Diff File Expand all files Collapse all files

72
index.js
View File

@@ -122,6 +122,7 @@ function RedisClient (options, stream) {
} }
this.command_queue = new Queue(); // Holds sent commands to de-pipeline them this.command_queue = new Queue(); // Holds sent commands to de-pipeline them
this.offline_queue = new Queue(); // Holds commands issued but not able to be sent this.offline_queue = new Queue(); // Holds commands issued but not able to be sent
this.pipeline_queue = new Queue(); // Holds all pipelined commands
// ATTENTION: connect_timeout should change in v.3.0 so it does not count towards ending reconnection attempts after x seconds // ATTENTION: connect_timeout should change in v.3.0 so it does not count towards ending reconnection attempts after x seconds
// This should be done by the retry_strategy. Instead it should only be the timeout for connecting to redis // This should be done by the retry_strategy. Instead it should only be the timeout for connecting to redis
this.connect_timeout = +options.connect_timeout || 3600000; // 60 * 60 * 1000 ms this.connect_timeout = +options.connect_timeout || 3600000; // 60 * 60 * 1000 ms
@@ -144,8 +145,8 @@ function RedisClient (options, stream) {
this.auth_pass = options.auth_pass || options.password; this.auth_pass = options.auth_pass || options.password;
this.selected_db = options.db; // Save the selected db here, used when reconnecting this.selected_db = options.db; // Save the selected db here, used when reconnecting
this.old_state = null; this.old_state = null;
this.send_anyway = false; this.fire_strings = true; // Determine if strings or buffers should be written to the stream
this.pipeline = 0; this.pipeline = false;
this.times_connected = 0; this.times_connected = 0;
this.options = options; this.options = options;
this.buffers = options.return_buffers || options.detect_buffers; this.buffers = options.return_buffers || options.detect_buffers;
@@ -374,23 +375,25 @@ RedisClient.prototype.on_ready = function () {
debug('on_ready called ' + this.address + ' id ' + this.connection_id); debug('on_ready called ' + this.address + ' id ' + this.connection_id);
this.ready = true; this.ready = true;
var cork; this.cork = function () {
if (!this.stream.cork) { self.pipeline = true;
cork = function (len) { if (self.stream.cork) {
self.pipeline = len;
self.pipeline_queue = new Queue(len);
};
} else {
cork = function (len) {
self.pipeline = len;
self.pipeline_queue = new Queue(len);
self.stream.cork(); self.stream.cork();
}; }
this.uncork = function () { };
this.uncork = function () {
if (self.fire_strings) {
self.write_strings();
} else {
self.write_buffers();
}
self.pipeline = false;
self.fire_strings = true;
if (self.stream.uncork) {
// TODO: Consider using next tick here. See https://github.com/NodeRedis/node_redis/issues/1033
self.stream.uncork(); self.stream.uncork();
}; }
} };
this.cork = cork;
// Restore modal commands from previous connection. The order of the commands is important // Restore modal commands from previous connection. The order of the commands is important
if (this.selected_db !== undefined) { if (this.selected_db !== undefined) {
@@ -523,7 +526,8 @@ RedisClient.prototype.connection_gone = function (why, error) {
this.ready = false; this.ready = false;
// Deactivate cork to work with the offline queue // Deactivate cork to work with the offline queue
this.cork = noop; this.cork = noop;
this.pipeline = 0; this.uncork = noop;
this.pipeline = false;
var state = { var state = {
monitoring: this.monitoring, monitoring: this.monitoring,
@@ -792,10 +796,6 @@ RedisClient.prototype.internal_send_command = function (command, args, callback)
if (args[i].length > 30000) { if (args[i].length > 30000) {
big_data = true; big_data = true;
args_copy[i] = new Buffer(args[i], 'utf8'); args_copy[i] = new Buffer(args[i], 'utf8');
if (this.pipeline !== 0) {
this.pipeline += 2;
this.writeDefault = this.writeBuffers;
}
} else { } else {
args_copy[i] = args[i]; args_copy[i] = args[i];
} }
@@ -813,10 +813,6 @@ RedisClient.prototype.internal_send_command = function (command, args, callback)
args_copy[i] = args[i]; args_copy[i] = args[i];
buffer_args = true; buffer_args = true;
big_data = true; big_data = true;
if (this.pipeline !== 0) {
this.pipeline += 2;
this.writeDefault = this.writeBuffers;
}
} else { } else {
this.warn( this.warn(
'Deprecated: The ' + command.toUpperCase() + ' command contains a argument of type ' + args[i].constructor.name + '.\n' + 'Deprecated: The ' + command.toUpperCase() + ' command contains a argument of type ' + args[i].constructor.name + '.\n' +
@@ -870,6 +866,7 @@ RedisClient.prototype.internal_send_command = function (command, args, callback)
this.write(command_str); this.write(command_str);
} else { } else {
debug('Send command (' + command_str + ') has Buffer arguments'); debug('Send command (' + command_str + ') has Buffer arguments');
this.fire_strings = false;
this.write(command_str); this.write(command_str);
for (i = 0; i < len; i += 1) { for (i = 0; i < len; i += 1) {
@@ -887,40 +884,33 @@ RedisClient.prototype.internal_send_command = function (command, args, callback)
return !this.should_buffer; return !this.should_buffer;
}; };
RedisClient.prototype.writeDefault = RedisClient.prototype.writeStrings = function (data) { RedisClient.prototype.write_strings = function () {
var str = ''; var str = '';
for (var command = this.pipeline_queue.shift(); command; command = this.pipeline_queue.shift()) { for (var command = this.pipeline_queue.shift(); command; command = this.pipeline_queue.shift()) {
// Write to stream if the string is bigger than 4mb. The biggest string may be Math.pow(2, 28) - 15 chars long // Write to stream if the string is bigger than 4mb. The biggest string may be Math.pow(2, 28) - 15 chars long
if (str.length + command.length > 4 * 1024 * 1024) { if (str.length + command.length > 4 * 1024 * 1024) {
this.stream.write(str); this.should_buffer = !this.stream.write(str);
str = ''; str = '';
} }
str += command; str += command;
} }
this.should_buffer = !this.stream.write(str + data); if (str !== '') {
this.should_buffer = !this.stream.write(str);
}
}; };
RedisClient.prototype.writeBuffers = function (data) { RedisClient.prototype.write_buffers = function () {
for (var command = this.pipeline_queue.shift(); command; command = this.pipeline_queue.shift()) { for (var command = this.pipeline_queue.shift(); command; command = this.pipeline_queue.shift()) {
this.stream.write(command); this.should_buffer = !this.stream.write(command);
} }
this.should_buffer = !this.stream.write(data);
}; };
RedisClient.prototype.write = function (data) { RedisClient.prototype.write = function (data) {
if (this.pipeline === 0) { if (this.pipeline === false) {
this.should_buffer = !this.stream.write(data); this.should_buffer = !this.stream.write(data);
return; return;
} }
this.pipeline--;
if (this.pipeline === 0) {
this.writeDefault(data);
return;
}
this.pipeline_queue.push(data); this.pipeline_queue.push(data);
return;
}; };
Object.defineProperty(exports, 'debugMode', { Object.defineProperty(exports, 'debugMode', {

7
lib/multi.js
View File

@@ -126,7 +126,7 @@ Multi.prototype.exec_transaction = function exec_transaction (callback) {
var len = self.queue.length; var len = self.queue.length;
self.errors = []; self.errors = [];
self.callback = callback; self.callback = callback;
self._client.cork(len + 2); self._client.cork();
self.wants_buffers = new Array(len); self.wants_buffers = new Array(len);
pipeline_transaction_command(self, 'multi', []); pipeline_transaction_command(self, 'multi', []);
// Drain queue, callback will catch 'QUEUED' or error // Drain queue, callback will catch 'QUEUED' or error
@@ -151,7 +151,6 @@ Multi.prototype.exec_transaction = function exec_transaction (callback) {
multi_callback(self, err, replies); multi_callback(self, err, replies);
}); });
self._client.uncork(); self._client.uncork();
self._client.writeDefault = self._client.writeStrings;
return !self._client.should_buffer; return !self._client.should_buffer;
}; };
@@ -198,7 +197,7 @@ Multi.prototype.exec = Multi.prototype.EXEC = Multi.prototype.exec_batch = funct
return true; return true;
} }
self.results = []; self.results = [];
self._client.cork(len); self._client.cork();
while (args = self.queue.shift()) { while (args = self.queue.shift()) {
var command = args[0]; var command = args[0];
var cb; var cb;
@@ -213,9 +212,7 @@ Multi.prototype.exec = Multi.prototype.EXEC = Multi.prototype.exec_batch = funct
self._client.internal_send_command(command, args[1], cb); self._client.internal_send_command(command, args[1], cb);
index++; index++;
} }
self.queue = new Queue();
self._client.uncork(); self._client.uncork();
self._client.writeDefault = self._client.writeStrings;
return !self._client.should_buffer; return !self._client.should_buffer;
}; };

12
test/multi.spec.js
View File

@@ -125,9 +125,8 @@ describe("The 'multi' method", function () {
describe('when connected', function () { describe('when connected', function () {
beforeEach(function (done) { beforeEach(function () {
client = redis.createClient.apply(null, args); client = redis.createClient.apply(null, args);
client.once('connect', done);
}); });
it('executes a pipelined multi properly in combination with the offline queue', function (done) { it('executes a pipelined multi properly in combination with the offline queue', function (done) {
@@ -135,6 +134,7 @@ describe("The 'multi' method", function () {
multi1.set('m1', '123'); multi1.set('m1', '123');
multi1.get('m1'); multi1.get('m1');
multi1.exec(done); multi1.exec(done);
assert.strictEqual(client.offline_queue.length, 4);
}); });
it('executes a pipelined multi properly after a reconnect in combination with the offline queue', function (done) { it('executes a pipelined multi properly after a reconnect in combination with the offline queue', function (done) {
@@ -612,11 +612,17 @@ describe("The 'multi' method", function () {
}); });
it('emits error once if reconnecting after multi has been executed but not yet returned without callback', function (done) { it('emits error once if reconnecting after multi has been executed but not yet returned without callback', function (done) {
// NOTE: If uncork is called async by postponing it to the next tick, this behavior is going to change.
// The command won't be processed anymore two errors are returned instead of one
client.on('error', function (err) { client.on('error', function (err) {
assert.strictEqual(err.code, 'UNCERTAIN_STATE'); assert.strictEqual(err.code, 'UNCERTAIN_STATE');
done(); client.get('foo', function (err, res) {
assert.strictEqual(res, 'bar');
done();
});
}); });
// The commands should still be fired, no matter that the socket is destroyed on the same tick
client.multi().set('foo', 'bar').get('foo').exec(); client.multi().set('foo', 'bar').get('foo').exec();
// Abort connection before the value returned // Abort connection before the value returned
client.stream.destroy(); client.stream.destroy();

10
test/node_redis.spec.js
View File

@@ -121,12 +121,12 @@ describe('The node_redis client', function () {
str += str; str += str;
} }
var called = false; var called = false;
var temp = client.writeBuffers.bind(client); var temp = client.write_buffers.bind(client);
assert(String(client.writeBuffers) !== String(client.writeDefault)); assert(client.fire_strings);
client.writeBuffers = function (data) { client.write_buffers = function (data) {
called = true; called = true;
// To increase write performance for strings the value is converted to a buffer // To increase write performance for strings the value is converted to a buffer
assert(String(client.writeBuffers) === String(client.writeDefault)); assert(!client.fire_strings);
temp(data); temp(data);
}; };
client.multi().set('foo', str).get('foo', function (err, res) { client.multi().set('foo', str).get('foo', function (err, res) {
@@ -136,7 +136,7 @@ describe('The node_redis client', function () {
assert.strictEqual(res[1], str); assert.strictEqual(res[1], str);
done(); done();
}); });
assert(String(client.writeBuffers) !== String(client.writeDefault)); assert(client.fire_strings);
}); });
}); });