1
0
mirror of https://github.com/mariadb-corporation/mariadb-columnstore-engine.git synced 2025-07-29 08:21:15 +03:00

MCOL-529 Pool DBRM connections

DBRM connections are reused so that we don't have a huge amount of
TIME_WAIT sockets when there are large amounts of DML. Also applied to
i_s.columnstore_files
This commit is contained in:
Andrew Hutchings
2017-04-14 14:13:15 +01:00
parent 3e85b6ef07
commit 830b24c1fa
11 changed files with 349 additions and 56 deletions

View File

@ -33,6 +33,7 @@
#include "bytestream.h" #include "bytestream.h"
#include "liboamcpp.h" #include "liboamcpp.h"
#include "messagequeue.h" #include "messagequeue.h"
#include "messagequeuepool.h"
#include "we_messages.h" #include "we_messages.h"
// Required declaration as it isn't in a MairaDB include // Required declaration as it isn't in a MairaDB include
@ -81,15 +82,6 @@ static bool get_file_sizes(messageqcpp::MessageQueueClient *msgQueueClient, cons
} }
} }
static void cleanup(std::map<int, messageqcpp::MessageQueueClient*> &clients)
{
for(std::map<int, messageqcpp::MessageQueueClient*>::iterator itr = clients.begin(); itr != clients.end(); itr++)
{
delete itr->second;
}
}
static int is_columnstore_files_fill(THD *thd, TABLE_LIST *tables, COND *cond) static int is_columnstore_files_fill(THD *thd, TABLE_LIST *tables, COND *cond)
{ {
BRM::DBRM *emp = new BRM::DBRM(); BRM::DBRM *emp = new BRM::DBRM();
@ -105,7 +97,6 @@ static int is_columnstore_files_fill(THD *thd, TABLE_LIST *tables, COND *cond)
off_t fileSize = 0; off_t fileSize = 0;
off_t compressedFileSize = 0; off_t compressedFileSize = 0;
we_config.initConfigCache(); we_config.initConfigCache();
std::map<int, messageqcpp::MessageQueueClient*> clients;
messageqcpp::MessageQueueClient *msgQueueClient; messageqcpp::MessageQueueClient *msgQueueClient;
oam::Oam oam_instance; oam::Oam oam_instance;
int pmId = 0; int pmId = 0;
@ -143,37 +134,15 @@ static int is_columnstore_files_fill(THD *thd, TABLE_LIST *tables, COND *cond)
std::string DbRootPath = config->getConfig("SystemConfig", DbRootName.str()); std::string DbRootPath = config->getConfig("SystemConfig", DbRootName.str());
fileSize = compressedFileSize = 0; fileSize = compressedFileSize = 0;
snprintf(fullFileName, WriteEngine::FILE_NAME_SIZE, "%s/%s", DbRootPath.c_str(), oidDirName); snprintf(fullFileName, WriteEngine::FILE_NAME_SIZE, "%s/%s", DbRootPath.c_str(), oidDirName);
try oam_instance.getDbrootPmConfig(iter->dbRoot, pmId);
{ std::ostringstream oss;
msgQueueClient = clients.at(iter->dbRoot); oss << "pm" << pmId << "_WriteEngineServer";
} std::string client = oss.str();
catch (...) msgQueueClient = messageqcpp::MessageQueueClientPool::getInstance(oss.str());
{
msgQueueClient = NULL;
}
if (!msgQueueClient)
{
oam_instance.getDbrootPmConfig(iter->dbRoot, pmId);
std::ostringstream oss;
oss << "pm" << pmId << "_WriteEngineServer";
try
{
msgQueueClient = new messageqcpp::MessageQueueClient(oss.str());
}
catch (...)
{
delete msgQueueClient;
cleanup(clients);
delete emp;
return 1;
}
clients[iter->dbRoot] = msgQueueClient;
}
if (!get_file_sizes(msgQueueClient, fullFileName, &fileSize, &compressedFileSize)) if (!get_file_sizes(msgQueueClient, fullFileName, &fileSize, &compressedFileSize))
{ {
cleanup(clients); messageqcpp::MessageQueueClientPool::releaseInstance(msgQueueClient);
delete emp; delete emp;
return 1; return 1;
} }
@ -201,11 +170,13 @@ static int is_columnstore_files_fill(THD *thd, TABLE_LIST *tables, COND *cond)
if (schema_table_store_record(thd, table)) if (schema_table_store_record(thd, table))
{ {
cleanup(clients); messageqcpp::MessageQueueClientPool::releaseInstance(msgQueueClient);
delete emp; delete emp;
return 1; return 1;
} }
iter++; iter++;
messageqcpp::MessageQueueClientPool::releaseInstance(msgQueueClient);
msgQueueClient = NULL;
} }
} }
delete emp; delete emp;

View File

@ -5,6 +5,7 @@ include_directories( ${ENGINE_COMMON_INCLUDES} )
set(messageqcpp_LIB_SRCS set(messageqcpp_LIB_SRCS
messagequeue.cpp messagequeue.cpp
messagequeuepool.cpp
bytestream.cpp bytestream.cpp
socketparms.cpp socketparms.cpp
inetstreamsocket.cpp inetstreamsocket.cpp

View File

@ -70,6 +70,7 @@ either expressed or implied, of the FreeBSD Project.
#include <arpa/inet.h> #include <arpa/inet.h>
#include <netinet/tcp.h> #include <netinet/tcp.h>
#include <fcntl.h> #include <fcntl.h>
#include <sys/ioctl.h>
#endif #endif
#include <sys/types.h> #include <sys/types.h>
#include <sys/time.h> #include <sys/time.h>
@ -1070,5 +1071,44 @@ int InetStreamSocket::ping(const std::string& ipaddr, const struct timespec* tim
return 0; return 0;
} }
bool InetStreamSocket::isConnected() const
{
int error = 0;
socklen_t len = sizeof(error);
int retval = getsockopt(fSocketParms.sd(), SOL_SOCKET, SO_ERROR, &error, &len);
if (error || retval)
return false;
struct pollfd pfd[1];
pfd[0].fd = fSocketParms.sd();
pfd[0].events = POLLIN;
pfd[0].revents = 0;
error = poll(pfd, 1, 0);
if ((error < 0) || (pfd[0].revents & (POLLHUP | POLLNVAL | POLLERR))) {
return false;
}
return true;
}
bool InetStreamSocket::hasData() const
{
int count;
char buf[1];
ssize_t retval;
ioctl(fSocketParms.sd(), FIONREAD, &count);
if (count)
return true;
// EAGAIN | EWOULDBLOCK means the socket is clear. Anything else is data or error
retval = recv(fSocketParms.sd(), buf, 1, MSG_DONTWAIT);
if (retval & (EAGAIN | EWOULDBLOCK))
return false;
return true;
}
} //namespace messageqcpp } //namespace messageqcpp

View File

@ -196,6 +196,13 @@ public:
*/ */
EXPORT static int ping(const std::string& ipaddr, const struct timespec* timeout=0); EXPORT static int ping(const std::string& ipaddr, const struct timespec* timeout=0);
// Check if we are still connected
virtual bool isConnected() const;
// Check if the socket still has data pending
virtual bool hasData() const;
/* /*
* allow test suite access to private data for OOB test * allow test suite access to private data for OOB test
*/ */

View File

@ -174,7 +174,9 @@ public:
*/ */
virtual void connectionTimeout(const struct timespec* timeout) { fSocket->connectionTimeout(timeout); } virtual void connectionTimeout(const struct timespec* timeout) { fSocket->connectionTimeout(timeout); }
inline virtual bool isConnected() const;
inline virtual bool hasData() const;
friend class ::MessageQTestSuite; friend class ::MessageQTestSuite;
protected: protected:
@ -208,6 +210,8 @@ inline const SocketParms IOSocket::socketParms() const { idbassert(fSocket); ret
inline void IOSocket::socketParms(const SocketParms& socketParms) { idbassert(fSocket); fSocket->socketParms(socketParms); } inline void IOSocket::socketParms(const SocketParms& socketParms) { idbassert(fSocket); fSocket->socketParms(socketParms); }
inline void IOSocket::setSocketImpl(Socket* socket) { delete fSocket; fSocket = socket; } inline void IOSocket::setSocketImpl(Socket* socket) { delete fSocket; fSocket = socket; }
inline const int IOSocket::getConnectionNum() const { return fSocket->getConnectionNum(); } inline const int IOSocket::getConnectionNum() const { return fSocket->getConnectionNum(); }
inline bool IOSocket::isConnected() const { return fSocket->isConnected(); }
inline bool IOSocket::hasData() const { return fSocket->hasData(); }
/** /**
* stream an IOSocket rep to any ostream * stream an IOSocket rep to any ostream

View File

@ -249,6 +249,9 @@ public:
*/ */
inline const bool isSameAddr(const MessageQueueClient& rhs) const; inline const bool isSameAddr(const MessageQueueClient& rhs) const;
bool isConnected() { return fClientSock.isConnected(); }
bool hasData() { return fClientSock.hasData(); }
/* /*
* allow test suite access to private data for OOB test * allow test suite access to private data for OOB test
*/ */

View File

@ -0,0 +1,205 @@
/* Copyright (C) 2017 MariaDB Corporaton
This program is free software; you can redistribute it and/or
modify it under the terms of the GNU General Public License
as published by the Free Software Foundation; version 2 of
the License.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program; if not, write to the Free Software
Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
MA 02110-1301, USA. */
#include <sstream>
#include <map>
#include <boost/thread/mutex.hpp>
#include <time.h>
#include "messagequeuepool.h"
#include "messagequeue.h"
namespace messageqcpp {
boost::mutex queueMutex;
// Make linker happy
std::multimap<std::string, ClientObject*> MessageQueueClientPool::clientMap;
// 300 seconds idle until cleanup
#define MAX_IDLE_TIME 300
static uint64_t TimeSpecToSeconds(struct timespec* ts)
{
return (uint64_t)ts->tv_sec + (uint64_t)ts->tv_nsec / 1000000000;
}
MessageQueueClient *MessageQueueClientPool::getInstance(const std::string &ip, uint64_t port)
{
boost::mutex::scoped_lock lock(queueMutex);
std::ostringstream oss;
oss << ip << "_" << port;
std::string searchString = oss.str();
MessageQueueClient *returnClient = MessageQueueClientPool::findInPool(searchString);
// We found one, return it
if (returnClient != NULL)
{
return returnClient;
}
// We didn't find one, create new one
ClientObject *newClientObject = new ClientObject();
struct timespec now;
clock_gettime(CLOCK_MONOTONIC, &now);
uint64_t nowSeconds = TimeSpecToSeconds(&now);
newClientObject->client = new MessageQueueClient(ip, port);
newClientObject->inUse = true;
newClientObject->lastUsed = nowSeconds;
clientMap.insert(std::pair<std::string, ClientObject*>(searchString, newClientObject));
return newClientObject->client;
}
MessageQueueClient *MessageQueueClientPool::getInstance(const std::string &module)
{
boost::mutex::scoped_lock lock(queueMutex);
MessageQueueClient *returnClient = MessageQueueClientPool::findInPool(module);
// We found one, return it
if (returnClient != NULL)
{
return returnClient;
}
// We didn't find one, create new one
ClientObject *newClientObject = new ClientObject();
struct timespec now;
clock_gettime(CLOCK_MONOTONIC, &now);
uint64_t nowSeconds = TimeSpecToSeconds(&now);
newClientObject->client = new MessageQueueClient(module);
newClientObject->inUse = true;
newClientObject->lastUsed = nowSeconds;
clientMap.insert(std::pair<std::string, ClientObject*>(module, newClientObject));
return newClientObject->client;
}
MessageQueueClient *MessageQueueClientPool::findInPool(const std::string &search)
{
struct timespec now;
clock_gettime(CLOCK_MONOTONIC, &now);
uint64_t nowSeconds = TimeSpecToSeconds(&now);
MessageQueueClient *returnClient = NULL;
std::multimap<std::string,ClientObject*>::iterator it=clientMap.begin();
// Scan pool
while (it!=clientMap.end())
{
ClientObject *clientObject = it->second;
uint64_t elapsedTime = nowSeconds - clientObject->lastUsed;
// If connection hasn't been used for MAX_IDLE_TIME we probably don't need it so drop it
// Don't drop in use connections that have been in use a long time
if ((elapsedTime >= MAX_IDLE_TIME) && (!clientObject->inUse))
{
delete clientObject->client;
delete clientObject;
// Do this so we don't invalidate current interator
std::multimap<std::string,ClientObject*>::iterator toDelete = it;
it++;
clientMap.erase(toDelete);
continue;
}
if (!clientObject->inUse)
{
MessageQueueClient *client = clientObject->client;
// If the unused socket isn't connected or has data pending read, destroy it
if (!client->isConnected() || client->hasData())
{
delete client;
delete clientObject;
// Do this so we don't invalidate current interator
std::multimap<std::string,ClientObject*>::iterator toDelete = it;
it++;
clientMap.erase(toDelete);
continue;
}
}
// If connection matches store it for later, but keep scanning the pool for more timeout prunes
if (it->first.compare(search) == 0)
{
if ((returnClient == NULL) && (!clientObject->inUse))
{
returnClient = clientObject->client;
clientObject->inUse = true;
return returnClient;
}
}
it++;
}
return NULL;
}
void MessageQueueClientPool::releaseInstance(MessageQueueClient * client)
{
// Scan pool for pointer and release
// Set the last used and mark as not in use
if (client == NULL)
return;
boost::mutex::scoped_lock lock(queueMutex);
std::multimap<std::string,ClientObject*>::iterator it=clientMap.begin();
while (it!=clientMap.end())
{
if (it->second->client == client)
{
struct timespec now;
clock_gettime(CLOCK_MONOTONIC, &now);
uint64_t nowSeconds = TimeSpecToSeconds(&now);
it->second->inUse = false;
it->second->lastUsed = nowSeconds;
return;
}
it++;
}
}
// WriteEngine needs this as it forces connections closed and can't reuse. Also good for connection errors
void MessageQueueClientPool::deleteInstance(MessageQueueClient * client)
{
// Scan pool for pointer and delete
// Set the last used and mark as not in use
if (client == NULL)
return;
boost::mutex::scoped_lock lock(queueMutex);
std::multimap<std::string,ClientObject*>::iterator it=clientMap.begin();
while (it!=clientMap.end())
{
if (it->second->client == client)
{
delete it->second->client;
delete it->second;
clientMap.erase(it);
return;
}
it++;
}
}
}

View File

@ -0,0 +1,57 @@
/* Copyright (C) 2017 MariaDB Corporaton
This program is free software; you can redistribute it and/or
modify it under the terms of the GNU General Public License
as published by the Free Software Foundation; version 2 of
the License.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program; if not, write to the Free Software
Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
MA 02110-1301, USA. */
#ifndef MESSAGEQCPP_MESSAGEQUEUECLIENT_H
#define MESSAGEQCPP_MESSAGEQUEUECLIENT_H
#include <map>
#include "messagequeue.h"
namespace messageqcpp {
struct ClientObject
{
MessageQueueClient *client;
uint64_t lastUsed;
bool inUse;
ClientObject() :
client(NULL),
lastUsed(0),
inUse(false)
{}
};
class MessageQueueClientPool
{
public:
static MessageQueueClient *getInstance(const std::string &module);
static MessageQueueClient *getInstance(const std::string &ip, uint64_t port);
static void releaseInstance(MessageQueueClient * client);
static void deleteInstance(MessageQueueClient * client);
static MessageQueueClient *findInPool(const std::string &search);
private:
MessageQueueClientPool() { };
~MessageQueueClientPool() { };
static std::multimap<std::string, ClientObject*> clientMap;
};
}
#endif //MESSAGEQCPP_MESSAGEQUEUECLIENT_H

View File

@ -158,6 +158,9 @@ public:
*/ */
virtual const bool isSameAddr(const Socket* rhs) const = 0; virtual const bool isSameAddr(const Socket* rhs) const = 0;
virtual bool isConnected() const = 0;
virtual bool hasData() const = 0;
/* /*
* allow test suite access to private data for OOB test * allow test suite access to private data for OOB test
*/ */

View File

@ -40,6 +40,7 @@
#include "socketclosed.h" #include "socketclosed.h"
#include "configcpp.h" #include "configcpp.h"
#include "sessionmanagerserver.h" #include "sessionmanagerserver.h"
#include "messagequeuepool.h"
#define DBRM_DLLEXPORT #define DBRM_DLLEXPORT
#include "dbrm.h" #include "dbrm.h"
#undef DBRM_DLLEXPORT #undef DBRM_DLLEXPORT
@ -53,7 +54,7 @@
#endif #endif
#define DO_ERR_NETWORK \ #define DO_ERR_NETWORK \
delete msgClient; \ MessageQueueClientPool::releaseInstance(msgClient); \
msgClient = NULL; \ msgClient = NULL; \
mutex.unlock(); \ mutex.unlock(); \
return ERR_NETWORK; return ERR_NETWORK;
@ -97,8 +98,7 @@ DBRM::DBRM(const DBRM& brm)
DBRM::~DBRM() throw() DBRM::~DBRM() throw()
{ {
if (msgClient != NULL) if (msgClient != NULL)
msgClient->shutdown(); MessageQueueClientPool::releaseInstance(msgClient);
delete msgClient;
} }
DBRM& DBRM::operator=(const DBRM& brm) DBRM& DBRM::operator=(const DBRM& brm)
@ -742,7 +742,7 @@ reconnect:
if (msgClient == NULL) if (msgClient == NULL)
try { try {
msgClient = new MessageQueueClient(masterName); msgClient = MessageQueueClientPool::getInstance(masterName);
} }
catch(exception &e) { catch(exception &e) {
cerr << "class DBRM failed to create a MessageQueueClient: " << cerr << "class DBRM failed to create a MessageQueueClient: " <<
@ -766,7 +766,7 @@ reconnect:
cerr << "DBRM::send_recv caught: " << e.what() << endl; cerr << "DBRM::send_recv caught: " << e.what() << endl;
if (firstAttempt) { if (firstAttempt) {
firstAttempt = false; firstAttempt = false;
delete msgClient; MessageQueueClientPool::releaseInstance(msgClient);
msgClient = NULL; msgClient = NULL;
goto reconnect; goto reconnect;
} }
@ -776,7 +776,7 @@ reconnect:
cerr << "DBRM::send_recv: controller node closed the connection" << endl; cerr << "DBRM::send_recv: controller node closed the connection" << endl;
if (firstAttempt) { if (firstAttempt) {
firstAttempt = false; firstAttempt = false;
delete msgClient; MessageQueueClientPool::releaseInstance(msgClient);
msgClient = NULL; msgClient = NULL;
sleep(10); sleep(10);
goto reconnect; goto reconnect;
@ -3222,7 +3222,7 @@ bool DBRM::isDBRMReady() throw()
{ {
if (msgClient == NULL) if (msgClient == NULL)
{ {
msgClient = new MessageQueueClient(masterName); msgClient = MessageQueueClientPool::getInstance(masterName);
} }
if (msgClient->connect()) if (msgClient->connect())
{ {
@ -3232,7 +3232,7 @@ bool DBRM::isDBRMReady() throw()
catch (...) catch (...)
{ {
} }
delete msgClient; MessageQueueClientPool::releaseInstance(msgClient);
msgClient = NULL; msgClient = NULL;
sleep(1); sleep(1);
} }

View File

@ -32,7 +32,7 @@
#include "liboamcpp.h" #include "liboamcpp.h"
#include "stopwatch.h" #include "stopwatch.h"
#include "masterdbrmnode.h" #include "masterdbrmnode.h"
#include "messagequeuepool.h"
// #define BRM_VERBOSE // #define BRM_VERBOSE
// minor improvement to code clarity... // minor improvement to code clarity...
@ -163,7 +163,8 @@ void MasterDBRMNode::initMsgQueues(config::Config *config)
serverLock.unlock(); serverLock.unlock();
for (i = 1; i <= NumWorkers; i++) { for (i = 1; i <= NumWorkers; i++) {
snprintf(ctmp, 50, "DBRM_Worker%d", i); snprintf(ctmp, 50, "DBRM_Worker%d", i);
slaves.push_back(new MessageQueueClient(ctmp, config)); std::string module(ctmp);
slaves.push_back(MessageQueueClientPool::getInstance(module));
} }
} }
@ -882,8 +883,8 @@ void MasterDBRMNode::finalCleanup()
cerr << "Closing connections" << endl; cerr << "Closing connections" << endl;
#endif #endif
for (sIt = slaves.begin(); sIt != slaves.end(); sIt++) { for (sIt = slaves.begin(); sIt != slaves.end(); sIt++) {
(*sIt)->shutdown(); MessageQueueClientPool::releaseInstance(*sIt);
delete *sIt; *sIt = NULL;
} }
slaves.clear(); slaves.clear();
@ -1036,8 +1037,8 @@ void MasterDBRMNode::doReload(messageqcpp::IOSocket *sock)
} }
for (i = 0; i < (int) slaves.size(); i++) { for (i = 0; i < (int) slaves.size(); i++) {
slaves[i]->shutdown(); MessageQueueClientPool::releaseInstance(slaves[i]);
delete slaves[i]; slaves[i] = NULL;
} }
slaves.clear(); slaves.clear();
@ -1045,7 +1046,8 @@ void MasterDBRMNode::doReload(messageqcpp::IOSocket *sock)
for (i = 1; i <= NumWorkers; i++) { for (i = 1; i <= NumWorkers; i++) {
snprintf(ctmp, 50, "DBRM_Worker%d", i); snprintf(ctmp, 50, "DBRM_Worker%d", i);
slaves.push_back(new MessageQueueClient(ctmp, config)); std::string module(ctmp);
slaves.push_back(MessageQueueClientPool::getInstance(module));
} }
iSlave = slaves.end(); iSlave = slaves.end();