MCOL-529 Pool DBRM connections

DBRM connections are reused so that we don't have a huge amount of
TIME_WAIT sockets when there are large amounts of DML. Also applied to
i_s.columnstore_files

utils/messageqcpp/CMakeLists.txt

@@ -5,6 +5,7 @@ include_directories( ${ENGINE_COMMON_INCLUDES} )
 
 set(messageqcpp_LIB_SRCS
     messagequeue.cpp
+    messagequeuepool.cpp
     bytestream.cpp
     socketparms.cpp
     inetstreamsocket.cpp

utils/messageqcpp/inetstreamsocket.cpp

@@ -70,6 +70,7 @@ either expressed or implied, of the FreeBSD Project.
 #include <arpa/inet.h>
 #include <netinet/tcp.h>
 #include <fcntl.h>
+#include <sys/ioctl.h>
 #endif
 #include <sys/types.h>
 #include <sys/time.h>
@@ -1070,5 +1071,44 @@ int InetStreamSocket::ping(const std::string& ipaddr, const struct timespec* tim
 	return 0;
 }
 
+bool InetStreamSocket::isConnected() const
+{
+    int error = 0;
+    socklen_t len = sizeof(error);
+    int retval = getsockopt(fSocketParms.sd(), SOL_SOCKET, SO_ERROR, &error, &len);
+    
+    if (error || retval)
+        return false;
+        
+    struct pollfd pfd[1];
+    pfd[0].fd = fSocketParms.sd();
+    pfd[0].events = POLLIN;
+    pfd[0].revents = 0;
+
+    error = poll(pfd, 1, 0);
+    if ((error < 0) || (pfd[0].revents & (POLLHUP | POLLNVAL | POLLERR))) {
+        return false;
+    }
+    
+    return true;
+}
+
+bool InetStreamSocket::hasData() const
+{
+    int count;
+    char buf[1];
+    ssize_t retval;
+    ioctl(fSocketParms.sd(), FIONREAD, &count);
+    if (count)
+        return true;
+        
+    // EAGAIN | EWOULDBLOCK means the socket is clear. Anything else is data or error
+    retval = recv(fSocketParms.sd(), buf, 1, MSG_DONTWAIT);
+    if (retval & (EAGAIN | EWOULDBLOCK))
+        return false;
+        
+    return true;
+}
+
 } //namespace messageqcpp
 

utils/messageqcpp/inetstreamsocket.h

@@ -196,6 +196,13 @@ public:
 	 */
 	EXPORT static int ping(const std::string& ipaddr, const struct timespec* timeout=0);
 
+    // Check if we are still connected
+    virtual bool isConnected() const;
+    
+    // Check if the socket still has data pending
+    
+    virtual bool hasData() const;
+    
 	/*
 	 * allow test suite access to private data for OOB test
 	 */

utils/messageqcpp/iosocket.h

@@ -174,7 +174,9 @@ public:
 	 */
 	virtual void connectionTimeout(const struct timespec* timeout) { fSocket->connectionTimeout(timeout); }
 	
-	
+	inline virtual bool isConnected() const;
+    inline virtual bool hasData() const;
+    
 	friend class ::MessageQTestSuite;
 
 protected:
@@ -208,6 +210,8 @@ inline const SocketParms IOSocket::socketParms() const { idbassert(fSocket); ret
 inline void IOSocket::socketParms(const SocketParms& socketParms) { idbassert(fSocket); fSocket->socketParms(socketParms); }
 inline void IOSocket::setSocketImpl(Socket* socket) { delete fSocket; fSocket = socket; }
 inline const int IOSocket::getConnectionNum() const { return fSocket->getConnectionNum(); }
+inline bool IOSocket::isConnected() const { return fSocket->isConnected(); }
+inline bool IOSocket::hasData() const { return fSocket->hasData(); }
 
 /**
  * stream an IOSocket rep to any ostream

utils/messageqcpp/messagequeue.h

@@ -249,6 +249,9 @@ public:
 	 */
 	inline const bool isSameAddr(const MessageQueueClient& rhs) const;
 
+    bool isConnected() { return fClientSock.isConnected(); }
+    
+    bool hasData() { return fClientSock.hasData(); }
 	/*
 	 * allow test suite access to private data for OOB test
 	 */

utils/messageqcpp/messagequeuepool.cpp

@@ -0,0 +1,205 @@
+/* Copyright (C) 2017 MariaDB Corporaton
+
+   This program is free software; you can redistribute it and/or
+   modify it under the terms of the GNU General Public License
+   as published by the Free Software Foundation; version 2 of
+   the License.
+
+   This program is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+   GNU General Public License for more details.
+
+   You should have received a copy of the GNU General Public License
+   along with this program; if not, write to the Free Software
+   Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
+   MA 02110-1301, USA. */
+
+#include <sstream>
+#include <map>
+#include <boost/thread/mutex.hpp>
+#include <time.h>
+#include "messagequeuepool.h"
+#include "messagequeue.h"
+
+namespace messageqcpp { 
+
+boost::mutex queueMutex;
+// Make linker happy
+std::multimap<std::string, ClientObject*> MessageQueueClientPool::clientMap;
+
+// 300 seconds idle until cleanup
+#define MAX_IDLE_TIME 300
+
+static uint64_t TimeSpecToSeconds(struct timespec* ts)
+{
+    return (uint64_t)ts->tv_sec + (uint64_t)ts->tv_nsec / 1000000000;
+}
+
+MessageQueueClient *MessageQueueClientPool::getInstance(const std::string &ip, uint64_t port)
+{
+    boost::mutex::scoped_lock lock(queueMutex);
+
+    std::ostringstream oss;
+    oss << ip << "_" << port;
+    std::string searchString = oss.str();
+    
+    MessageQueueClient *returnClient = MessageQueueClientPool::findInPool(searchString);
+    
+    // We found one, return it
+    if (returnClient != NULL)
+    {
+        return returnClient;
+    }
+    
+    // We didn't find one, create new one
+    ClientObject *newClientObject = new ClientObject();
+    struct timespec now;
+    clock_gettime(CLOCK_MONOTONIC, &now);
+    uint64_t nowSeconds = TimeSpecToSeconds(&now);
+
+    newClientObject->client = new MessageQueueClient(ip, port);
+    newClientObject->inUse = true;
+    newClientObject->lastUsed = nowSeconds;
+    clientMap.insert(std::pair<std::string, ClientObject*>(searchString, newClientObject));
+    return newClientObject->client;
+}
+
+MessageQueueClient *MessageQueueClientPool::getInstance(const std::string &module)
+{
+    boost::mutex::scoped_lock lock(queueMutex);
+
+    MessageQueueClient *returnClient = MessageQueueClientPool::findInPool(module);
+    
+    // We found one, return it
+    if (returnClient != NULL)
+    {
+        return returnClient;
+    }
+    
+    // We didn't find one, create new one
+    ClientObject *newClientObject = new ClientObject();
+    struct timespec now;
+    clock_gettime(CLOCK_MONOTONIC, &now);
+    uint64_t nowSeconds = TimeSpecToSeconds(&now);
+
+    newClientObject->client = new MessageQueueClient(module);
+    newClientObject->inUse = true;
+    newClientObject->lastUsed = nowSeconds;
+    clientMap.insert(std::pair<std::string, ClientObject*>(module, newClientObject));
+    return newClientObject->client;
+}
+
+MessageQueueClient *MessageQueueClientPool::findInPool(const std::string &search)
+{
+    struct timespec now;
+    clock_gettime(CLOCK_MONOTONIC, &now);
+    uint64_t nowSeconds = TimeSpecToSeconds(&now);
+    MessageQueueClient *returnClient = NULL;
+        
+    std::multimap<std::string,ClientObject*>::iterator it=clientMap.begin();
+    
+    // Scan pool
+    while (it!=clientMap.end())
+    {
+        ClientObject *clientObject = it->second;
+        uint64_t elapsedTime = nowSeconds - clientObject->lastUsed;
+        
+        // If connection hasn't been used for MAX_IDLE_TIME we probably don't need it so drop it
+        // Don't drop in use connections that have been in use a long time
+        if ((elapsedTime >= MAX_IDLE_TIME) && (!clientObject->inUse))
+        {
+            delete clientObject->client;
+            delete clientObject;
+            // Do this so we don't invalidate current interator
+            std::multimap<std::string,ClientObject*>::iterator toDelete = it;
+            it++;
+            clientMap.erase(toDelete);
+            continue;
+        }
+        
+        if (!clientObject->inUse)
+        {
+            MessageQueueClient *client = clientObject->client;
+            
+            // If the unused socket isn't connected or has data pending read, destroy it
+            if (!client->isConnected() || client->hasData())
+            {
+                delete client;
+                delete clientObject;
+                // Do this so we don't invalidate current interator
+                std::multimap<std::string,ClientObject*>::iterator toDelete = it;
+                it++;
+                clientMap.erase(toDelete);
+                continue;
+            }
+            
+        }
+        
+        // If connection matches store it for later, but keep scanning the pool for more timeout prunes
+        if (it->first.compare(search) == 0)
+        {
+            if ((returnClient == NULL) && (!clientObject->inUse))
+            {
+                returnClient = clientObject->client;
+                clientObject->inUse = true;
+                return returnClient;
+            }
+        }
+        it++;
+    }
+    return NULL;
+}
+
+void MessageQueueClientPool::releaseInstance(MessageQueueClient * client)
+{
+    // Scan pool for pointer and release
+    // Set the last used and mark as not in use
+    
+    if (client == NULL)
+        return;
+    
+    boost::mutex::scoped_lock lock(queueMutex);
+    std::multimap<std::string,ClientObject*>::iterator it=clientMap.begin();
+    
+    while (it!=clientMap.end())
+    {
+        if (it->second->client == client)
+        {
+            struct timespec now;
+            clock_gettime(CLOCK_MONOTONIC, &now);
+            uint64_t nowSeconds = TimeSpecToSeconds(&now);
+            it->second->inUse = false;
+            it->second->lastUsed = nowSeconds;
+            return;
+        }
+        it++;
+    }
+}
+
+// WriteEngine needs this as it forces connections closed and can't reuse. Also good for connection errors
+void MessageQueueClientPool::deleteInstance(MessageQueueClient * client)
+{
+    // Scan pool for pointer and delete
+    // Set the last used and mark as not in use
+    
+    if (client == NULL)
+        return;
+    
+    boost::mutex::scoped_lock lock(queueMutex);
+    std::multimap<std::string,ClientObject*>::iterator it=clientMap.begin();
+    
+    while (it!=clientMap.end())
+    {
+        if (it->second->client == client)
+        {
+            delete it->second->client;
+            delete it->second;
+            clientMap.erase(it);
+            return;
+        }
+        it++;
+    }
+    
+}
+}

utils/messageqcpp/messagequeuepool.h

@@ -0,0 +1,57 @@
+/* Copyright (C) 2017 MariaDB Corporaton
+
+   This program is free software; you can redistribute it and/or
+   modify it under the terms of the GNU General Public License
+   as published by the Free Software Foundation; version 2 of
+   the License.
+
+   This program is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+   GNU General Public License for more details.
+
+   You should have received a copy of the GNU General Public License
+   along with this program; if not, write to the Free Software
+   Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
+   MA 02110-1301, USA. */
+
+#ifndef MESSAGEQCPP_MESSAGEQUEUECLIENT_H
+#define MESSAGEQCPP_MESSAGEQUEUECLIENT_H
+
+#include <map>
+#include "messagequeue.h"
+
+namespace messageqcpp { 
+
+
+struct ClientObject
+{
+    MessageQueueClient *client;
+    uint64_t lastUsed;
+    bool inUse;
+    
+    ClientObject() :
+        client(NULL),
+        lastUsed(0),
+        inUse(false)
+    {}
+};
+
+class MessageQueueClientPool
+{
+    public:
+        static MessageQueueClient *getInstance(const std::string &module);
+        static MessageQueueClient *getInstance(const std::string &ip, uint64_t port);
+        static void releaseInstance(MessageQueueClient * client);
+        static void deleteInstance(MessageQueueClient * client);
+        static MessageQueueClient *findInPool(const std::string &search);
+    
+    private:
+        MessageQueueClientPool() { };
+        ~MessageQueueClientPool() { };
+        
+        static std::multimap<std::string, ClientObject*> clientMap;
+};
+
+}
+#endif //MESSAGEQCPP_MESSAGEQUEUECLIENT_H

utils/messageqcpp/socket.h

@@ -158,6 +158,9 @@ public:
 	 */
 	virtual const bool isSameAddr(const Socket* rhs) const = 0;
 
+	virtual bool isConnected() const = 0;
+    virtual bool hasData() const = 0;
+
 	/*
 	 * allow test suite access to private data for OOB test
 	 */