From cfce821513dc9d99010f96509a25f06bc9ab60a8 Mon Sep 17 00:00:00 2001 From: Roman Nozdrin Date: Thu, 8 Oct 2020 12:09:35 +0000 Subject: [PATCH] MCOL-4337 Controllernode now tries to establish connections to all workernodes when initialising There is a new corresponding XML config options DBRM_Controller.WorkerConnectionTimeout that controls how long controllernode waits for wns before it proceeds --- versioning/BRM/controllernode.rc | 102 -------------------- versioning/BRM/libbrm.rc | 102 -------------------- versioning/BRM/masterdbrmnode.cpp | 152 +++++++++++++++++------------- versioning/BRM/masterdbrmnode.h | 9 ++ versioning/BRM/workernode.rc | 102 -------------------- 5 files changed, 96 insertions(+), 371 deletions(-) delete mode 100644 versioning/BRM/controllernode.rc delete mode 100644 versioning/BRM/libbrm.rc delete mode 100644 versioning/BRM/workernode.rc diff --git a/versioning/BRM/controllernode.rc b/versioning/BRM/controllernode.rc deleted file mode 100644 index 34c481af2..000000000 --- a/versioning/BRM/controllernode.rc +++ /dev/null @@ -1,102 +0,0 @@ -// Microsoft Visual C++ generated resource script. -// -#include "resource.h" - -#define APSTUDIO_READONLY_SYMBOLS -///////////////////////////////////////////////////////////////////////////// -// -// Generated from the TEXTINCLUDE 2 resource. -// -#include "afxres.h" - -///////////////////////////////////////////////////////////////////////////// -#undef APSTUDIO_READONLY_SYMBOLS - -///////////////////////////////////////////////////////////////////////////// -// English (U.S.) resources - -#if !defined(AFX_RESOURCE_DLL) || defined(AFX_TARG_ENU) -#ifdef _WIN32 -LANGUAGE LANG_ENGLISH, SUBLANG_ENGLISH_US -#pragma code_page(1252) -#endif //_WIN32 - -#ifdef APSTUDIO_INVOKED -///////////////////////////////////////////////////////////////////////////// -// -// TEXTINCLUDE -// - -1 TEXTINCLUDE -BEGIN - "resource.h\0" -END - -2 TEXTINCLUDE -BEGIN - "#include ""afxres.h""\r\n" - "\0" -END - -3 TEXTINCLUDE -BEGIN - "\r\n" - "\0" -END - -#endif // APSTUDIO_INVOKED - - -///////////////////////////////////////////////////////////////////////////// -// -// Version -// - -VS_VERSION_INFO VERSIONINFO - FILEVERSION 4,6,0,0 - PRODUCTVERSION 4,6,0,0 - FILEFLAGSMASK 0x17L -#ifdef _DEBUG - FILEFLAGS 0x1L -#else - FILEFLAGS 0x0L -#endif - FILEOS 0x4L - FILETYPE 0x1L - FILESUBTYPE 0x0L -BEGIN - BLOCK "StringFileInfo" - BEGIN - BLOCK "040904b0" - BEGIN - VALUE "CompanyName", "InfiniDB, Inc." - VALUE "FileDescription", "InfiniDB DBRM Controller Node" - VALUE "FileVersion", "4.6.0-0" - VALUE "InternalName", "controllernode" - VALUE "LegalCopyright", "Copyright (C) 2014" - VALUE "OriginalFilename", "controllernode.exe" - VALUE "ProductName", "InfiniDB" - VALUE "ProductVersion", "4.6.0.0 Beta" - END - END - BLOCK "VarFileInfo" - BEGIN - VALUE "Translation", 0x409, 1200 - END -END - -#endif // English (U.S.) resources -///////////////////////////////////////////////////////////////////////////// - - - -#ifndef APSTUDIO_INVOKED -///////////////////////////////////////////////////////////////////////////// -// -// Generated from the TEXTINCLUDE 3 resource. -// - - -///////////////////////////////////////////////////////////////////////////// -#endif // not APSTUDIO_INVOKED - diff --git a/versioning/BRM/libbrm.rc b/versioning/BRM/libbrm.rc deleted file mode 100644 index 29b77ad8c..000000000 --- a/versioning/BRM/libbrm.rc +++ /dev/null @@ -1,102 +0,0 @@ -// Microsoft Visual C++ generated resource script. -// -#include "resource.h" - -#define APSTUDIO_READONLY_SYMBOLS -///////////////////////////////////////////////////////////////////////////// -// -// Generated from the TEXTINCLUDE 2 resource. -// -#include "afxres.h" - -///////////////////////////////////////////////////////////////////////////// -#undef APSTUDIO_READONLY_SYMBOLS - -///////////////////////////////////////////////////////////////////////////// -// English (U.S.) resources - -#if !defined(AFX_RESOURCE_DLL) || defined(AFX_TARG_ENU) -#ifdef _WIN32 -LANGUAGE LANG_ENGLISH, SUBLANG_ENGLISH_US -#pragma code_page(1252) -#endif //_WIN32 - -#ifdef APSTUDIO_INVOKED -///////////////////////////////////////////////////////////////////////////// -// -// TEXTINCLUDE -// - -1 TEXTINCLUDE -BEGIN - "resource.h\0" -END - -2 TEXTINCLUDE -BEGIN - "#include ""afxres.h""\r\n" - "\0" -END - -3 TEXTINCLUDE -BEGIN - "\r\n" - "\0" -END - -#endif // APSTUDIO_INVOKED - - -///////////////////////////////////////////////////////////////////////////// -// -// Version -// - -VS_VERSION_INFO VERSIONINFO - FILEVERSION 4,6,0,0 - PRODUCTVERSION 4,6,0,0 - FILEFLAGSMASK 0x17L -#ifdef _DEBUG - FILEFLAGS 0x1L -#else - FILEFLAGS 0x0L -#endif - FILEOS 0x4L - FILETYPE 0x2L - FILESUBTYPE 0x0L -BEGIN - BLOCK "StringFileInfo" - BEGIN - BLOCK "040904b0" - BEGIN - VALUE "CompanyName", "InfiniDB, Inc." - VALUE "FileDescription", "InfiniDB BRM API" - VALUE "FileVersion", "4.6.0-0" - VALUE "InternalName", "libbrm" - VALUE "LegalCopyright", "Copyright (C) 2014" - VALUE "OriginalFilename", "libbrm.dll" - VALUE "ProductName", "InfiniDB" - VALUE "ProductVersion", "4.6.0.0 Beta" - END - END - BLOCK "VarFileInfo" - BEGIN - VALUE "Translation", 0x409, 1200 - END -END - -#endif // English (U.S.) resources -///////////////////////////////////////////////////////////////////////////// - - - -#ifndef APSTUDIO_INVOKED -///////////////////////////////////////////////////////////////////////////// -// -// Generated from the TEXTINCLUDE 3 resource. -// - - -///////////////////////////////////////////////////////////////////////////// -#endif // not APSTUDIO_INVOKED - diff --git a/versioning/BRM/masterdbrmnode.cpp b/versioning/BRM/masterdbrmnode.cpp index 68add1c8b..c4328a22e 100644 --- a/versioning/BRM/masterdbrmnode.cpp +++ b/versioning/BRM/masterdbrmnode.cpp @@ -140,22 +140,9 @@ MasterDBRMNode& MasterDBRMNode::operator=(const MasterDBRMNode& m) void MasterDBRMNode::initMsgQueues(config::Config* config) { - string stmp; - int ltmp; - char ctmp[50]; - int i; - - stmp = config->getConfig("DBRM_Controller", "NumWorkers"); - - if (stmp.length() == 0) - throw runtime_error("MasterDBRMNode::initMsgQueues(): config file error looking for "); - - ltmp = static_cast(config::Config::fromText(stmp)); - - if (ltmp < 1) - throw runtime_error("MasterDBRMNode::initMsgQueues(): Bad NumWorkers value"); - - NumWorkers = ltmp; + std::string methodName("MasterDBRMNode::initMsgQueues()"); + size_t connectTimeoutSecs = 0; + getNumWorkersAndTimeout(connectTimeoutSecs, methodName, config); serverLock.lock(); @@ -171,11 +158,80 @@ void MasterDBRMNode::initMsgQueues(config::Config* config) serverLock.unlock(); - for (i = 1; i <= NumWorkers; i++) + connectToWorkers(connectTimeoutSecs); +} + +void MasterDBRMNode::getNumWorkersAndTimeout(size_t& connectTimeoutSecs, + const std::string& methodName, + config::Config* config) +{ + string stmp; + int ltmp; + connectTimeoutSecs = 30; // default + + stmp = config->getConfig("DBRM_Controller", "NumWorkers"); + + if (stmp.length() == 0) + throw runtime_error(methodName + + ": config file error looking for "); + + ltmp = static_cast(config::Config::fromText(stmp)); + + if (ltmp < 1) + throw runtime_error(methodName + ": Bad NumWorkers value"); + + NumWorkers = ltmp; + stmp = config->getConfig("DBRM_Controller", "WorkerConnectionTimeout"); + if (stmp.length() > 0) { - snprintf(ctmp, 50, "DBRM_Worker%d", i); - std::string module(ctmp); - slaves.push_back(MessageQueueClientPool::getInstance(module)); + ltmp = static_cast(config::Config::fromText(stmp)); + if (ltmp > 1) + connectTimeoutSecs = ltmp; + } +} + +void MasterDBRMNode::connectToWorkers(const size_t connectTimeoutSecs) +{ + size_t timeoutMsecs = connectTimeoutSecs * 1000000; + size_t timeSpent = 0; + int workersOnline = 0; + bool initialRun = true; + while (timeoutMsecs > timeSpent && workersOnline < NumWorkers) + { + char ctmp[50]; + for (int i = 0; i < NumWorkers; i++) + { + snprintf(ctmp, sizeof(ctmp), "DBRM_Worker%d", i+1); + std::string module(ctmp); + if (static_cast(slaves.size()) < NumWorkers) + { + slaves.push_back(MessageQueueClientPool::getInstance(module)); + } + + if (!slaves[i]->isConnected()) + { + if (!slaves[i]->connect()) + { + // first iteration + if (initialRun) + log("DBRM Controller: Warning: could not connect to " + module, + logging::LOG_TYPE_WARNING); + } + else + { + log("DBRM Controller: Connected to " + module, + logging::LOG_TYPE_DEBUG); + workersOnline++; + } + } + } + if (initialRun) + initialRun = false; + if (workersOnline < NumWorkers) + { + usleep(connectTimeoutStep); + timeSpent += connectTimeoutStep; + } } } @@ -1238,17 +1294,17 @@ void MasterDBRMNode::doReload(messageqcpp::IOSocket* sock) */ ByteStream reply; - string stmp; - int ltmp; - char ctmp[50]; - int i; config::Config* config = config::Config::makeConfig(); log("Reloading", LOG_TYPE_INFO); - stmp = config->getConfig("DBRM_Controller", "NumWorkers"); - - if (stmp.length() == 0) + std::string methodName("MasterDBRMNode::doReload()"); + size_t connectTimeoutSecs = 0; + try + { + getNumWorkersAndTimeout(connectTimeoutSecs, methodName, config); + } + catch (std::exception&) { reply << (uint8_t) ERR_FAILURE; @@ -1258,40 +1314,18 @@ void MasterDBRMNode::doReload(messageqcpp::IOSocket* sock) } catch (exception&) { } - throw runtime_error("MasterDBRMNode::doReload(): config file error looking for "); + throw; } - ltmp = static_cast(config::Config::fromText(stmp)); - - if (ltmp < 1) - { - reply << (uint8_t) ERR_FAILURE; - - try - { - sock->write(reply); - } - catch (exception&) { } - - throw runtime_error("MasterDBRMNode::doReload(): Bad NumWorkers value"); - } - - for (i = 0; i < (int) slaves.size(); i++) + for (int i = 0; i < (int) slaves.size(); i++) { MessageQueueClientPool::deleteInstance(slaves[i]); - slaves[i] = NULL; + slaves[i] = nullptr; } slaves.clear(); - NumWorkers = ltmp; - - for (i = 1; i <= NumWorkers; i++) - { - snprintf(ctmp, 50, "DBRM_Worker%d", i); - std::string module(ctmp); - slaves.push_back(MessageQueueClientPool::getInstance(module)); - } + connectToWorkers(connectTimeoutSecs); iSlave = slaves.end(); undo(); @@ -1306,18 +1340,6 @@ void MasterDBRMNode::doReload(messageqcpp::IOSocket* sock) } catch (exception&) { } - /* Asynchronous version - ByteStream reply; - - reply << (uint8_t) ERR_OK; - try { - sock->write(reply); - } - catch (exception&) { } - // sock->close(); - - reloadCmd = true; - */ } void MasterDBRMNode::doVerID(ByteStream& msg, ThreadParams* p) diff --git a/versioning/BRM/masterdbrmnode.h b/versioning/BRM/masterdbrmnode.h index 852017e93..f5fbf2200 100644 --- a/versioning/BRM/masterdbrmnode.h +++ b/versioning/BRM/masterdbrmnode.h @@ -81,6 +81,8 @@ namespace BRM * \endcode */ +constexpr size_t connectTimeoutStep = 50000; + class MasterDBRMNode { public: @@ -140,6 +142,13 @@ public: { return readOnly; } + /** @brief Connects to the all workers */ + void connectToWorkers(const size_t connectTimeoutSecs); + + /** @brief Extracts number of workers and connection timeout from the config */ + void getNumWorkersAndTimeout(size_t& connectTimeoutSecs, + const std::string& methodName, + config::Config* config); private: diff --git a/versioning/BRM/workernode.rc b/versioning/BRM/workernode.rc deleted file mode 100644 index 954998609..000000000 --- a/versioning/BRM/workernode.rc +++ /dev/null @@ -1,102 +0,0 @@ -// Microsoft Visual C++ generated resource script. -// -#include "resource.h" - -#define APSTUDIO_READONLY_SYMBOLS -///////////////////////////////////////////////////////////////////////////// -// -// Generated from the TEXTINCLUDE 2 resource. -// -#include "afxres.h" - -///////////////////////////////////////////////////////////////////////////// -#undef APSTUDIO_READONLY_SYMBOLS - -///////////////////////////////////////////////////////////////////////////// -// English (U.S.) resources - -#if !defined(AFX_RESOURCE_DLL) || defined(AFX_TARG_ENU) -#ifdef _WIN32 -LANGUAGE LANG_ENGLISH, SUBLANG_ENGLISH_US -#pragma code_page(1252) -#endif //_WIN32 - -#ifdef APSTUDIO_INVOKED -///////////////////////////////////////////////////////////////////////////// -// -// TEXTINCLUDE -// - -1 TEXTINCLUDE -BEGIN - "resource.h\0" -END - -2 TEXTINCLUDE -BEGIN - "#include ""afxres.h""\r\n" - "\0" -END - -3 TEXTINCLUDE -BEGIN - "\r\n" - "\0" -END - -#endif // APSTUDIO_INVOKED - - -///////////////////////////////////////////////////////////////////////////// -// -// Version -// - -VS_VERSION_INFO VERSIONINFO - FILEVERSION 4,6,0,0 - PRODUCTVERSION 4,6,0,0 - FILEFLAGSMASK 0x17L -#ifdef _DEBUG - FILEFLAGS 0x1L -#else - FILEFLAGS 0x0L -#endif - FILEOS 0x4L - FILETYPE 0x1L - FILESUBTYPE 0x0L -BEGIN - BLOCK "StringFileInfo" - BEGIN - BLOCK "040904b0" - BEGIN - VALUE "CompanyName", "InfiniDB, Inc." - VALUE "FileDescription", "InfiniDB DBRM Worker Node" - VALUE "FileVersion", "4.6.0-0" - VALUE "InternalName", "workernode" - VALUE "LegalCopyright", "Copyright (C) 2014" - VALUE "OriginalFilename", "workernode.exe" - VALUE "ProductName", "InfiniDB" - VALUE "ProductVersion", "4.6.0.0 Beta" - END - END - BLOCK "VarFileInfo" - BEGIN - VALUE "Translation", 0x409, 1200 - END -END - -#endif // English (U.S.) resources -///////////////////////////////////////////////////////////////////////////// - - - -#ifndef APSTUDIO_INVOKED -///////////////////////////////////////////////////////////////////////////// -// -// Generated from the TEXTINCLUDE 3 resource. -// - - -///////////////////////////////////////////////////////////////////////////// -#endif // not APSTUDIO_INVOKED -