Deep build refactoring phase 2 (#3564)

* configcpp refactored

* chore(build): massive removals, auto add files to debian install file

* chore(build): configure before autobake

* chore(build): use custom cmake commands for components, mariadb-plugin-columnstore.install generated

* chore(build): install deps as separate step for build-packages

* more deps

* chore(codemanagement, build): build refactoring stage2

* chore(safety): Locked Map for MessageqCpp with a simpler way

 Please enter the commit message for your changes. Lines starting

* chore(codemanagement, ci): better coredumps handling, deps fixed

* Delete build/bootstrap_mcs.py

* Update charset.cpp (add license)

primitives/blockcache/CMakeLists.txt

@@ -14,4 +14,4 @@ set(dbbc_STAT_SRCS
     fsutils.cpp
 )
 columnstore_static_library(dbbc ${dbbc_STAT_SRCS})
-columnstore_link(dbbc ${NETSNMP_LIBRARIES} loggingcpp)
+columnstore_link(dbbc loggingcpp)

primitives/linux-port/CMakeLists.txt

@@ -5,4 +5,4 @@ include_directories(${ENGINE_COMMON_INCLUDES} ../blockcache ../primproc)
 set(processor_STAT_SRCS primitiveprocessor.cpp dictionary.cpp column.cpp)
 
 columnstore_static_library(processor ${processor_STAT_SRCS})
-columnstore_link(processor ${NETSNMP_LIBRARIES} loggingcpp)
+columnstore_link(processor loggingcpp)

primitives/linux-port/column.cpp

@@ -40,7 +40,6 @@ using namespace boost;
 #include "simd_sse.h"
 #include "simd_arm.h"
 #include "utils/common/columnwidth.h"
-#include "utils/common/bit_cast.h"
 
 #include "exceptclasses.h"
 

primitives/primproc/CMakeLists.txt

@@ -3,25 +3,26 @@ include_directories(${ENGINE_COMMON_INCLUDES} ../blockcache ../linux-port)
 # ########## next target ###############
 
 set(PrimProc_SRCS
-    primproc.cpp
+    activestatementcounter.cpp
     batchprimitiveprocessor.cpp
     bppseeder.cpp
     bppsendthread.cpp
     columncommand.cpp
     command.cpp
     dictstep.cpp
+    femsghandler.cpp
     filtercommand.cpp
     logger.cpp
     passthrucommand.cpp
     primitiveserver.cpp
+    primproc.cpp
     pseudocc.cpp
+    rssmonfcn.cpp
     rtscommand.cpp
-    umsocketselector.cpp
+    samenodepseudosocket.cpp
     serviceexemgr.cpp
     sqlfrontsessionthread.cpp
-    rssmonfcn.cpp
-    activestatementcounter.cpp
-    femsghandler.cpp
+    umsocketselector.cpp
     ../../utils/common/crashtrace.cpp
 )
 
@@ -31,11 +32,11 @@ target_include_directories(PrimProc PRIVATE ${Boost_INCLUDE_DIRS})
 columnstore_link(
     PrimProc
     ${ENGINE_LDFLAGS}
-    ${NETSNMP_LIBRARIES}
     ${ENGINE_WRITE_LIBS}
     threadpool
     cacheutils
     dbbc
     processor
     loggingcpp
+    statistics_manager
 )

primitives/primproc/samenodepseudosocket.cpp

@@ -0,0 +1,127 @@
+/* Copyright (C) 2024 MariaDB Corp.
+
+   This program is free software; you can redistribute it and/or
+   modify it under the terms of the GNU General Public License
+   as published by the Free Software Foundation; version 2 of
+   the License.
+
+   This program is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+   GNU General Public License for more details.
+
+   You should have received a copy of the GNU General Public License
+   along with this program; if not, write to the Free Software
+   Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
+   MA 02110-1301, USA. */
+
+#include <string>
+
+#include "samenodepseudosocket.h"
+#include "iosocket.h"
+
+namespace messageqcpp
+{
+SameNodePseudoSocket::SameNodePseudoSocket(joblist::DistributedEngineComm* exeMgrDecPtr) : dec_(exeMgrDecPtr)
+{
+  assert(dec_);
+}
+
+SameNodePseudoSocket::~SameNodePseudoSocket()
+{
+}
+
+void SameNodePseudoSocket::open()
+{
+}
+
+void SameNodePseudoSocket::close()
+{
+}
+
+Socket* SameNodePseudoSocket::clone() const
+{
+  return nullptr;
+}
+
+SameNodePseudoSocket::SameNodePseudoSocket(const SameNodePseudoSocket& /*rhs*/)
+{
+}
+
+SameNodePseudoSocket& SameNodePseudoSocket::operator=(const SameNodePseudoSocket& /*rhs*/)
+{
+  return *this;
+}
+
+const SBS SameNodePseudoSocket::read(const struct ::timespec* /*timeout*/, bool* /*isTimeOut*/, Stats* /*stats*/) const
+{
+  return nullptr;
+}
+
+// This is the only working method of this class. It puts SBS directly into DEC queue.
+void SameNodePseudoSocket::write(SBS msg, Stats* /*stats*/)
+{
+  dec_->addDataToOutput(msg);
+}
+
+void SameNodePseudoSocket::write(const ByteStream& /*msg*/, Stats* /*stats*/)
+{
+}
+
+void SameNodePseudoSocket::write_raw(const ByteStream& /*msg*/, Stats* /*stats*/) const
+{
+}
+
+void SameNodePseudoSocket::connect(const sockaddr* /*serv_addr*/)
+{
+}
+
+void SameNodePseudoSocket::bind(const sockaddr* /*serv_addr*/)
+{
+}
+
+const IOSocket SameNodePseudoSocket::accept(const struct timespec* /*timeout*/)
+{
+  return IOSocket();
+}
+
+void SameNodePseudoSocket::listen(int /*backlog*/)
+{
+}
+
+const std::string SameNodePseudoSocket::toString() const
+{
+  return "";
+}
+
+const std::string SameNodePseudoSocket::addr2String() const
+{
+  return "";
+}
+
+bool SameNodePseudoSocket::isSameAddr(const Socket* /*rhs*/) const
+{
+  return false;
+}
+
+bool SameNodePseudoSocket::isSameAddr(const struct in_addr& /*ipv4Addr*/) const
+{
+  return false;
+}
+
+int SameNodePseudoSocket::ping(const std::string& /*ipaddr*/, const struct timespec* /*timeout*/)
+{
+  return 0;
+}
+
+bool SameNodePseudoSocket::isConnected() const
+{
+  return true;
+}
+
+bool SameNodePseudoSocket::hasData() const
+{
+  return false;
+}
+
+}  // namespace messageqcpp

primitives/primproc/samenodepseudosocket.h

@@ -0,0 +1,99 @@
+/* Copyright (C) 2024 MariaDB Corp.
+
+   This program is free software; you can redistribute it and/or
+   modify it under the terms of the GNU General Public License
+   as published by the Free Software Foundation; version 2 of
+   the License.
+
+   This program is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+   GNU General Public License for more details.
+
+   You should have received a copy of the GNU General Public License
+   along with this program; if not, write to the Free Software
+   Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
+   MA 02110-1301, USA. */
+
+#pragma once
+
+#include "../../dbcon/joblist/distributedenginecomm.h"
+
+#include "socket.h"
+#include "socketparms.h"
+#include "bytestream.h"
+
+namespace messageqcpp
+{
+class IOSocket;
+
+// This class is a dummy replacement for a TCP socket
+// wrapper to communicate with the same node.
+class SameNodePseudoSocket : public Socket
+{
+ public:
+  explicit SameNodePseudoSocket(joblist::DistributedEngineComm* exeMgrDecPtr);
+  ~SameNodePseudoSocket() override;
+  void write(SBS msg, Stats* stats = nullptr) override;
+
+ private:
+  void bind(const sockaddr* serv_addr) override;
+  SameNodePseudoSocket(const SameNodePseudoSocket& rhs);
+  virtual SameNodePseudoSocket& operator=(const SameNodePseudoSocket& rhs);
+
+  void connectionTimeout(const struct ::timespec* /*timeout*/) override
+  {
+  }
+
+  void syncProto(bool /*use*/) override
+  {
+  }
+
+  int getConnectionNum() const override
+  {
+    return 1;
+  }
+
+  inline void socketParms(const SocketParms& /*socket*/) override
+  {
+  }
+
+  inline const SocketParms socketParms() const override
+  {
+    return SocketParms();
+  }
+
+  // all these virtual methods are to stay inaccessable.
+  inline void sa(const sockaddr* sa) override;
+  void open() override;
+  void close() override;
+  inline bool isOpen() const override;
+  const SBS read(const struct timespec* timeout = nullptr, bool* isTimeOut = nullptr,
+                 Stats* stats = nullptr) const override;
+  void write(const ByteStream& msg, Stats* stats = nullptr) override;
+  void write_raw(const ByteStream& msg, Stats* stats = nullptr) const override;
+  void listen(int backlog = 5) override;
+  const IOSocket accept(const struct timespec* timeout = nullptr) override;
+  void connect(const sockaddr* serv_addr) override;
+  Socket* clone() const override;
+  virtual const std::string toString() const;
+  const std::string addr2String() const override;
+  bool isSameAddr(const Socket* rhs) const override;
+  bool isSameAddr(const struct in_addr& ipv4Addr) const override;
+  static int ping(const std::string& ipaddr, const struct timespec* timeout = nullptr);
+  bool isConnected() const override;
+  bool hasData() const override;
+
+  joblist::DistributedEngineComm* dec_ = nullptr;
+};
+
+inline bool SameNodePseudoSocket::isOpen() const
+{
+  return true;
+}
+
+inline void SameNodePseudoSocket::sa(const sockaddr* /*sa*/)
+{
+}
+
+}  // namespace messageqcpp

primitives/primproc/serviceexemgr.cpp

@@ -78,7 +78,7 @@
 #include "dbrm.h"
 
 #include "mariadb_my_sys.h"
-#include "statistics.h"
+#include "statistics_manager/statistics.h"
 #include "serviceexemgr.h"
 #include "sqlfrontsessionthread.h"
 

primitives/primproc/serviceexemgr.h

@@ -58,7 +58,7 @@
 #include "dbrm.h"
 
 #include "mariadb_my_sys.h"
-#include "statistics.h"
+#include "statistics_manager/statistics.h"
 
 namespace exemgr
 {
@@ -69,7 +69,7 @@ class Opt
   int m_debug;
   bool m_e;
   bool m_fg;
-  Opt() : m_debug(0), m_e(false), m_fg(false){};
+  Opt() : m_debug(0), m_e(false), m_fg(false) {};
   Opt(int argc, char* argv[]) : m_debug(0), m_e(false), m_fg(false)
   {
     int c;

primitives/primproc/sqlfrontsessionthread.h

@@ -56,76 +56,77 @@
 #include "dbrm.h"
 
 #include "mariadb_my_sys.h"
-#include "statistics.h"
+#include "statistics_manager/statistics.h"
 #include "serviceexemgr.h"
 
 namespace exemgr
 {
-  class SQLFrontSessionThread
+class SQLFrontSessionThread
+{
+ public:
+  SQLFrontSessionThread(const messageqcpp::IOSocket& ios, joblist::DistributedEngineComm* ec,
+                        joblist::ResourceManager* rm)
+   : fIos(ios)
+   , fEc(ec)
+   , fRm(rm)
+   , fStatsRetrieved(false)
+   , fTeleClient(globServiceExeMgr->getTeleServerParms())
+   , fOamCachePtr(oam::OamCache::makeOamCache())
   {
-  public:
-    SQLFrontSessionThread(const messageqcpp::IOSocket& ios, joblist::DistributedEngineComm* ec,
-                  joblist::ResourceManager* rm)
-    : fIos(ios)
-    , fEc(ec)
-    , fRm(rm)
-    , fStatsRetrieved(false)
-    , fTeleClient(globServiceExeMgr->getTeleServerParms())
-    , fOamCachePtr(oam::OamCache::makeOamCache())
-    {
-    }
+  }
 
-  private:
-    messageqcpp::IOSocket fIos;
-    joblist::DistributedEngineComm* fEc;
-    joblist::ResourceManager* fRm;
-    querystats::QueryStats fStats;
+ private:
+  messageqcpp::IOSocket fIos;
+  joblist::DistributedEngineComm* fEc;
+  joblist::ResourceManager* fRm;
+  querystats::QueryStats fStats;
 
-    // Variables used to store return stats
-    bool fStatsRetrieved;
+  // Variables used to store return stats
+  bool fStatsRetrieved;
 
-    querytele::QueryTeleClient fTeleClient;
+  querytele::QueryTeleClient fTeleClient;
 
-    oam::OamCache* fOamCachePtr;  // this ptr is copyable...
+  oam::OamCache* fOamCachePtr;  // this ptr is copyable...
 
-    //...Reinitialize stats for start of a new query
-    void initStats(uint32_t sessionId, std::string& sqlText)
-    {
-      initMaxMemPct(sessionId);
+  //...Reinitialize stats for start of a new query
+  void initStats(uint32_t sessionId, std::string& sqlText)
+  {
+    initMaxMemPct(sessionId);
 
-      fStats.reset();
-      fStats.setStartTime();
-      fStats.fSessionID = sessionId;
-      fStats.fQuery = sqlText;
-      fStatsRetrieved = false;
-    }
-    //...Get % memory usage during latest query for sesssionId.
-    //...SessionId >= 0x80000000 is system catalog query we can ignore.
-    static uint64_t getMaxMemPct(uint32_t sessionId);
-    //...Delete sessionMemMap entry for the specified session's memory % use.
-    //...SessionId >= 0x80000000 is system catalog query we can ignore.
-    static void deleteMaxMemPct(uint32_t sessionId);
-    //...Get and log query stats to specified output stream
-    const std::string formatQueryStats(
-        joblist::SJLP& jl,         // joblist associated with query
-        const std::string& label,  // header label to print in front of log output
-        bool includeNewLine,       // include line breaks in query stats std::string
-        bool vtableModeOn, bool wantExtendedStats, uint64_t rowsReturned);
-    static void incThreadCntPerSession(uint32_t sessionId);
-    static void decThreadCntPerSession(uint32_t sessionId);
-    //...Init sessionMemMap entry for specified session to 0 memory %.
-    //...SessionId >= 0x80000000 is system catalog query we can ignore.
-    static void initMaxMemPct(uint32_t sessionId);
-    //... Round off to human readable format (KB, MB, or GB).
-    const std::string roundBytes(uint64_t value) const;
-    void setRMParms(const execplan::CalpontSelectExecutionPlan::RMParmVec& parms);
-    void buildSysCache(const execplan::CalpontSelectExecutionPlan& csep,
-                      boost::shared_ptr<execplan::CalpontSystemCatalog> csc);
-    void writeCodeAndError(messageqcpp::ByteStream::quadbyte code, const std::string emsg);
-    void analyzeTableExecute(messageqcpp::ByteStream& bs, joblist::SJLP& jl, bool& stmtCounted);
-    void analyzeTableHandleStats(messageqcpp::ByteStream& bs);
-    uint64_t roundMB(uint64_t value) const;
-  public:
-    void operator()();
-  };
-}
+    fStats.reset();
+    fStats.setStartTime();
+    fStats.fSessionID = sessionId;
+    fStats.fQuery = sqlText;
+    fStatsRetrieved = false;
+  }
+  //...Get % memory usage during latest query for sesssionId.
+  //...SessionId >= 0x80000000 is system catalog query we can ignore.
+  static uint64_t getMaxMemPct(uint32_t sessionId);
+  //...Delete sessionMemMap entry for the specified session's memory % use.
+  //...SessionId >= 0x80000000 is system catalog query we can ignore.
+  static void deleteMaxMemPct(uint32_t sessionId);
+  //...Get and log query stats to specified output stream
+  const std::string formatQueryStats(
+      joblist::SJLP& jl,         // joblist associated with query
+      const std::string& label,  // header label to print in front of log output
+      bool includeNewLine,       // include line breaks in query stats std::string
+      bool vtableModeOn, bool wantExtendedStats, uint64_t rowsReturned);
+  static void incThreadCntPerSession(uint32_t sessionId);
+  static void decThreadCntPerSession(uint32_t sessionId);
+  //...Init sessionMemMap entry for specified session to 0 memory %.
+  //...SessionId >= 0x80000000 is system catalog query we can ignore.
+  static void initMaxMemPct(uint32_t sessionId);
+  //... Round off to human readable format (KB, MB, or GB).
+  const std::string roundBytes(uint64_t value) const;
+  void setRMParms(const execplan::CalpontSelectExecutionPlan::RMParmVec& parms);
+  void buildSysCache(const execplan::CalpontSelectExecutionPlan& csep,
+                     boost::shared_ptr<execplan::CalpontSystemCatalog> csc);
+  void writeCodeAndError(messageqcpp::ByteStream::quadbyte code, const std::string emsg);
+  void analyzeTableExecute(messageqcpp::ByteStream& bs, joblist::SJLP& jl, bool& stmtCounted);
+  void analyzeTableHandleStats(messageqcpp::ByteStream& bs);
+  uint64_t roundMB(uint64_t value) const;
+
+ public:
+  void operator()();
+};
+}  // namespace exemgr