This patch disables FairThreadPool to double check if this feature contributes to multiple strange side-effects and ocassional failed MTR tests

primitives/primproc/bppseeder.h

@@ -47,7 +47,7 @@
 
 namespace primitiveprocessor
 {
-class BPPSeeder : public threadpool::FairThreadPool::Functor
+class BPPSeeder : public threadpool::PriorityThreadPool::Functor
 {
  public:
   BPPSeeder(const messageqcpp::SBS&, const SP_UM_MUTEX& wLock, const SP_UM_IOSOCK& ios, const int pmThreads,

primitives/primproc/bppsendthread.h

@@ -24,13 +24,12 @@
 
 #pragma once
 
-#include "fair_threadpool.h"
 #include "umsocketselector.h"
 #include <queue>
 #include <set>
 #include <condition_variable>
 #include "threadnaming.h"
-#include "fair_threadpool.h"
+#include "prioritythreadpool.h"
 
 namespace primitiveprocessor
 {
@@ -85,7 +84,7 @@ class BPPSendThread
   {
     return die;
   }
-  void setProcessorPool(boost::shared_ptr<threadpool::FairThreadPool> processorPool)
+  void setProcessorPool(boost::shared_ptr<threadpool::PriorityThreadPool> processorPool)
   {
     fProcessorPool = processorPool;
   }
@@ -149,7 +148,7 @@ class BPPSendThread
   uint64_t maxByteSize;
   // Used to tell the ThreadPool It should consider additional threads because a
   // queue full event has happened and a thread has been blocked.
-  boost::shared_ptr<threadpool::FairThreadPool> fProcessorPool;
+  boost::shared_ptr<threadpool::PriorityThreadPool> fProcessorPool;
 };
 
 }  // namespace primitiveprocessor

primitives/primproc/primitiveserver.cpp

@@ -1,5 +1,5 @@
 /* Copyright (C) 2014 InfiniDB, Inc.
-   Copyright (C) 2016 MariaDB Corporation
+   Copyright (C) 2016-2022 MariaDB Corporation
 
    This program is free software; you can redistribute it and/or
    modify it under the terms of the GNU General Public License
@@ -124,7 +124,7 @@ oam::OamCache* oamCache = oam::OamCache::makeOamCache();
 // FIXME: there is an anon ns burried later in between 2 named namespaces...
 namespace primitiveprocessor
 {
-
+boost::shared_ptr<threadpool::PriorityThreadPool> OOBPool;
 BlockRequestProcessor** BRPp;
 #ifndef _MSC_VER
 dbbc::Stats stats;
@@ -1049,7 +1049,7 @@ using namespace primitiveprocessor;
 /** @brief The job type to process a dictionary scan (pDictionaryScan class on the UM)
  * TODO: Move this & the impl into different files
  */
-class DictScanJob : public threadpool::FairThreadPool::Functor
+class DictScanJob : public threadpool::PriorityThreadPool::Functor
 {
  public:
   DictScanJob(SP_UM_IOSOCK ios, SBS bs, SP_UM_MUTEX writeLock);
@@ -1240,7 +1240,7 @@ struct BPPHandler
     scoped.unlock();
   }
 
-  struct BPPHandlerFunctor : public FairThreadPool::Functor
+  struct BPPHandlerFunctor : public PriorityThreadPool::Functor
   {
     BPPHandlerFunctor(boost::shared_ptr<BPPHandler> r, SBS b) : bs(b)
     {
@@ -1706,7 +1706,7 @@ return 0;
   PrimitiveServer* fPrimitiveServerPtr;
 };
 
-class DictionaryOp : public FairThreadPool::Functor
+class DictionaryOp : public PriorityThreadPool::Functor
 {
  public:
   DictionaryOp(SBS cmd) : bs(cmd)
@@ -1943,7 +1943,8 @@ struct ReadThread
   void operator()()
   {
     utils::setThreadName("PPReadThread");
-    boost::shared_ptr<threadpool::FairThreadPool> procPoolPtr = fPrimitiveServerPtr->getProcessorThreadPool();
+    boost::shared_ptr<threadpool::PriorityThreadPool> procPoolPtr =
+        fPrimitiveServerPtr->getProcessorThreadPool();
     SBS bs;
     UmSocketSelector* pUmSocketSelector = UmSocketSelector::instance();
 
@@ -1994,9 +1995,6 @@ struct ReadThread
           idbassert(bs->length() >= sizeof(ISMPacketHeader));
 
           const ISMPacketHeader* ismHdr = reinterpret_cast<const ISMPacketHeader*>(bs->buf());
-          // uint64_t someVal = ismHdr->Command;
-          // std::cout << " PP read thread Command " << someVal << std::endl;
-
           /* This switch is for the OOB commands */
           switch (ismHdr->Command)
           {
@@ -2056,7 +2054,7 @@ struct ReadThread
               const uint32_t weight = 1;
               const uint32_t priority = 0;
               uint32_t id = 0;
-              boost::shared_ptr<FairThreadPool::Functor> functor;
+              boost::shared_ptr<PriorityThreadPool::Functor> functor;
               if (ismHdr->Command == DICT_CREATE_EQUALITY_FILTER)
               {
                 functor.reset(new CreateEqualityFilter(bs));
@@ -2088,7 +2086,7 @@ struct ReadThread
                 id = fBPPHandler->getUniqueID(bs, ismHdr->Command);
                 functor.reset(new BPPHandler::Abort(fBPPHandler, bs));
               }
-              FairThreadPool::Job job(uniqueID, stepID, txnId, functor, outIos, weight, priority, id);
+              PriorityThreadPool::Job job(uniqueID, stepID, txnId, functor, outIos, weight, priority, id);
               procPoolPtr->addJob(job);
               break;
             }
@@ -2097,7 +2095,7 @@ struct ReadThread
             case BATCH_PRIMITIVE_RUN:
             {
               TokenByScanRequestHeader* hdr = nullptr;
-              boost::shared_ptr<FairThreadPool::Functor> functor;
+              boost::shared_ptr<PriorityThreadPool::Functor> functor;
               uint32_t id = 0;
               uint32_t weight = 0;
               uint32_t priority = 0;
@@ -2137,9 +2135,8 @@ struct ReadThread
               }
               else if (ismHdr->Command == BATCH_PRIMITIVE_RUN)
               {
-                functor.reset(new BPPSeeder(bs, writeLock, outIos,
-                  fPrimitiveServerPtr->ProcessorThreads(),
-                  fPrimitiveServerPtr->PTTrace()));
+                functor.reset(new BPPSeeder(bs, writeLock, outIos, fPrimitiveServerPtr->ProcessorThreads(),
+                                            fPrimitiveServerPtr->PTTrace()));
                 BPPSeeder* bpps = dynamic_cast<BPPSeeder*>(functor.get());
                 id = bpps->getID();
                 priority = bpps->priority();
@@ -2150,7 +2147,7 @@ struct ReadThread
                 uniqueID = *((uint32_t*)&buf[pos + 10]);
                 weight = ismHdr->Size + *((uint32_t*)&buf[pos + 18]);
               }
-              FairThreadPool::Job job(uniqueID, stepID, txnId, functor, outIos, weight, priority, id);
+              PriorityThreadPool::Job job(uniqueID, stepID, txnId, functor, outIos, weight, priority, id);
               procPoolPtr->addJob(job);
 
               break;
@@ -2310,8 +2307,13 @@ PrimitiveServer::PrimitiveServer(int serverThreads, int serverQueueSize, int pro
   fServerpool.setQueueSize(fServerQueueSize);
   fServerpool.setName("PrimitiveServer");
 
-  fProcessorPool.reset(new threadpool::FairThreadPool(fProcessorWeight, highPriorityThreads,
-                                                      medPriorityThreads, lowPriorityThreads, 0));
+  fProcessorPool.reset(new threadpool::PriorityThreadPool(fProcessorWeight, highPriorityThreads,
+                                                          medPriorityThreads, lowPriorityThreads, 0));
+  // We're not using either the priority or the job-clustering features, just need a threadpool
+  // that can reschedule jobs, and an unlimited non-blocking queue
+  OOBPool.reset(new threadpool::PriorityThreadPool(1, 5, 0, 0, 1));
+  // Initialize a local pointer.
+  fOOBPool = OOBPool;
 
   asyncCounter = 0;
 

primitives/primproc/primitiveserver.h

@@ -36,7 +36,7 @@
 #include <boost/thread.hpp>
 
 #include "threadpool.h"
-#include "fair_threadpool.h"
+#include "prioritythreadpool.h"
 #include "messagequeue.h"
 #include "blockrequestprocessor.h"
 #include "batchprimitiveprocessor.h"
@@ -48,6 +48,7 @@ extern oam::OamCache* oamCache;
 
 namespace primitiveprocessor
 {
+extern boost::shared_ptr<threadpool::PriorityThreadPool> OOBPool;
 extern dbbc::BlockRequestProcessor** BRPp;
 extern BRM::DBRM* brm;
 extern boost::mutex bppLock;
@@ -129,11 +130,16 @@ class PrimitiveServer
 
   /** @brief get a pointer the shared processor thread pool
    */
-  inline boost::shared_ptr<threadpool::FairThreadPool> getProcessorThreadPool() const
+  inline boost::shared_ptr<threadpool::PriorityThreadPool> getProcessorThreadPool() const
   {
     return fProcessorPool;
   }
 
+  inline boost::shared_ptr<threadpool::PriorityThreadPool> getOOBThreadPool() const
+  {
+    return fOOBPool;
+  }
+
   int ReadAheadBlocks() const
   {
     return fReadAheadBlocks;
@@ -165,7 +171,8 @@ class PrimitiveServer
   /** @brief the thread pool used to process
    * primitive commands
    */
-  boost::shared_ptr<threadpool::FairThreadPool> fProcessorPool;
+  boost::shared_ptr<threadpool::PriorityThreadPool> fProcessorPool;
+  boost::shared_ptr<threadpool::PriorityThreadPool> fOOBPool;
 
   int fServerThreads;
   int fServerQueueSize;

primitives/primproc/primitiveserverthreadpools.h

@@ -18,22 +18,29 @@
 #pragma once
 
 #include <boost/shared_ptr.hpp>
-#include "fair_threadpool.h"
+#include "prioritythreadpool.h"
 
 class PrimitiveServerThreadPools
 {
  public:
   PrimitiveServerThreadPools() = default;
-  PrimitiveServerThreadPools(boost::shared_ptr<threadpool::FairThreadPool> primServerThreadPool)
-   : fPrimServerThreadPool(primServerThreadPool)
+  PrimitiveServerThreadPools(boost::shared_ptr<threadpool::PriorityThreadPool> primServerThreadPool,
+                             boost::shared_ptr<threadpool::PriorityThreadPool> OOBThreadPool)
+   : fPrimServerThreadPool(primServerThreadPool), fOOBThreadPool(OOBThreadPool)
   {
   }
 
-  boost::shared_ptr<threadpool::FairThreadPool> getPrimitiveServerThreadPool()
+  boost::shared_ptr<threadpool::PriorityThreadPool> getPrimitiveServerThreadPool()
   {
     return fPrimServerThreadPool;
   }
 
+  boost::shared_ptr<threadpool::PriorityThreadPool> getOOBThreadPool()
+  {
+    return fOOBThreadPool;
+  }
+
  private:
-  boost::shared_ptr<threadpool::FairThreadPool> fPrimServerThreadPool;
+  boost::shared_ptr<threadpool::PriorityThreadPool> fPrimServerThreadPool;
+  boost::shared_ptr<threadpool::PriorityThreadPool> fOOBThreadPool;
 };

primitives/primproc/primproc.cpp

@@ -725,6 +725,7 @@ int ServicePrimProc::Child()
 #endif
 
   primServerThreadPool = server.getProcessorThreadPool();
+  OOBThreadPool = server.getOOBThreadPool();
 
   server.start(this, startupRaceLock);
 

primitives/primproc/primproc.h

@@ -35,7 +35,7 @@
 #include <map>
 
 #include "service.h"
-#include "fair_threadpool.h"
+#include "prioritythreadpool.h"
 #include "pp_logger.h"
 
 namespace primitiveprocessor
@@ -155,12 +155,12 @@ class ServicePrimProc : public Service, public Opt
 
   void LogErrno() override
   {
-    cerr << strerror(errno) << endl;
+    std::cerr << strerror(errno) << std::endl;
   }
 
   void ParentLogChildMessage(const std::string& str) override
   {
-    cout << str << endl;
+    std::cout << str << std::endl;
   }
   int Child() override;
   int Run()
@@ -172,12 +172,12 @@ class ServicePrimProc : public Service, public Opt
     return startupRaceFlag_;
   }
 
-  boost::shared_ptr<threadpool::FairThreadPool> getPrimitiveServerThreadPool()
+  boost::shared_ptr<threadpool::PriorityThreadPool> getPrimitiveServerThreadPool()
   {
     return primServerThreadPool;
   }
 
-  boost::shared_ptr<threadpool::FairThreadPool> getOOBThreadPool()
+  boost::shared_ptr<threadpool::PriorityThreadPool> getOOBThreadPool()
   {
     return OOBThreadPool;
   }
@@ -190,6 +190,6 @@ class ServicePrimProc : public Service, public Opt
   static ServicePrimProc* fInstance;
   // Since C++20 flag's init value is false.
   std::atomic_flag startupRaceFlag_{false};
-  boost::shared_ptr<threadpool::FairThreadPool> primServerThreadPool;
-  boost::shared_ptr<threadpool::FairThreadPool> OOBThreadPool;
+  boost::shared_ptr<threadpool::PriorityThreadPool> primServerThreadPool;
+  boost::shared_ptr<threadpool::PriorityThreadPool> OOBThreadPool;
 };

primitives/primproc/sqlfrontsessionthread.cpp

@@ -520,7 +520,8 @@ namespace exemgr
       statementsRunningCount->incr(stmtCounted);
 
       PrimitiveServerThreadPools primitiveServerThreadPools(
-          ServicePrimProc::instance()->getPrimitiveServerThreadPool());
+        ServicePrimProc::instance()->getPrimitiveServerThreadPool(),
+        ServicePrimProc::instance()->getOOBThreadPool());
 
       if (tryTuples)
       {