mariadb-columnstore-engine/writeengine/redistribute/we_redistributecontrolthread.cpp

/* Copyright (C) 2014 InfiniDB, Inc.
   Copyright (C) 2016 MariaDB Corporaton

   This program is free software; you can redistribute it and/or
   modify it under the terms of the GNU General Public License
   as published by the Free Software Foundation; version 2 of
   the License.

   This program is distributed in the hope that it will be useful,
   but WITHOUT ANY WARRANTY; without even the implied warranty of
   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
   GNU General Public License for more details.

   You should have received a copy of the GNU General Public License
   along with this program; if not, write to the Free Software
   Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
   MA 02110-1301, USA. */

/*
* $Id: we_redistributecontrolthread.cpp 4450 2013-01-21 14:13:24Z rdempsey $
*/

#include <iostream>
#include <set>
#include <vector>
#include <cassert>
#include <stdexcept>
#include <sstream>
#include <string>
#include <unistd.h>
using namespace std;

#include "boost/scoped_ptr.hpp"
#include "boost/scoped_array.hpp"
#include "boost/thread/mutex.hpp"
#include "boost/filesystem/path.hpp"
#include "boost/filesystem/operations.hpp"
using namespace boost;

#include "installdir.h"

#include "configcpp.h"
using namespace config;

#include "liboamcpp.h"
#include "oamcache.h"
using namespace oam;

#include "dbrm.h"
using namespace BRM;

#include "messagequeue.h"
#include "bytestream.h"
using namespace messageqcpp;

#include "calpontsystemcatalog.h"
using namespace execplan;

#include "we_messages.h"
#include "we_redistributedef.h"
#include "we_redistributecontrol.h"
#include "we_redistributecontrolthread.h"

namespace redistribute
{

// static variables
boost::mutex RedistributeControlThread::fActionMutex;
volatile bool RedistributeControlThread::fStopAction = false;
string RedistributeControlThread::fWesInUse;


void RedistributeControlThread::setStopAction(bool s)
{
	mutex::scoped_lock lock(fActionMutex);
	fStopAction = s;
}


RedistributeControlThread::RedistributeControlThread(uint32_t act) :
	fAction(act), fMaxDbroot(0), fEntryCount(0), fErrorCode(RED_EC_OK)
{
}


RedistributeControlThread::~RedistributeControlThread()
{
//	fWEClient->removeQueue(uniqueId);
}


void RedistributeControlThread::operator()()
{
	if (fAction == RED_CNTL_START)
		doRedistribute();
	else if (fAction == RED_CNTL_STOP)
		doStopAction();
}


void RedistributeControlThread::doRedistribute()
{
	if (setup() != 0)
		fErrorCode = RED_EC_CNTL_SETUP_FAIL;
	else if (makeRedistributePlan() != 0)
		fErrorCode = RED_EC_MAKEPLAN_FAIL;

	try
	{
		if (fErrorCode == RED_EC_OK && !fStopAction && fEntryCount > 0)
			executeRedistributePlan();
	}
	catch (const std::exception& ex)
	{
		fErrorMsg += ex.what();
		fErrorCode = RED_EC_EXECUTE_FAIL;
	}
	catch (...)
	{
		fErrorMsg += "Error when executing the plan.";
		fErrorCode = RED_EC_EXECUTE_FAIL;
	}

	uint32_t state = RED_STATE_FINISH;
	if (fErrorCode != RED_EC_OK)
		state = RED_STATE_FAILED;

	try
	{
		if (!fStopAction)
			fControl->updateState(state);
	}
	catch (const std::exception& ex)
	{
		fErrorMsg += ex.what();
		if (fErrorCode == RED_EC_OK)
			fErrorCode = RED_EC_UPDATE_STATE;
	}
	catch (...)
	{
		fErrorMsg += "Error when updating state.";
		if (fErrorCode == RED_EC_OK)
			fErrorCode = RED_EC_UPDATE_STATE;
	}

	if (fErrorMsg.empty())
		fControl->logMessage("finished @controlThread::doRedistribute");
	else
		fControl->logMessage(fErrorMsg + " @controlThread::doRedistribute");

	{
		mutex::scoped_lock lock(fActionMutex);
		fWesInUse.clear();
	}
}


int RedistributeControlThread::setup()
{
	int ret = 0;

	try
	{
//		fUniqueId = fDbrm.getUnique64();
//		fWEClient = WriteEngine::WEClients::instance(WriteEngine::WEClients::REDISTRIBUTE);
//		fWEClient->addQueue(uniqueId);
		fConfig = Config::makeConfig();
		fOamCache = oam::OamCache::makeOamCache();
		fControl = RedistributeControl::instance();
//		fOam.reset(new oam::Oam);
//		fDbrm.reset(new BRM::DBRM);

		vector<int>::iterator i = fControl->fSourceList.begin();
		for (; i != fControl->fSourceList.end(); i++)
		{
			fSourceSet.insert(*i);
			fDbrootSet.insert(*i);
			if (*i > fMaxDbroot)
				fMaxDbroot = *i;
		}

		vector<int>::iterator j = fControl->fDestinationList.begin();
		for (; j != fControl->fDestinationList.end(); j++)
		{
			fTargetSet.insert(*j);
			if (fDbrootSet.find(*j) == fDbrootSet.end())
			{
				fDbrootSet.insert(*j);
			}
//			if (*j > fMaxDbroot)
//				fMaxDbroot = *j;
		}
	}
	catch (const std::exception& ex)
	{
		fErrorMsg += ex.what();
		ret = 1;
	}
	catch (...)
	{
		ret = 1;
	}

	return ret;
}


int RedistributeControlThread::makeRedistributePlan()
{
	int ret = 0;
	try
	{
		if (fControl->fPlanFilePtr != NULL)
		{
			// should not happen, just in case.
			fclose(fControl->fPlanFilePtr);
			fControl->fPlanFilePtr = NULL;
		}

		// get all user table oids
		boost::shared_ptr<CalpontSystemCatalog> csc = CalpontSystemCatalog::makeCalpontSystemCatalog(0);
		vector<pair<CalpontSystemCatalog::OID, CalpontSystemCatalog::TableName> >
			tables = csc->getTables();
		vector<pair<CalpontSystemCatalog::OID, CalpontSystemCatalog::TableName> >::iterator i;
		for (i = tables.begin(); i != tables.end(); i++)
		{
			// in case, action is cancelled.
			if (fStopAction)
				break;

			// column oids
			CalpontSystemCatalog::RIDList cols = csc->columnRIDs(i->second, true);
			typedef std::map<PartitionInfo, RedistributeExtentEntry>  PartitionExtentMap;
			PartitionExtentMap   partitionMap;
			vector<EMEntry> entries;

			// sample the first column
			int rc = fControl->fDbrm->getExtents(cols[0].objnum, entries, false, false, true);
			if (rc != 0 || entries.size() == 0)
			{
				ostringstream oss;
				oss << "Error in DBRM getExtents; oid:" << cols[0].objnum << "; returnCode: " << rc;
				throw runtime_error(oss.str());
			}

			for (vector<EMEntry>::iterator j = entries.begin(); j != entries.end(); j++)
			{
				RedistributeExtentEntry redEntry;
				redEntry.oid = cols[0].objnum;
				redEntry.dbroot = j->dbRoot;
				redEntry.partition = j->partitionNum;
				redEntry.segment = j->segmentNum;
				redEntry.lbid = j->range.start;
				redEntry.range = j->range.size * 1024;

				PartitionInfo partInfo(j->dbRoot, j->partitionNum);
				partitionMap.insert(make_pair(partInfo, redEntry));
			}

			// sort partitions by dbroot
			vector<vector<int> > dbPartVec(fMaxDbroot + 1);
			uint64_t totalPartitionCount = 0;
			int      maxPartitionId = 0;
			for (PartitionExtentMap::iterator j = partitionMap.begin();
					j != partitionMap.end(); j++)
			{
				int dbroot = j->first.dbroot;
				if (fSourceSet.find(dbroot) != fSourceSet.end())
				{
					// only dbroot in source list needs attention
					dbPartVec[dbroot].push_back(j->first.partition);

					if (j->first.partition > maxPartitionId)
						maxPartitionId = j->first.partition;

					totalPartitionCount++;
				}
			}

			// sort the partitions
			for (vector<vector<int> >::iterator k = dbPartVec.begin(); k != dbPartVec.end(); k++)
				sort(k->begin(), k->end());

			// divide the dbroots into the source and target sets
			uint64_t average = totalPartitionCount / fTargetSet.size();
			// Remainder is the number of partitions that must be spread across some
			// of the dbroots such that no dbroot has more than average+1 partitions.
			uint64_t remainder = totalPartitionCount % fTargetSet.size();
			set<int> sourceDbroots;
			set<int> targetDbroots;
//			list<pair<size_t, int> > targetList;  // to be ordered by partition size
			int64_t extra = remainder;
			for (set<int>::iterator j = fDbrootSet.begin(); j != fDbrootSet.end(); ++j)
			{
				if (fTargetSet.find(*j) == fTargetSet.end())
				{
					// Not a target (removed on command line). Always a source.
					sourceDbroots.insert(*j);
					continue;
				}
				// If a dbroot has exactly average+1 partitions and there's extras to be had,
				// then it is neither a source nor a target.
				if ((dbPartVec[*j].size() == average+1) && extra)
				{
					--extra;
					continue;
				}
				if (dbPartVec[*j].size() > average)
				{
					// Sources are those dbroots with more than average partitions
					sourceDbroots.insert(*j);
				}
				else
				{
					// Targets are those with room ( <= average ) 
					targetDbroots.insert(*j);
				}
			}
			// At this point, there are two concepts of target. (1)Those in fTargetSet, which is the
			// set of dbroots the user wants partitions on and (2) those in targetDbroots, a subset of
			// fTargetSet, which is those that actually have room, based on average, for more data.

			// After redistribution, partition count for each dbroot is average or average+1.
			// When remainder > 0,  some targets will have (average+1) partitions.

			// loop through target dbroots and find partitions from sources to move to each.
			set<int>::iterator sourceDbroot = sourceDbroots.begin();
			int  sourceCnt = sourceDbroots.size();
			for (set<int>::iterator targetDbroot = targetDbroots.begin(); 
				                    targetDbroot != targetDbroots.end(); 
									++targetDbroot)
			{
				// check if this target will have average + 1 partitions.
				uint64_t e = 0;
				if (extra-- > 0)
					e = 1;

				// A set of the partitions already on the target. We try not to move the same partition here.
				set<int> targetParts(dbPartVec[*targetDbroot].begin(), dbPartVec[*targetDbroot].end());
				if (targetParts.size() >= (average+e))
					continue; // Don't move any partitions to this target

				// partitions to be moved to this target
				vector<PartitionInfo> planVec;

				// looking for source partitions start from partition 0
				bool done = false; // if target got enough partitions, set to true.
				int  loop = 0;     // avoid infinite loop. maxPartitionId is the last partition of one of the dbroots. It's a place to stop if all else fails.
				while (!done && loop <= maxPartitionId)
				{
					for (int p = loop++; p <= maxPartitionId && !done; ++p)
					{
						bool found = false;
						if (targetParts.find(p) == targetParts.end())  // True if the partition is not on the target already
						{
							// try to find partition p in one of the source dbroots
							for (int x = 0; x < sourceCnt && !found; ++x)
							{
								vector<int>& sourceParts = dbPartVec[*sourceDbroot];
								// This partition needs to move if:
								// 1) source still has more than average partitions, or
								// 2) source is not a listed target (we want to empty source).
								bool bNotTarget = fTargetSet.find(*sourceDbroot) == fTargetSet.end(); // true if source not in target list
								if (sourceParts.size() >= average || bNotTarget)
								{
									vector<int>::iterator y = find(sourceParts.begin(), sourceParts.end(), p);
									if ((y != sourceParts.end()) &&
										(sourceParts.size() > targetParts.size() || bNotTarget))
									{
										targetParts.insert(p);
										planVec.push_back(PartitionInfo(*sourceDbroot, p));
										found = true;

										// update the source
										sourceParts.erase(y);
									}
								}

								if (++sourceDbroot == sourceDbroots.end())
									sourceDbroot = sourceDbroots.begin();
							} // for source

							if (targetParts.size() == (average+e))
								done = true;
						} // !find p
					} // for p
				} // while loop

				// dump the plan for the target to file
				dumpPlanToFile(i->first, planVec, *targetDbroot);
			} // for target

			// It's possible that a source that is "removed" on the command line is not empty.
			// This can happen if a partition exists on all dbroots.

			// WCOL-786: use nextDbroot to start the loop looking for a suitible target 
			// where we left off with the previous partition. This gives each target 
			// an opportunity to get some of the data. In the case of dbroot removal,
			// there is often the same number of partitions on each remaining dbroot.
			// This logic tends to roundrobin which dbroot gets the next batch.
			set<int>::iterator nextDbroot = targetDbroots.begin();
			set<int>::iterator targetDbroot;
			int targetCnt = (int)targetDbroots.size();
			// Loop through the sources, looking for dbroots that are not targets that also still contain partitions
			for (set<int>::iterator sourceDbroot = sourceDbroots.begin(); 
									sourceDbroot != sourceDbroots.end(); 
									++sourceDbroot)
			{
				// Is this source in target list? If so, do nothing.
				if (fTargetSet.find(*sourceDbroot) != fTargetSet.end())
					continue;
				vector<int>& sourceParts = dbPartVec[*sourceDbroot]; // Partitions still on source

				// We can't erase from a vector we're iterating, so we need a kludge:
				for (int p = 0; p <= maxPartitionId; ++p)
				{
					vector<int>::iterator sourcePart = find(sourceParts.begin(), sourceParts.end(), p);
					if (sourcePart == sourceParts.end())
					{
						continue;
					}
					// Look through targets to see which can accept this partition. Find the one with the least
					// number of partitions. Someday we want to put with the dbroot having the fewest segments of
					// the partition.
					uint64_t partCount = std::numeric_limits<uint64_t>::max();
					int tdbroot = 0;
					targetDbroot = nextDbroot;
					// MCOL-786. Start at targetDbroot and loop around back to the same spot.
					for (int tbd=0; tbd < targetCnt; ++tbd)
					{
						if (targetDbroot == targetDbroots.end())
						{
							targetDbroot == targetDbroots.begin();
						}
						if (dbPartVec[*targetDbroot].size() < partCount)
						{
							tdbroot = *targetDbroot;
							partCount = dbPartVec[*targetDbroot].size();
							nextDbroot = targetDbroot;
							++nextDbroot;
						}
						++targetDbroot;
					}
					if (tdbroot == 0)
					{
						continue;
					}
					set<int> targetParts(dbPartVec[tdbroot].begin(), dbPartVec[tdbroot].end());
					vector<PartitionInfo> planVec; // partitions to be moved to this target
					targetParts.insert(p);
					planVec.push_back(PartitionInfo(*sourceDbroot, p));
					sourceParts.erase(sourcePart);
					dumpPlanToFile(i->first, planVec, tdbroot);
				} // for sourceParts
			} // for source
		} // for tables
	}
	catch (const std::exception& ex)
	{
		fErrorMsg += ex.what();
		ret = 2;
	}
	catch (...)
	{
		ret = 2;
	}
	displayPlan();
	return ret;
}

void RedistributeControlThread::dumpPlanToFile(uint64_t oid, vector<PartitionInfo>& vec, int target)
{
	// open the plan file, if not already opened, to write.
	if (fControl->fPlanFilePtr == NULL)
	{
		errno = 0;
		fControl->fPlanFilePtr = fopen(fControl->fPlanFilePath.c_str(), "w+");
		if (fControl->fPlanFilePtr == NULL)
		{
			int e = errno;
			ostringstream oss;
			oss << "Failed to open redistribute.plan: " << strerror(e) << " (" << e << ")";
			throw runtime_error(oss.str());
		}
	}

	size_t entryNum = vec.size();
	scoped_array<RedistributePlanEntry> entries(new RedistributePlanEntry[entryNum]);
	for (uint64_t i = 0; i < entryNum; ++i)
	{
		entries[i].table = oid;
		entries[i].source = vec[i].dbroot;
		entries[i].partition = vec[i].partition;
		entries[i].destination = target;
		entries[i].status = RED_TRANS_READY;
	}

	errno = 0;
	size_t n = fwrite(entries.get(), sizeof(RedistributePlanEntry), entryNum, fControl->fPlanFilePtr);
	if (n != entryNum)  // need retry
	{
		int e = errno;
		ostringstream oss;
		oss << "Failed to write into redistribute.plan: " << strerror(e) << " (" << e << ")";
		throw runtime_error(oss.str());
	}

	fEntryCount += entryNum;
}

void RedistributeControlThread::displayPlan()
{
	// start from the first entry
	try
	{
		if (!fControl->fPlanFilePtr)
		{
			ostringstream oss;
			oss << "No data is schefuled to be moved" << endl;
			fControl->logMessage(oss.str());
			return;
		}
		rewind(fControl->fPlanFilePtr);

		ByteStream bs;
		uint32_t entryId = 0;
		long entrySize = sizeof(RedistributePlanEntry);
		fControl->logMessage(string("Redistribution Plan:"));
		while (entryId++ < fEntryCount)
		{
			RedistributePlanEntry entry;
			errno = 0;
			size_t n = fread(&entry, entrySize, 1, fControl->fPlanFilePtr);
			if (n != 1)
			{
				int e = errno;
				ostringstream oss;
				oss << "Failed to read from redistribute.plan: " << strerror(e) << " (" << e << ")";
				throw runtime_error(oss.str());
			}

			// Print this plan entry
			ostringstream oss;
			oss << "table oid " << entry.table << " partition " << entry.partition
				 << " moves from dbroot " << entry.source << " to " << entry.destination << endl;
			fControl->logMessage(oss.str());
		}
	}
	catch (std::exception &e)
	{
		ostringstream oss;
		oss << "exception during display of plan: " << e.what() << endl;
		fControl->logMessage(oss.str());
	}
	catch (...)
	{
		ostringstream oss;
		oss << "exception during display of plan" << endl;
		fControl->logMessage(oss.str());
	}
}

int RedistributeControlThread::executeRedistributePlan()
{
	// update the info with total partitions to move
	fControl->setEntryCount(fEntryCount);

	// start from the first entry
	rewind(fControl->fPlanFilePtr);

	ByteStream bs;
	uint32_t entryId = 0;
	long entrySize = sizeof(RedistributePlanEntry);
	while (entryId++ < fEntryCount)
	{
		try
		{
// skip system status check in case no OAM
#if !defined(_MSC_VER) && !defined(SKIP_OAM_INIT)
			// make sure system is in active state
			bool isActive = false;
			while (!isActive)
			{
				bool noExcept = true;
				SystemStatus systemstatus;
				try
				{
					fControl->fOam->getSystemStatus(systemstatus);
				}
				catch (const std::exception& ex)
				{
					fErrorMsg += ex.what();
					noExcept = false;
				}
				catch (...)
				{
					noExcept = false;
				}

				if (noExcept && ((isActive = (systemstatus.SystemOpState == oam::ACTIVE)) == false))
					sleep(1);;
			}
#endif

			if (fStopAction)
				return RED_EC_USER_STOP;


			RedistributePlanEntry entry;
			errno = 0;
			size_t n = fread(&entry, entrySize, 1, fControl->fPlanFilePtr);
			if (n != 1)
			{
				int e = errno;
				ostringstream oss;
				oss << "Failed to read from redistribute.plan: " << strerror(e) << " (" << e << ")";
				throw runtime_error(oss.str());
			}

			if (entry.status != (int) RED_TRANS_READY)
				continue;

			// send the job to source dbroot
			size_t headerSize = sizeof(RedistributeMsgHeader);
			size_t entrySize = sizeof(RedistributePlanEntry);
			RedistributeMsgHeader header(entry.destination,entry.source,entryId,RED_ACTN_REQUEST);
			if (connectToWes(header.source) == 0)
			{
				bs.restart();
				entry.starttime = time(NULL);
				bs << (ByteStream::byte) WriteEngine::WE_SVR_REDISTRIBUTE;
				bs.append((const ByteStream::byte*) &header, headerSize);
				bs.append((const ByteStream::byte*) &entry, entrySize);
				fMsgQueueClient->write(bs);

				SBS sbs = fMsgQueueClient->read();
				entry.status = RED_TRANS_FAILED;
				if (sbs->length() == 0)
				{
					ostringstream oss;
					oss << "Zero byte read, Network error.  entryID=" << entryId;
					fErrorMsg = oss.str();
				}
				else if (sbs->length() < (headerSize + entrySize + 1))
				{
					ostringstream oss;
					oss << "Short message, length=" << sbs->length() << ". entryID=" << entryId;
					fErrorMsg = oss.str();
				}
				else
				{
					ByteStream::byte wesMsgId;
					*sbs >> wesMsgId;
					// Need check header info
					//const RedistributeMsgHeader* h = (const RedistributeMsgHeader*) sbs->buf();
					sbs->advance(headerSize);
					const RedistributePlanEntry* e = (const RedistributePlanEntry*) sbs->buf();
					sbs->advance(entrySize);
					entry.status = e->status;
					entry.endtime = time(NULL);
//					if (entry.status == (int32_t) RED_TRANS_FAILED)
//						*sbs >> fErrorMsg;
				}

				// done with this connection, may consider to reuse.
				fMsgQueueClient.reset();
			}
			else
			{
				entry.status = RED_TRANS_FAILED;
				ostringstream oss;
				oss << "Connect to PM failed." << ". entryID=" << entryId;
				fErrorMsg += oss.str();
			}

			if (!fErrorMsg.empty())
				throw runtime_error(fErrorMsg);

			errno = 0;
			int rc = fseek(fControl->fPlanFilePtr, -((long)entrySize), SEEK_CUR);
			if (rc != 0)
			{
				int e = errno;
				ostringstream oss;
				oss << "fseek is failed: " << strerror(e) << " (" << e << "); entry id=" << entryId;
				throw runtime_error(oss.str());
			}

			errno = 0;
			n = fwrite(&entry, entrySize, 1, fControl->fPlanFilePtr);
			if (n != 1)  // need retry
			{
				int e = errno;
				ostringstream oss;
				oss << "Failed to update redistribute.plan: " << strerror(e) << " (" << e
					<< "); entry id=" << entryId;
				throw runtime_error(oss.str());
			}

			fflush(fControl->fPlanFilePtr);

			fControl->updateProgressInfo(entry.status, entry.endtime);

		}
		catch (const std::exception& ex)
		{
			fControl->logMessage(string("got exception when executing plan:") + ex.what());
		}
		catch (...)
		{
			fControl->logMessage("got unknown exception when executing plan.");
		}
	}

	return 0;
}


int  RedistributeControlThread::connectToWes(int dbroot)
{
	int ret = 0;
	OamCache::dbRootPMMap_t dbrootToPM = fOamCache->getDBRootToPMMap();
	int pmId = (*dbrootToPM)[dbroot];
	ostringstream oss;
	oss << "pm" << pmId << "_WriteEngineServer";
	try
	{
		mutex::scoped_lock lock(fActionMutex);
		fWesInUse = oss.str();
		fMsgQueueClient.reset(new MessageQueueClient(fWesInUse, fConfig));
	}
	catch (const std::exception& ex)
	{
		fErrorMsg = "Caught exception when connecting to " + oss.str() + " -- " + ex.what();
		ret = 1;
	}
	catch (...)
	{
		fErrorMsg = "Caught exception when connecting to " + oss.str() + " -- unknown";
		ret = 2;
	}

	if (ret != 0)
	{
		mutex::scoped_lock lock(fActionMutex);
		fWesInUse.clear();

		fMsgQueueClient.reset();
	}

	return ret;
}


void RedistributeControlThread::doStopAction()
{
	fConfig = Config::makeConfig();
	fControl = RedistributeControl::instance();

	mutex::scoped_lock lock(fActionMutex);
	if (!fWesInUse.empty())
	{
		// send the stop message to dbroots
		size_t headerSize = sizeof(RedistributeMsgHeader);
		RedistributeMsgHeader header(-1, -1, -1, RED_ACTN_STOP);

		try
		{
			fMsgQueueClient.reset(new MessageQueueClient(fWesInUse, fConfig));
			ByteStream bs;
			bs << (ByteStream::byte) WriteEngine::WE_SVR_REDISTRIBUTE;
			bs.append((const ByteStream::byte*) &header, headerSize);
			fMsgQueueClient->write(bs);

			SBS sbs;
			sbs = fMsgQueueClient->read();
			// no retry yet.
		}
		catch (const std::exception& ex)
		{
			fErrorMsg = "Caught exception when connecting to " + fWesInUse + " -- " + ex.what();
		}
		catch (...)
		{
			fErrorMsg = "Caught exception when connecting to " + fWesInUse + " -- unknown";
		}
	}

	if (!fErrorMsg.empty())
		fControl->logMessage(fErrorMsg + " @controlThread::doStop");
	else
		fControl->logMessage("User stop @controlThread::doStop");

	fWesInUse.clear();
	fMsgQueueClient.reset();
}


} // namespace

// vim:ts=4 sw=4: