MCOL-379 - fix false critical memory usage alarm

oam/install_scripts/post-install

@@ -155,6 +155,7 @@ rm -f $installdir/data/bulk/tmpjob/* >/dev/null 2>&1
 
 #create columnstore temp file directory
 $SUDO chmod 777 /tmp
+$SUDO rm -f /tmp/* > /dev/null 2>&1
 mkdir -p /tmp/columnstore_tmp_files >/dev/null 2>&1
 
 #setup core file directory and link
@@ -213,7 +214,6 @@ if [ $user = "root" ]; then
 	$installdir/bin/syslogSetup.sh install > /tmp/syslog_install.log 2>&1
 	rm -f /etc/default/columnstore
 else
-	$SUDO rm -fr /tmp/* > /dev/null 2>&1
 	$installdir/bin/syslogSetup.sh --installdir=$installdir install > /tmp/syslog_install.log 2>&1
 	$SUDO chown $user:$user $installdir/etc/Columnstore.xml
 	$SUDO chmod -R 777 /dev/shm

oamapps/serverMonitor/memoryMonitor.cpp

@@ -32,10 +32,11 @@ using namespace logging;
 using namespace servermonitor;
 //using namespace procheartbeat;
 
-unsigned long totalMem;
 ProcessMemoryList pml;
 int swapFlag = 0;
 
+uint64_t totalMem;
+
 pthread_mutex_t MEMORY_LOCK;
 
 /*****************************************************************************************
@@ -62,7 +63,7 @@ void memoryMonitor()
 
 	//set monitoring period to 60 seconds
 	int monitorPeriod = MONITOR_PERIOD;
-    utils::CGroupConfigurator cg;
+	utils::CGroupConfigurator cg;
 
 	while(true)
 	{
@@ -87,12 +88,11 @@ void memoryMonitor()
 		//get memory stats
 		totalMem = cg.getTotalMemory();
 		uint64_t freeMem = cg.getFreeMemory();
-        uint64_t usedMem = totalMem - freeMem;
+		uint64_t usedMem = totalMem - freeMem;
 
 		//get swap stats
-        uint64_t totalSwap = cg.getTotalSwapSpace();
+		uint64_t totalSwap = cg.getTotalSwapSpace();
-        uint64_t usedSwap = cg.getSwapInUse();
+		uint64_t usedSwap = cg.getSwapInUse();
-
 
 		if ( totalSwap == 0 ) {
 			swapUsagePercent = 0;
@@ -136,13 +136,34 @@ void memoryMonitor()
 		else
 			memoryUsagePercent =  (usedMem / (totalMem / 100)) + 1;
 
+		/*LoggingID lid(SERVER_MONITOR_LOG_ID);
+		MessageLog ml(lid);
+		Message msg;
+		Message::Args args;
+		args.add("memoryUsagePercent ");
+		args.add((uint64_t) memoryUsagePercent);
+		args.add("usedMem ");
+		args.add((uint64_t) usedMem);
+		args.add("totalMem ");
+		args.add((uint64_t) totalMem);
+		msg.format(args);
+		ml.logInfoMessage(msg);
+*/
+		//first time called, log
+		//adjust if over 100%
+		if ( swapUsagePercent < 0 )
+			swapUsagePercent = 0;
+		if ( swapUsagePercent > 100 )
+			swapUsagePercent = 100;
+
+		if ( memoryUsagePercent < 0 )
+			memoryUsagePercent = 0;
+		if ( memoryUsagePercent > 100 )
+			memoryUsagePercent = 100;
+
 		// check for Memory alarms
 		if (memoryUsagePercent >= memoryCritical && memoryCritical > 0 ) {
 			if ( monitorPeriod == MONITOR_PERIOD ) {
-				//first time called, log
-				//adjust if over 100%
-				if ( memoryUsagePercent > 100 )
-					memoryUsagePercent = 100;
 
 				LoggingID lid(SERVER_MONITOR_LOG_ID);
 				MessageLog ml(lid);

procmgr/processmanager.cpp

@@ -3783,7 +3783,6 @@ void ProcessManager::setSystemState(uint16_t state)
 	{
 //		log.writeLog(__LINE__, "EXCEPTION ERROR on MessageQueueClient: Caught unknown exception!", LOG_TYPE_ERROR);
 	}
-	pthread_mutex_unlock(&STATUS_LOCK);
 
 	// Process Alarms
 	string system = "System";
@@ -3798,7 +3797,11 @@ void ProcessManager::setSystemState(uint16_t state)
 		else
 			if ( state == oam::AUTO_OFFLINE )
 				aManager.sendAlarmReport(system.c_str(), SYSTEM_DOWN_AUTO, SET);
+		//this alarm doesnt get clear by reporter, so clear on stopage
+		aManager.sendAlarmReport(system.c_str(), CONN_FAILURE, CLEAR);
 	}
+	
+	pthread_mutex_unlock(&STATUS_LOCK);
 }
 
 /******************************************************************************************

utils/common/cgroupconfigurator.cpp

@@ -173,9 +173,6 @@ uint64_t CGroupConfigurator::getTotalMemory()
 {
     uint64_t ret;
 
-    if (totalMemory != 0)
-        return totalMemory;
-
     if (!cGroupDefined)
         ret = getTotalMemoryFromProc();
     else {
@@ -183,7 +180,11 @@ uint64_t CGroupConfigurator::getTotalMemory()
         if (ret == 0)
             ret = getTotalMemoryFromProc();
     }
-    //cout << "Total mem available is " << ret << endl;
+    //ostringstream os;
+    //os << "Total mem available is " << ret;
+    //cerr << os.str() << endl;
+    //log(logging::LOG_TYPE_WARNING, os.str());
+
     totalMemory = ret;
     return totalMemory;
 }
@@ -216,17 +217,31 @@ uint64_t CGroupConfigurator::getTotalMemoryFromProc()
 	input[79] = '\0';
 	pclose(cmdPipe);
 	memTot = atoi(input);
+
+	//ostringstream os;
+	//os << "FreeBSD Total mem available is " << memTot;
+	//cerr << os.str() << endl;
+	//log(logging::LOG_TYPE_WARNING, os.str());
 #else
 	ifstream in("/proc/meminfo");
 	string x;
 
 	in >> x;
 	in >> memTot;
+
+	//ostringstream os;
+	//os << "meminfo Total mem available is " << memTot;
+	//cerr << os.str() << endl;
+	//log(logging::LOG_TYPE_WARNING, os.str());
 #endif
 
 	//memTot is now in KB, convert to bytes
 	memTot *= 1024;
 
+	//os << "meminfo Total bytes mem available is " << memTot;
+	//cerr << os.str() << endl;
+	//log(logging::LOG_TYPE_WARNING, os.str());
+
 	return memTot;
 }
 
@@ -307,9 +322,8 @@ uint64_t CGroupConfigurator::getMemUsageFromCGroup()
 uint64_t CGroupConfigurator::getFreeMemoryFromProc()
 {
 	uint64_t memFree  = 0;
-	uint64_t buffers  = 0;
+	uint64_t memAvailable  = 0;
-	uint64_t cached   = 0;
+	uint64_t memTotal = 0;
-    uint64_t memTotal = 0;
 
 #if defined(_MSC_VER)
 	MEMORYSTATUSEX memStat;
@@ -338,18 +352,15 @@ uint64_t CGroupConfigurator::getFreeMemoryFromProc()
 	in >> memFree;
 	in >> x;         // kB
 
-	in >> x;         // Buffers:
+	in >> x;         // MemAvailable:
-	in >> buffers;
+	in >> memAvailable;
 	in >> x;         // kB
 
-	in >> x;         // Cached:
-	in >> cached;
 #endif
 
 	// amount available for application
-	memFree = memFree + buffers + cached;
+	memAvailable *= 1024;
-	memFree *= 1024;
+    return memAvailable;
-    return memFree;
 }
 
 uint64_t CGroupConfigurator::getTotalSwapSpace()