mariadb-columnstore-engine/writeengine/server/we_getfilesizes.cpp

/* Copyright (C) 2014 InfiniDB, Inc.

   This program is free software; you can redistribute it and/or
   modify it under the terms of the GNU General Public License
   as published by the Free Software Foundation; version 2 of
   the License.

   This program is distributed in the hope that it will be useful,
   but WITHOUT ANY WARRANTY; without even the implied warranty of
   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
   GNU General Public License for more details.

   You should have received a copy of the GNU General Public License
   along with this program; if not, write to the Free Software
   Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
   MA 02110-1301, USA. */

/*****************************************************************************
 * $Id: we_getfilesizes.cpp 4450 2013-01-21 14:13:24Z rdempsey $
 *
 ****************************************************************************/
#include "we_getfilesizes.h"

#include <iostream>
#include <stdexcept>
using namespace std;

#include "calpontsystemcatalog.h"
using namespace execplan;

#include "threadpool.h"
using namespace threadpool;

#include "bytestream.h"
using namespace messageqcpp;
#include "we_fileop.h"

#include "idbcompress.h"
using namespace compress;

#include "IDBFileSystem.h"
#include "IDBPolicy.h"
using namespace idbdatafile;

namespace WriteEngine
{
struct FileInfo                             
{
        uint32_t      partition;          /** @brief Partition for a file*/
        uint16_t      segment;            /** @brief Segment for a file */
        uint16_t      dbRoot;             /** @brief DbRoot for a file */
        std::string    segFileName;        /** @brief seg file path */
		double  	   fileSize;		   /** @brief seg file size in giga bytes */
		void serialize(messageqcpp::ByteStream& bs)
		{
			bs << partition;
			bs << segment;
			bs << dbRoot;
			bs << segFileName;
			bs << (*(uint64_t*)(&fileSize));
		}
};
typedef std::vector<FileInfo> Files;
typedef std::map<uint32_t, Files> columnMap;
typedef std::map<int, columnMap*> allColumnMap;
allColumnMap wholeMap;
boost::mutex columnMapLock;	
ActiveThreadCounter *activeThreadCounter;

size_t readFillBuffer(
	idbdatafile::IDBDataFile* pFile,
	char*   buffer,
	size_t  bytesReq)
{
	char*   pBuf = buffer;
	ssize_t nBytes;
	size_t  bytesToRead = bytesReq;
	size_t  totalBytesRead = 0;

	while (1)
	{
		nBytes = pFile->read(pBuf, bytesToRead);
		if (nBytes > 0)
			totalBytesRead += nBytes;
		else
			break;

		if ((size_t)nBytes == bytesToRead)
			break;

		pBuf        += nBytes;
		bytesToRead  =  bytesToRead - (size_t)nBytes;
	}

	return totalBytesRead;
}

off64_t getCompressedDataSize(string& fileName)
{
		off64_t dataSize = 0;
		IDBDataFile* pFile = 0;
		size_t nBytes;
		// Some IDBPolicy functions can throw exceptions, caller will catch it
		IDBPolicy::configIDBPolicy();
		bool bHdfsFile = IDBPolicy::useHdfs();
		if (bHdfsFile)
			pFile = IDBDataFile::open(IDBDataFile::HDFS,
				fileName.c_str(), "r", 0);
		else
			pFile = IDBDataFile::open(IDBDataFile::BUFFERED,
				fileName.c_str(), "r", 0);
		if (!pFile)
		{
			std::ostringstream oss;
			oss << "Cannot open file " << fileName << " for read.";
			throw std::runtime_error(oss.str());
		}
		
		IDBCompressInterface decompressor;
		//--------------------------------------------------------------------------
		// Read headers and extract compression pointers
		//--------------------------------------------------------------------------
		char hdr1[IDBCompressInterface::HDR_BUF_LEN];
		nBytes = readFillBuffer( pFile,hdr1,IDBCompressInterface::HDR_BUF_LEN);
		if ( nBytes != IDBCompressInterface::HDR_BUF_LEN )
		{
			std::ostringstream oss;
			oss << "Error reading first header from file " << fileName;
			throw std::runtime_error(oss.str());
		}

		int64_t ptrSecSize = decompressor.getHdrSize(hdr1) - IDBCompressInterface::HDR_BUF_LEN;
		char* hdr2 = new char[ptrSecSize];
		nBytes = readFillBuffer( pFile,hdr2,ptrSecSize);
		if ( (int64_t)nBytes != ptrSecSize )
		{
			std::ostringstream oss;
			oss << "Error reading second header from file " << fileName;
			throw std::runtime_error(oss.str());
		}

		CompChunkPtrList chunkPtrs;
		int rc = decompressor.getPtrList(hdr2, ptrSecSize, chunkPtrs);
		delete[] hdr2;
		if (rc != 0)
		{
			std::ostringstream oss;
			oss << "Error decompressing second header from file " << fileName;
			throw std::runtime_error(oss.str());
		}

		unsigned k = chunkPtrs.size();
		// last header's offset + length will be the data bytes
		dataSize = chunkPtrs[k-1].first + chunkPtrs[k-1].second;
		delete pFile;
		return dataSize;
}
struct ColumnThread
{
    ColumnThread(uint32_t oid, int32_t compressionType, bool reportRealUse, int key) 
    : fOid(oid), fCompressionType(compressionType), fReportRealUse(reportRealUse), fKey(key) 
    {}
    void operator()()
    {
		Config config;
		config.initConfigCache();
		std::vector<uint16_t> rootList;
		config.getRootIdList( rootList );
		FileOp fileOp;
		Files aFiles;

		// This function relies on IDBPolicy being initialized by
		// IDBPolicy::init().  This is done when WriteEngineServer main() calls
		// IDBPolicy::configIDBPolicy();
		IDBDataFile::Types fileType;
		bool bUsingHdfs = IDBPolicy::useHdfs();
		if (bUsingHdfs)
			fileType = IDBDataFile::HDFS;
		else
			fileType = IDBDataFile::UNBUFFERED;
		IDBFileSystem& fs = IDBFileSystem::getFs( fileType );
		for (uint32_t i=0; i < rootList.size(); i++)
		{
			std::vector<struct BRM::EMEntry> entries;
			(void)BRMWrapper::getInstance()->getExtents_dbroot(fOid, entries, rootList[i]);
			std::vector<struct BRM::EMEntry>::const_iterator iter = entries.begin();
			while ( iter != entries.end() ) //organize extents into files
			{	
				//Find the size of this file
				//string fileName;
				char fileName[200];
				(void)fileOp.getFileName( fOid, fileName, rootList[i], entries[0].partitionNum, entries[0].segmentNum);
				string aFile(fileName); //convert between char* and string
				off64_t fileSize = 0;
				if (fReportRealUse && (fCompressionType > 0))
				{
					try {
						fileSize = getCompressedDataSize(aFile);
					}
					catch (std::exception& ex)
					{
						cerr << ex.what();
					}
				}
				else
					fileSize = fs.size( fileName );
					
				if (fileSize > 0) // File exists, add to list
				{
					FileInfo aFileInfo;
					aFileInfo.partition = entries[0].partitionNum;
					aFileInfo.segment = entries[0].segmentNum;
					aFileInfo.dbRoot = rootList[i];
					aFileInfo.segFileName = aFile;
					aFileInfo.fileSize = (double)fileSize / (1024 * 1024 * 1024);
					aFiles.push_back(aFileInfo);
					//cout.precision(15);
					//cout << "The file " << aFileInfo.segFileName << " has size " << fixed << aFileInfo.fileSize << "GB" << endl;
				}
				
				//erase the entries from this dbroot.
				std::vector<struct BRM::EMEntry> entriesTrimed;
				for (uint32_t m=0; m<entries.size(); m++)
				{
					if ((entries[0].partitionNum != entries[m].partitionNum) || (entries[0].segmentNum != entries[m].segmentNum))
					entriesTrimed.push_back(entries[m]);
				}
				entriesTrimed.swap(entries);
				iter = entries.begin();
			}
		}
		
		boost::mutex::scoped_lock lk(columnMapLock);
		//cout << "Current size of columnsMap is " << columnsMap.size() << endl;
		allColumnMap::iterator colMapiter = wholeMap.find(fKey);
		if (colMapiter != wholeMap.end())
		{
			(colMapiter->second)->insert(make_pair(fOid,aFiles));	
			activeThreadCounter->decr();
		//cout << "Added to columnsMap aFiles with size " << aFiles.size() << " for oid " << fOid << endl;		
		}
	}
	uint32_t fOid;
	int32_t fCompressionType;
	bool fReportRealUse;
	int fKey;
};

//------------------------------------------------------------------------------
// Get file size from file name in bytestream object
//------------------------------------------------------------------------------
int WE_GetFileSizes::processFileName(
    messageqcpp::ByteStream& bs,
    std::string& errMsg, int key)
{
    uint8_t rc = 0;
    off_t fileSize;
    off_t compressedFileSize;
    errMsg.clear();

    try
    {
        std::string fileName;

        bs >> fileName;
        fileSize = IDBPolicy::size(fileName.c_str());
        compressedFileSize = IDBPolicy::compressedSize(fileName.c_str());
    }
	catch(std::exception& ex)
	{
		errMsg = ex.what();
		rc     = 1;
	}
    bs.reset();
    bs << fileSize;
    bs << compressedFileSize;
    return rc;
}

//------------------------------------------------------------------------------
// Process a table size based on input from the
// bytestream object.
//------------------------------------------------------------------------------
int WE_GetFileSizes::processTable(
	messageqcpp::ByteStream& bs,
	std::string& errMsg, int key)
{
	uint8_t rc = 0;
	errMsg.clear();

	try {
		std::string aTableName;
		std::string schemaName;
		bool reportRealUse = false;
		ByteStream::byte tmp8;
		bs >> schemaName;
		//cout << "schema: "<< schemaName << endl;

		bs >> aTableName;
		//cout << "tableName: " << aTableName << endl;
		bs >> tmp8;
		reportRealUse = (tmp8 != 0);
		
		//get column oids
		boost::shared_ptr<CalpontSystemCatalog> systemCatalogPtr = CalpontSystemCatalog::makeCalpontSystemCatalog(0);
		CalpontSystemCatalog::TableName tableName;
		tableName.schema = schemaName;
		tableName.table = aTableName;
		CalpontSystemCatalog::RIDList columnList = systemCatalogPtr->columnRIDs(tableName);
		CalpontSystemCatalog::ColType colType;
		CalpontSystemCatalog::DictOIDList dictOidList = systemCatalogPtr->dictOIDs(tableName);
		int serverThreads = 20;
		int serverQueueSize = serverThreads * 100;
		threadpool::ThreadPool tp(serverThreads,serverQueueSize);
		int totalSize = columnList.size() + dictOidList.size();
		activeThreadCounter = new ActiveThreadCounter(totalSize);
		
		columnMap *columnsMap = new columnMap();
		{	
			boost::mutex::scoped_lock lk(columnMapLock);
			wholeMap[key] = columnsMap;
		}
		for (uint32_t i=0; i < columnList.size(); i++)
		{
			colType = systemCatalogPtr->colType(columnList[i].objnum);
			tp.invoke(ColumnThread(columnList[i].objnum, colType.compressionType, reportRealUse, key));	
			if (colType.ddn.dictOID > 0)
				tp.invoke(ColumnThread(colType.ddn.dictOID, colType.compressionType, reportRealUse, key));	
		}
	/*	for (uint32_t i=0; i < dictOidList.size(); i++)
		{
			tp.invoke(ColumnThread(dictOidList[i].dictOID));		
		} */
		//check whether all threads finish
		int sleepTime = 100; // sleep 100 milliseconds between checks
		struct timespec rm_ts;

		rm_ts.tv_sec = sleepTime/1000;
		rm_ts.tv_nsec = sleepTime%1000 *1000000;
		uint32_t currentActiveThreads = 10;
		while (currentActiveThreads > 0)
		{
#ifdef _MSC_VER
			Sleep(sleepTime);
#else
			struct timespec abs_ts;
			do
			{
				abs_ts.tv_sec = rm_ts.tv_sec;
				abs_ts.tv_nsec = rm_ts.tv_nsec;
			}
			while(nanosleep(&abs_ts,&rm_ts) < 0);
#endif
			currentActiveThreads = activeThreadCounter->cur();
		}
	}
	catch(std::exception& ex)
	{
		//cout << "WE_GetFileSizes got exception-" << ex.what() <<
		//	std::endl;
		errMsg = ex.what();
		rc     = 1;
	}
	//Build the message to send to the caller
	bs.reset();
	boost::mutex::scoped_lock lk(columnMapLock);
	allColumnMap::iterator colMapiter = wholeMap.find(key);
	if (colMapiter != wholeMap.end())
	{
		columnMap::iterator iter = colMapiter->second->begin();
		uint64_t size;
		Files::iterator it;
		while ( iter != colMapiter->second->end())
		{
			bs << iter->first;
			//cout << "processTable::coloid = " << iter->first << endl;

			size = iter->second.size();
			bs << size;
			for (it = iter->second.begin(); it != iter->second.end(); it++)
				it->serialize(bs);
			//cout << "length now is " << bs.length() << endl;
			iter++;
		}
		wholeMap.erase(colMapiter);
	}
	return rc;
}


}