diff --git a/writeengine/bulk/we_colopbulk.cpp b/writeengine/bulk/we_colopbulk.cpp index ced669d7d..eb9bc6637 100644 --- a/writeengine/bulk/we_colopbulk.cpp +++ b/writeengine/bulk/we_colopbulk.cpp @@ -63,7 +63,7 @@ ColumnOpBulk::~ColumnOpBulk() // @bug 5572 - HDFS usage: add *.tmp file backup flag IDBDataFile* ColumnOpBulk::openFile(const WriteEngine::Column& column, uint16_t dbRoot, uint32_t partition, uint16_t segment, std::string& segFile, bool useTmpSuffix, - const char* mode, int ioBuffSize) const + const char* mode, int ioBuffSize, bool isReadOnly) const { return FileOp::openFile(column.dataFile.fid, dbRoot, partition, segment, segFile, mode, column.colWidth, useTmpSuffix); @@ -92,7 +92,7 @@ int ColumnOpBulk::blocksInFile(IDBDataFile*) const //------------------------------------------------------------------------------ // Stub for readBlock //------------------------------------------------------------------------------ -int ColumnOpBulk::readBlock(IDBDataFile*, unsigned char*, const uint64_t, bool) +int ColumnOpBulk::readBlock(IDBDataFile*, unsigned char*, const uint64_t) { throw std::logic_error("Unauthorized use of ColumnOpBulk::readBlock"); diff --git a/writeengine/bulk/we_colopbulk.h b/writeengine/bulk/we_colopbulk.h index b455f9f7a..618594e79 100644 --- a/writeengine/bulk/we_colopbulk.h +++ b/writeengine/bulk/we_colopbulk.h @@ -50,8 +50,9 @@ class ColumnOpBulk : public ColumnOp virtual int blocksInFile(IDBDataFile*) const; virtual IDBDataFile* openFile(const WriteEngine::Column& column, uint16_t dbRoot, uint32_t partition, uint16_t segment, std::string& segFile, bool useTmpSuffix, - const char* mode = "r+b", int ioBuffSize = DEFAULT_BUFSIZ) const; - virtual int readBlock(IDBDataFile*, unsigned char*, const uint64_t, bool isReadOnly = false); + const char* mode = "r+b", int ioBuffSize = DEFAULT_BUFSIZ, + bool isReadOnly = false) const; + virtual int readBlock(IDBDataFile*, unsigned char*, const uint64_t); virtual int saveBlock(IDBDataFile*, const unsigned char*, const uint64_t); }; diff --git a/writeengine/shared/we_chunkmanager.cpp b/writeengine/shared/we_chunkmanager.cpp index 6830c4625..82de77777 100644 --- a/writeengine/shared/we_chunkmanager.cpp +++ b/writeengine/shared/we_chunkmanager.cpp @@ -252,10 +252,11 @@ int ChunkManager::removeBackups(TxnID txnId) // @bug 5572 - HDFS usage: add *.tmp file backup flag IDBDataFile* ChunkManager::getFilePtr(const Column& column, uint16_t root, uint32_t partition, uint16_t segment, string& filename, const char* mode, int size, - bool useTmpSuffix) const + bool useTmpSuffix, bool isReadOnly) const { CompFileData* fileData = getFileData(column.dataFile.fid, root, partition, segment, filename, mode, size, - column.colDataType, column.colWidth, useTmpSuffix); + column.colDataType, column.colWidth, useTmpSuffix, + false, isReadOnly); return (fileData ? fileData->fFilePtr : NULL); } @@ -313,7 +314,8 @@ IDBDataFile* ChunkManager::getFilePtrByName(const std::string& filename, FID& fi CompFileData* ChunkManager::getFileData(const FID& fid, uint16_t root, uint32_t partition, uint16_t segment, string& filename, const char* mode, int size, const CalpontSystemCatalog::ColDataType colDataType, int colWidth, - bool useTmpSuffix, bool dctnry) const + bool useTmpSuffix, bool dctnry, + bool isReadOnly) const { FileID fileID(fid, root, partition, segment); map::const_iterator mit = fFileMap.find(fileID); @@ -335,7 +337,7 @@ CompFileData* ChunkManager::getFileData(const FID& fid, uint16_t root, uint32_t // Initialize the given `filename`. filename = name; - return getFileData_(fileID, filename, mode, size, colDataType, colWidth, useTmpSuffix, dctnry); + return getFileData_(fileID, filename, mode, size, colDataType, colWidth, useTmpSuffix, dctnry, isReadOnly); } CompFileData* ChunkManager::getFileDataByName(const std::string& filename, const FID& fid, uint16_t root, @@ -359,9 +361,9 @@ CompFileData* ChunkManager::getFileDataByName(const std::string& filename, const CompFileData* ChunkManager::getFileData_(const FileID& fileID, const string& filename, const char* mode, int size, const CalpontSystemCatalog::ColDataType colDataType, - int colWidth, bool useTmpSuffix, bool dctnry) const + int colWidth, bool useTmpSuffix, bool dctnry, bool isReadOnly) const { - CompFileData* fileData = new CompFileData(fileID, fileID.fFid, colDataType, colWidth); + CompFileData* fileData = new CompFileData(fileID, fileID.fFid, colDataType, colWidth, isReadOnly); fileData->fFileName = filename; if (openFile(fileData, mode, colWidth, useTmpSuffix, __LINE__) != NO_ERROR) @@ -483,7 +485,7 @@ IDBDataFile* ChunkManager::createDctnryFile(const FID& fid, int64_t width, uint1 // Read the block for the specified fbo, from pFile's applicable chunk, and // into readBuf. //------------------------------------------------------------------------------ -int ChunkManager::readBlock(IDBDataFile* pFile, unsigned char* readBuf, uint64_t fbo, bool isReadOnly) +int ChunkManager::readBlock(IDBDataFile* pFile, unsigned char* readBuf, uint64_t fbo) { map::iterator fpIt = fFilePtrMap.find(pFile); @@ -504,7 +506,7 @@ int ChunkManager::readBlock(IDBDataFile* pFile, unsigned char* readBuf, uint64_t // chunk is not already uncompressed if (chunkData == NULL) - rc = fetchChunkFromFile(pFile, offset.quot, chunkData, isReadOnly); + rc = fetchChunkFromFile(pFile, offset.quot, chunkData); if (rc == NO_ERROR) { @@ -639,7 +641,7 @@ int ChunkManager::flushChunks(int rc, const std::map& columOids) break; // finally update the header - if ((rc = writeHeader(fileData, __LINE__)) != NO_ERROR) + if (!fileData->fReadOnly && (rc = writeHeader(fileData, __LINE__)) != NO_ERROR) break; //@Bug 4977 remove log file @@ -678,8 +680,7 @@ int ChunkManager::flushChunks(int rc, const std::map& columOids) // If the header ptr for the requested chunk is 0 (or has length 0), then // chunkData is initialized with a new empty chunk. //------------------------------------------------------------------------------ -int ChunkManager::fetchChunkFromFile(IDBDataFile* pFile, int64_t id, ChunkData*& chunkData, - bool isReadOnly) +int ChunkManager::fetchChunkFromFile(IDBDataFile* pFile, int64_t id, ChunkData*& chunkData) { // return value int rc = NO_ERROR; @@ -725,7 +726,7 @@ int ChunkManager::fetchChunkFromFile(IDBDataFile* pFile, int64_t id, ChunkData*& return rc; } - if (!isReadOnly && (rc = writeHeader(fIt->second, __LINE__)) != NO_ERROR) + if (!fIt->second->fReadOnly && (rc = writeHeader(fIt->second, __LINE__)) != NO_ERROR) { // logged by writeHeader return rc; diff --git a/writeengine/shared/we_chunkmanager.h b/writeengine/shared/we_chunkmanager.h index 484777488..957fc726f 100644 --- a/writeengine/shared/we_chunkmanager.h +++ b/writeengine/shared/we_chunkmanager.h @@ -139,7 +139,8 @@ class CompFileData { public: CompFileData(const FileID& id, const FID& fid, - const execplan::CalpontSystemCatalog::ColDataType colDataType, int colWidth) + const execplan::CalpontSystemCatalog::ColDataType colDataType, int colWidth, + bool readOnly = false) : fFileID(id) , fFid(fid) , fColDataType(colDataType) @@ -147,6 +148,7 @@ class CompFileData , fDctnryCol(false) , fFilePtr(NULL) , fCompressionType(1) + , fReadOnly(readOnly) { } @@ -163,6 +165,7 @@ class CompFileData CompFileHeader fFileHeader; std::list fChunkList; uint32_t fCompressionType; + bool fReadOnly; friend class ChunkManager; }; @@ -179,7 +182,8 @@ class ChunkManager // @brief Retrieve a file pointer in the chunk manager. // for column file IDBDataFile* getFilePtr(const Column& column, uint16_t root, uint32_t partition, uint16_t segment, - std::string& filename, const char* mode, int size, bool useTmpSuffix) const; + std::string& filename, const char* mode, int size, bool useTmpSuffix, + bool isReadOnly = false) const; // @brief Retrieve a file pointer in the chunk manager. // for dictionary file @@ -200,7 +204,7 @@ class ChunkManager // @brief Read a block from pFile at offset fbo. // The data may copied from memory if the chunk it belongs to is already available. - int readBlock(IDBDataFile* pFile, unsigned char* readBuf, uint64_t fbo, bool isReadOnly = false); + int readBlock(IDBDataFile* pFile, unsigned char* readBuf, uint64_t fbo); // @brief Save a block to a chunk in pFile. // The block is not written to disk immediately, will be delayed until flush. @@ -279,7 +283,7 @@ class ChunkManager CompFileData* getFileData(const FID& fid, uint16_t root, uint32_t partition, uint16_t segment, std::string& filename, const char* mode, int size, const execplan::CalpontSystemCatalog::ColDataType colDataType, int colWidth, - bool useTmpSuffix, bool dictnry = false) const; + bool useTmpSuffix, bool dictnry = false, bool isReadOnly = false) const; CompFileData* getFileDataByName(const std::string& filename, const FID& fid, uint16_t root, uint32_t partition, uint16_t segment, const char* mode, int size, @@ -287,7 +291,7 @@ class ChunkManager bool useTmpSuffix, bool dctnry) const; // @brief Retrieve a chunk of pFile from disk. - int fetchChunkFromFile(IDBDataFile* pFile, int64_t id, ChunkData*& chunkData, bool isReadOnly = false); + int fetchChunkFromFile(IDBDataFile* pFile, int64_t id, ChunkData*& chunkData); // @brief Compress a chunk and write it to file. int writeChunkToFile(CompFileData* fileData, int64_t id); @@ -370,7 +374,7 @@ class ChunkManager private: CompFileData* getFileData_(const FileID& fid, const std::string& filename, const char* mode, int size, const execplan::CalpontSystemCatalog::ColDataType colDataType, int colWidth, - bool useTmpSuffix, bool dictnry = false) const; + bool useTmpSuffix, bool dictnry = false, bool isReadOnly = false) const; }; } // namespace WriteEngine diff --git a/writeengine/wrapper/we_colop.cpp b/writeengine/wrapper/we_colop.cpp index 4a9c60adf..fc3a22a2b 100644 --- a/writeengine/wrapper/we_colop.cpp +++ b/writeengine/wrapper/we_colop.cpp @@ -1407,15 +1407,22 @@ bool ColumnOp::isValid(Column& column) const ***********************************************************/ // @bug 5572 - HDFS usage: add *.tmp file backup flag int ColumnOp::openColumnFile(Column& column, std::string& segFile, bool useTmpSuffix, - int ioBuffSize) const + int ioBuffSize, bool isReadOnly) const { if (!isValid(column)) return ERR_INVALID_PARAM; + std::string mode; + + if (isReadOnly) + mode = "r"; + else + mode = "r+b"; + // open column data file column.dataFile.pFile = openFile(column, column.dataFile.fDbRoot, column.dataFile.fPartition, column.dataFile.fSegment, - column.dataFile.fSegFileName, useTmpSuffix, "r+b", ioBuffSize); + column.dataFile.fSegFileName, useTmpSuffix, mode.c_str(), ioBuffSize, isReadOnly); segFile = column.dataFile.fSegFileName; if (column.dataFile.pFile == NULL) @@ -1824,7 +1831,7 @@ int ColumnOp::writeRowsReadOnly(Column& curCol, uint64_t totalRow, const RIDList { curDataFbo = dataFbo; //@Bug 4849. need to check error code to prevent disk error - rc = readBlock(curCol.dataFile.pFile, dataBuf, curDataFbo, true); + rc = readBlock(curCol.dataFile.pFile, dataBuf, curDataFbo); if (rc != NO_ERROR) return rc; diff --git a/writeengine/wrapper/we_colop.h b/writeengine/wrapper/we_colop.h index 1be5d6398..d2f85428e 100644 --- a/writeengine/wrapper/we_colop.h +++ b/writeengine/wrapper/we_colop.h @@ -197,7 +197,7 @@ class ColumnOp : public DbFileOp * segment file that is opened. */ EXPORT virtual int openColumnFile(Column& column, std::string& segFile, bool useTmpSuffix, - int ioBuffSize = DEFAULT_BUFSIZ) const; + int ioBuffSize = DEFAULT_BUFSIZ, bool isReadOnly = false) const; /** * @brief Open table file @@ -264,7 +264,7 @@ class ColumnOp : public DbFileOp */ virtual IDBDataFile* openFile(const Column& column, uint16_t dbRoot, uint32_t partition, uint16_t segment, std::string& segFile, bool useTmpSuffix, const char* mode = "r+b", - int ioBuffSize = DEFAULT_BUFSIZ) const = 0; + int ioBuffSize = DEFAULT_BUFSIZ, bool isReadOnly = false) const = 0; /** * @brief backup blocks to version buffer @@ -287,8 +287,7 @@ class ColumnOp : public DbFileOp /** * @brief populate readBuf with data in block #lbid */ - virtual int readBlock(IDBDataFile* pFile, unsigned char* readBuf, const uint64_t fbo, - bool isReadOnly = false) = 0; + virtual int readBlock(IDBDataFile* pFile, unsigned char* readBuf, const uint64_t fbo) = 0; /** * @brief output writeBuf to pFile starting at position fbo diff --git a/writeengine/wrapper/we_colopcompress.cpp b/writeengine/wrapper/we_colopcompress.cpp index f2a3d22ba..c426d547e 100644 --- a/writeengine/wrapper/we_colopcompress.cpp +++ b/writeengine/wrapper/we_colopcompress.cpp @@ -63,7 +63,7 @@ ColumnOpCompress0::~ColumnOpCompress0() IDBDataFile* ColumnOpCompress0::openFile(const Column& column, const uint16_t dbRoot, const uint32_t partition, const uint16_t segment, std::string& segFile, bool useTmpSuffix, const char* mode, - const int ioBuffSize) const + const int ioBuffSize, bool isReadOnly) const { return FileOp::openFile(column.dataFile.fid, dbRoot, partition, segment, segFile, mode, column.colWidth, useTmpSuffix); @@ -96,7 +96,7 @@ int ColumnOpCompress0::blocksInFile(IDBDataFile* pFile) const } int ColumnOpCompress0::readBlock(IDBDataFile* pFile, unsigned char* readBuf, - const uint64_t fbo, bool isReadOnly) + const uint64_t fbo) { return readDBFile(pFile, readBuf, fbo, true); } @@ -144,10 +144,10 @@ ColumnOpCompress1::~ColumnOpCompress1() IDBDataFile* ColumnOpCompress1::openFile(const Column& column, const uint16_t dbRoot, const uint32_t partition, const uint16_t segment, std::string& segFile, bool useTmpSuffix, const char* mode, - const int ioBuffSize) const + const int ioBuffSize, bool isReadOnly) const { return m_chunkManager->getFilePtr(column, dbRoot, partition, segment, segFile, mode, ioBuffSize, - useTmpSuffix); + useTmpSuffix, isReadOnly); } bool ColumnOpCompress1::abbreviatedExtent(IDBDataFile* pFile, int colWidth) const @@ -161,9 +161,9 @@ int ColumnOpCompress1::blocksInFile(IDBDataFile* pFile) const } int ColumnOpCompress1::readBlock(IDBDataFile* pFile, unsigned char* readBuf, - const uint64_t fbo, bool isReadOnly) + const uint64_t fbo) { - return m_chunkManager->readBlock(pFile, readBuf, fbo, isReadOnly); + return m_chunkManager->readBlock(pFile, readBuf, fbo); } int ColumnOpCompress1::saveBlock(IDBDataFile* pFile, const unsigned char* writeBuf, const uint64_t fbo) diff --git a/writeengine/wrapper/we_colopcompress.h b/writeengine/wrapper/we_colopcompress.h index 7fc04582c..5c92ed885 100644 --- a/writeengine/wrapper/we_colopcompress.h +++ b/writeengine/wrapper/we_colopcompress.h @@ -56,7 +56,7 @@ class ColumnOpCompress0 : public ColumnOp */ IDBDataFile* openFile(const Column& column, uint16_t dbRoot, uint32_t partition, uint16_t segment, std::string& segFile, bool useTmpSuffix, const char* mode = "r+b", - int ioBuffSize = DEFAULT_BUFSIZ) const; + int ioBuffSize = DEFAULT_BUFSIZ, bool isReadOnly = false) const; /** * @brief virtual method in ColumnOp @@ -72,7 +72,7 @@ class ColumnOpCompress0 : public ColumnOp /** * @brief virtual method in ColumnOp */ - int readBlock(IDBDataFile* pFile, unsigned char* readBuf, const uint64_t fbo, bool isReadOnly = false); + int readBlock(IDBDataFile* pFile, unsigned char* readBuf, const uint64_t fbo); /** * @brief virtual method in ColumnOp @@ -112,7 +112,7 @@ class ColumnOpCompress1 : public ColumnOp */ IDBDataFile* openFile(const Column& column, uint16_t dbRoot, uint32_t partition, uint16_t segment, std::string& segFile, bool useTmpSuffix, const char* mode = "r+b", - int ioBuffSize = DEFAULT_BUFSIZ) const; + int ioBuffSize = DEFAULT_BUFSIZ, bool isReadOnly = false) const; /** * @brief virtual method in ColumnOp @@ -161,7 +161,7 @@ class ColumnOpCompress1 : public ColumnOp /** * @brief virtual method in ColumnOp */ - int readBlock(IDBDataFile* pFile, unsigned char* readBuf, const uint64_t fbo, bool isReadOnly = false); + int readBlock(IDBDataFile* pFile, unsigned char* readBuf, const uint64_t fbo); /** * @brief virtual method in ColumnOp diff --git a/writeengine/wrapper/writeengine.cpp b/writeengine/wrapper/writeengine.cpp index c920d717d..465df250f 100644 --- a/writeengine/wrapper/writeengine.cpp +++ b/writeengine/wrapper/writeengine.cpp @@ -5803,9 +5803,9 @@ int WriteEngineWrapper::writeColumnRecUpdate(const TxnID& txnid, const CSCTypesL } string segFile; - bool isFlush = (m_opType != DELETE || !hasAUXCol || (i == colStructList.size() - 1)); + bool isReadOnly = (m_opType == DELETE && hasAUXCol && (i != colStructList.size() - 1)); - rc = colOp->openColumnFile(curCol, segFile, true, IO_BUFF_SIZE); // @bug 5572 HDFS tmp file + rc = colOp->openColumnFile(curCol, segFile, true, IO_BUFF_SIZE, isReadOnly); // @bug 5572 HDFS tmp file if (rc != NO_ERROR) break; @@ -5933,7 +5933,7 @@ int WriteEngineWrapper::writeColumnRecUpdate(const TxnID& txnid, const CSCTypesL #ifdef PROFILE timer.start("writeRows "); #endif - if (!hasAUXCol || (i == colStructList.size() - 1)) + if (!isReadOnly) rc = colOp->writeRows(curCol, totalRow, ridList, valArray, oldValArray, true); else rc = colOp->writeRowsReadOnly(curCol, totalRow, ridList, oldValArray); @@ -5945,7 +5945,7 @@ int WriteEngineWrapper::writeColumnRecUpdate(const TxnID& txnid, const CSCTypesL updateMaxMinRange(1, totalRow, cscColTypeList[i], curColStruct.colType, m_opType == DELETE ? NULL : valArray, oldValArray, cpInfo, false); - colOp->clearColumn(curCol, isFlush); + colOp->clearColumn(curCol, !isReadOnly); if (valArray != NULL) {