database/mariadb-columnstore-engine
1
0
Fork 0
mirror of https://github.com/mariadb-corporation/mariadb-columnstore-engine.git synced 2025-07-30 19:23:07 +03:00
Code Activity

MCOL-5021 non-AUX column files are opened in read-only mode during

Browse Source 
the DELETE operation. ColumnOp::readBlock() calls can cause writes
to database files when the active chunk list in ChunkManager is full.
Since non-AUX columns are read-only for the DELETE operation, we prevent
writes of compressed chunks and header for these columns by passing
an isReadOnly flag to CompFileData which indicates whether the column
is read-only or read-write.
This commit is contained in:
Gagan Goel
2022-04-22 17:06:08 -04:00
parent 35a3a93964
commit 60eb0f86ec
9 changed files with 55 additions and 43 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
writeengine/bulk/we_colopbulk.cpp
View File

@ -63,7 +63,7 @@ ColumnOpBulk::~ColumnOpBulk()
// @bug 5572 - HDFS usage: add *.tmp file backup flag
IDBDataFile* ColumnOpBulk::openFile(const WriteEngine::Column& column, uint16_t dbRoot, uint32_t partition,
uint16_t segment, std::string& segFile, bool useTmpSuffix,
const char* mode, int ioBuffSize) const
const char* mode, int ioBuffSize, bool isReadOnly) const
{
return FileOp::openFile(column.dataFile.fid, dbRoot, partition, segment, segFile, mode, column.colWidth,
useTmpSuffix);
@ -92,7 +92,7 @@ int ColumnOpBulk::blocksInFile(IDBDataFile*) const
//------------------------------------------------------------------------------
// Stub for readBlock
//------------------------------------------------------------------------------
int ColumnOpBulk::readBlock(IDBDataFile*, unsigned char*, const uint64_t, bool)
int ColumnOpBulk::readBlock(IDBDataFile*, unsigned char*, const uint64_t)
{
throw std::logic_error("Unauthorized use of ColumnOpBulk::readBlock");

5
writeengine/bulk/we_colopbulk.h
View File

@ -50,8 +50,9 @@ class ColumnOpBulk : public ColumnOp
virtual int blocksInFile(IDBDataFile*) const;
virtual IDBDataFile* openFile(const WriteEngine::Column& column, uint16_t dbRoot, uint32_t partition,
uint16_t segment, std::string& segFile, bool useTmpSuffix,
const char* mode = "r+b", int ioBuffSize = DEFAULT_BUFSIZ) const;
virtual int readBlock(IDBDataFile*, unsigned char*, const uint64_t, bool isReadOnly = false);
const char* mode = "r+b", int ioBuffSize = DEFAULT_BUFSIZ,
bool isReadOnly = false) const;
virtual int readBlock(IDBDataFile*, unsigned char*, const uint64_t);
virtual int saveBlock(IDBDataFile*, const unsigned char*, const uint64_t);
};

25
writeengine/shared/we_chunkmanager.cpp
View File

@ -252,10 +252,11 @@ int ChunkManager::removeBackups(TxnID txnId)
// @bug 5572 - HDFS usage: add *.tmp file backup flag
IDBDataFile* ChunkManager::getFilePtr(const Column& column, uint16_t root, uint32_t partition,
uint16_t segment, string& filename, const char* mode, int size,
bool useTmpSuffix) const
bool useTmpSuffix, bool isReadOnly) const
{
CompFileData* fileData = getFileData(column.dataFile.fid, root, partition, segment, filename, mode, size,
column.colDataType, column.colWidth, useTmpSuffix);
column.colDataType, column.colWidth, useTmpSuffix,
false, isReadOnly);
return (fileData ? fileData->fFilePtr : NULL);
}
@ -313,7 +314,8 @@ IDBDataFile* ChunkManager::getFilePtrByName(const std::string& filename, FID& fi
CompFileData* ChunkManager::getFileData(const FID& fid, uint16_t root, uint32_t partition, uint16_t segment,
string& filename, const char* mode, int size,
const CalpontSystemCatalog::ColDataType colDataType, int colWidth,
bool useTmpSuffix, bool dctnry) const
bool useTmpSuffix, bool dctnry,
bool isReadOnly) const
{
FileID fileID(fid, root, partition, segment);
map<FileID, CompFileData*>::const_iterator mit = fFileMap.find(fileID);
@ -335,7 +337,7 @@ CompFileData* ChunkManager::getFileData(const FID& fid, uint16_t root, uint32_t
// Initialize the given `filename`.
filename = name;
return getFileData_(fileID, filename, mode, size, colDataType, colWidth, useTmpSuffix, dctnry);
return getFileData_(fileID, filename, mode, size, colDataType, colWidth, useTmpSuffix, dctnry, isReadOnly);
}
CompFileData* ChunkManager::getFileDataByName(const std::string& filename, const FID& fid, uint16_t root,
@ -359,9 +361,9 @@ CompFileData* ChunkManager::getFileDataByName(const std::string& filename, const
CompFileData* ChunkManager::getFileData_(const FileID& fileID, const string& filename, const char* mode,
int size, const CalpontSystemCatalog::ColDataType colDataType,
int colWidth, bool useTmpSuffix, bool dctnry) const
int colWidth, bool useTmpSuffix, bool dctnry, bool isReadOnly) const
{
CompFileData* fileData = new CompFileData(fileID, fileID.fFid, colDataType, colWidth);
CompFileData* fileData = new CompFileData(fileID, fileID.fFid, colDataType, colWidth, isReadOnly);
fileData->fFileName = filename;
if (openFile(fileData, mode, colWidth, useTmpSuffix, __LINE__) != NO_ERROR)
@ -483,7 +485,7 @@ IDBDataFile* ChunkManager::createDctnryFile(const FID& fid, int64_t width, uint1
// Read the block for the specified fbo, from pFile's applicable chunk, and
// into readBuf.
//------------------------------------------------------------------------------
int ChunkManager::readBlock(IDBDataFile* pFile, unsigned char* readBuf, uint64_t fbo, bool isReadOnly)
int ChunkManager::readBlock(IDBDataFile* pFile, unsigned char* readBuf, uint64_t fbo)
{
map<IDBDataFile*, CompFileData*>::iterator fpIt = fFilePtrMap.find(pFile);
@ -504,7 +506,7 @@ int ChunkManager::readBlock(IDBDataFile* pFile, unsigned char* readBuf, uint64_t
// chunk is not already uncompressed
if (chunkData == NULL)
rc = fetchChunkFromFile(pFile, offset.quot, chunkData, isReadOnly);
rc = fetchChunkFromFile(pFile, offset.quot, chunkData);
if (rc == NO_ERROR)
{
@ -639,7 +641,7 @@ int ChunkManager::flushChunks(int rc, const std::map<FID, FID>& columOids)
break;
// finally update the header
if ((rc = writeHeader(fileData, __LINE__)) != NO_ERROR)
if (!fileData->fReadOnly && (rc = writeHeader(fileData, __LINE__)) != NO_ERROR)
break;
//@Bug 4977 remove log file
@ -678,8 +680,7 @@ int ChunkManager::flushChunks(int rc, const std::map<FID, FID>& columOids)
// If the header ptr for the requested chunk is 0 (or has length 0), then
// chunkData is initialized with a new empty chunk.
//------------------------------------------------------------------------------
int ChunkManager::fetchChunkFromFile(IDBDataFile* pFile, int64_t id, ChunkData*& chunkData,
bool isReadOnly)
int ChunkManager::fetchChunkFromFile(IDBDataFile* pFile, int64_t id, ChunkData*& chunkData)
{
// return value
int rc = NO_ERROR;
@ -725,7 +726,7 @@ int ChunkManager::fetchChunkFromFile(IDBDataFile* pFile, int64_t id, ChunkData*&
return rc;
}
if (!isReadOnly && (rc = writeHeader(fIt->second, __LINE__)) != NO_ERROR)
if (!fIt->second->fReadOnly && (rc = writeHeader(fIt->second, __LINE__)) != NO_ERROR)
{
// logged by writeHeader
return rc;

16
writeengine/shared/we_chunkmanager.h
View File

@ -139,7 +139,8 @@ class CompFileData
{
public:
CompFileData(const FileID& id, const FID& fid,
const execplan::CalpontSystemCatalog::ColDataType colDataType, int colWidth)
const execplan::CalpontSystemCatalog::ColDataType colDataType, int colWidth,
bool readOnly = false)
: fFileID(id)
, fFid(fid)
, fColDataType(colDataType)
@ -147,6 +148,7 @@ class CompFileData
, fDctnryCol(false)
, fFilePtr(NULL)
, fCompressionType(1)
, fReadOnly(readOnly)
{
}
@ -163,6 +165,7 @@ class CompFileData
CompFileHeader fFileHeader;
std::list<ChunkData*> fChunkList;
uint32_t fCompressionType;
bool fReadOnly;
friend class ChunkManager;
};
@ -179,7 +182,8 @@ class ChunkManager
// @brief Retrieve a file pointer in the chunk manager.
// for column file
IDBDataFile* getFilePtr(const Column& column, uint16_t root, uint32_t partition, uint16_t segment,
std::string& filename, const char* mode, int size, bool useTmpSuffix) const;
std::string& filename, const char* mode, int size, bool useTmpSuffix,
bool isReadOnly = false) const;
// @brief Retrieve a file pointer in the chunk manager.
// for dictionary file
@ -200,7 +204,7 @@ class ChunkManager
// @brief Read a block from pFile at offset fbo.
// The data may copied from memory if the chunk it belongs to is already available.
int readBlock(IDBDataFile* pFile, unsigned char* readBuf, uint64_t fbo, bool isReadOnly = false);
int readBlock(IDBDataFile* pFile, unsigned char* readBuf, uint64_t fbo);
// @brief Save a block to a chunk in pFile.
// The block is not written to disk immediately, will be delayed until flush.
@ -279,7 +283,7 @@ class ChunkManager
CompFileData* getFileData(const FID& fid, uint16_t root, uint32_t partition, uint16_t segment,
std::string& filename, const char* mode, int size,
const execplan::CalpontSystemCatalog::ColDataType colDataType, int colWidth,
bool useTmpSuffix, bool dictnry = false) const;
bool useTmpSuffix, bool dictnry = false, bool isReadOnly = false) const;
CompFileData* getFileDataByName(const std::string& filename, const FID& fid, uint16_t root,
uint32_t partition, uint16_t segment, const char* mode, int size,
@ -287,7 +291,7 @@ class ChunkManager
bool useTmpSuffix, bool dctnry) const;
// @brief Retrieve a chunk of pFile from disk.
int fetchChunkFromFile(IDBDataFile* pFile, int64_t id, ChunkData*& chunkData, bool isReadOnly = false);
int fetchChunkFromFile(IDBDataFile* pFile, int64_t id, ChunkData*& chunkData);
// @brief Compress a chunk and write it to file.
int writeChunkToFile(CompFileData* fileData, int64_t id);
@ -370,7 +374,7 @@ class ChunkManager
private:
CompFileData* getFileData_(const FileID& fid, const std::string& filename, const char* mode, int size,
const execplan::CalpontSystemCatalog::ColDataType colDataType, int colWidth,
bool useTmpSuffix, bool dictnry = false) const;
bool useTmpSuffix, bool dictnry = false, bool isReadOnly = false) const;
};
} // namespace WriteEngine

13
writeengine/wrapper/we_colop.cpp
View File

@ -1407,15 +1407,22 @@ bool ColumnOp::isValid(Column& column) const
***********************************************************/
// @bug 5572 - HDFS usage: add *.tmp file backup flag
int ColumnOp::openColumnFile(Column& column, std::string& segFile, bool useTmpSuffix,
int ioBuffSize) const
int ioBuffSize, bool isReadOnly) const
{
if (!isValid(column))
return ERR_INVALID_PARAM;
std::string mode;
if (isReadOnly)
mode = "r";
else
mode = "r+b";
// open column data file
column.dataFile.pFile =
openFile(column, column.dataFile.fDbRoot, column.dataFile.fPartition, column.dataFile.fSegment,
column.dataFile.fSegFileName, useTmpSuffix, "r+b", ioBuffSize);
column.dataFile.fSegFileName, useTmpSuffix, mode.c_str(), ioBuffSize, isReadOnly);
segFile = column.dataFile.fSegFileName;
if (column.dataFile.pFile == NULL)
@ -1824,7 +1831,7 @@ int ColumnOp::writeRowsReadOnly(Column& curCol, uint64_t totalRow, const RIDList
{
curDataFbo = dataFbo;
//@Bug 4849. need to check error code to prevent disk error
rc = readBlock(curCol.dataFile.pFile, dataBuf, curDataFbo, true);
rc = readBlock(curCol.dataFile.pFile, dataBuf, curDataFbo);
if (rc != NO_ERROR)
return rc;

7
writeengine/wrapper/we_colop.h
View File

@ -197,7 +197,7 @@ class ColumnOp : public DbFileOp
* segment file that is opened.
*/
EXPORT virtual int openColumnFile(Column& column, std::string& segFile, bool useTmpSuffix,
int ioBuffSize = DEFAULT_BUFSIZ) const;
int ioBuffSize = DEFAULT_BUFSIZ, bool isReadOnly = false) const;
/**
* @brief Open table file
@ -264,7 +264,7 @@ class ColumnOp : public DbFileOp
*/
virtual IDBDataFile* openFile(const Column& column, uint16_t dbRoot, uint32_t partition, uint16_t segment,
std::string& segFile, bool useTmpSuffix, const char* mode = "r+b",
int ioBuffSize = DEFAULT_BUFSIZ) const = 0;
int ioBuffSize = DEFAULT_BUFSIZ, bool isReadOnly = false) const = 0;
/**
* @brief backup blocks to version buffer
@ -287,8 +287,7 @@ class ColumnOp : public DbFileOp
/**
* @brief populate readBuf with data in block #lbid
*/
virtual int readBlock(IDBDataFile* pFile, unsigned char* readBuf, const uint64_t fbo,
bool isReadOnly = false) = 0;
virtual int readBlock(IDBDataFile* pFile, unsigned char* readBuf, const uint64_t fbo) = 0;
/**
* @brief output writeBuf to pFile starting at position fbo

12
writeengine/wrapper/we_colopcompress.cpp
View File

@ -63,7 +63,7 @@ ColumnOpCompress0::~ColumnOpCompress0()
IDBDataFile* ColumnOpCompress0::openFile(const Column& column, const uint16_t dbRoot,
const uint32_t partition, const uint16_t segment,
std::string& segFile, bool useTmpSuffix, const char* mode,
const int ioBuffSize) const
const int ioBuffSize, bool isReadOnly) const
{
return FileOp::openFile(column.dataFile.fid, dbRoot, partition, segment, segFile, mode, column.colWidth,
useTmpSuffix);
@ -96,7 +96,7 @@ int ColumnOpCompress0::blocksInFile(IDBDataFile* pFile) const
}
int ColumnOpCompress0::readBlock(IDBDataFile* pFile, unsigned char* readBuf,
const uint64_t fbo, bool isReadOnly)
const uint64_t fbo)
{
return readDBFile(pFile, readBuf, fbo, true);
}
@ -144,10 +144,10 @@ ColumnOpCompress1::~ColumnOpCompress1()
IDBDataFile* ColumnOpCompress1::openFile(const Column& column, const uint16_t dbRoot,
const uint32_t partition, const uint16_t segment,
std::string& segFile, bool useTmpSuffix, const char* mode,
const int ioBuffSize) const
const int ioBuffSize, bool isReadOnly) const
{
return m_chunkManager->getFilePtr(column, dbRoot, partition, segment, segFile, mode, ioBuffSize,
useTmpSuffix);
useTmpSuffix, isReadOnly);
}
bool ColumnOpCompress1::abbreviatedExtent(IDBDataFile* pFile, int colWidth) const
@ -161,9 +161,9 @@ int ColumnOpCompress1::blocksInFile(IDBDataFile* pFile) const
}
int ColumnOpCompress1::readBlock(IDBDataFile* pFile, unsigned char* readBuf,
const uint64_t fbo, bool isReadOnly)
const uint64_t fbo)
{
return m_chunkManager->readBlock(pFile, readBuf, fbo, isReadOnly);
return m_chunkManager->readBlock(pFile, readBuf, fbo);
}
int ColumnOpCompress1::saveBlock(IDBDataFile* pFile, const unsigned char* writeBuf, const uint64_t fbo)

8
writeengine/wrapper/we_colopcompress.h
View File

@ -56,7 +56,7 @@ class ColumnOpCompress0 : public ColumnOp
*/
IDBDataFile* openFile(const Column& column, uint16_t dbRoot, uint32_t partition, uint16_t segment,
std::string& segFile, bool useTmpSuffix, const char* mode = "r+b",
int ioBuffSize = DEFAULT_BUFSIZ) const;
int ioBuffSize = DEFAULT_BUFSIZ, bool isReadOnly = false) const;
/**
* @brief virtual method in ColumnOp
@ -72,7 +72,7 @@ class ColumnOpCompress0 : public ColumnOp
/**
* @brief virtual method in ColumnOp
*/
int readBlock(IDBDataFile* pFile, unsigned char* readBuf, const uint64_t fbo, bool isReadOnly = false);
int readBlock(IDBDataFile* pFile, unsigned char* readBuf, const uint64_t fbo);
/**
* @brief virtual method in ColumnOp
@ -112,7 +112,7 @@ class ColumnOpCompress1 : public ColumnOp
*/
IDBDataFile* openFile(const Column& column, uint16_t dbRoot, uint32_t partition, uint16_t segment,
std::string& segFile, bool useTmpSuffix, const char* mode = "r+b",
int ioBuffSize = DEFAULT_BUFSIZ) const;
int ioBuffSize = DEFAULT_BUFSIZ, bool isReadOnly = false) const;
/**
* @brief virtual method in ColumnOp
@ -161,7 +161,7 @@ class ColumnOpCompress1 : public ColumnOp
/**
* @brief virtual method in ColumnOp
*/
int readBlock(IDBDataFile* pFile, unsigned char* readBuf, const uint64_t fbo, bool isReadOnly = false);
int readBlock(IDBDataFile* pFile, unsigned char* readBuf, const uint64_t fbo);
/**
* @brief virtual method in ColumnOp

8
writeengine/wrapper/writeengine.cpp
View File

@ -5803,9 +5803,9 @@ int WriteEngineWrapper::writeColumnRecUpdate(const TxnID& txnid, const CSCTypesL
}
string segFile;
bool isFlush = (m_opType != DELETE || !hasAUXCol || (i == colStructList.size() - 1));
bool isReadOnly = (m_opType == DELETE && hasAUXCol && (i != colStructList.size() - 1));
rc = colOp->openColumnFile(curCol, segFile, true, IO_BUFF_SIZE); // @bug 5572 HDFS tmp file
rc = colOp->openColumnFile(curCol, segFile, true, IO_BUFF_SIZE, isReadOnly); // @bug 5572 HDFS tmp file
if (rc != NO_ERROR)
break;
@ -5933,7 +5933,7 @@ int WriteEngineWrapper::writeColumnRecUpdate(const TxnID& txnid, const CSCTypesL
#ifdef PROFILE
timer.start("writeRows ");
#endif
if (!hasAUXCol || (i == colStructList.size() - 1))
if (!isReadOnly)
rc = colOp->writeRows(curCol, totalRow, ridList, valArray, oldValArray, true);
else
rc = colOp->writeRowsReadOnly(curCol, totalRow, ridList, oldValArray);
@ -5945,7 +5945,7 @@ int WriteEngineWrapper::writeColumnRecUpdate(const TxnID& txnid, const CSCTypesL
updateMaxMinRange(1, totalRow, cscColTypeList[i], curColStruct.colType,
m_opType == DELETE ? NULL : valArray, oldValArray, cpInfo, false);
colOp->clearColumn(curCol, isFlush);
colOp->clearColumn(curCol, !isReadOnly);
if (valArray != NULL)
{