database/mariadb-columnstore-engine
1
0
Fork 0
mirror of https://github.com/mariadb-corporation/mariadb-columnstore-engine.git synced 2025-07-30 19:23:07 +03:00
Code Activity

Merge branch 'develop-1.2' into develop-merge-up-20190514

Browse Source
This commit is contained in:
Andrew Hutchings
2019-05-14 13:58:33 +01:00
parent 6c4afdf18b 0dd33c6f75
commit 020b211bb7
83 changed files with 469 additions and 638 deletions
Download Patch File Download Diff File Expand all files Collapse all files

11
writeengine/bulk/cpimport.cpp
View File

@ -830,10 +830,11 @@ void printInputSource(
if (alternateImportDir == IMPORT_PATH_CWD)
{
char cwdBuf[4096];
::getcwd(cwdBuf, sizeof(cwdBuf));
char *bufPtr = &cwdBuf[0];
bufPtr = ::getcwd(cwdBuf, sizeof(cwdBuf));
if (!(BulkLoad::disableConsoleOutput()))
cout << "Input file(s) will be read from : " << cwdBuf << endl;
cout << "Input file(s) will be read from : " << bufPtr << endl;
}
else
{
@ -1021,7 +1022,11 @@ int main(int argc, char** argv)
#ifdef _MSC_VER
_setmaxstdio(2048);
#else
setuid( 0 ); // set effective ID to root; ignore return status
// set effective ID to root
if( setuid( 0 ) < 0 )
{
std::cerr << " cpimport: setuid failed " << std::endl;
}
#endif
setupSignalHandlers();

11
writeengine/bulk/we_tableinfo.cpp
View File

@ -1460,9 +1460,11 @@ void TableInfo::writeBadRows( const std::vector<std::string>* errorDatRows,
if (!p.has_root_path())
{
// We could fail here having fixed size buffer
char cwdPath[4096];
getcwd(cwdPath, sizeof(cwdPath));
boost::filesystem::path rejectFileName2( cwdPath );
char* buffPtr = &cwdPath[0];
buffPtr = getcwd(cwdPath, sizeof(cwdPath));
boost::filesystem::path rejectFileName2( buffPtr );
rejectFileName2 /= fRejectDataFileName;
fBadFiles.push_back( rejectFileName2.string() );
@ -1567,8 +1569,9 @@ void TableInfo::writeErrReason( const std::vector< std::pair<RID,
if (!p.has_root_path())
{
char cwdPath[4096];
getcwd(cwdPath, sizeof(cwdPath));
boost::filesystem::path errFileName2( cwdPath );
char* buffPtr = &cwdPath[0];
buffPtr = getcwd(cwdPath, sizeof(cwdPath));
boost::filesystem::path errFileName2( buffPtr );
errFileName2 /= fRejectErrFileName;
fErrFiles.push_back( errFileName2.string() );

10
writeengine/dictionary/we_dctnry.cpp
View File

@ -807,7 +807,8 @@ int Dctnry::insertDctnry(const char* buf,
while (startPos < totalRow)
{
found = false;
memset(&curSig, 0, sizeof(curSig));
void *curSigPtr = static_cast<void*>(&curSig);
memset(curSigPtr, 0, sizeof(curSig));
curSig.size = pos[startPos][col].offset;
// Strip trailing null bytes '\0' (by adjusting curSig.size) if import-
@ -1321,7 +1322,8 @@ void Dctnry::preLoadStringCache( const DataBlock& fileBlock )
int op = 1; // ordinal position of the string within the block
Signature aSig;
memset( &aSig, 0, sizeof(Signature));
void *aSigPtr = static_cast<void*>(&aSig);
memset(aSigPtr, 0, sizeof(aSig));
while ((offBeg != DCTNRY_END_HEADER) &&
(op <= MAX_STRING_CACHE_SIZE))
@ -1365,8 +1367,10 @@ void Dctnry::preLoadStringCache( const DataBlock& fileBlock )
******************************************************************************/
void Dctnry::addToStringCache( const Signature& newSig )
{
// We better add constructors that sets everything to 0;
Signature asig;
memset(&asig, 0, sizeof(Signature));
void *aSigPtr = static_cast<void*>(&asig);
memset(aSigPtr, 0, sizeof(asig));
asig.signature = new unsigned char[newSig.size];
memcpy(asig.signature, newSig.signature, newSig.size );
asig.size = newSig.size;

6
writeengine/server/we_ddlcommandproc.cpp
View File

@ -2469,7 +2469,8 @@ uint8_t WE_DDLCommandProc::updateSyscolumnTablename(ByteStream& bs, std::string&
//It's the same string for each column, so we just need one dictionary struct
memset(&dictTuple, 0, sizeof(dictTuple));
void *dictTuplePtr = static_cast<void*>(&dictTuple);
memset(dictTuplePtr, 0, sizeof(dictTuple));
dictTuple.sigValue = (unsigned char*)newTablename.c_str();
dictTuple.sigSize = newTablename.length();
dictTuple.isNull = false;
@ -3292,7 +3293,8 @@ uint8_t WE_DDLCommandProc::updateSystablesTablename(ByteStream& bs, std::string&
//It's the same string for each column, so we just need one dictionary struct
memset(&dictTuple, 0, sizeof(dictTuple));
void *dictTuplePtr = static_cast<void*>(&dictTuple);
memset(dictTuplePtr, 0, sizeof(dictTuple));
dictTuple.sigValue = (unsigned char*)newTablename.c_str();
dictTuple.sigSize = newTablename.length();
dictTuple.isNull = false;

46
writeengine/server/we_dmlcommandproc.cpp
View File

@ -722,6 +722,10 @@ uint8_t WE_DMLCommandProc::processSingleInsert(messageqcpp::ByteStream& bs, std:
}
}
// MCOL-1495 Remove fCatalogMap entries CS won't use anymore.
CalpontSystemCatalog::removeCalpontSystemCatalog(sessionId);
CalpontSystemCatalog::removeCalpontSystemCatalog(sessionId | 0x80000000);
return rc;
}
@ -1361,7 +1365,9 @@ uint8_t WE_DMLCommandProc::processBatchInsert(messageqcpp::ByteStream& bs, std::
}
}
//cout << "Batch insert return code " << rc << endl;
// MCOL-1495 Remove fCatalogMap entries CS won't use anymore.
CalpontSystemCatalog::removeCalpontSystemCatalog(sessionId);
CalpontSystemCatalog::removeCalpontSystemCatalog(sessionId | 0x80000000);
return rc;
}
@ -2087,18 +2093,11 @@ uint8_t WE_DMLCommandProc::processBatchInsertBinary(messageqcpp::ByteStream& bs,
args.add(cols);
err = IDBErrorInfo::instance()->errorMsg(WARN_DATA_TRUNC, args);
// Strict mode enabled, so rollback on warning
// NOTE: This doesn't seem to be a possible code path yet
/*if (insertPkg.get_isWarnToError())
{
string applName ("BatchInsert");
fWEWrapper.bulkRollback(tblOid,txnid.id,tableName.toString(),
applName, false, err);
BulkRollbackMgr::deleteMetaFile( tblOid );
}*/
}
}
//cout << "Batch insert return code " << rc << endl;
// MCOL-1495 Remove fCatalogMap entries CS won't use anymore.
CalpontSystemCatalog::removeCalpontSystemCatalog(sessionId);
CalpontSystemCatalog::removeCalpontSystemCatalog(sessionId | 0x80000000);
return rc;
}
@ -2240,6 +2239,9 @@ uint8_t WE_DMLCommandProc::commitBatchAutoOn(messageqcpp::ByteStream& bs, std::s
fWEWrapper.getDictMap().erase(txnID);
}
// MCOL-1495 Remove fCatalogMap entries CS won't use anymore.
CalpontSystemCatalog::removeCalpontSystemCatalog(sessionId);
CalpontSystemCatalog::removeCalpontSystemCatalog(sessionId | 0x80000000);
return rc;
}
@ -2847,16 +2849,14 @@ uint8_t WE_DMLCommandProc::processUpdate(messageqcpp::ByteStream& bs,
convertToRelativeRid (rid, extentNum, blockNum);
rowIDLists.push_back(rid);
uint32_t colWidth = (colTypes[j].colWidth > 8 ? 8 : colTypes[j].colWidth);
// populate stats.blocksChanged
for (unsigned int k = 0; k < columnsUpdated.size(); k++)
int rrid = (int) relativeRID / (BYTE_PER_BLOCK / colWidth);
// populate stats.blocksChanged
if (rrid > preBlkNums[j])
{
if ((int)(relativeRID / (BYTE_PER_BLOCK / colWidth)) > preBlkNums[j])
{
preBlkNums[j] = rrid ;
blocksChanged++;
preBlkNums[j] = relativeRID / (BYTE_PER_BLOCK / colWidth);
}
}
}
}
ridsFetched = true;
@ -3778,7 +3778,6 @@ uint8_t WE_DMLCommandProc::processUpdate(messageqcpp::ByteStream& bs,
err = IDBErrorInfo::instance()->errorMsg(WARN_DATA_TRUNC, args);
}
//cout << "finished update" << endl;
return rc;
}
@ -3961,6 +3960,10 @@ uint8_t WE_DMLCommandProc::processFlushFiles(messageqcpp::ByteStream& bs, std::s
//if (idbdatafile::IDBPolicy::useHdfs())
// cacheutils::dropPrimProcFdCache();
TableMetaData::removeTableMetaData(tableOid);
// MCOL-1495 Remove fCatalogMap entries CS won't use anymore.
CalpontSystemCatalog::removeCalpontSystemCatalog(txnId);
CalpontSystemCatalog::removeCalpontSystemCatalog(txnId | 0x80000000);
return rc;
}
@ -4136,7 +4139,6 @@ uint8_t WE_DMLCommandProc::processDelete(messageqcpp::ByteStream& bs,
}
}
//cout << "WES return rc " << (int)rc << " with msg " << err << endl;
return rc;
}

4
writeengine/server/we_getfilesizes.cpp
View File

@ -269,8 +269,8 @@ int WE_GetFileSizes::processFileName(
std::string& errMsg, int key)
{
uint8_t rc = 0;
off_t fileSize;
off_t compressedFileSize;
off_t fileSize = 0;
off_t compressedFileSize = 0;
errMsg.clear();
try

27
writeengine/shared/we_chunkmanager.cpp
View File

@ -1,4 +1,5 @@
/* Copyright (C) 2014 InfiniDB, Inc.
Copyright (C) 2019 MariaDB Corporation.
This program is free software; you can redistribute it and/or
modify it under the terms of the GNU General Public License
@ -65,6 +66,8 @@ namespace WriteEngine
extern int NUM_BLOCKS_PER_INITIAL_EXTENT; // defined in we_dctnry.cpp
extern WErrorCodes ec; // defined in we_log.cpp
const int COMPRESSED_CHUNK_SIZE = compress::IDBCompressInterface::maxCompressedSize(UNCOMPRESSED_CHUNK_SIZE) + 64 + 3 + 8 * 1024;
//------------------------------------------------------------------------------
// Search for the specified chunk in fChunkList.
//------------------------------------------------------------------------------
@ -1923,10 +1926,22 @@ int ChunkManager::reallocateChunks(CompFileData* fileData)
struct tm ltm;
localtime_r(reinterpret_cast<time_t*>(&tv.tv_sec), &ltm);
char tmText[24];
// this snprintf call causes a compiler warning b/c buffer size is less
// then maximum string size.
#if defined(__GNUC__) && __GNUC__ >= 6
#pragma GCC diagnostic push
#pragma GCC diagnostic ignored "-Wformat-truncation="
snprintf(tmText, sizeof(tmText), ".%04d%02d%02d%02d%02d%02d%06ld",
ltm.tm_year + 1900, ltm.tm_mon + 1,
ltm.tm_mday, ltm.tm_hour, ltm.tm_min,
ltm.tm_sec, tv.tv_usec);
#pragma GCC diagnostic pop
#else
snprintf(tmText, sizeof(tmText), ".%04d%02d%02d%02d%02d%02d%06ld",
ltm.tm_year + 1900, ltm.tm_mon + 1,
ltm.tm_mday, ltm.tm_hour, ltm.tm_min,
ltm.tm_sec, tv.tv_usec);
#endif
string dbgFileName(rlcFileName + tmText);
ostringstream oss;
@ -2106,10 +2121,22 @@ int ChunkManager::reallocateChunks(CompFileData* fileData)
struct tm ltm;
localtime_r(reinterpret_cast<time_t*>(&tv.tv_sec), &ltm);
char tmText[24];
// this snprintf call causes a compiler warning b/c buffer size is less
// then maximum string size.
#if defined(__GNUC__) && __GNUC__ >= 6
#pragma GCC diagnostic push
#pragma GCC diagnostic ignored "-Wformat-truncation="
snprintf(tmText, sizeof(tmText), ".%04d%02d%02d%02d%02d%02d%06ld",
ltm.tm_year + 1900, ltm.tm_mon + 1,
ltm.tm_mday, ltm.tm_hour, ltm.tm_min,
ltm.tm_sec, tv.tv_usec);
#pragma GCC diagnostic pop
#else
snprintf(tmText, sizeof(tmText), ".%04d%02d%02d%02d%02d%02d%06ld",
ltm.tm_year + 1900, ltm.tm_mon + 1,
ltm.tm_mday, ltm.tm_hour, ltm.tm_min,
ltm.tm_sec, tv.tv_usec);
#endif
string dbgFileName(rlcFileName + tmText);
ostringstream oss;

1
writeengine/shared/we_chunkmanager.h
View File

@ -68,7 +68,6 @@ const int UNCOMPRESSED_CHUNK_SIZE = compress::IDBCompressInterface::UNCOMPRESSED
const int COMPRESSED_FILE_HEADER_UNIT = compress::IDBCompressInterface::HDR_BUF_LEN;
// assume UNCOMPRESSED_CHUNK_SIZE > 0xBFFF (49151), 8 * 1024 bytes padding
const int COMPRESSED_CHUNK_SIZE = compress::IDBCompressInterface::maxCompressedSize(UNCOMPRESSED_CHUNK_SIZE) + 64 + 3 + 8 * 1024;
const int BLOCKS_IN_CHUNK = UNCOMPRESSED_CHUNK_SIZE / BYTE_PER_BLOCK;
const int MAXOFFSET_PER_CHUNK = 511 * BYTE_PER_BLOCK;

20
writeengine/shared/we_fileop.cpp
View File

@ -330,12 +330,15 @@ int FileOp::deleteFile( FID fid ) const
std::vector<std::string> dbRootPathList;
Config::getDBRootPathList( dbRootPathList );
int rc;
for (unsigned i = 0; i < dbRootPathList.size(); i++)
{
char rootOidDirName[FILE_NAME_SIZE];
sprintf(rootOidDirName, "%s/%s", dbRootPathList[i].c_str(), oidDirName);
rc = snprintf(rootOidDirName, FILE_NAME_SIZE, "%s/%s",
dbRootPathList[i].c_str(), oidDirName);
if ( IDBPolicy::remove( rootOidDirName ) != 0 )
if ( rc == FILE_NAME_SIZE || IDBPolicy::remove( rootOidDirName ) != 0 )
{
ostringstream oss;
oss << "Unable to remove " << rootOidDirName;
@ -363,6 +366,7 @@ int FileOp::deleteFiles( const std::vector<int32_t>& fids ) const
char dbDir [MAX_DB_DIR_LEVEL][MAX_DB_DIR_NAME_SIZE];
std::vector<std::string> dbRootPathList;
Config::getDBRootPathList( dbRootPathList );
int rc;
for ( unsigned n = 0; n < fids.size(); n++ )
{
@ -376,10 +380,10 @@ int FileOp::deleteFiles( const std::vector<int32_t>& fids ) const
for (unsigned i = 0; i < dbRootPathList.size(); i++)
{
char rootOidDirName[FILE_NAME_SIZE];
sprintf(rootOidDirName, "%s/%s", dbRootPathList[i].c_str(),
rc = snprintf(rootOidDirName, FILE_NAME_SIZE, "%s/%s", dbRootPathList[i].c_str(),
oidDirName);
if ( IDBPolicy::remove( rootOidDirName ) != 0 )
if ( rc == FILE_NAME_SIZE || IDBPolicy::remove( rootOidDirName ) != 0 )
{
ostringstream oss;
oss << "Unable to remove " << rootOidDirName;
@ -410,6 +414,7 @@ int FileOp::deletePartitions( const std::vector<OID>& fids,
char dbDir [MAX_DB_DIR_LEVEL][MAX_DB_DIR_NAME_SIZE];
char rootOidDirName[FILE_NAME_SIZE];
char partitionDirName[FILE_NAME_SIZE];
int rcd, rcp;
for (uint32_t i = 0; i < partitions.size(); i++)
{
@ -420,12 +425,13 @@ int FileOp::deletePartitions( const std::vector<OID>& fids,
dbDir[0], dbDir[1], dbDir[2], dbDir[3], dbDir[4]);
// config expects dbroot starting from 0
std::string rt( Config::getDBRootByNum(partitions[i].lp.dbroot) );
sprintf(rootOidDirName, "%s/%s",
rcd = snprintf(rootOidDirName, FILE_NAME_SIZE, "%s/%s",
rt.c_str(), tempFileName);
sprintf(partitionDirName, "%s/%s",
rcp = snprintf(partitionDirName, FILE_NAME_SIZE, "%s/%s",
rt.c_str(), oidDirName);
if ( IDBPolicy::remove( rootOidDirName ) != 0 )
if ( rcd == FILE_NAME_SIZE || rcp == FILE_NAME_SIZE
|| IDBPolicy::remove( rootOidDirName ) != 0 )
{
ostringstream oss;
oss << "Unable to remove " << rootOidDirName;

2
writeengine/shared/we_index.h
View File

@ -393,7 +393,7 @@ struct IdxMultiColKey
curMask.reset();
curLevel = maxLevel = 0;
totalBit = 0;
memset( testbitArray, 0, IDX_MAX_MULTI_COL_IDX_LEVEL);
memset( testbitArray, 0, IDX_MAX_MULTI_COL_IDX_LEVEL * sizeof(testbitArray[0]));
memset( keyBuf, 0, IDX_MAX_MULTI_COL_BIT / 8 );
curMask = 0x1F;
curMask = curMask << (IDX_MAX_MULTI_COL_BIT - 5);

2
writeengine/shared/we_type.h
View File

@ -489,7 +489,7 @@ struct CacheControl /** @brief Cache control structure */
int checkInterval; /** @brief A check point interval in seconds */
CacheControl()
{
totalBlock = pctFree = checkInterval; /** @brief constructor */
totalBlock = pctFree = checkInterval = 0; /** @brief constructor */
}
};

2
writeengine/splitter/we_filereadthread.cpp
View File

@ -356,7 +356,7 @@ unsigned int WEFileReadThread::readDataFile(messageqcpp::SBS& Sbs)
//char aBuff[1024*1024]; // TODO May have to change it later
//char*pStart = aBuff;
unsigned int aIdx = 0;
unsigned int aLen = 0;
int aLen = 0;
*Sbs << (ByteStream::byte)(WE_CLT_SRV_DATA);
while ((!fInFile.eof()) && (aIdx < getBatchQty()))

9
writeengine/splitter/we_splitterapp.cpp
View File

@ -1,4 +1,5 @@
/* Copyright (C) 2014 InfiniDB, Inc.
Copyright (C) 2019 MariaDB Corporation.
This program is free software; you can redistribute it and/or
modify it under the terms of the GNU General Public License
@ -604,7 +605,13 @@ void WESplitterApp::updateWithJobFile(int aIdx)
int main(int argc, char** argv)
{
std::string err;
setuid(0); //@BUG 4343 set effective userid to root.
// Why do we need this if we don't care about f()'s rc ?
// @BUG4343
if( setuid( 0 ) < 0 )
{
std::cerr << " we_splitterapp: setuid failed " << std::endl;
}
std::cin.sync_with_stdio(false);
try

3
writeengine/wrapper/we_colop.cpp
View File

@ -239,7 +239,8 @@ int ColumnOp::allocRowId(const TxnID& txnid, bool useStartingExtent,
for (i = 0; i < dbRootExtentTrackers.size(); i++)
{
if (i != column.colNo)
uint32_t colNo = column.colNo;
if (i != colNo)
dbRootExtentTrackers[i]->nextSegFile(dbRoot, partition, segment, newHwm, startLbid);
// Round up HWM to the end of the current extent

12
writeengine/xml/we_xmlgenproc.cpp
View File

@ -38,9 +38,9 @@ namespace
{
const char* DICT_TYPE("D");
const char* ENCODING("UTF-8");
const char* JOBNAME("Job_");
const char* LOGNAME("Jobxml_");
const std::string LOGDIR("/log/");
const char* JOBNAME("Job_");
}
namespace WriteEngine
@ -438,10 +438,11 @@ void XMLGenProc::getColumnsForTable(
throw std::runtime_error( oss.str() );
}
}
//------------------------------------------------------------------------------
// Generate Job XML File Name
//------------------------------------------------------------------------------
std::string XMLGenProc::genJobXMLFileName( ) const
{
std::string xmlFileName;
@ -465,7 +466,10 @@ std::string XMLGenProc::genJobXMLFileName( ) const
if (!p.has_root_path())
{
char cwdPath[4096];
getcwd(cwdPath, sizeof(cwdPath));
char *buf;
buf = getcwd(cwdPath, sizeof(cwdPath));
if (buf == NULL)
throw std::runtime_error("Failed to get the current working directory!");
boost::filesystem::path p2(cwdPath);
p2 /= p;
xmlFileName = p2.string();
@ -479,7 +483,7 @@ std::string XMLGenProc::genJobXMLFileName( ) const
return xmlFileName;
}
//------------------------------------------------------------------------------
// writeXMLFile
//------------------------------------------------------------------------------

8
writeengine/xml/we_xmljob.cpp
View File

@ -1313,7 +1313,13 @@ int XMLJob::genJobXMLFileName(
// nothing else to do
#else
char cwdPath[4096];
getcwd(cwdPath, sizeof(cwdPath));
char *err;
err = getcwd(cwdPath, sizeof(cwdPath));
if (err == NULL)
{
errMsg = "Failed to get the current working directory.";
return -1;
}
string trailingPath(xmlFilePath.string());
xmlFilePath = cwdPath;
xmlFilePath /= trailingPath;