database/mariadb-columnstore-engine
1
0
Fork 0
mirror of https://github.com/mariadb-corporation/mariadb-columnstore-engine.git synced 2025-07-29 08:21:15 +03:00
Code Activity

Fixes of bugs from ASAN warnings, part one (#2796)

Browse Source
This commit is contained in:
Leonid Fedorov
2023-03-30 18:29:04 +03:00
committed by GitHub
parent a1d20d82d5
commit 2f153184c3
71 changed files with 591 additions and 2038 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
writeengine/build/startdbrm
View File

@ -1,2 +0,0 @@
nohup controlernode &
nohup workernode DBRM_Worker1 &

48
writeengine/build/we_rules.mak
View File

@ -1,48 +0,0 @@
IPCS_CLEANUP=$(EXPORT_ROOT)/bin/dbrm stop; ipcs-pat -d > /dev/null; sleep 2; $(EXPORT_ROOT)/bin/dbrm start;
CPPFLAGS=-I. -I$(EXPORT_ROOT)/include -I/usr/include/libxml2
CXXFLAGS+=$(DEBUG_FLAGS) -D_FILE_OFFSET_BITS=64 -Wall -fpic
LLIBS=-L. -L$(EXPORT_ROOT)/lib -lxml2 -lrwlock -lconfigcpp -lmessageqcpp -lbrm -lboost_thread \
-lloggingcpp -lboost_date_time -lboost_filesystem
TLIBS=$(LLIBS) -lcppunit -ldl
GLIBS=$(TLIBS:-lcppunit=)
LIBDIR=../obj
LOBJS_SHARED= \
$(LIBDIR)/we_blockop.o \
$(LIBDIR)/we_brm.o \
$(LIBDIR)/we_bulkrollbackmgr.o \
$(LIBDIR)/we_bulkrollbackfile.o \
$(LIBDIR)/we_bulkrollbackfilecompressed.o \
$(LIBDIR)/we_bulkrollbackfilecompressedhdfs.o \
$(LIBDIR)/we_cache.o \
$(LIBDIR)/we_chunkmanager.o \
$(LIBDIR)/we_config.o \
$(LIBDIR)/we_confirmhdfsdbfile.o \
$(LIBDIR)/we_convertor.o \
$(LIBDIR)/we_dbfileop.o \
$(LIBDIR)/we_dbrootextenttracker.o \
$(LIBDIR)/we_define.o \
$(LIBDIR)/we_fileop.o \
$(LIBDIR)/we_log.o \
$(LIBDIR)/we_rbmetawriter.o \
$(LIBDIR)/we_simplesyslog.o \
$(LIBDIR)/we_stats.o
LOBJS_INDEX= \
$(LIBDIR)/we_freemgr.o \
$(LIBDIR)/we_indexlist.o \
$(LIBDIR)/we_indexlist_common.o \
$(LIBDIR)/we_indexlist_find_delete.o \
$(LIBDIR)/we_indexlist_multiple_narray.o \
$(LIBDIR)/we_indexlist_narray.o \
$(LIBDIR)/we_indexlist_update_hdr_sub.o \
$(LIBDIR)/we_indextree.o
LOBJS_DCTNRY= $(LIBDIR)/we_dctnry.o
LOBJS_XML= $(LIBDIR)/we_xmlop.o \
$(LIBDIR)/we_xmljob.o \
$(LIBDIR)/we_xmlgendata.o \
$(LIBDIR)/we_xmlgenproc.o

100
writeengine/bulk/cpimport.rc
View File

@ -1,100 +0,0 @@
// Microsoft Visual C++ generated resource script.
//
#include "resource.h"
#define APSTUDIO_READONLY_SYMBOLS
/////////////////////////////////////////////////////////////////////////////
//
// Generated from the TEXTINCLUDE 2 resource.
//
#include "afxres.h"
/////////////////////////////////////////////////////////////////////////////
#undef APSTUDIO_READONLY_SYMBOLS
/////////////////////////////////////////////////////////////////////////////
// English (United States) resources
#if !defined(AFX_RESOURCE_DLL) || defined(AFX_TARG_ENU)
LANGUAGE LANG_ENGLISH, SUBLANG_ENGLISH_US
#pragma code_page(1252)
#ifdef APSTUDIO_INVOKED
/////////////////////////////////////////////////////////////////////////////
//
// TEXTINCLUDE
//
1 TEXTINCLUDE
BEGIN
"resource.h\0"
END
2 TEXTINCLUDE
BEGIN
"#include ""afxres.h""\r\n"
"\0"
END
3 TEXTINCLUDE
BEGIN
"\r\n"
"\0"
END
#endif // APSTUDIO_INVOKED
/////////////////////////////////////////////////////////////////////////////
//
// Version
//
VS_VERSION_INFO VERSIONINFO
FILEVERSION 4,6,0,0
PRODUCTVERSION 4,6,0,0
FILEFLAGSMASK 0x17L
#ifdef _DEBUG
FILEFLAGS 0x1L
#else
FILEFLAGS 0x0L
#endif
FILEOS 0x4L
FILETYPE 0x1L
FILESUBTYPE 0x0L
BEGIN
BLOCK "StringFileInfo"
BEGIN
BLOCK "040904b0"
BEGIN
VALUE "CompanyName", "InfiniDB"
VALUE "FileDescription", "InfiniDB Bulk Loader"
VALUE "FileVersion", "4.6.0-0"
VALUE "InternalName", "cpimport"
VALUE "LegalCopyright", "Copyright (C) 2014"
VALUE "OriginalFilename", "cpimport.exe"
VALUE "ProductName", "InfiniDB"
VALUE "ProductVersion", "4.6.0.0 Beta"
END
END
BLOCK "VarFileInfo"
BEGIN
VALUE "Translation", 0x409, 1200
END
END
#endif // English (United States) resources
/////////////////////////////////////////////////////////////////////////////
#ifndef APSTUDIO_INVOKED
/////////////////////////////////////////////////////////////////////////////
//
// Generated from the TEXTINCLUDE 3 resource.
//
/////////////////////////////////////////////////////////////////////////////
#endif // not APSTUDIO_INVOKED

10
writeengine/bulk/we_bulkloadbuffer.cpp
View File

@ -2415,12 +2415,10 @@ void BulkLoadBuffer::tokenize(const boost::ptr_vector<ColumnInfo>& columnsInfo,
//------------------------------------------------------------------
case FLD_PARSE_ENCLOSED_STATE:
{
char next = *(p + 1);
if ((p + 1 < pEndOfData) &&
(((c == ESCAPE_CHAR) && ((next == STRING_ENCLOSED_CHAR) || (next == ESCAPE_CHAR) ||
(next == LINE_FEED) || (next == CARRIAGE_RETURN))) ||
((c == STRING_ENCLOSED_CHAR) && (next == STRING_ENCLOSED_CHAR))))
(((c == ESCAPE_CHAR) && ((*(p + 1) == STRING_ENCLOSED_CHAR) || (*(p + 1) == ESCAPE_CHAR) ||
(*(p + 1) == LINE_FEED) || (*(p + 1) == CARRIAGE_RETURN))) ||
((c == STRING_ENCLOSED_CHAR) && (*(p + 1) == STRING_ENCLOSED_CHAR))))
{
// Create/save original data before stripping out bytes
if (rawDataRowLength == 0)
@ -2515,7 +2513,7 @@ void BulkLoadBuffer::tokenize(const boost::ptr_vector<ColumnInfo>& columnsInfo,
// cout << "triming ... " << endl;
char* tmp = p;
while (*(--tmp) == ' ')
while (tmp != lastRowHead && *(--tmp) == ' ')
{
// cout << "offset is " << offset <<endl;
offset--;

102
writeengine/libwriteengine.rc
View File

@ -1,102 +0,0 @@
// Microsoft Visual C++ generated resource script.
//
#include "resource.h"
#define APSTUDIO_READONLY_SYMBOLS
/////////////////////////////////////////////////////////////////////////////
//
// Generated from the TEXTINCLUDE 2 resource.
//
#include "afxres.h"
/////////////////////////////////////////////////////////////////////////////
#undef APSTUDIO_READONLY_SYMBOLS
/////////////////////////////////////////////////////////////////////////////
// English (U.S.) resources
#if !defined(AFX_RESOURCE_DLL) || defined(AFX_TARG_ENU)
#ifdef _WIN32
LANGUAGE LANG_ENGLISH, SUBLANG_ENGLISH_US
#pragma code_page(1252)
#endif //_WIN32
#ifdef APSTUDIO_INVOKED
/////////////////////////////////////////////////////////////////////////////
//
// TEXTINCLUDE
//
1 TEXTINCLUDE
BEGIN
"resource.h\0"
END
2 TEXTINCLUDE
BEGIN
"#include ""afxres.h""\r\n"
"\0"
END
3 TEXTINCLUDE
BEGIN
"\r\n"
"\0"
END
#endif // APSTUDIO_INVOKED
/////////////////////////////////////////////////////////////////////////////
//
// Version
//
VS_VERSION_INFO VERSIONINFO
FILEVERSION 4,6,0,0
PRODUCTVERSION 4,6,0,0
FILEFLAGSMASK 0x17L
#ifdef _DEBUG
FILEFLAGS 0x1L
#else
FILEFLAGS 0x0L
#endif
FILEOS 0x4L
FILETYPE 0x2L
FILESUBTYPE 0x0L
BEGIN
BLOCK "StringFileInfo"
BEGIN
BLOCK "040904b0"
BEGIN
VALUE "CompanyName", "InfiniDB, Inc."
VALUE "FileDescription", "InfiniDB Write Engine API"
VALUE "FileVersion", "4.6.0-0"
VALUE "InternalName", "libwriteengine"
VALUE "LegalCopyright", "Copyright (C) 2014"
VALUE "OriginalFilename", "libwriteengine.dll"
VALUE "ProductName", "InfiniDB"
VALUE "ProductVersion", "4.6.0.0 Beta"
END
END
BLOCK "VarFileInfo"
BEGIN
VALUE "Translation", 0x409, 1200
END
END
#endif // English (U.S.) resources
/////////////////////////////////////////////////////////////////////////////
#ifndef APSTUDIO_INVOKED
/////////////////////////////////////////////////////////////////////////////
//
// Generated from the TEXTINCLUDE 3 resource.
//
/////////////////////////////////////////////////////////////////////////////
#endif // not APSTUDIO_INVOKED

102
writeengine/server/WriteEngineServer.rc
View File

@ -1,102 +0,0 @@
// Microsoft Visual C++ generated resource script.
//
#include "resource.h"
#define APSTUDIO_READONLY_SYMBOLS
/////////////////////////////////////////////////////////////////////////////
//
// Generated from the TEXTINCLUDE 2 resource.
//
#include "afxres.h"
/////////////////////////////////////////////////////////////////////////////
#undef APSTUDIO_READONLY_SYMBOLS
/////////////////////////////////////////////////////////////////////////////
// English (U.S.) resources
#if !defined(AFX_RESOURCE_DLL) || defined(AFX_TARG_ENU)
#ifdef _WIN32
LANGUAGE LANG_ENGLISH, SUBLANG_ENGLISH_US
#pragma code_page(1252)
#endif //_WIN32
#ifdef APSTUDIO_INVOKED
/////////////////////////////////////////////////////////////////////////////
//
// TEXTINCLUDE
//
1 TEXTINCLUDE
BEGIN
"resource.h\0"
END
2 TEXTINCLUDE
BEGIN
"#include ""afxres.h""\r\n"
"\0"
END
3 TEXTINCLUDE
BEGIN
"\r\n"
"\0"
END
#endif // APSTUDIO_INVOKED
/////////////////////////////////////////////////////////////////////////////
//
// Version
//
VS_VERSION_INFO VERSIONINFO
FILEVERSION 4,6,0,0
PRODUCTVERSION 4,6,0,0
FILEFLAGSMASK 0x17L
#ifdef _DEBUG
FILEFLAGS 0x1L
#else
FILEFLAGS 0x0L
#endif
FILEOS 0x4L
FILETYPE 0x1L
FILESUBTYPE 0x0L
BEGIN
BLOCK "StringFileInfo"
BEGIN
BLOCK "040904b0"
BEGIN
VALUE "CompanyName", "InfiniDB, Inc."
VALUE "FileDescription", "InfiniDB Write Engine Server"
VALUE "FileVersion", "4.6.0-0"
VALUE "InternalName", "WriteEngineServer"
VALUE "LegalCopyright", "Copyright (C) 2014"
VALUE "OriginalFilename", "WriteEngineServer.exe"
VALUE "ProductName", "InfiniDB"
VALUE "ProductVersion", "4.6.0.0 Beta"
END
END
BLOCK "VarFileInfo"
BEGIN
VALUE "Translation", 0x409, 1200
END
END
#endif // English (U.S.) resources
/////////////////////////////////////////////////////////////////////////////
#ifndef APSTUDIO_INVOKED
/////////////////////////////////////////////////////////////////////////////
//
// Generated from the TEXTINCLUDE 3 resource.
//
/////////////////////////////////////////////////////////////////////////////
#endif // not APSTUDIO_INVOKED

32
writeengine/shared/we_fileop.cpp
View File

@ -60,7 +60,7 @@ namespace WriteEngine
{
/*static*/ boost::mutex FileOp::m_createDbRootMutexes;
/*static*/ boost::mutex FileOp::m_mkdirMutex;
/*static*/ std::map<int, boost::mutex*> FileOp::m_DbRootAddExtentMutexes;
/*static*/ std::map<int, boost::mutex> FileOp::m_DbRootAddExtentMutexes;
// in 1 call to fwrite(), during initialization
// StopWatch timer;
@ -993,7 +993,7 @@ int FileOp::initColumnExtent(IDBDataFile* pFile, uint16_t dbRoot, int nBlocks, c
Stats::startParseEvent(WE_STATS_WAIT_TO_CREATE_COL_EXTENT);
#endif
boost::mutex::scoped_lock lk(*m_DbRootAddExtentMutexes[dbRoot]);
boost::mutex::scoped_lock lk(m_DbRootAddExtentMutexes[dbRoot]);
#ifdef PROFILE
if (bExpandExtent)
@ -1714,7 +1714,7 @@ int FileOp::initDctnryExtent(IDBDataFile* pFile, uint16_t dbRoot, int nBlocks, u
Stats::startParseEvent(WE_STATS_WAIT_TO_CREATE_DCT_EXTENT);
#endif
boost::mutex::scoped_lock lk(*m_DbRootAddExtentMutexes[dbRoot]);
boost::mutex::scoped_lock lk(m_DbRootAddExtentMutexes[dbRoot]);
#ifdef PROFILE
if (bExpandExtent)
@ -1809,33 +1809,13 @@ void FileOp::initDbRootExtentMutexes()
for (size_t i = 0; i < rootIds.size(); i++)
{
boost::mutex* pM = new boost::mutex;
m_DbRootAddExtentMutexes[rootIds[i]] = pM;
m_DbRootAddExtentMutexes.emplace(std::piecewise_construct,
std::forward_as_tuple(rootIds[i]),
std::forward_as_tuple());
}
}
}
/***********************************************************
* DESCRIPTION:
* Cleans up memory allocated to the DBRoot extent mutexes. Calling
* this function is not necessary, but it is provided for completeness,
* to complement initDbRootExtentMutexes(), and to provide a way to
* free up memory at the end of program execution.
***********************************************************/
/* static */
void FileOp::removeDbRootExtentMutexes()
{
boost::mutex::scoped_lock lk(m_createDbRootMutexes);
std::map<int, boost::mutex*>::iterator k = m_DbRootAddExtentMutexes.begin();
while (k != m_DbRootAddExtentMutexes.end())
{
delete k->second;
++k;
}
}
/***********************************************************
* DESCRIPTION:
* Write out (reinitialize) a partial extent in a column file.

3
writeengine/shared/we_fileop.h
View File

@ -443,7 +443,6 @@ class FileOp : public BlockOp, public WeUIDGID
execplan::CalpontSystemCatalog::ColDataType colDataType);
static void initDbRootExtentMutexes();
static void removeDbRootExtentMutexes();
int writeInitialCompColumnChunk(IDBDataFile* pFile, int nBlocksAllocated, int nRows,
const uint8_t* emptyVal, int width, BRM::LBID_t lbid,
@ -457,7 +456,7 @@ class FileOp : public BlockOp, public WeUIDGID
static boost::mutex m_createDbRootMutexes;
// Mutexes used to serialize extent creation within each DBRoot
static std::map<int, boost::mutex*> m_DbRootAddExtentMutexes;
static std::map<int, boost::mutex> m_DbRootAddExtentMutexes;
// protect race condition in creating directories
static boost::mutex m_mkdirMutex;

4
writeengine/shared/we_type.h
View File

@ -399,8 +399,8 @@ struct JobColumn /** @brief Job Column Structure */
int compressionType; /** @brief compression type */
bool autoIncFlag; /** @brief auto increment flag */
DctnryStruct dctnry; /** @brief dictionary structure */
int64_t fMinIntSat; /** @brief For integer type, the min saturation value */
uint64_t fMaxIntSat; /** @brief For integer type, the max saturation value */
int128_t fMinIntSat; /** @brief For integer type, the min saturation value */
uint128_t fMaxIntSat; /** @brief For integer type, the max saturation value */
double fMinDblSat; /** @brief for float/double, the min saturation value */
double fMaxDblSat; /** @brief for float/double, the max saturation value */
bool fWithDefault; /** @brief With default */

101
writeengine/splitter/splitter.rc
View File

@ -1,101 +0,0 @@
// Microsoft Visual C++ generated resource script.
//
#include "resource.h"
#define APSTUDIO_READONLY_SYMBOLS
/////////////////////////////////////////////////////////////////////////////
//
// Generated from the TEXTINCLUDE 2 resource.
//
#include "afxres.h"
/////////////////////////////////////////////////////////////////////////////
#undef APSTUDIO_READONLY_SYMBOLS
/////////////////////////////////////////////////////////////////////////////
// English (U.S.) resources
#if !defined(AFX_RESOURCE_DLL) || defined(AFX_TARG_ENU)
#ifdef _WIN32
LANGUAGE LANG_ENGLISH, SUBLANG_ENGLISH_US
#pragma code_page(1252)
#endif //_WIN32
#ifdef APSTUDIO_INVOKED
/////////////////////////////////////////////////////////////////////////////
//
// TEXTINCLUDE
//
1 TEXTINCLUDE
BEGIN
"resource.h\0"
END
2 TEXTINCLUDE
BEGIN
"#include ""afxres.h""\r\n"
"\0"
END
3 TEXTINCLUDE
BEGIN
"\r\n"
"\0"
END
#endif // APSTUDIO_INVOKED
/////////////////////////////////////////////////////////////////////////////
//
// Version
//
VS_VERSION_INFO VERSIONINFO
FILEVERSION 4,6,0,0
PRODUCTVERSION 4,6,0,0
FILEFLAGSMASK 0x17L
#ifdef _DEBUG
FILEFLAGS 0x1L
#else
FILEFLAGS 0x0L
#endif
FILEOS 0x4L
FILETYPE 0x1L
FILESUBTYPE 0x0L
BEGIN
BLOCK "StringFileInfo"
BEGIN
BLOCK "040904b0"
BEGIN
VALUE "FileDescription", "InfiniDB Bulk Loader"
VALUE "FileVersion", "4.6.0-0"
VALUE "InternalName", "splitter"
VALUE "LegalCopyright", "Copyright (C) 2014"
VALUE "OriginalFilename", "cpimport.exe"
VALUE "ProductName", "InfiniDB"
VALUE "ProductVersion", "4.6.0.0 Beta"
END
END
BLOCK "VarFileInfo"
BEGIN
VALUE "Translation", 0x409, 1200
END
END
#endif // English (U.S.) resources
/////////////////////////////////////////////////////////////////////////////
#ifndef APSTUDIO_INVOKED
/////////////////////////////////////////////////////////////////////////////
//
// Generated from the TEXTINCLUDE 3 resource.
//
/////////////////////////////////////////////////////////////////////////////
#endif // not APSTUDIO_INVOKED

2
writeengine/splitter/we_cmdargs.cpp
View File

@ -464,7 +464,7 @@ bool WECmdArgs::str2PmList(std::string& PmList, VecInts& V)
if (aLen > 0)
{
strncpy(aBuff, PmList.c_str(), BUFLEN);
strncpy(aBuff, PmList.c_str(), BUFLEN - 1);
aBuff[BUFLEN - 1] = 0;
}
else

31
writeengine/wrapper/writeengine.cpp
View File

@ -65,6 +65,7 @@ using namespace idbdatafile;
#include "dataconvert.h"
#include "string_prefixes.h"
#include "mcs_decimal.h"
namespace WriteEngine
//#define PROFILE 1
@ -917,7 +918,7 @@ int WriteEngineWrapper::fillColumn(const TxnID& txnid, const OID& dataOid,
Column refCol;
ColType newColType;
ColType refColType;
boost::scoped_array<char> defVal(new char[MAX_COLUMN_BOUNDARY]);
boost::scoped_array<char> defVal(new char[datatypes::MAXDECIMALWIDTH]);
ColumnOp* colOpNewCol = m_colOp[op(compressionType)];
ColumnOp* refColOp = m_colOp[op(refCompressionType)];
Dctnry* dctnry = m_dctnry[op(compressionType)];
@ -992,6 +993,29 @@ int WriteEngineWrapper::fillColumn(const TxnID& txnid, const OID& dataOid,
return rc;
}
// TODO: Get rid of this
void emptyValueToAny(boost::any* any, const uint8_t* emptyValue, int colWidth)
{
switch (colWidth)
{
case 16:
*any = *(uint128_t*)emptyValue;
break;
case 8:
*any = *(uint64_t*)emptyValue;
break;
case 4:
*any = *(uint32_t*)emptyValue;
break;
case 2:
*any = *(uint16_t*)emptyValue;
break;
default:
*any = *emptyValue;
}
}
int WriteEngineWrapper::deleteRow(const TxnID& txnid, const vector<CSCTypesList>& colExtentsColType,
vector<ColStructList>& colExtentsStruct, vector<void*>& colOldValueList,
vector<RIDList>& ridLists, const int32_t tableOid, bool hasAUXCol)
@ -1032,10 +1056,7 @@ int WriteEngineWrapper::deleteRow(const TxnID& txnid, const vector<CSCTypesList>
const uint8_t* emptyVal = m_colOp[op(curColStruct.fCompressionType)]->getEmptyRowValue(
curColStruct.colDataType, curColStruct.colWidth);
if (curColStruct.colWidth == datatypes::MAXDECIMALWIDTH)
curTuple.data = *(int128_t*)emptyVal;
else
curTuple.data = *(int64_t*)emptyVal;
emptyValueToAny(&curTuple.data, emptyVal, curColStruct.colWidth);
curTupleList.push_back(curTuple);
colValueList.push_back(curTupleList);

30
writeengine/xml/we_xmljob.cpp
View File

@ -21,6 +21,7 @@
*******************************************************************************/
/** @file */
#include "mcs_basic_types.h"
#define WRITEENGINEXMLJOB_DLLEXPORT
#include "we_xmljob.h"
#undef WRITEENGINEXMLJOB_DLLEXPORT
@ -46,28 +47,6 @@ using namespace execplan;
namespace WriteEngine
{
// Maximum saturation value for DECIMAL types based on precision
// TODO MCOL-641 add support here. see dataconvert.cpp
const long long columnstore_precision[19] = {0,
9,
99,
999,
9999,
99999,
999999,
9999999,
99999999,
999999999,
9999999999LL,
99999999999LL,
999999999999LL,
9999999999999LL,
99999999999999LL,
999999999999999LL,
9999999999999999LL,
99999999999999999LL,
999999999999999999LL};
//------------------------------------------------------------------------------
// Constructor
//------------------------------------------------------------------------------
@ -695,13 +674,14 @@ void XMLJob::initSatLimits(JobColumn& curColumn) const
}
else if (curColumn.typeName == ColDataTypeStr[CalpontSystemCatalog::DECIMAL])
{
curColumn.fMinIntSat = -columnstore_precision[curColumn.precision];
curColumn.fMaxIntSat = columnstore_precision[curColumn.precision];
curColumn.fMaxIntSat = dataconvert::decimalRangeUp<int128_t>(curColumn.precision);
curColumn.fMinIntSat = -curColumn.fMaxIntSat;
}
else if (curColumn.typeName == ColDataTypeStr[CalpontSystemCatalog::UDECIMAL])
{
curColumn.fMinIntSat = 0;
curColumn.fMaxIntSat = columnstore_precision[curColumn.precision];
curColumn.fMaxIntSat = dataconvert::decimalRangeUp<int128_t>(curColumn.precision);
}
else if (curColumn.typeName == ColDataTypeStr[CalpontSystemCatalog::FLOAT])
{