From b25fee320ac6c9e06901fcdd527e3f24302cd950 Mon Sep 17 00:00:00 2001 From: Alexey Antipovsky Date: Wed, 11 Nov 2020 05:37:51 +0000 Subject: [PATCH] Remove variable-length arrays (-Wvla) --- dbcon/joblist/tuplehashjoin.cpp | 3 +- dbcon/mysql/ha_mcs_execplan.cpp | 6 +- dbcon/mysql/ha_window_function.cpp | 7 +- oam/oamcpp/liboamcpp.cpp | 7 +- oamapps/serverMonitor/cpuMonitor.cpp | 2 +- primitives/linux-port/dictionary.cpp | 6 +- .../primproc/batchprimitiveprocessor.cpp | 5 +- storage-manager/src/IOCoordinator.cpp | 16 +- storage-manager/src/unit_tests.cpp | 186 +++++++++--------- utils/common/vlarray.h | 100 ++++++++++ utils/funcexp/func_char.cpp | 4 +- utils/funcexp/func_strcmp.cpp | 2 +- utils/idbdatafile/IDBPolicy.cpp | 9 +- utils/joiner/tuplejoiner.cpp | 15 +- utils/rowgroup/rowaggregation.cpp | 6 +- utils/windowfunction/wf_udaf.cpp | 11 +- 16 files changed, 250 insertions(+), 135 deletions(-) create mode 100644 utils/common/vlarray.h diff --git a/dbcon/joblist/tuplehashjoin.cpp b/dbcon/joblist/tuplehashjoin.cpp index 3b6c0639d..e7ddbb235 100644 --- a/dbcon/joblist/tuplehashjoin.cpp +++ b/dbcon/joblist/tuplehashjoin.cpp @@ -44,6 +44,7 @@ using namespace std; #include "tupleaggregatestep.h" #include "errorids.h" #include "diskjoinstep.h" +#include "vlarray.h" using namespace execplan; using namespace joiner; @@ -295,7 +296,7 @@ void TupleHashJoinStep::startSmallRunners(uint index) */ stopMemTracking = false; - uint64_t jobs[numCores]; + utils::VLArray jobs(numCores); uint64_t memMonitor = jobstepThreadPool.invoke([this, index] { this->trackMem(index); }); // starting 1 thread when in PM mode, since it's only inserting into a // vector of rows. The rest will be started when converted to UM mode. diff --git a/dbcon/mysql/ha_mcs_execplan.cpp b/dbcon/mysql/ha_mcs_execplan.cpp index cd7b92b32..8a78fbddd 100755 --- a/dbcon/mysql/ha_mcs_execplan.cpp +++ b/dbcon/mysql/ha_mcs_execplan.cpp @@ -86,6 +86,7 @@ using namespace execplan; using namespace funcexp; #include "collation.h" +#include "vlarray.h" const uint64_t AGG_BIT = 0x01; const uint64_t SUB_BIT = 0x02; @@ -5026,8 +5027,7 @@ ReturnedColumn* buildAggregateColumn(Item* item, gp_walk_info& gwi) context.setPrecision(udafc->resultType().precision); context.setParamCount(udafc->aggParms().size()); - mcsv1sdk::ColumnDatum colType; - mcsv1sdk::ColumnDatum colTypes[udafc->aggParms().size()]; + utils::VLArray colTypes(udafc->aggParms().size()); // Build the column type vector. // Modified for MCOL-1201 multi-argument aggregate @@ -5035,11 +5035,11 @@ ReturnedColumn* buildAggregateColumn(Item* item, gp_walk_info& gwi) { const execplan::CalpontSystemCatalog::ColType& resultType = udafc->aggParms()[i]->resultType(); + mcsv1sdk::ColumnDatum& colType = colTypes[i]; colType.dataType = resultType.colDataType; colType.precision = resultType.precision; colType.scale = resultType.scale; colType.charsetNumber = resultType.charsetNumber; - colTypes[i] = colType; } // Call the user supplied init() diff --git a/dbcon/mysql/ha_window_function.cpp b/dbcon/mysql/ha_window_function.cpp index e76a161fa..6a15d30fb 100644 --- a/dbcon/mysql/ha_window_function.cpp +++ b/dbcon/mysql/ha_window_function.cpp @@ -49,6 +49,8 @@ using namespace funcexp; #include "mcsv1_udaf.h" using namespace mcsv1sdk; +#include "vlarray.h" + namespace cal_impl_if { @@ -389,8 +391,7 @@ ReturnedColumn* buildWindowFunctionColumn(Item* item, gp_walk_info& gwi, bool& n context.setPrecision(rt.precision); context.setParamCount(funcParms.size()); - mcsv1sdk::ColumnDatum colType; - mcsv1sdk::ColumnDatum colTypes[funcParms.size()]; + utils::VLArray colTypes(funcParms.size()); // Turn on the Analytic flag so the function is aware it is being called // as a Window Function. @@ -402,11 +403,11 @@ ReturnedColumn* buildWindowFunctionColumn(Item* item, gp_walk_info& gwi, bool& n { const execplan::CalpontSystemCatalog::ColType& resultType = funcParms[i]->resultType(); + mcsv1sdk::ColumnDatum& colType = colTypes[i]; colType.dataType = resultType.colDataType; colType.precision = resultType.precision; colType.scale = resultType.scale; colType.charsetNumber = resultType.charsetNumber; - colTypes[i] = colType; } // Call the user supplied init() diff --git a/oam/oamcpp/liboamcpp.cpp b/oam/oamcpp/liboamcpp.cpp index 98655210b..b772a6871 100644 --- a/oam/oamcpp/liboamcpp.cpp +++ b/oam/oamcpp/liboamcpp.cpp @@ -64,6 +64,7 @@ #include "sessionmanager.h" #include "IDBPolicy.h" #include "IDBDataFile.h" +#include "vlarray.h" #if defined(__GNUC__) #include @@ -8954,8 +8955,8 @@ int Oam::glusterctl(GLUSTER_COMMANDS command, std::string argument1, std::string int numberNewDBRoots = (dbrootCount - dbrootID) + 1; int numberDBRootsPerPM = numberNewDBRoots / numberNewPMs; - std::vector dbrootPms[dbrootCount]; - DataRedundancySetup DataRedundancyConfigs[numberPMs]; + utils::VLArray > dbrootPms(dbrootCount); + utils::VLArray DataRedundancyConfigs(numberPMs); int startDBRootID = dbrootID; for (int pm = (pmID - 1); pm < numberPMs; pm++, startDBRootID++) @@ -9067,7 +9068,7 @@ int Oam::glusterctl(GLUSTER_COMMANDS command, std::string argument1, std::string //Need to wait since peer probe success does not always mean it is ready for volume create command sleep(10); - int pmnextbrick[numberPMs]; + utils::VLArray pmnextbrick(numberPMs); for (int pm = (pmID - 1); pm < numberPMs; pm++) { diff --git a/oamapps/serverMonitor/cpuMonitor.cpp b/oamapps/serverMonitor/cpuMonitor.cpp index 3d0c53a5f..4d7bc02d5 100644 --- a/oamapps/serverMonitor/cpuMonitor.cpp +++ b/oamapps/serverMonitor/cpuMonitor.cpp @@ -87,7 +87,7 @@ void cpuMonitor() ml.logErrorMessage(msg); } */ - int periodCount = 5; + const int periodCount = 5; float cpuPeriod[periodCount]; int periodCounter = 0; float averageCpuUsage = 0; diff --git a/primitives/linux-port/dictionary.cpp b/primitives/linux-port/dictionary.cpp index b197a6948..7404af7d4 100644 --- a/primitives/linux-port/dictionary.cpp +++ b/primitives/linux-port/dictionary.cpp @@ -604,11 +604,13 @@ int PrimitiveProcessor::convertToRegexp(idb_regex_t* regex, const p_DataValue* s bool PrimitiveProcessor::isLike(const p_DataValue* dict, const idb_regex_t* regex) throw() { #ifdef POSIX_REGEX - char cBuf[dict->len + 1]; + char* cBuf = new char[dict->len + 1]; memcpy(cBuf, dict->data, dict->len); cBuf[dict->len] = '\0'; - return (regexec(®ex->regex, cBuf, 0, NULL, 0) == 0); + bool ret = (regexec(®ex->regex, cBuf, 0, NULL, 0) == 0); + delete [] cBuf; + return ret; #else /* Note, the passed-in pointers are effectively begin() and end() iterators */ return regex_match(dict->data, dict->data + dict->len, regex->regex); diff --git a/primitives/primproc/batchprimitiveprocessor.cpp b/primitives/primproc/batchprimitiveprocessor.cpp index 6cd1ccec4..3750c0cde 100644 --- a/primitives/primproc/batchprimitiveprocessor.cpp +++ b/primitives/primproc/batchprimitiveprocessor.cpp @@ -53,6 +53,7 @@ using namespace boost; #include "blockcacheclient.h" #include "MonitorProcMem.h" #include "threadnaming.h" +#include "vlarray.h" #define MAX64 0x7fffffffffffffffLL #define MIN64 0x8000000000000000LL @@ -589,7 +590,7 @@ void BatchPrimitiveProcessor::addToJoiner(ByteStream& bs) // properly-named functions for clarity. if (typelessJoin[joinerNum]) { - vector > tmpBuckets[processorThreads]; + utils::VLArray > > tmpBuckets(processorThreads); TypelessData tlLargeKey; uint8_t nullFlag; PoolAllocator &storedKeyAllocator = storedKeyAllocators[joinerNum]; @@ -652,7 +653,7 @@ void BatchPrimitiveProcessor::addToJoiner(ByteStream& bs) uint64_t nullValue = joinNullValues[joinerNum]; bool &l_doMatchNulls = doMatchNulls[joinerNum]; joblist::JoinType joinType = joinTypes[joinerNum]; - vector > tmpBuckets[processorThreads]; + utils::VLArray > > tmpBuckets(processorThreads); if (joinType & MATCHNULLS) { diff --git a/storage-manager/src/IOCoordinator.cpp b/storage-manager/src/IOCoordinator.cpp index 25a856b75..a84a76d0a 100644 --- a/storage-manager/src/IOCoordinator.cpp +++ b/storage-manager/src/IOCoordinator.cpp @@ -30,6 +30,7 @@ #include #include #include "checks.h" +#include "vlarray.h" #define max(x, y) (x > y ? x : y) #define min(x, y) (x < y ? x : y) @@ -192,12 +193,13 @@ ssize_t IOCoordinator::read(const char *_filename, uint8_t *data, off_t offset, vector relevants = meta.metadataRead(offset, length); map journalFDs, objectFDs; map keyToJournalName, keyToObjectName; - ScopedCloser fdMinders[relevants.size() * 2]; + utils::VLArray fdMinders(relevants.size() * 2, -1); int mindersIndex = 0; char buf[80]; // load them into the cache vector keys; + keys.reserve(relevants.size()); for (const auto &object : relevants) keys.push_back(object.key); cache->read(firstDir, keys); @@ -414,12 +416,12 @@ ssize_t IOCoordinator::_write(const boost::filesystem::path &filename, const uin objects = metadata.metadataRead(currentEndofData,1); if (objects.size() == 1) { - // last objeect needs data - metadataObject lastObject = objects[0]; - uint64_t nullJournalSize = (objectSize - lastObject.length); - uint8_t nullData[nullJournalSize]; - memset(nullData,0,nullJournalSize); - err = replicator->addJournalEntry((firstDir/lastObject.key),nullData,lastObject.length,nullJournalSize); + // last objeect needs data + metadataObject lastObject = objects[0]; + uint64_t nullJournalSize = (objectSize - lastObject.length); + utils::VLArray nullData(nullJournalSize); + memset(nullData, 0, nullJournalSize); + err = replicator->addJournalEntry((firstDir/lastObject.key),nullData.data(),lastObject.length,nullJournalSize); if (err < 0) { l_errno = errno; diff --git a/storage-manager/src/unit_tests.cpp b/storage-manager/src/unit_tests.cpp index b3460f447..f66c1c470 100644 --- a/storage-manager/src/unit_tests.cpp +++ b/storage-manager/src/unit_tests.cpp @@ -236,33 +236,31 @@ bool opentask(bool connectionTest=false) sleep(1); close(sessionSock); close(clientSock); - err = ::recv(sessionSock, buf, 1024, MSG_DONTWAIT); + err = ::recv(sessionSock, buf, sizeof(buf), MSG_DONTWAIT); assert(err == -1); t.join(); } else { t.join(); - // read the response - err = ::recv(sessionSock, buf, 1024, MSG_DONTWAIT); - sm_response *resp = (sm_response *) buf; - assert(err == sizeof(struct stat) + sizeof(sm_response)); - assert(resp->header.type == SM_MSG_START); - assert(resp->header.payloadLen == sizeof(struct stat) + sizeof(ssize_t)); - assert(resp->header.flags == 0); - assert(resp->returnCode == 0); - struct stat *_stat = (struct stat *) resp->payload; + // read the response + err = ::recv(sessionSock, buf, sizeof(buf), MSG_DONTWAIT); + sm_response *resp = (sm_response *) buf; + assert(err == sizeof(struct stat) + sizeof(sm_response)); + assert(resp->header.type == SM_MSG_START); + assert(resp->header.payloadLen == sizeof(struct stat) + sizeof(ssize_t)); + assert(resp->header.flags == 0); + assert(resp->returnCode == 0); + struct stat *_stat = (struct stat *) resp->payload; - // what can we verify about the stat... - assert(_stat->st_uid == getuid()); - assert(_stat->st_gid == getgid()); - assert(_stat->st_size == 0); - /* verify the file is there */ - string metaPath = Config::get()->getValue("ObjectStorage", "metadata_path"); - assert(!metaPath.empty()); - metaPath += string("/" + prefix + "/" + testFile + ".meta"); + // what can we verify about the stat... + assert(_stat->st_size == 0); + /* verify the file is there */ + string metaPath = Config::get()->getValue("ObjectStorage", "metadata_path"); + assert(!metaPath.empty()); + metaPath += string("/" + prefix + "/" + testFile + ".meta"); - assert(boost::filesystem::exists(metaPath)); + assert(boost::filesystem::exists(metaPath)); } cout << "opentask OK" << endl; @@ -293,7 +291,7 @@ bool replicatorTest() //check file contents fd = ::open(newObjectCacheFullPath.c_str(), O_RDONLY); - err = ::read(fd, buf, 1024); + err = ::read(fd, buf, sizeof(buf)); assert(err == 10); buf[10] = 0; assert(!strcmp("1234567890", (const char *) buf)); @@ -304,7 +302,7 @@ bool replicatorTest() repli->addJournalEntry(newobject,data,0,10); fd = ::open(newObjectJournalFullPath.c_str(), O_RDONLY); - err = ::read(fd, buf, 1024); + err = ::read(fd, buf, sizeof(buf)); assert((uint) err == (header.length() + 1 + 16 + 10)); buf[err] = 0; assert(!strcmp("1234567890", (const char *) buf + header.length() + 1 + 16)); @@ -323,10 +321,10 @@ void metadataJournalTest(std::size_t size, off_t offset) // make an empty file to write to bf::path fullPath = homepath / prefix / "metadataJournalTest"; const char *filename = fullPath.string().c_str(); - uint8_t buf[(sizeof(write_cmd)+std::strlen(filename)+size)]; + std::vector buf(sizeof(write_cmd)+std::strlen(filename)+size); uint64_t *data; - sm_msg_header *hdr = (sm_msg_header *) buf; + sm_msg_header *hdr = (sm_msg_header *) buf.data(); write_cmd *cmd = (write_cmd *) &hdr[1]; cmd->opcode = WRITE; cmd->offset = offset; @@ -348,8 +346,9 @@ void metadataJournalTest(std::size_t size, off_t offset) w.run(); // verify response - err = ::recv(sessionSock, buf, 1024, MSG_DONTWAIT); - sm_response *resp = (sm_response *) buf; + uint8_t bufRead[1024]; + err = ::recv(sessionSock, bufRead, sizeof(bufRead), MSG_DONTWAIT); + sm_response *resp = (sm_response *) bufRead; assert(err == sizeof(*resp)); assert(resp->header.type == SM_MSG_START); assert(resp->header.payloadLen == sizeof(ssize_t)); @@ -363,10 +362,10 @@ void metadataJournalTest_append(std::size_t size) bf::path fullPath = homepath / prefix / "metadataJournalTest"; const char *filename = fullPath.string().c_str(); - uint8_t buf[(sizeof(write_cmd)+std::strlen(filename)+size)]; + std::vector buf(sizeof(write_cmd)+std::strlen(filename)+size); uint64_t *data; - sm_msg_header *hdr = (sm_msg_header *) buf; + sm_msg_header *hdr = (sm_msg_header *) buf.data(); append_cmd *cmd = (append_cmd *) &hdr[1]; cmd->opcode = APPEND; cmd->count = size; @@ -387,8 +386,9 @@ void metadataJournalTest_append(std::size_t size) a.run(); // verify response - err = ::recv(sessionSock, buf, 1024, MSG_DONTWAIT); - sm_response *resp = (sm_response *) buf; + uint8_t bufRead[1024]; + err = ::recv(sessionSock, bufRead, sizeof(bufRead), MSG_DONTWAIT); + sm_response *resp = (sm_response *) bufRead; assert(err == sizeof(*resp)); assert(resp->header.type == SM_MSG_START); assert(resp->header.payloadLen == sizeof(ssize_t)); @@ -438,7 +438,7 @@ bool writetask() w.run(); // verify response - int err = ::recv(sessionSock, buf, 1024, MSG_DONTWAIT); + int err = ::recv(sessionSock, buf, sizeof(buf), MSG_DONTWAIT); sm_response *resp = (sm_response *) buf; assert(err == sizeof(*resp)); assert(resp->header.type == SM_MSG_START); @@ -447,7 +447,7 @@ bool writetask() assert(resp->returnCode == 9); //check file contents - err = ::read(fd, buf, 1024); + err = ::read(fd, buf, sizeof(buf)); assert(err == 9); buf[9] = 0; assert(!strcmp("123456789", (const char *) buf)); @@ -488,7 +488,7 @@ bool appendtask() a.run(); // verify response - err = ::recv(sessionSock, buf, 1024, MSG_DONTWAIT); + err = ::recv(sessionSock, buf, sizeof(buf), MSG_DONTWAIT); sm_response *resp = (sm_response *) buf; assert(err == sizeof(*resp)); assert(resp->header.type == SM_MSG_START); @@ -498,7 +498,7 @@ bool appendtask() //check file contents ::lseek(fd, 0, SEEK_SET); - err = ::read(fd, buf, 1024); + err = ::read(fd, buf, sizeof(buf)); assert(err == 17); buf[17] = 0; assert(!strcmp("testjunk123456789", (const char *) buf)); @@ -557,23 +557,23 @@ void unlinktask(bool connectionTest=false) sleep(1); close(sessionSock); close(clientSock); - err = ::recv(sessionSock, buf, 1024, MSG_DONTWAIT); + err = ::recv(sessionSock, buf, sizeof(buf), MSG_DONTWAIT); assert(err == -1); t.join(); } else { t.join(); - // read the response - err = ::recv(sessionSock, buf, 1024, MSG_DONTWAIT); - sm_response *resp = (sm_response *) buf; - assert(err == sizeof(*resp)); - assert(resp->header.type == SM_MSG_START); - assert(resp->header.payloadLen == sizeof(ssize_t)); - assert(resp->header.flags == 0); - assert(resp->returnCode == 0); - // confirm it no longer exists - assert(!bf::exists(fullPathMeta)); + // read the response + err = ::recv(sessionSock, buf, sizeof(buf), MSG_DONTWAIT); + sm_response *resp = (sm_response *) buf; + assert(err == sizeof(*resp)); + assert(resp->header.type == SM_MSG_START); + assert(resp->header.payloadLen == sizeof(ssize_t)); + assert(resp->header.flags == 0); + assert(resp->returnCode == 0); + // confirm it no longer exists + assert(!bf::exists(fullPathMeta)); } // delete it again, make sure we get an error message & reasonable error code @@ -610,7 +610,7 @@ void unlinktask(bool connectionTest=false) bool stattask(bool connectionTest=false) { - int err=0; + int err=0; bf::path fullPath = homepath / prefix / "stattest1"; string filename = fullPath.string(); string Metafilename = prefix + "/stattest1"; @@ -652,7 +652,7 @@ bool stattask(bool connectionTest=false) sleep(1); close(sessionSock); close(clientSock); - err = ::recv(sessionSock, buf, 1024, MSG_DONTWAIT); + err = ::recv(sessionSock, buf, sizeof(buf), MSG_DONTWAIT); assert(err == -1); t.join(); } @@ -660,7 +660,7 @@ bool stattask(bool connectionTest=false) { t.join(); // read the response - err = ::recv(sessionSock, buf, 1024, MSG_DONTWAIT); + err = ::recv(sessionSock, buf, sizeof(buf), MSG_DONTWAIT); sm_response *resp = (sm_response *) buf; assert(err == sizeof(struct stat) + sizeof(sm_response)); assert(resp->header.type == SM_MSG_START); @@ -782,10 +782,10 @@ bool IOCTruncate() meta.writeMetadata(); // make sure there are 16k bytes, and the data is valid before going forward - memset(buf, 0, 16384); - err = ioc->read(testFile, buf, 0, 16384); - assert(err == 16384); - for (int i = 0; i < 16384/4; i++) + memset(buf, 0, sizeof(buf)); + err = ioc->read(testFile, buf, 0, sizeof(buf)); + assert(err == sizeof(buf)); + for (int i = 0; i < (int)sizeof(buf)/4; i++) assert(buf32[i] == (i % 2048)); assert(bf::exists(cachedSecondObject)); assert(bf::exists(cachedObjectPath)); @@ -795,7 +795,7 @@ bool IOCTruncate() assert(!err); meta = MetadataFile(metaTestFile); assert(meta.getLength() == 10240); - memset(buf, 0, 16384); + memset(buf, 0, sizeof(buf)); err = ioc->read(testFile, buf, 0, 10240); for (int i = 0; i < 10240/4; i++) assert(buf32[i] == (i % 2048)); @@ -870,7 +870,7 @@ bool truncatetask(bool connectionTest=false) sleep(1); close(sessionSock); close(clientSock); - err = ::recv(sessionSock, buf, 1024, MSG_DONTWAIT); + err = ::recv(sessionSock, buf, sizeof(buf), MSG_DONTWAIT); assert(err == -1); t.join(); } @@ -878,7 +878,7 @@ bool truncatetask(bool connectionTest=false) { t.join(); // read the response - err = ::recv(sessionSock, buf, 1024, MSG_DONTWAIT); + err = ::recv(sessionSock, buf, sizeof(buf), MSG_DONTWAIT); sm_response *resp = (sm_response *) buf; assert(err == sizeof(sm_response)); assert(resp->header.type == SM_MSG_START); @@ -914,7 +914,6 @@ bool listdirtask(bool connectionTest=false) bf::create_directories(tmpPath); for (int i = 0; i < 10; i++) { - string file(tmpPath.string() + "/dummy" + to_string(i)); files.insert(file); file += ".meta"; @@ -924,7 +923,7 @@ bool listdirtask(bool connectionTest=false) } uint8_t buf[8192]; - memset(buf,0,8192); + memset(buf,0,sizeof(buf)); listdir_cmd *cmd = (listdir_cmd *) buf; cmd->opcode = LIST_DIRECTORY; @@ -956,7 +955,7 @@ bool listdirtask(bool connectionTest=false) sleep(1); close(sessionSock); close(clientSock); - err = ::recv(sessionSock, buf, 1024, MSG_DONTWAIT); + err = ::recv(sessionSock, buf, sizeof(buf), MSG_DONTWAIT); assert(err == -1); t.join(); } @@ -965,7 +964,7 @@ bool listdirtask(bool connectionTest=false) t.join(); /* going to keep this simple. Don't run this in a big dir. */ /* maybe later I'll make a dir, put a file in it, and etc. For now run it in a small dir. */ - err = ::recv(sessionSock, buf, 8192, MSG_DONTWAIT); + err = ::recv(sessionSock, buf, sizeof(buf), MSG_DONTWAIT); sm_response *resp = (sm_response *) buf; assert(err > 0); assert(resp->header.type == SM_MSG_START); @@ -998,7 +997,7 @@ bool listdirtask(bool connectionTest=false) void pingtask() { - int err=0; + int err=0; uint8_t buf[1024]; ping_cmd *cmd = (ping_cmd *) buf; cmd->opcode = PING; @@ -1017,7 +1016,7 @@ void pingtask() t.join(); // read the response - err = ::recv(sessionSock, buf, 1024, MSG_DONTWAIT); + err = ::recv(sessionSock, buf, sizeof(buf), MSG_DONTWAIT); sm_response *resp = (sm_response *) buf; assert(err == sizeof(sm_response)); assert(resp->header.type == SM_MSG_START); @@ -1086,7 +1085,7 @@ bool copytask(bool connectionTest=false) sleep(1); close(sessionSock); close(clientSock); - err = ::recv(sessionSock, buf, 1024, MSG_DONTWAIT); + err = ::recv(sessionSock, buf, sizeof(buf), MSG_DONTWAIT); assert(err == -1); t.join(); } @@ -1094,7 +1093,7 @@ bool copytask(bool connectionTest=false) { t.join(); // read the response - err = ::recv(sessionSock, buf, 1024, MSG_DONTWAIT); + err = ::recv(sessionSock, buf, sizeof(buf), MSG_DONTWAIT); sm_response *resp = (sm_response *) buf; assert(err == sizeof(sm_response)); assert(resp->header.type == SM_MSG_START); @@ -1649,10 +1648,10 @@ void IOCCopyFile1() int err = ioc->copyFile(l_sourceFile.string().c_str(), l_destFile.string().c_str()); assert(!err); uint8_t buf1[8192], buf2[8192]; - err = ioc->read(l_sourceFile.string().c_str(), buf1, 0, 8192); - assert(err == 8192); - err = ioc->read(l_destFile.string().c_str(), buf2, 0, 8192); - assert(err == 8192); + err = ioc->read(l_sourceFile.string().c_str(), buf1, 0, sizeof(buf1)); + assert(err == sizeof(buf1)); + err = ioc->read(l_destFile.string().c_str(), buf2, 0, sizeof(buf2)); + assert(err == sizeof(buf2)); assert(memcmp(buf1, buf2, 8192) == 0); assert(ioc->unlink(l_sourceFile.string().c_str()) == 0); @@ -1718,10 +1717,10 @@ void IOCCopyFile3() int err = ioc->copyFile(l_sourceFile.string().c_str(), l_destFile.string().c_str()); assert(!err); uint8_t buf1[8192], buf2[8192]; - err = ioc->read(l_sourceFile.string().c_str(), buf1, 0, 8192); - assert(err == 8192); - err = ioc->read(l_destFile.string().c_str(), buf2, 0, 8192); - assert(err == 8192); + err = ioc->read(l_sourceFile.string().c_str(), buf1, 0, sizeof(buf1)); + assert(err == sizeof(buf1)); + err = ioc->read(l_destFile.string().c_str(), buf2, 0, sizeof(buf2)); + assert(err == sizeof(buf2)); assert(memcmp(buf1, buf2, 8192) == 0); assert(ioc->unlink(l_sourceFile.string().c_str()) == 0); @@ -1786,9 +1785,9 @@ void shortMsg() ioc->open(filename,O_WRONLY | O_CREAT,&_stat); size_t size = 27; - uint8_t bufWrite[(sizeof(write_cmd)+std::strlen(filename)+size)]; + std::vector bufWrite(sizeof(write_cmd)+std::strlen(filename)+size); - sm_msg_header *hdrWrite = (sm_msg_header *) bufWrite; + sm_msg_header *hdrWrite = (sm_msg_header *) bufWrite.data(); write_cmd *cmdWrite = (write_cmd *) &hdrWrite[1]; uint8_t *dataWrite; @@ -1810,8 +1809,9 @@ void shortMsg() w.run(); // verify response - int err = ::recv(sessionSock, bufWrite, 1024, MSG_DONTWAIT); - sm_response *resp = (sm_response *) bufWrite; + uint8_t bufRead[1024]; + int err = ::recv(sessionSock, bufRead, sizeof(bufRead), MSG_DONTWAIT); + sm_response *resp = (sm_response *) bufRead; assert(err == sizeof(*resp)); assert(resp->header.type == SM_MSG_START); assert(resp->header.payloadLen == sizeof(ssize_t)); @@ -1819,33 +1819,33 @@ void shortMsg() assert(resp->returnCode == 9); - uint8_t bufAppend[(sizeof(append_cmd)+std::strlen(filename)+size)]; + std::vector bufAppend(sizeof(append_cmd)+std::strlen(filename)+size); uint8_t *dataAppend; - sm_msg_header *hdrAppend = (sm_msg_header *) bufAppend; - append_cmd *cmdAppend = (append_cmd *) &hdrAppend[1]; - cmdAppend->opcode = APPEND; - cmdAppend->count = size; - cmdAppend->flen = std::strlen(filename); - memcpy(&cmdAppend->filename, filename, cmdAppend->flen); - dataAppend = (uint8_t *) &cmdAppend->filename[cmdAppend->flen]; + sm_msg_header *hdrAppend = (sm_msg_header *) bufAppend.data(); + append_cmd *cmdAppend = (append_cmd *) &hdrAppend[1]; + cmdAppend->opcode = APPEND; + cmdAppend->count = size; + cmdAppend->flen = std::strlen(filename); + memcpy(&cmdAppend->filename, filename, cmdAppend->flen); + dataAppend = (uint8_t *) &cmdAppend->filename[cmdAppend->flen]; memcpy(dataAppend, "123456789123456789123456789", cmdAppend->count); hdrAppend->type = SM_MSG_START; hdrAppend->payloadLen = sizeof(*cmdAppend) + cmdAppend->flen + 9; - AppendTask a(clientSock, hdrAppend->payloadLen); - err = ::write(sessionSock, cmdAppend, hdrAppend->payloadLen); + AppendTask a(clientSock, hdrAppend->payloadLen); + err = ::write(sessionSock, cmdAppend, hdrAppend->payloadLen); - a.run(); + a.run(); - // verify response - err = ::recv(sessionSock, bufAppend, 1024, MSG_DONTWAIT); - resp = (sm_response *) bufAppend; - assert(err == sizeof(*resp)); - assert(resp->header.type == SM_MSG_START); - assert(resp->header.payloadLen == sizeof(ssize_t)); - assert(resp->header.flags == 0); - assert(resp->returnCode == 9); + // verify response + err = ::recv(sessionSock, bufRead, sizeof(bufRead), MSG_DONTWAIT); + resp = (sm_response *) bufRead; + assert(err == sizeof(*resp)); + assert(resp->header.type == SM_MSG_START); + assert(resp->header.payloadLen == sizeof(ssize_t)); + assert(resp->header.flags == 0); + assert(resp->returnCode == 9); ioc->unlink(fullPath.string().c_str()); cout << "shortWriteMsg Test OK" << endl; } diff --git a/utils/common/vlarray.h b/utils/common/vlarray.h new file mode 100644 index 000000000..c803046c0 --- /dev/null +++ b/utils/common/vlarray.h @@ -0,0 +1,100 @@ +/* Copyright (C) 2020 MariaDB Corporation + + This program is free software; you can redistribute it and/or + modify it under the terms of the GNU General Public License + as published by the Free Software Foundation; version 2 of + the License. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program; if not, write to the Free Software + Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, + MA 02110-1301, USA. */ +#ifndef UTILS_COMMON_VLARRAY_H +#define UTILS_COMMON_VLARRAY_H + +namespace utils { + +template +class VLArray +{ +public: + VLArray(size_t sz) : + sz(sz), + stack_storage(NULL), + dyn_storage(NULL), + ptr(NULL) + { + if (sz > SIZE) { + dyn_storage = new T[sz]; + ptr = dyn_storage; + } else { + stack_storage = new (stack) T[sz]; + ptr = stack_storage; + } + } + + VLArray(size_t sz, const T& initval) : VLArray(sz) + { + for (size_t i= 0; i < sz; ++i) + ptr[i]= initval; + } + + VLArray(const VLArray&) = delete; + VLArray(VLArray&&) = delete; + VLArray& operator=(const VLArray&) = delete; + VLArray& operator=(VLArray&&) = delete; + + ~VLArray() { + if (dyn_storage) { + delete [] dyn_storage; + } else { + // we cannot use `delete [] stack_storage` here so call d-tors explicitly + if (!std::is_trivially_destructible::value) { + for (size_t i = 0; i < sz; ++i) + stack_storage[i].~T(); + } + } + } + + size_t size() const { return sz; } + const T* data() const { return ptr; } + T* data() { return ptr; } + + const T& operator[](size_t i) const { return ptr[i]; } + T& operator[](size_t i) { return ptr[i]; } + + const T& at(size_t i) const { + if (i >= sz) { + throw std::out_of_range("index out of range: " + std::to_string(i) + + " >= size " + std::to_string(sz)); + } + return ptr[i]; + } + + T& at(size_t i) { + if (i >= sz) { + throw std::out_of_range("index out of range: " + std::to_string(i) + + " >= size " + std::to_string(sz)); + } + return ptr[i]; + } + + operator const T* () const { return ptr; } + operator T* () { return ptr; } + +private: + const size_t sz; + alignas(T) char stack[SIZE * sizeof(T)]; + T* stack_storage; + T* dyn_storage; + T* ptr; +}; + +} // namespace utils + +#endif // UTILS_COMMON_VLARRAY_H diff --git a/utils/funcexp/func_char.cpp b/utils/funcexp/func_char.cpp index 88d0e01ff..c4a27730e 100644 --- a/utils/funcexp/func_char.cpp +++ b/utils/funcexp/func_char.cpp @@ -40,6 +40,8 @@ using namespace logging; #include "collation.h" +#include "vlarray.h" + namespace { @@ -89,7 +91,7 @@ string Func_char::getStrVal(Row& row, CalpontSystemCatalog::ColType& ct) { const int BUF_SIZE = 4 * parm.size(); - char buf[BUF_SIZE]; + utils::VLArray buf(BUF_SIZE); buf[0]= 0; char* pBuf = buf; CHARSET_INFO* cs = ct.getCharset(); diff --git a/utils/funcexp/func_strcmp.cpp b/utils/funcexp/func_strcmp.cpp index 50d031279..474357704 100644 --- a/utils/funcexp/func_strcmp.cpp +++ b/utils/funcexp/func_strcmp.cpp @@ -75,7 +75,7 @@ std::string Func_strcmp::getStrVal(rowgroup::Row& row, bool& isNull, execplan::CalpontSystemCatalog::ColType& type) { - uint64_t val = getIntVal(row, fp, isNull, type); + int64_t val = getIntVal(row, fp, isNull, type); if (val > 0) return string("1"); diff --git a/utils/idbdatafile/IDBPolicy.cpp b/utils/idbdatafile/IDBPolicy.cpp index 9d2766fcf..514681859 100644 --- a/utils/idbdatafile/IDBPolicy.cpp +++ b/utils/idbdatafile/IDBPolicy.cpp @@ -36,6 +36,7 @@ #endif #include "installdir.h" +#include "vlarray.h" using namespace std; @@ -246,7 +247,7 @@ void IDBPolicy::configIDBPolicy() // The feature is used in the FileOp code and enabled by default. char configSectionPref[] = "DBRoot"; int confSectionLen = sizeof(configSectionPref)+oam::MAX_MODULE_ID_SIZE; - char configSection[confSectionLen]; + utils::VLArray configSection(confSectionLen); IDBPolicy::init( idblog, bUseRdwrMemBuffer, hdfsRdwrScratch, hdfsRdwrBufferMaxSize ); s_configed = true; @@ -283,8 +284,8 @@ void IDBPolicy::configIDBPolicy() oam::DBRootConfigList::iterator dbRootIter = dbRootVec.begin(); for(; dbRootIter != dbRootVec.end(); dbRootIter++) { - ::memset(configSection + sizeof(configSectionPref), 0, oam::MAX_MODULE_ID_SIZE); - rc = snprintf(configSection, confSectionLen, "%s%d", configSectionPref, *dbRootIter); + ::memset(configSection.data() + sizeof(configSectionPref), 0, oam::MAX_MODULE_ID_SIZE); + rc = snprintf(configSection.data(), confSectionLen, "%s%d", configSectionPref, *dbRootIter); // gcc 8.2 warnings if ( rc < 0 || rc >= confSectionLen) { @@ -292,7 +293,7 @@ void IDBPolicy::configIDBPolicy() oss << "IDBPolicy::configIDBPolicy: failed to parse DBRootX section."; throw runtime_error(oss.str()); } - string setting = cf->getConfig(configSection, "PreallocSpace"); + string setting = cf->getConfig(configSection.data(), "PreallocSpace"); if ( setting.length() != 0 ) { diff --git a/utils/joiner/tuplejoiner.cpp b/utils/joiner/tuplejoiner.cpp index dc35f8ca9..30ad8c3dc 100644 --- a/utils/joiner/tuplejoiner.cpp +++ b/utils/joiner/tuplejoiner.cpp @@ -28,6 +28,7 @@ #include "hasher.h" #include "lbidlist.h" #include "spinlock.h" +#include "vlarray.h" using namespace std; using namespace rowgroup; @@ -279,8 +280,8 @@ void TupleJoiner::bucketsToTables(buckets_t *buckets, hash_table_t *tables) void TupleJoiner::um_insertTypeless(uint threadID, uint rowCount, Row &r) { - TypelessData td[rowCount]; - vector > v[bucketCount]; + utils::VLArray td(rowCount); + utils::VLArray > > v(bucketCount); uint i; FixedAllocator *alloc = &storedKeyAlloc[threadID]; @@ -298,7 +299,7 @@ void TupleJoiner::um_insertTypeless(uint threadID, uint rowCount, Row &r) void TupleJoiner::um_insertLongDouble(uint rowCount, Row &r) { - vector > v[bucketCount]; + utils::VLArray > > v(bucketCount); uint i; uint smallKeyColumn = smallKeyColumns[0]; @@ -318,7 +319,7 @@ void TupleJoiner::um_insertInlineRows(uint rowCount, Row &r) { uint i; int64_t smallKey; - vector > v[bucketCount]; + utils::VLArray > > v(bucketCount); uint smallKeyColumn = smallKeyColumns[0]; for (i = 0; i < rowCount; i++, r.nextRow()) @@ -340,7 +341,7 @@ void TupleJoiner::um_insertStringTable(uint rowCount, Row &r) { int64_t smallKey; uint i; - vector > v[bucketCount]; + utils::VLArray > > v(bucketCount); uint smallKeyColumn = smallKeyColumns[0]; for (i = 0; i < rowCount; i++, r.nextRow()) @@ -810,7 +811,7 @@ void TupleJoiner::setInUM() size = rows.size(); size_t chunkSize = ((size / numCores) + 1 < 50000 ? 50000 : (size / numCores) + 1); // don't start a thread to process < 50k rows - uint64_t jobs[numCores]; + utils::VLArray jobs(numCores); i = 0; for (size_t firstRow = 0; i < (uint) numCores && firstRow < size; i++, firstRow += chunkSize) jobs[i] = jobstepThreadPool->invoke([this, firstRow, chunkSize, size] { @@ -862,7 +863,7 @@ void TupleJoiner::setInUM(vector &rgs) size = rgs.size(); size_t chunkSize = ((size / numCores) + 1 < 10 ? 10 : (size / numCores) + 1); // don't issue jobs for < 10 rowgroups - uint64_t jobs[numCores]; + utils::VLArray jobs(numCores); i = 0; for (size_t firstRow = 0; i < (uint) numCores && firstRow < size; i++, firstRow += chunkSize) jobs[i] = jobstepThreadPool->invoke([this, firstRow, chunkSize, size, i, &rgs] { diff --git a/utils/rowgroup/rowaggregation.cpp b/utils/rowgroup/rowaggregation.cpp index b59ce6542..c73e751af 100755 --- a/utils/rowgroup/rowaggregation.cpp +++ b/utils/rowgroup/rowaggregation.cpp @@ -50,6 +50,7 @@ #include "rowaggregation.h" #include "calpontsystemcatalog.h" #include "utils_utf8.h" +#include "vlarray.h" #include "collation.h" @@ -1935,8 +1936,8 @@ void RowAggregation::doUDAF(const Row& rowIn, int64_t colIn, int64_t colOut, { uint32_t paramCount = fRGContext.getParameterCount(); // The vector of parameters to be sent to the UDAF - mcsv1sdk::ColumnDatum valsIn[paramCount]; - uint32_t dataFlags[paramCount]; + utils::VLArray valsIn(paramCount); + utils::VLArray dataFlags(paramCount); execplan::ConstantColumn* cc; bool bIsNull = false; execplan::CalpontSystemCatalog::ColDataType colDataType; @@ -2204,6 +2205,7 @@ void RowAggregation::doUDAF(const Row& rowIn, int64_t colIn, int64_t colOut, ++funcColsIdx; colIn = fFunctionCols[funcColsIdx]->fInputColumnIndex; colOut = fFunctionCols[funcColsIdx]->fOutputColumnIndex; + (void)colOut; } else { diff --git a/utils/windowfunction/wf_udaf.cpp b/utils/windowfunction/wf_udaf.cpp index af45d7c5f..2b8d7a4a9 100644 --- a/utils/windowfunction/wf_udaf.cpp +++ b/utils/windowfunction/wf_udaf.cpp @@ -49,6 +49,8 @@ using namespace joblist; #include "wf_udaf.h" +#include "vlarray.h" + namespace windowfunction { @@ -118,7 +120,7 @@ bool WF_udaf::dropValues(int64_t b, int64_t e) getContext().setContextFlag(mcsv1sdk::CONTEXT_IS_ANALYTIC); // Put the parameter metadata (type, scale, precision) into valsIn - mcsv1sdk::ColumnDatum valsIn[getContext().getParameterCount()]; + utils::VLArray valsIn(getContext().getParameterCount()); ConstantColumn* cc = NULL; for (uint32_t i = 0; i < getContext().getParameterCount(); ++i) @@ -141,6 +143,7 @@ bool WF_udaf::dropValues(int64_t b, int64_t e) } } + utils::VLArray flags(getContext().getParameterCount()); for (int64_t i = b; i < e; i++) { if (i % 1000 == 0 && fStep->cancelled()) @@ -149,7 +152,6 @@ bool WF_udaf::dropValues(int64_t b, int64_t e) fRow.setData(getPointer(fRowData->at(i))); // NULL flags - uint32_t flags[getContext().getParameterCount()]; bool bSkipIt = false; for (uint32_t k = 0; k < getContext().getParameterCount(); ++k) @@ -786,7 +788,7 @@ void WF_udaf::operator()(int64_t b, int64_t e, int64_t c) getContext().setContextFlag(mcsv1sdk::CONTEXT_IS_ANALYTIC); // Put the parameter metadata (type, scale, precision) into valsIn - mcsv1sdk::ColumnDatum valsIn[getContext().getParameterCount()]; + utils::VLArray valsIn(getContext().getParameterCount()); ConstantColumn* cc = NULL; for (uint32_t i = 0; i < getContext().getParameterCount(); ++i) @@ -815,7 +817,7 @@ void WF_udaf::operator()(int64_t b, int64_t e, int64_t c) getContext().clearContextFlag(mcsv1sdk::CONTEXT_HAS_CURRENT_ROW); bool bSkipIt = false; - + utils::VLArray flags(getContext().getParameterCount()); for (int64_t i = b; i <= e; i++) { if (i % 1000 == 0 && fStep->cancelled()) @@ -824,7 +826,6 @@ void WF_udaf::operator()(int64_t b, int64_t e, int64_t c) fRow.setData(getPointer(fRowData->at(i))); // NULL flags - uint32_t flags[getContext().getParameterCount()]; bSkipIt = false; for (uint32_t k = 0; k < getContext().getParameterCount(); ++k)