From 72eb1d7345c547bcfa6a6b9aafe5c07b32f7e447 Mon Sep 17 00:00:00 2001 From: David Hall Date: Mon, 29 Oct 2018 12:18:15 -0500 Subject: [PATCH 01/34] MCOL-1793 Window functions fail when current row outside of window --- utils/udfsdk/avg_mode.cpp | 108 +---------------- utils/udfsdk/avgx.cpp | 112 +----------------- utils/udfsdk/docs/source/usage/sourcefile.rst | 20 +--- utils/udfsdk/median.cpp | 108 +---------------- utils/udfsdk/ssq.cpp | 108 +---------------- utils/windowfunction/windowfunction.cpp | 38 ++++-- 6 files changed, 36 insertions(+), 458 deletions(-) mode change 100644 => 100755 utils/udfsdk/docs/source/usage/sourcefile.rst diff --git a/utils/udfsdk/avg_mode.cpp b/utils/udfsdk/avg_mode.cpp index 5429183d9..dba0859fb 100644 --- a/utils/udfsdk/avg_mode.cpp +++ b/utils/udfsdk/avg_mode.cpp @@ -69,65 +69,13 @@ mcsv1_UDAF::ReturnCode avg_mode::nextValue(mcsv1Context* context, ColumnDatum* v { static_any::any& valIn = valsIn[0].columnData; MODE_DATA& data = static_cast(context->getUserData())->mData; - DATATYPE val = 0.0; if (valIn.empty()) { return mcsv1_UDAF::SUCCESS; // Ought not happen when UDAF_IGNORE_NULLS is on. } - if (valIn.compatible(charTypeId)) - { - val = valIn.cast(); - } - else if (valIn.compatible(scharTypeId)) - { - val = valIn.cast(); - } - else if (valIn.compatible(shortTypeId)) - { - val = valIn.cast(); - } - else if (valIn.compatible(intTypeId)) - { - val = valIn.cast(); - } - else if (valIn.compatible(longTypeId)) - { - val = valIn.cast(); - } - else if (valIn.compatible(llTypeId)) - { - val = valIn.cast(); - } - else if (valIn.compatible(ucharTypeId)) - { - val = valIn.cast(); - } - else if (valIn.compatible(ushortTypeId)) - { - val = valIn.cast(); - } - else if (valIn.compatible(uintTypeId)) - { - val = valIn.cast(); - } - else if (valIn.compatible(ulongTypeId)) - { - val = valIn.cast(); - } - else if (valIn.compatible(ullTypeId)) - { - val = valIn.cast(); - } - else if (valIn.compatible(floatTypeId)) - { - val = valIn.cast(); - } - else if (valIn.compatible(doubleTypeId)) - { - val = valIn.cast(); - } + DATATYPE val = convertAnyTo(valIn); // For decimal types, we need to move the decimal point. uint32_t scale = valsIn[0].scale; @@ -190,65 +138,13 @@ mcsv1_UDAF::ReturnCode avg_mode::dropValue(mcsv1Context* context, ColumnDatum* v { static_any::any& valIn = valsDropped[0].columnData; MODE_DATA& data = static_cast(context->getUserData())->mData; - DATATYPE val = 0.0; if (valIn.empty()) { return mcsv1_UDAF::SUCCESS; // Ought not happen when UDAF_IGNORE_NULLS is on. } - if (valIn.compatible(charTypeId)) - { - val = valIn.cast(); - } - else if (valIn.compatible(scharTypeId)) - { - val = valIn.cast(); - } - else if (valIn.compatible(shortTypeId)) - { - val = valIn.cast(); - } - else if (valIn.compatible(intTypeId)) - { - val = valIn.cast(); - } - else if (valIn.compatible(longTypeId)) - { - val = valIn.cast(); - } - else if (valIn.compatible(llTypeId)) - { - val = valIn.cast(); - } - else if (valIn.compatible(ucharTypeId)) - { - val = valIn.cast(); - } - else if (valIn.compatible(ushortTypeId)) - { - val = valIn.cast(); - } - else if (valIn.compatible(uintTypeId)) - { - val = valIn.cast(); - } - else if (valIn.compatible(ulongTypeId)) - { - val = valIn.cast(); - } - else if (valIn.compatible(ullTypeId)) - { - val = valIn.cast(); - } - else if (valIn.compatible(floatTypeId)) - { - val = valIn.cast(); - } - else if (valIn.compatible(doubleTypeId)) - { - val = valIn.cast(); - } + DATATYPE val = convertAnyTo(valIn); // For decimal types, we need to move the decimal point. uint32_t scale = valsDropped[0].scale; diff --git a/utils/udfsdk/avgx.cpp b/utils/udfsdk/avgx.cpp index 5af852967..15548db36 100644 --- a/utils/udfsdk/avgx.cpp +++ b/utils/udfsdk/avgx.cpp @@ -75,69 +75,13 @@ mcsv1_UDAF::ReturnCode avgx::nextValue(mcsv1Context* context, ColumnDatum* valsI { static_any::any& valIn_x = valsIn[0].columnData; struct avgx_data* data = (struct avgx_data*)context->getUserData()->data; - DATATYPE val = 0.0; if (valIn_x.empty()) { return mcsv1_UDAF::SUCCESS; // Ought not happen when UDAF_IGNORE_NULLS is on. } - if (valIn_x.compatible(longTypeId)) - { - val = valIn_x.cast(); - } - else if (valIn_x.compatible(charTypeId)) - { - val = valIn_x.cast(); - } - else if (valIn_x.compatible(scharTypeId)) - { - val = valIn_x.cast(); - } - else if (valIn_x.compatible(shortTypeId)) - { - val = valIn_x.cast(); - } - else if (valIn_x.compatible(intTypeId)) - { - val = valIn_x.cast(); - } - else if (valIn_x.compatible(longTypeId)) - { - val = valIn_x.cast(); - } - else if (valIn_x.compatible(llTypeId)) - { - val = valIn_x.cast(); - } - else if (valIn_x.compatible(ucharTypeId)) - { - val = valIn_x.cast(); - } - else if (valIn_x.compatible(ushortTypeId)) - { - val = valIn_x.cast(); - } - else if (valIn_x.compatible(uintTypeId)) - { - val = valIn_x.cast(); - } - else if (valIn_x.compatible(ulongTypeId)) - { - val = valIn_x.cast(); - } - else if (valIn_x.compatible(ullTypeId)) - { - val = valIn_x.cast(); - } - else if (valIn_x.compatible(floatTypeId)) - { - val = valIn_x.cast(); - } - else if (valIn_x.compatible(doubleTypeId)) - { - val = valIn_x.cast(); - } + DATATYPE val = convertAnyTo(valIn_x); // For decimal types, we need to move the decimal point. uint32_t scale = valsIn[0].scale; @@ -183,65 +127,13 @@ mcsv1_UDAF::ReturnCode avgx::dropValue(mcsv1Context* context, ColumnDatum* valsD { static_any::any& valIn_x = valsDropped[0].columnData; struct avgx_data* data = (struct avgx_data*)context->getUserData()->data; - DATATYPE val = 0.0; if (valIn_x.empty()) { return mcsv1_UDAF::SUCCESS; // Ought not happen when UDAF_IGNORE_NULLS is on. } - if (valIn_x.compatible(charTypeId)) - { - val = valIn_x.cast(); - } - else if (valIn_x.compatible(scharTypeId)) - { - val = valIn_x.cast(); - } - else if (valIn_x.compatible(shortTypeId)) - { - val = valIn_x.cast(); - } - else if (valIn_x.compatible(intTypeId)) - { - val = valIn_x.cast(); - } - else if (valIn_x.compatible(longTypeId)) - { - val = valIn_x.cast(); - } - else if (valIn_x.compatible(llTypeId)) - { - val = valIn_x.cast(); - } - else if (valIn_x.compatible(ucharTypeId)) - { - val = valIn_x.cast(); - } - else if (valIn_x.compatible(ushortTypeId)) - { - val = valIn_x.cast(); - } - else if (valIn_x.compatible(uintTypeId)) - { - val = valIn_x.cast(); - } - else if (valIn_x.compatible(ulongTypeId)) - { - val = valIn_x.cast(); - } - else if (valIn_x.compatible(ullTypeId)) - { - val = valIn_x.cast(); - } - else if (valIn_x.compatible(floatTypeId)) - { - val = valIn_x.cast(); - } - else if (valIn_x.compatible(doubleTypeId)) - { - val = valIn_x.cast(); - } + DATATYPE val = convertAnyTo(valIn_x); // For decimal types, we need to move the decimal point. uint32_t scale = valsDropped[0].scale; diff --git a/utils/udfsdk/docs/source/usage/sourcefile.rst b/utils/udfsdk/docs/source/usage/sourcefile.rst old mode 100644 new mode 100755 index 5c43f29e4..ce0aa0d6f --- a/utils/udfsdk/docs/source/usage/sourcefile.rst +++ b/utils/udfsdk/docs/source/usage/sourcefile.rst @@ -124,9 +124,9 @@ nextValue() nextValue() is called from the PM for aggregate usage and the UM for Analytic usage. -valsIn contains a vector of all the parameters from the function call in the SQL query (In Columndtore 1.1, this will always contain exactly one entry). +valsIn contains a vector of all the parameters from the function call in the SQL query. -Depending on your function, you may wish to be able to handle many different types of input. A good way to handle this is to have a series of if..else..if statements comparing the input type and dealing with each separately. For instace, if you want to handle multiple numeric types, you might use:: +Depending on your function, you may wish to be able to handle many different types of input. There's a helper template function convertAnyTo() which will convert the input static:any value to the designated type. For Example, if your internal accumulater is of type double, you might use:: static_any::any& valIn = valsDropped[0].columnData; AVGData& data = static_cast(context->getUserData())->mData; @@ -137,21 +137,7 @@ Depending on your function, you may wish to be able to handle many different typ return mcsv1_UDAF::SUCCESS; // Ought not happen when UDAF_IGNORE_NULLS is on. } - if (valIn.compatible(charTypeId)) - { - val = valIn.cast(); - } - else if (valIn.compatible(scharTypeId)) - { - val = valIn.cast(); - } - else if (valIn.compatible(shortTypeId)) - { - val = valIn.cast(); - } - . - . - . + val = convertAnyTo(valIn); Once you've gotten your data in a format you like, then do your aggregation. For AVG, you might see:: diff --git a/utils/udfsdk/median.cpp b/utils/udfsdk/median.cpp index 9c7e72dc3..2d4750e4d 100644 --- a/utils/udfsdk/median.cpp +++ b/utils/udfsdk/median.cpp @@ -69,65 +69,13 @@ mcsv1_UDAF::ReturnCode median::nextValue(mcsv1Context* context, ColumnDatum* val { static_any::any& valIn = valsIn[0].columnData; MEDIAN_DATA& data = static_cast(context->getUserData())->mData; - DATATYPE val = 0.0; if (valIn.empty()) { return mcsv1_UDAF::SUCCESS; // Ought not happen when UDAF_IGNORE_NULLS is on. } - if (valIn.compatible(charTypeId)) - { - val = valIn.cast(); - } - else if (valIn.compatible(scharTypeId)) - { - val = valIn.cast(); - } - else if (valIn.compatible(shortTypeId)) - { - val = valIn.cast(); - } - else if (valIn.compatible(intTypeId)) - { - val = valIn.cast(); - } - else if (valIn.compatible(longTypeId)) - { - val = valIn.cast(); - } - else if (valIn.compatible(llTypeId)) - { - val = valIn.cast(); - } - else if (valIn.compatible(ucharTypeId)) - { - val = valIn.cast(); - } - else if (valIn.compatible(ushortTypeId)) - { - val = valIn.cast(); - } - else if (valIn.compatible(uintTypeId)) - { - val = valIn.cast(); - } - else if (valIn.compatible(ulongTypeId)) - { - val = valIn.cast(); - } - else if (valIn.compatible(ullTypeId)) - { - val = valIn.cast(); - } - else if (valIn.compatible(floatTypeId)) - { - val = valIn.cast(); - } - else if (valIn.compatible(doubleTypeId)) - { - val = valIn.cast(); - } + DATATYPE val = convertAnyTo(valIn); // For decimal types, we need to move the decimal point. uint32_t scale = valsIn[0].scale; @@ -215,65 +163,13 @@ mcsv1_UDAF::ReturnCode median::dropValue(mcsv1Context* context, ColumnDatum* val { static_any::any& valIn = valsDropped[0].columnData; MEDIAN_DATA& data = static_cast(context->getUserData())->mData; - DATATYPE val = 0.0; if (valIn.empty()) { return mcsv1_UDAF::SUCCESS; // Ought not happen when UDAF_IGNORE_NULLS is on. } - if (valIn.compatible(charTypeId)) - { - val = valIn.cast(); - } - else if (valIn.compatible(scharTypeId)) - { - val = valIn.cast(); - } - else if (valIn.compatible(shortTypeId)) - { - val = valIn.cast(); - } - else if (valIn.compatible(intTypeId)) - { - val = valIn.cast(); - } - else if (valIn.compatible(longTypeId)) - { - val = valIn.cast(); - } - else if (valIn.compatible(llTypeId)) - { - val = valIn.cast(); - } - else if (valIn.compatible(ucharTypeId)) - { - val = valIn.cast(); - } - else if (valIn.compatible(ushortTypeId)) - { - val = valIn.cast(); - } - else if (valIn.compatible(uintTypeId)) - { - val = valIn.cast(); - } - else if (valIn.compatible(ulongTypeId)) - { - val = valIn.cast(); - } - else if (valIn.compatible(ullTypeId)) - { - val = valIn.cast(); - } - else if (valIn.compatible(floatTypeId)) - { - val = valIn.cast(); - } - else if (valIn.compatible(doubleTypeId)) - { - val = valIn.cast(); - } + DATATYPE val = convertAnyTo(valIn); // For decimal types, we need to move the decimal point. uint32_t scale = valsDropped[0].scale; diff --git a/utils/udfsdk/ssq.cpp b/utils/udfsdk/ssq.cpp index 20fdc33db..74b60b5f6 100644 --- a/utils/udfsdk/ssq.cpp +++ b/utils/udfsdk/ssq.cpp @@ -85,65 +85,13 @@ mcsv1_UDAF::ReturnCode ssq::nextValue(mcsv1Context* context, ColumnDatum* valsIn { static_any::any& valIn = valsIn[0].columnData; struct ssq_data* data = (struct ssq_data*)context->getUserData()->data; - DATATYPE val = 0.0; if (context->isParamNull(0) || valIn.empty()) { return mcsv1_UDAF::SUCCESS; } - if (valIn.compatible(charTypeId)) - { - val = valIn.cast(); - } - else if (valIn.compatible(scharTypeId)) - { - val = valIn.cast(); - } - else if (valIn.compatible(shortTypeId)) - { - val = valIn.cast(); - } - else if (valIn.compatible(intTypeId)) - { - val = valIn.cast(); - } - else if (valIn.compatible(longTypeId)) - { - val = valIn.cast(); - } - else if (valIn.compatible(llTypeId)) - { - val = valIn.cast(); - } - else if (valIn.compatible(ucharTypeId)) - { - val = valIn.cast(); - } - else if (valIn.compatible(ushortTypeId)) - { - val = valIn.cast(); - } - else if (valIn.compatible(uintTypeId)) - { - val = valIn.cast(); - } - else if (valIn.compatible(ulongTypeId)) - { - val = valIn.cast(); - } - else if (valIn.compatible(ullTypeId)) - { - val = valIn.cast(); - } - else if (valIn.compatible(floatTypeId)) - { - val = valIn.cast(); - } - else if (valIn.compatible(doubleTypeId)) - { - val = valIn.cast(); - } + DATATYPE val = convertAnyTo(valIn); // For decimal types, we need to move the decimal point. uint32_t scale = valsIn[0].scale; @@ -186,65 +134,13 @@ mcsv1_UDAF::ReturnCode ssq::dropValue(mcsv1Context* context, ColumnDatum* valsDr { static_any::any& valIn = valsDropped[0].columnData; struct ssq_data* data = (struct ssq_data*)context->getUserData()->data; - DATATYPE val = 0.0; if (valIn.empty()) { return mcsv1_UDAF::SUCCESS; // Ought not happen when UDAF_IGNORE_NULLS is on. } - if (valIn.compatible(charTypeId)) - { - val = valIn.cast(); - } - else if (valIn.compatible(scharTypeId)) - { - val = valIn.cast(); - } - else if (valIn.compatible(shortTypeId)) - { - val = valIn.cast(); - } - else if (valIn.compatible(intTypeId)) - { - val = valIn.cast(); - } - else if (valIn.compatible(longTypeId)) - { - val = valIn.cast(); - } - else if (valIn.compatible(llTypeId)) - { - val = valIn.cast(); - } - else if (valIn.compatible(ucharTypeId)) - { - val = valIn.cast(); - } - else if (valIn.compatible(ushortTypeId)) - { - val = valIn.cast(); - } - else if (valIn.compatible(uintTypeId)) - { - val = valIn.cast(); - } - else if (valIn.compatible(ulongTypeId)) - { - val = valIn.cast(); - } - else if (valIn.compatible(ullTypeId)) - { - val = valIn.cast(); - } - else if (valIn.compatible(floatTypeId)) - { - val = valIn.cast(); - } - else if (valIn.compatible(doubleTypeId)) - { - val = valIn.cast(); - } + DATATYPE val = convertAnyTo(valIn); // For decimal types, we need to move the decimal point. uint32_t scale = valsDropped[0].scale; diff --git a/utils/windowfunction/windowfunction.cpp b/utils/windowfunction/windowfunction.cpp index a62d42042..e8d9a4d2a 100644 --- a/utils/windowfunction/windowfunction.cpp +++ b/utils/windowfunction/windowfunction.cpp @@ -187,21 +187,33 @@ void WindowFunction::operator()() prevFrame = w; } - // UDAnF functions may have a dropValue function implemented. - // If they do, we can optimize by calling dropValue() for those - // values leaving the window and nextValue for those entering, rather - // than a resetData() and then iterating over the entire window. - // Built-in functions may have this functionality added in the future. - if (fFunctionType->dropValues(prevFrame.first, w.first)) + // If b > e then the frame is entirely outside of the partition + // and there's no values to add + if (b <= e) { - b = firstTime ? w.first : prevFrame.second + 1; - } - else - { - fFunctionType->resetData(); - } + // UDAnF functions may have a dropValue function implemented. + // If they do, we can optimize by calling dropValue() for those + // values leaving the window and nextValue for those entering, rather + // than a resetData() and then iterating over the entire window. + // Built-in functions may have this functionality added in the future. + // If b > e, then nothing to drop. + if (!firstTime) + { + if (fFunctionType->dropValues(prevFrame.first, w.first)) + { + // Adjust the beginning of the frame for nextValue + // to start where the previous frame left off. + b = prevFrame.second + 1; + } + else + { + // dropValues failed so do the entire frame. + fFunctionType->resetData(); + } + } - fFunctionType->operator()(b, e, i); + fFunctionType->operator()(b, e, i); // Calls nextValue + } prevFrame = w; firstTime = false; } From d01fe36fdc6509f3526123a5f424432a80a8c432 Mon Sep 17 00:00:00 2001 From: David Hall Date: Tue, 6 Nov 2018 10:11:45 -0600 Subject: [PATCH 02/34] MCOL-1793 Window functions return garbage if current row outside of window frame. --- utils/udfsdk/udfsdk.vpj | 2 ++ utils/windowfunction/wf_udaf.cpp | 7 +++++++ utils/windowfunction/windowfunction.cpp | 10 +++++----- 3 files changed, 14 insertions(+), 5 deletions(-) diff --git a/utils/udfsdk/udfsdk.vpj b/utils/udfsdk/udfsdk.vpj index 1096f8431..65db73610 100755 --- a/utils/udfsdk/udfsdk.vpj +++ b/utils/udfsdk/udfsdk.vpj @@ -205,6 +205,7 @@ + @@ -217,6 +218,7 @@ + diff --git a/utils/windowfunction/wf_udaf.cpp b/utils/windowfunction/wf_udaf.cpp index f9e38d9a1..ef61ca89e 100644 --- a/utils/windowfunction/wf_udaf.cpp +++ b/utils/windowfunction/wf_udaf.cpp @@ -481,6 +481,7 @@ bool WF_udaf::dropValues(int64_t b, int64_t e) } } + WindowFunctionType::resetData(); return true; } @@ -708,6 +709,12 @@ void WF_udaf::operator()(int64_t b, int64_t e, int64_t c) uint64_t colOut = fFieldIndex[0]; bool isNull = false; + // Initialize result to NULL. If no values are found, NULL is the result. +// if (getContext().getRunFlag(mcsv1sdk::UDAF_DEFAULT_NULL)) +// { +// getNullValueAny(fValOut, getContext().getResultType(), getContext().getColWidth()); +// } + if ((fFrameUnit == WF__FRAME_ROWS) || (fPrev == -1) || (!fPeer->operator()(getPointer(fRowData->at(c)), getPointer(fRowData->at(fPrev))))) diff --git a/utils/windowfunction/windowfunction.cpp b/utils/windowfunction/windowfunction.cpp index e8d9a4d2a..d89c0fdc4 100644 --- a/utils/windowfunction/windowfunction.cpp +++ b/utils/windowfunction/windowfunction.cpp @@ -188,7 +188,7 @@ void WindowFunction::operator()() } // If b > e then the frame is entirely outside of the partition - // and there's no values to add + // and there's no values to drop if (b <= e) { // UDAnF functions may have a dropValue function implemented. @@ -207,15 +207,15 @@ void WindowFunction::operator()() } else { - // dropValues failed so do the entire frame. + // dropValues failed or doesn't exist + // so do the entire frame. fFunctionType->resetData(); } } - - fFunctionType->operator()(b, e, i); // Calls nextValue } - prevFrame = w; + fFunctionType->operator()(b, e, i); // UDAnF: Calls nextValue and evaluate firstTime = false; + prevFrame = w; } } } From e58c4c3381a3cdf18778008229a2c2864a6106c4 Mon Sep 17 00:00:00 2001 From: David Hall Date: Tue, 6 Nov 2018 10:16:34 -0600 Subject: [PATCH 03/34] MCOL-1793 remove commented experimental code --- utils/windowfunction/wf_udaf.cpp | 6 ------ 1 file changed, 6 deletions(-) diff --git a/utils/windowfunction/wf_udaf.cpp b/utils/windowfunction/wf_udaf.cpp index ef61ca89e..eabd121db 100644 --- a/utils/windowfunction/wf_udaf.cpp +++ b/utils/windowfunction/wf_udaf.cpp @@ -709,12 +709,6 @@ void WF_udaf::operator()(int64_t b, int64_t e, int64_t c) uint64_t colOut = fFieldIndex[0]; bool isNull = false; - // Initialize result to NULL. If no values are found, NULL is the result. -// if (getContext().getRunFlag(mcsv1sdk::UDAF_DEFAULT_NULL)) -// { -// getNullValueAny(fValOut, getContext().getResultType(), getContext().getColWidth()); -// } - if ((fFrameUnit == WF__FRAME_ROWS) || (fPrev == -1) || (!fPeer->operator()(getPointer(fRowData->at(c)), getPointer(fRowData->at(fPrev))))) From 1426b4d99bf324a68fb23197b3512402c0ee248c Mon Sep 17 00:00:00 2001 From: David Hall Date: Tue, 6 Nov 2018 10:30:38 -0600 Subject: [PATCH 04/34] MCOL-1793-regr_* functions get wrong answer. Pretty up some code. --- utils/windowfunction/windowfunction.cpp | 38 +++++++++++-------------- 1 file changed, 17 insertions(+), 21 deletions(-) diff --git a/utils/windowfunction/windowfunction.cpp b/utils/windowfunction/windowfunction.cpp index d89c0fdc4..677809f16 100644 --- a/utils/windowfunction/windowfunction.cpp +++ b/utils/windowfunction/windowfunction.cpp @@ -187,30 +187,26 @@ void WindowFunction::operator()() prevFrame = w; } + // UDAnF functions may have a dropValue function implemented. + // If they do, we can optimize by calling dropValue() for those + // values leaving the window and nextValue for those entering, rather + // than a resetData() and then iterating over the entire window. + // Built-in functions may have this functionality added in the future. // If b > e then the frame is entirely outside of the partition // and there's no values to drop - if (b <= e) + if (!firstTime && b <= e) { - // UDAnF functions may have a dropValue function implemented. - // If they do, we can optimize by calling dropValue() for those - // values leaving the window and nextValue for those entering, rather - // than a resetData() and then iterating over the entire window. - // Built-in functions may have this functionality added in the future. - // If b > e, then nothing to drop. - if (!firstTime) + if (fFunctionType->dropValues(prevFrame.first, w.first)) { - if (fFunctionType->dropValues(prevFrame.first, w.first)) - { - // Adjust the beginning of the frame for nextValue - // to start where the previous frame left off. - b = prevFrame.second + 1; - } - else - { - // dropValues failed or doesn't exist - // so do the entire frame. - fFunctionType->resetData(); - } + // Adjust the beginning of the frame for nextValue + // to start where the previous frame left off. + b = prevFrame.second + 1; + } + else + { + // dropValues failed or doesn't exist + // so calculate the entire frame. + fFunctionType->resetData(); } } fFunctionType->operator()(b, e, i); // UDAnF: Calls nextValue and evaluate @@ -230,7 +226,7 @@ void WindowFunction::operator()() } catch (...) { - fStep->handleException("unknow exception", logging::ERR_EXECUTE_WINDOW_FUNCTION); + fStep->handleException("unknown exception", logging::ERR_EXECUTE_WINDOW_FUNCTION); } } From df6d97ca046d665ed6a21c98a7729a07ae5c3f85 Mon Sep 17 00:00:00 2001 From: David Hall Date: Tue, 6 Nov 2018 10:48:37 -0600 Subject: [PATCH 05/34] MCOL-1793 Handle transition of Window Frame from outside of Partition to inside --- utils/windowfunction/windowfunction.cpp | 27 ++++++++++++++----------- 1 file changed, 15 insertions(+), 12 deletions(-) diff --git a/utils/windowfunction/windowfunction.cpp b/utils/windowfunction/windowfunction.cpp index 677809f16..aa4dcb4a8 100644 --- a/utils/windowfunction/windowfunction.cpp +++ b/utils/windowfunction/windowfunction.cpp @@ -194,23 +194,26 @@ void WindowFunction::operator()() // Built-in functions may have this functionality added in the future. // If b > e then the frame is entirely outside of the partition // and there's no values to drop - if (!firstTime && b <= e) + if (b <= e) { - if (fFunctionType->dropValues(prevFrame.first, w.first)) + if (!firstTime) { - // Adjust the beginning of the frame for nextValue - // to start where the previous frame left off. - b = prevFrame.second + 1; - } - else - { - // dropValues failed or doesn't exist - // so calculate the entire frame. - fFunctionType->resetData(); + if (fFunctionType->dropValues(prevFrame.first, w.first)) + { + // Adjust the beginning of the frame for nextValue + // to start where the previous frame left off. + b = prevFrame.second + 1; + } + else + { + // dropValues failed or doesn't exist + // so calculate the entire frame. + fFunctionType->resetData(); + } } + firstTime = false; } fFunctionType->operator()(b, e, i); // UDAnF: Calls nextValue and evaluate - firstTime = false; prevFrame = w; } } From 06d1c9fcbe98f5d3caee5d80a58277d3eaff95fa Mon Sep 17 00:00:00 2001 From: David Hall Date: Tue, 6 Nov 2018 11:07:49 -0600 Subject: [PATCH 06/34] MCOL-1793 Handle transition of Window Frame from outside of Partition to inside part 2 --- utils/windowfunction/windowfunction.cpp | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/utils/windowfunction/windowfunction.cpp b/utils/windowfunction/windowfunction.cpp index aa4dcb4a8..c46259e4a 100644 --- a/utils/windowfunction/windowfunction.cpp +++ b/utils/windowfunction/windowfunction.cpp @@ -211,7 +211,11 @@ void WindowFunction::operator()() fFunctionType->resetData(); } } - firstTime = false; + else + { + fFunctionType->resetData(); + firstTime = false; + } } fFunctionType->operator()(b, e, i); // UDAnF: Calls nextValue and evaluate prevFrame = w; From b1d5f54abea9e3507662b040f7fd22da2a0e57ec Mon Sep 17 00:00:00 2001 From: Andrew Hutchings Date: Wed, 14 Nov 2018 14:29:58 +0000 Subject: [PATCH 07/34] MCOL-1868 Fix error in unused code There was a a bad line in some code that we don't currently compile. This patch fixes that code in case we use it in the future. --- utils/funcexp/func_concat_ws.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/utils/funcexp/func_concat_ws.cpp b/utils/funcexp/func_concat_ws.cpp index 2718f20b3..d6253cf10 100644 --- a/utils/funcexp/func_concat_ws.cpp +++ b/utils/funcexp/func_concat_ws.cpp @@ -93,7 +93,7 @@ string Func_concat_ws::getStrVal(Row& row, string tmp; for ( uint32_t i = 1 ; i < parm.size() ; i++) { - string(stringValue(parm[i], row, isNull).c_str(), tmp); + stringValue(parm[i], row, isNull, tmp); str += tmp; if (isNull) From 5fd94e1438571347f0877a63e7f6e75177e58091 Mon Sep 17 00:00:00 2001 From: Patrick LeBlanc Date: Wed, 14 Nov 2018 17:15:14 -0600 Subject: [PATCH 08/34] MCOL-1844. Preserve user-added args in 'myCnf-include-args.text' across upgrades. --- oam/install_scripts/pre-uninstall | 1 + oamapps/postConfigure/mycnfUpgrade.cpp | 59 +++++++++++++++++++++++++- 2 files changed, 59 insertions(+), 1 deletion(-) diff --git a/oam/install_scripts/pre-uninstall b/oam/install_scripts/pre-uninstall index 2f58b5421..33a6328f2 100755 --- a/oam/install_scripts/pre-uninstall +++ b/oam/install_scripts/pre-uninstall @@ -138,6 +138,7 @@ if [ $quiet != 1 ]; then #make copy of Columnstore.xml /bin/cp -f $installdir/etc/Columnstore.xml $installdir/etc/Columnstore.xml.rpmsave > /dev/null 2>&1 /bin/cp -f $installdir/mysql/my.cnf $installdir/mysql/my.cnf.rpmsave > /dev/null 2>&1 + cp $installdir/bin/myCnf-include-args.text $installdir/bin/myCnf-include-args.text.rpmsave >& /dev/null rm -f $installdir/etc/AlarmConfig.xml.installSave fi diff --git a/oamapps/postConfigure/mycnfUpgrade.cpp b/oamapps/postConfigure/mycnfUpgrade.cpp index a8808e740..18b606a2b 100644 --- a/oamapps/postConfigure/mycnfUpgrade.cpp +++ b/oamapps/postConfigure/mycnfUpgrade.cpp @@ -54,6 +54,59 @@ using namespace std; using namespace oam; + +/* MCOL-1844. On an upgrade, the user may have customized options in their old + * myCnf-include-args.text file. Merge it with the packaged version, and then process as we + * have before. + */ +string rtrim(const string &in) { + string::const_reverse_iterator rbegin = in.rbegin(); + while (rbegin != in.rend() && isspace(*rbegin)) + ++rbegin; + return string(in.begin(), rbegin.base()); +} + +void mergeMycnfIncludeArgs() +{ + string userArgsFilename = startup::StartUp::installDir() + "/bin/myCnf-include-args.text.rpmsave"; + string packagedArgsFilename = startup::StartUp::installDir() + "/bin/myCnf-include-args.text"; + ifstream userArgs(userArgsFilename.c_str()); + fstream packagedArgs(packagedArgsFilename.c_str(), ios::in); + + if (!userArgs || !packagedArgs) + return; + + // de-dup the args and comments in both files + set argMerger; + set comments; + string line; + while (getline(packagedArgs, line)) { + line = rtrim(line); + if (line[0] == '#') + comments.insert(line); + else if (line.size() > 0) + argMerger.insert(line); + } + while (getline(userArgs, line)) { + line = rtrim(line); + if (line[0] == '#') + comments.insert(line); + else if (line.size() > 0) + argMerger.insert(line); + } + userArgs.close(); + packagedArgs.close(); + + // write the merged version, comments first. They'll get ordered + // alphabetically but, meh. + packagedArgs.open(packagedArgsFilename.c_str(), ios::out | ios::trunc); + for (set::iterator it = comments.begin(); it != comments.end(); it++) + packagedArgs << *it << endl; + for (set::iterator it = argMerger.begin(); it != argMerger.end(); it++) + packagedArgs << *it << endl; + packagedArgs.close(); +} + int main(int argc, char *argv[]) { Oam oam; @@ -84,6 +137,10 @@ int main(int argc, char *argv[]) exit (1); } + // MCOL-1844. The user may have added options to their myCnf-include-args file. Merge + // myCnf-include-args.text with myCnf-include-args.text.rpmsave, save in myCnf-include-args.text + mergeMycnfIncludeArgs(); + //include arguments file string includeFile = startup::StartUp::installDir() + "/bin/myCnf-include-args.text"; ifstream includefile (includeFile.c_str()); @@ -144,7 +201,7 @@ int main(int argc, char *argv[]) ofstream newFile (mycnfFile.c_str()); //create new file - int fd = open(mycnfFile.c_str(), O_RDWR|O_CREAT, 0666); + int fd = open(mycnfFile.c_str(), O_RDWR|O_CREAT, 0644); copy(lines.begin(), lines.end(), ostream_iterator(newFile, "\n")); newFile.close(); From f8267a54b694594c78085b3ee61f7c8ddaa9f5c0 Mon Sep 17 00:00:00 2001 From: David Hill Date: Fri, 16 Nov 2018 14:13:31 -0600 Subject: [PATCH 09/34] MCOL-1944 - correct the non-root user:group setup --- oam/install_scripts/syslogSetup.sh | 38 +++++++++++++----------------- 1 file changed, 17 insertions(+), 21 deletions(-) diff --git a/oam/install_scripts/syslogSetup.sh b/oam/install_scripts/syslogSetup.sh index e2d690fb4..703f62ed0 100755 --- a/oam/install_scripts/syslogSetup.sh +++ b/oam/install_scripts/syslogSetup.sh @@ -14,9 +14,22 @@ rsyslog7=0 user=`whoami 2>/dev/null` +#set default names groupname=adm username=syslog +# determine username/groupname + +if [ -f /var/log/messages ]; then + username=`stat -c "%U %G" /var/log/messages | awk '{print $1}'` + groupname=`stat -c "%U %G" /var/log/messages | awk '{print $2}'` +fi + +if [ -f /var/log/syslog ]; then + username=`stat -c "%U %G" /var/log/syslog | awk '{print $1}'` + groupname=`stat -c "%U %G" /var/log/syslog | awk '{print $2}'` +fi + for arg in "$@"; do if [ `expr -- "$arg" : '--prefix='` -eq 9 ]; then prefix="`echo $arg | awk -F= '{print $2}'`" @@ -161,8 +174,8 @@ makeDir() { test -d /var/log/mariadb/columnstore/archive || mkdir /var/log/mariadb/columnstore/archive >/dev/null 2>&1 test -d /var/log/mariadb/columnstore/corefiles || mkdir /var/log/mariadb/columnstore/corefiles >/dev/null 2>&1 test -d /var/log/mariadb/columnstore/trace || mkdir /var/log/mariadb/columnstore/trace >/dev/null 2>&1 - chmod 777 -R /var/log/mariadb/columnstore - chown $user:$user -R /var/log/mariadb + chmod 750 -R /var/log/mariadb/columnstore + chown $username:$groupname -R /var/log/mariadb } install() { @@ -170,9 +183,6 @@ makeDir checkSyslog if [ ! -z "$syslog_conf" ] ; then $installdir/bin/setConfig -d Installation SystemLogConfigFile ${syslog_conf} >/dev/null 2>&1 - if [ $user != "root" ]; then - chown $user:$user /home/$user/mariadb/columnstore/etc/* - fi if [ "$syslog_conf" == /etc/rsyslog.d/columnstore.conf ] || [ "$syslog_conf" == /etc/rsyslog.d/49-columnstore.conf ]; then @@ -188,21 +198,6 @@ if [ ! -z "$syslog_conf" ] ; then #set the syslog for ColumnStore logging # remove older version incase it was installed by previous build rm -rf /etc/rsyslog.d/columnstore.conf - - # determine username/groupname - - if [ -f /var/log/messages ]; then - user=`stat -c "%U %G" /var/log/messages | awk '{print $1}'` - group=`stat -c "%U %G" /var/log/messages | awk '{print $2}'` - fi - - if [ -f /var/log/syslog ]; then - user=`stat -c "%U %G" /var/log/syslog | awk '{print $1}'` - group=`stat -c "%U %G" /var/log/syslog | awk '{print $2}'` - fi - - # set permissions - chown $user:$group -R /var/log/mariadb > /dev/null 2>&1 if [ $rsyslog7 == 1 ]; then rm -f /etc/rsyslog.d/49-columnstore.conf @@ -210,6 +205,7 @@ if [ ! -z "$syslog_conf" ] ; then sed -i -e s/groupname/$groupname/g ${syslog_conf} sed -i -e s/username/$username/g ${syslog_conf} + chmod 644 ${syslog_conf} else cp ${columnstoreSyslogFile} ${syslog_conf} fi @@ -233,7 +229,7 @@ if [ ! -z "$syslog_conf" ] ; then if [ $? -eq 0 ]; then if [ -f ${syslog_conf}.columnstoreSave ] ; then #uninstall the syslog for ColumnStore logging - v -f ${syslog_conf} ${syslog_conf}.ColumnStoreBackup + mv -f ${syslog_conf} ${syslog_conf}.ColumnStoreBackup mv -f ${syslog_conf}.columnstoreSave ${syslog_conf} >/dev/null 2>&1 if [ ! -f ${syslog_conf} ] ; then cp ${syslog_conf}.ColumnStoreBackup ${syslog_conf} From 9bf8df6a6f120008281ff14ed937f08356fad390 Mon Sep 17 00:00:00 2001 From: David Hill Date: Mon, 19 Nov 2018 09:07:01 -0600 Subject: [PATCH 10/34] MCOL-1947 - change module and home alias --- oam/install_scripts/columnstoreAlias | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/oam/install_scripts/columnstoreAlias b/oam/install_scripts/columnstoreAlias index 255eb7e7e..623cf26b8 100644 --- a/oam/install_scripts/columnstoreAlias +++ b/oam/install_scripts/columnstoreAlias @@ -4,7 +4,7 @@ alias mcsmysql='/usr/local/mariadb/columnstore/mysql/bin/mysql --defaults-extra- alias ma=/usr/local/mariadb/columnstore/bin/mcsadmin alias mcsadmin=/usr/local/mariadb/columnstore/bin/mcsadmin alias cpimport=/usr/local/mariadb/columnstore/bin/cpimport -alias home='cd /usr/local/mariadb/columnstore' +alias mcshome='cd /usr/local/mariadb/columnstore' alias log='cd /var/log/mariadb/columnstore/' alias core='cd /var/log/mariadb/columnstore/corefiles' alias tmsg='tail -f /var/log/messages' @@ -14,4 +14,4 @@ alias terror='tail -f /var/log/mariadb/columnstore/err.log' alias twarning='tail -f /var/log/mariadb/columnstore/warning.log' alias tcrit='tail -f /var/log/mariadb/columnstore/crit.log' alias dbrm='cd /usr/local/mariadb/columnstore/data1/systemFiles/dbrm' -alias module='cat /usr/local/mariadb/columnstore/local/module' +alias mcsmodule='cat /usr/local/mariadb/columnstore/local/module' From 3bf269b3f7cf131c5195e353deda04f9edbf217a Mon Sep 17 00:00:00 2001 From: Roman Nozdrin Date: Tue, 20 Nov 2018 13:44:47 +0300 Subject: [PATCH 11/34] MCOL-1519 GROUP BY handler now uses an appropriate SELECT_LEX structure. Before that handler used an outter query SELECT_LEX that could give incorrect information for subquery. --- dbcon/mysql/ha_calpont.cpp | 14 +++++--------- dbcon/mysql/ha_calpont_execplan.cpp | 6 ++---- 2 files changed, 7 insertions(+), 13 deletions(-) diff --git a/dbcon/mysql/ha_calpont.cpp b/dbcon/mysql/ha_calpont.cpp index 03289ba60..e16a8ba83 100644 --- a/dbcon/mysql/ha_calpont.cpp +++ b/dbcon/mysql/ha_calpont.cpp @@ -1198,8 +1198,6 @@ void check_walk(const Item* item, void* arg) } } -#include - /*@brief create_calpont_group_by_handler- Creates handler*/ /*********************************************************** * DESCRIPTION: @@ -1221,19 +1219,19 @@ static group_by_handler* create_calpont_group_by_handler(THD* thd, Query* query) { ha_calpont_group_by_handler* handler = NULL; - LEX* lex = thd->lex; - SELECT_LEX *select_lex = &lex->select_lex; + SELECT_LEX *select_lex = query->from->select_lex; // Create a handler if query is valid. See comments for details. if ( thd->infinidb_vtable.vtable_state == THD::INFINIDB_DISABLE_VTABLE && ( thd->variables.infinidb_vtable_mode == 0 || thd->variables.infinidb_vtable_mode == 2 ) - && ( query->group_by || thd->lex->select_lex.with_sum_func ) ) + && ( query->group_by || select_lex->with_sum_func ) ) { bool unsupported_feature = false; // Impossible HAVING or WHERE - if ( ( select_lex->having && select_lex->having_value == Item::COND_FALSE ) - || ( select_lex->cond_value && select_lex->cond_value == Item::COND_FALSE ) ) + if ( ( query->having && select_lex->having_value == Item::COND_FALSE ) + || ( select_lex->cond_count > 0 + && select_lex->cond_value == Item::COND_FALSE ) ) { unsupported_feature = true; } @@ -1265,8 +1263,6 @@ create_calpont_group_by_handler(THD* thd, Query* query) } } - std::cerr << "create_calpont_group_by_handler handler " << handler << std::endl; - return handler; } diff --git a/dbcon/mysql/ha_calpont_execplan.cpp b/dbcon/mysql/ha_calpont_execplan.cpp index a872ace4f..6e1f223d5 100644 --- a/dbcon/mysql/ha_calpont_execplan.cpp +++ b/dbcon/mysql/ha_calpont_execplan.cpp @@ -8248,13 +8248,11 @@ int cp_get_table_plan(THD* thd, SCSEP& csep, cal_table_info& ti) int cp_get_group_plan(THD* thd, SCSEP& csep, cal_impl_if::cal_group_info& gi) { - LEX* lex = thd->lex; - idbassert(lex != 0); - SELECT_LEX select_lex = lex->select_lex; + SELECT_LEX *select_lex = gi.groupByTables->select_lex; gp_walk_info gwi; gwi.thd = thd; - int status = getGroupPlan(gwi, select_lex, csep, gi); + int status = getGroupPlan(gwi, *select_lex, csep, gi); #ifdef DEBUG_WALK_COND cerr << "---------------- cp_get_group_plan EXECUTION PLAN ----------------" << endl; From 575fdf22847cec41276fff8bf4530186d0eca803 Mon Sep 17 00:00:00 2001 From: David Hill Date: Tue, 20 Nov 2018 09:11:11 -0600 Subject: [PATCH 12/34] MCOL-1944 --- oam/install_scripts/syslogSetup.sh | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/oam/install_scripts/syslogSetup.sh b/oam/install_scripts/syslogSetup.sh index 703f62ed0..c1d758396 100755 --- a/oam/install_scripts/syslogSetup.sh +++ b/oam/install_scripts/syslogSetup.sh @@ -11,6 +11,7 @@ prefix=/usr/local installdir=$prefix/mariadb/columnstore syslog_conf=nofile rsyslog7=0 +non-root-user="no" user=`whoami 2>/dev/null` @@ -41,6 +42,7 @@ for arg in "$@"; do user="`echo $arg | awk -F= '{print $2}'`" groupname=$user username=$user + non-root-user="yes" elif [ `expr -- "$arg" : '--..*'` -ge 3 ]; then echo "ignoring unknown argument: $arg" 1>&2 elif [ `expr -- "$arg" : '--'` -eq 2 ]; then @@ -174,7 +176,9 @@ makeDir() { test -d /var/log/mariadb/columnstore/archive || mkdir /var/log/mariadb/columnstore/archive >/dev/null 2>&1 test -d /var/log/mariadb/columnstore/corefiles || mkdir /var/log/mariadb/columnstore/corefiles >/dev/null 2>&1 test -d /var/log/mariadb/columnstore/trace || mkdir /var/log/mariadb/columnstore/trace >/dev/null 2>&1 - chmod 750 -R /var/log/mariadb/columnstore + if [ $non-root-user == "yes" ]; then + chmod 777 /var/log/mariadb/columnstore + fi chown $username:$groupname -R /var/log/mariadb } @@ -248,7 +252,6 @@ if [ ! -z "$syslog_conf" ] ; then rm -f /etc/logrotate.d/columnstore restartSyslog - fi } From 332d07328e34498c3c01c9e8a85d49ab04f05249 Mon Sep 17 00:00:00 2001 From: David Hill Date: Tue, 20 Nov 2018 09:35:02 -0600 Subject: [PATCH 13/34] MCOL-1944 --- oam/install_scripts/syslogSetup.sh | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/oam/install_scripts/syslogSetup.sh b/oam/install_scripts/syslogSetup.sh index c1d758396..84980eef8 100755 --- a/oam/install_scripts/syslogSetup.sh +++ b/oam/install_scripts/syslogSetup.sh @@ -11,7 +11,7 @@ prefix=/usr/local installdir=$prefix/mariadb/columnstore syslog_conf=nofile rsyslog7=0 -non-root-user="no" +non_root_user=no user=`whoami 2>/dev/null` @@ -42,7 +42,7 @@ for arg in "$@"; do user="`echo $arg | awk -F= '{print $2}'`" groupname=$user username=$user - non-root-user="yes" + non_root_user=yes elif [ `expr -- "$arg" : '--..*'` -ge 3 ]; then echo "ignoring unknown argument: $arg" 1>&2 elif [ `expr -- "$arg" : '--'` -eq 2 ]; then @@ -176,7 +176,7 @@ makeDir() { test -d /var/log/mariadb/columnstore/archive || mkdir /var/log/mariadb/columnstore/archive >/dev/null 2>&1 test -d /var/log/mariadb/columnstore/corefiles || mkdir /var/log/mariadb/columnstore/corefiles >/dev/null 2>&1 test -d /var/log/mariadb/columnstore/trace || mkdir /var/log/mariadb/columnstore/trace >/dev/null 2>&1 - if [ $non-root-user == "yes" ]; then + if [ $non_root_user == "yes" ]; then chmod 777 /var/log/mariadb/columnstore fi chown $username:$groupname -R /var/log/mariadb From 9363504dd4838f8e63acb49e8861510e51927abc Mon Sep 17 00:00:00 2001 From: Patrick LeBlanc Date: Tue, 20 Nov 2018 12:43:05 -0600 Subject: [PATCH 14/34] MCOL-1847. Addressed feedback from Daniel. Instead of a seperate param for cache size, we decided to allow an override of the NumBlocksPct param instead. If NumBlocksPct ends in an 'm' or a 'g', it will be interpreted as megabytes or gigabytes instead of a %age. --- oam/etc/Columnstore.xml | 7 ++----- oam/etc/Columnstore.xml.singleserver | 7 ++----- primitives/primproc/primproc.cpp | 29 +++++++++++----------------- 3 files changed, 15 insertions(+), 28 deletions(-) diff --git a/oam/etc/Columnstore.xml b/oam/etc/Columnstore.xml index 79fb0cff5..b5610bf11 100644 --- a/oam/etc/Columnstore.xml +++ b/oam/etc/Columnstore.xml @@ -427,12 +427,9 @@ + - - - - 1 0 diff --git a/oam/etc/Columnstore.xml.singleserver b/oam/etc/Columnstore.xml.singleserver index 75e741ada..0c291579a 100644 --- a/oam/etc/Columnstore.xml.singleserver +++ b/oam/etc/Columnstore.xml.singleserver @@ -419,12 +419,9 @@ + 50 - - - - 1 0 diff --git a/primitives/primproc/primproc.cpp b/primitives/primproc/primproc.cpp index 3957cc783..4ec66b767 100644 --- a/primitives/primproc/primproc.cpp +++ b/primitives/primproc/primproc.cpp @@ -503,16 +503,10 @@ int main(int argc, char* argv[]) } string strBlockPct = cf->getConfig(dbbc, "NumBlocksPct"); - string strBlockAbs = cf->getConfig(dbbc, "NumBlocksInMB"); - bool usePct = !(strBlockPct.empty()); // which to use. Prefer Pct if both are specified. - - if (usePct) - temp = atoi(strBlockPct.c_str()); - else - temp = atoi(strBlockAbs.c_str()); + temp = atoi(strBlockPct.c_str()); #ifdef _MSC_VER - /* TODO: implement handling for NumBlocksInMB */ + /* TODO: implement handling for the 'm' or 'g' chars in NumBlocksPct */ if (temp > 0) BRPBlocksPct = temp; @@ -532,19 +526,18 @@ int main(int argc, char* argv[]) } #else - if (usePct) + if (temp > 0) + BRPBlocksPct = temp; + /* MCOL-1847. Did the user specify this in MB or GB? */ + int len = strBlockPct.length(); + if (strBlockPct[len-1] == 'g' || strBlockPct[len-1] == 'm') { - if (temp > 0) - BRPBlocksPct = temp; - BRPBlocks = ((BRPBlocksPct / 100.0) * (double) cg.getTotalMemory()) / 8192; + if (strBlockPct[len-1] == 'g') + BRPBlocksPct *= 1024; + BRPBlocks = BRPBlocksPct * 128; // 128 blocks per MB } else - { - if (temp > 0) - BRPBlocks = temp * 128; // 128 blocks per MB. - else - BRPBlocks = 131072; // 1GB, why not. - } + BRPBlocks = ((BRPBlocksPct / 100.0) * (double) cg.getTotalMemory()) / 8192; #endif #if 0 temp = toInt(cf->getConfig(dbbc, "NumThreads")); From de9dc56f21de76d36be5ef184efc103f0ef7f576 Mon Sep 17 00:00:00 2001 From: Patrick LeBlanc Date: Tue, 20 Nov 2018 13:00:36 -0600 Subject: [PATCH 15/34] MCOL-1847. Made the field parsing a little more paranoid. --- primitives/primproc/primproc.cpp | 20 ++++++++++++-------- 1 file changed, 12 insertions(+), 8 deletions(-) diff --git a/primitives/primproc/primproc.cpp b/primitives/primproc/primproc.cpp index 4ec66b767..0845410ff 100644 --- a/primitives/primproc/primproc.cpp +++ b/primitives/primproc/primproc.cpp @@ -526,16 +526,20 @@ int main(int argc, char* argv[]) } #else - if (temp > 0) + bool cacheInMB = false; + if (temp > 0) { BRPBlocksPct = temp; - /* MCOL-1847. Did the user specify this in MB or GB? */ - int len = strBlockPct.length(); - if (strBlockPct[len-1] == 'g' || strBlockPct[len-1] == 'm') - { - if (strBlockPct[len-1] == 'g') - BRPBlocksPct *= 1024; - BRPBlocks = BRPBlocksPct * 128; // 128 blocks per MB + /* MCOL-1847. Did the user specify this in MB or GB? */ + int len = strBlockPct.length(); + if (strBlockPct[len-1] == 'g' || strBlockPct[len-1] == 'm') + { + if (strBlockPct[len-1] == 'g') + BRPBlocksPct *= 1024; + cacheInMB = true; + } } + if (cacheInMB) + BRPBlocks = BRPBlocksPct * 128; // 128 blocks per MB else BRPBlocks = ((BRPBlocksPct / 100.0) * (double) cg.getTotalMemory()) / 8192; #endif From 4d70ccd679db034124345e2c12edc7bbd37f5912 Mon Sep 17 00:00:00 2001 From: David Hill Date: Tue, 20 Nov 2018 13:21:34 -0600 Subject: [PATCH 16/34] MCOL-1944 - fix issues with logging after upgrade --- oam/install_scripts/syslogSetup.sh | 20 ++++++++++++++------ 1 file changed, 14 insertions(+), 6 deletions(-) diff --git a/oam/install_scripts/syslogSetup.sh b/oam/install_scripts/syslogSetup.sh index 84980eef8..a900c2e84 100755 --- a/oam/install_scripts/syslogSetup.sh +++ b/oam/install_scripts/syslogSetup.sh @@ -172,14 +172,19 @@ fi } makeDir() { - test -d /var/log/mariadb/columnstore || mkdir -p /var/log/mariadb/columnstore >/dev/null 2>&1 + if [ ! -d /var/log/mariadb/columnstore ];then + mkdir -p /var/log/mariadb/columnstore >/dev/null 2>&1 + chown $username:$groupname -R /var/log/mariadb + fi + + if [ $non_root_user == "yes" ]; then + chmod 777 /var/log/mariadb + chmod 777 /var/log/mariadb/columnstore + fi + test -d /var/log/mariadb/columnstore/archive || mkdir /var/log/mariadb/columnstore/archive >/dev/null 2>&1 test -d /var/log/mariadb/columnstore/corefiles || mkdir /var/log/mariadb/columnstore/corefiles >/dev/null 2>&1 test -d /var/log/mariadb/columnstore/trace || mkdir /var/log/mariadb/columnstore/trace >/dev/null 2>&1 - if [ $non_root_user == "yes" ]; then - chmod 777 /var/log/mariadb/columnstore - fi - chown $username:$groupname -R /var/log/mariadb } install() { @@ -187,12 +192,15 @@ makeDir checkSyslog if [ ! -z "$syslog_conf" ] ; then $installdir/bin/setConfig -d Installation SystemLogConfigFile ${syslog_conf} >/dev/null 2>&1 + if [ $non_root_user == "yes" ]; then + chown $user:$user $installdir/etc/Columnstore.xml* + fi + rm -f ${syslog_conf}.columnstoreSave if [ "$syslog_conf" == /etc/rsyslog.d/columnstore.conf ] || [ "$syslog_conf" == /etc/rsyslog.d/49-columnstore.conf ]; then i=1 else - rm -f ${syslog_conf}.columnstoreSave cp ${syslog_conf} ${syslog_conf}.columnstoreSave >/dev/null 2>&1 sed -i '/# MariaDB/,$d' ${syslog_conf}.columnstoreSave > /dev/null 2>&1 fi From 8b6a6c9fcaca5b255de006037ed79a6f99371f53 Mon Sep 17 00:00:00 2001 From: David Hill Date: Tue, 20 Nov 2018 13:41:07 -0600 Subject: [PATCH 17/34] MCOL-1953 - removed unneeded 'done' --- .../clusterTester/columnstoreClusterTester.sh | 34 +++++++++---------- 1 file changed, 16 insertions(+), 18 deletions(-) diff --git a/utils/clusterTester/columnstoreClusterTester.sh b/utils/clusterTester/columnstoreClusterTester.sh index 887ed9af7..7e507dbff 100755 --- a/utils/clusterTester/columnstoreClusterTester.sh +++ b/utils/clusterTester/columnstoreClusterTester.sh @@ -334,7 +334,6 @@ checkSSH() checkRemoteDir() { - if [ "$USER" != "root" ]; then # Non-root User directory permissions check # @@ -342,23 +341,22 @@ checkRemoteDir() echo "** Run Non-root User directory permissions check on remote nodes (/dev/shm)" echo "" - `$COLUMNSTORE_INSTALL_DIR/bin/remote_command.sh $ipadd $PASSWORD 'touch /dev/shm/cs_check' 1 > ${tmpDir}/remote_command_check 2>&1` - rc="$?" - if [ $rc -eq 0 ] || ( [ $rc -eq 2 ] && [ $OS == "suse12" ] ) ; then - `grep "Permission denied" ${tmpDir}/remote_command_check > /dev/null 2>&1` - if [ "$?" -eq 0 ]; then - echo "$ipadd Node permission test on /dev/shm : ${bold}Failed${normal}, change permissions to 777 and re-test" - pass=false - REPORTPASS=false - else - echo "$ipadd Node permission test on /dev/shm : Passed" - fi - else - echo "Error running remote_command.sh to $ipadd Node, check ${tmpDir}/remote_command_check" - pass=false - REPORTPASS=false - fi - done + `$COLUMNSTORE_INSTALL_DIR/bin/remote_command.sh $ipadd $PASSWORD 'touch /dev/shm/cs_check' 1 > ${tmpDir}/remote_command_check 2>&1` + rc="$?" + if [ $rc -eq 0 ] || ( [ $rc -eq 2 ] && [ $OS == "suse12" ] ) ; then + `grep "Permission denied" ${tmpDir}/remote_command_check > /dev/null 2>&1` + if [ "$?" -eq 0 ]; then + echo "$ipadd Node permission test on /dev/shm : ${bold}Failed${normal}, change permissions to 777 and re-test" + pass=false + REPORTPASS=false + else + echo "$ipadd Node permission test on /dev/shm : Passed" + fi + else + echo "Error running remote_command.sh to $ipadd Node, check ${tmpDir}/remote_command_check" + pass=false + REPORTPASS=false + fi if ! $pass; then checkContinue From 63bdf451c520b96a8f99e5474abfef95a4a1c0ca Mon Sep 17 00:00:00 2001 From: Patrick LeBlanc Date: Tue, 20 Nov 2018 13:49:04 -0600 Subject: [PATCH 18/34] MCOL-1847. Now parsing NumBlocksPct with Config::fromText(). --- primitives/primproc/primproc.cpp | 19 +++++++++---------- 1 file changed, 9 insertions(+), 10 deletions(-) diff --git a/primitives/primproc/primproc.cpp b/primitives/primproc/primproc.cpp index 0845410ff..2e88493a0 100644 --- a/primitives/primproc/primproc.cpp +++ b/primitives/primproc/primproc.cpp @@ -391,7 +391,7 @@ int main(int argc, char* argv[]) int serverQueueSize = 10; int processorWeight = 8 * 1024; int processorQueueSize = 10 * 1024; - int BRPBlocksPct = 70; + int64_t BRPBlocksPct = 70; uint32_t BRPBlocks = 1887437; int BRPThreads = 16; int cacheCount = 1; @@ -526,20 +526,19 @@ int main(int argc, char* argv[]) } #else - bool cacheInMB = false; + bool absCache = false; if (temp > 0) { BRPBlocksPct = temp; - /* MCOL-1847. Did the user specify this in MB or GB? */ + /* MCOL-1847. Did the user specify this as an absolute? */ int len = strBlockPct.length(); - if (strBlockPct[len-1] == 'g' || strBlockPct[len-1] == 'm') - { - if (strBlockPct[len-1] == 'g') - BRPBlocksPct *= 1024; - cacheInMB = true; + if ((strBlockPct[len-1] >= 'a' && strBlockPct[len-1] <= 'z') || + (strBlockPct[len-1] >= 'A' && strBlockPct[len-1] <= 'Z')) { + absCache = true; + BRPBlocksPct = Config::fromText(strBlockPct); } } - if (cacheInMB) - BRPBlocks = BRPBlocksPct * 128; // 128 blocks per MB + if (absCache) + BRPBlocks = BRPBlocksPct / 8192; else BRPBlocks = ((BRPBlocksPct / 100.0) * (double) cg.getTotalMemory()) / 8192; #endif From 05ce8e24ef88cfcacdeea58e56da243a52434620 Mon Sep 17 00:00:00 2001 From: David Hill Date: Tue, 20 Nov 2018 15:13:20 -0600 Subject: [PATCH 19/34] MCOL-1944 - do chmod on all directories --- oam/install_scripts/syslogSetup.sh | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/oam/install_scripts/syslogSetup.sh b/oam/install_scripts/syslogSetup.sh index a900c2e84..31f457505 100755 --- a/oam/install_scripts/syslogSetup.sh +++ b/oam/install_scripts/syslogSetup.sh @@ -174,17 +174,21 @@ fi makeDir() { if [ ! -d /var/log/mariadb/columnstore ];then mkdir -p /var/log/mariadb/columnstore >/dev/null 2>&1 + test -d /var/log/mariadb/columnstore/archive || mkdir /var/log/mariadb/columnstore/archive >/dev/null 2>&1 + test -d /var/log/mariadb/columnstore/corefiles || mkdir /var/log/mariadb/columnstore/corefiles >/dev/null 2>&1 + test -d /var/log/mariadb/columnstore/trace || mkdir /var/log/mariadb/columnstore/trace >/dev/null 2>&1 chown $username:$groupname -R /var/log/mariadb + else + test -d /var/log/mariadb/columnstore/archive || mkdir /var/log/mariadb/columnstore/archive >/dev/null 2>&1 + test -d /var/log/mariadb/columnstore/corefiles || mkdir /var/log/mariadb/columnstore/corefiles >/dev/null 2>&1 + test -d /var/log/mariadb/columnstore/trace || mkdir /var/log/mariadb/columnstore/trace >/dev/null 2>&1 fi - + if [ $non_root_user == "yes" ]; then chmod 777 /var/log/mariadb chmod 777 /var/log/mariadb/columnstore fi - test -d /var/log/mariadb/columnstore/archive || mkdir /var/log/mariadb/columnstore/archive >/dev/null 2>&1 - test -d /var/log/mariadb/columnstore/corefiles || mkdir /var/log/mariadb/columnstore/corefiles >/dev/null 2>&1 - test -d /var/log/mariadb/columnstore/trace || mkdir /var/log/mariadb/columnstore/trace >/dev/null 2>&1 } install() { From 9c1bc910a56837000f2a139080a575dc7c8a80d3 Mon Sep 17 00:00:00 2001 From: David Hall Date: Tue, 20 Nov 2018 16:29:17 -0600 Subject: [PATCH 20/34] MCOL-1793, fix REGR_SLOPE calculations, change scale to DECIMAL_NOT_SPECIFIED (variable length) for most REGR_*** functions. --- utils/regr/corr.cpp | 4 ++-- utils/regr/covar_pop.cpp | 4 ++-- utils/regr/covar_samp.cpp | 11 ++++++++--- utils/regr/regr_intercept.cpp | 8 ++++---- utils/regr/regr_r2.cpp | 4 ++-- utils/regr/regr_slope.cpp | 12 +++++++----- utils/regr/regr_sxx.cpp | 4 ++-- utils/regr/regr_sxy.cpp | 4 ++-- utils/regr/regr_syy.cpp | 4 ++-- utils/regr/regrmysql.cpp | 5 +++-- utils/rowgroup/rowaggregation.cpp | 23 ++++++++++------------- 11 files changed, 44 insertions(+), 39 deletions(-) diff --git a/utils/regr/corr.cpp b/utils/regr/corr.cpp index f8d645ad4..3c819b86d 100644 --- a/utils/regr/corr.cpp +++ b/utils/regr/corr.cpp @@ -61,8 +61,8 @@ mcsv1_UDAF::ReturnCode corr::init(mcsv1Context* context, context->setUserDataSize(sizeof(corr_data)); context->setResultType(CalpontSystemCatalog::DOUBLE); context->setColWidth(8); - context->setScale(colTypes[0].scale + 8); - context->setPrecision(19); + context->setScale(DECIMAL_NOT_SPECIFIED); + context->setPrecision(0); context->setRunFlag(mcsv1sdk::UDAF_IGNORE_NULLS); return mcsv1_UDAF::SUCCESS; diff --git a/utils/regr/covar_pop.cpp b/utils/regr/covar_pop.cpp index 539497b6a..51d9a036f 100644 --- a/utils/regr/covar_pop.cpp +++ b/utils/regr/covar_pop.cpp @@ -59,8 +59,8 @@ mcsv1_UDAF::ReturnCode covar_pop::init(mcsv1Context* context, context->setUserDataSize(sizeof(covar_pop_data)); context->setResultType(CalpontSystemCatalog::DOUBLE); context->setColWidth(8); - context->setScale(colTypes[0].scale + 8); - context->setPrecision(19); + context->setScale(DECIMAL_NOT_SPECIFIED); + context->setPrecision(0); context->setRunFlag(mcsv1sdk::UDAF_IGNORE_NULLS); return mcsv1_UDAF::SUCCESS; diff --git a/utils/regr/covar_samp.cpp b/utils/regr/covar_samp.cpp index f3e16ffc4..b0ebb168b 100644 --- a/utils/regr/covar_samp.cpp +++ b/utils/regr/covar_samp.cpp @@ -59,8 +59,8 @@ mcsv1_UDAF::ReturnCode covar_samp::init(mcsv1Context* context, context->setUserDataSize(sizeof(covar_samp_data)); context->setResultType(CalpontSystemCatalog::DOUBLE); context->setColWidth(8); - context->setScale(colTypes[0].scale + 8); - context->setPrecision(19); + context->setScale(DECIMAL_NOT_SPECIFIED); + context->setPrecision(0); context->setRunFlag(mcsv1sdk::UDAF_IGNORE_NULLS); return mcsv1_UDAF::SUCCESS; @@ -136,7 +136,7 @@ mcsv1_UDAF::ReturnCode covar_samp::evaluate(mcsv1Context* context, static_any::a { struct covar_samp_data* data = (struct covar_samp_data*)context->getUserData()->data; double N = data->cnt; - if (N > 0) + if (N > 1) { double sumx = data->sumx; double sumy = data->sumy; @@ -145,6 +145,11 @@ mcsv1_UDAF::ReturnCode covar_samp::evaluate(mcsv1Context* context, static_any::a double covar_samp = (sumxy - ((sumx * sumy) / N)) / (N - 1); valOut = covar_samp; } + else + if (N == 1) + { + valOut = 0; + } return mcsv1_UDAF::SUCCESS; } diff --git a/utils/regr/regr_intercept.cpp b/utils/regr/regr_intercept.cpp index 7b0ccb943..3b1d78e18 100644 --- a/utils/regr/regr_intercept.cpp +++ b/utils/regr/regr_intercept.cpp @@ -60,8 +60,8 @@ mcsv1_UDAF::ReturnCode regr_intercept::init(mcsv1Context* context, context->setUserDataSize(sizeof(regr_intercept_data)); context->setResultType(CalpontSystemCatalog::DOUBLE); context->setColWidth(8); - context->setScale(colTypes[0].scale + 8); - context->setPrecision(19); + context->setScale(DECIMAL_NOT_SPECIFIED); + context->setPrecision(0); context->setRunFlag(mcsv1sdk::UDAF_IGNORE_NULLS); return mcsv1_UDAF::SUCCESS; @@ -145,13 +145,13 @@ mcsv1_UDAF::ReturnCode regr_intercept::evaluate(mcsv1Context* context, static_an double sumy = data->sumy; double sumx2 = data->sumx2; double sumxy = data->sumxy; - double slope = 0.0; + double slope = 0; double variance = (N * sumx2) - (sumx * sumx); if (variance != 0) { slope = ((N * sumxy) - (sumx * sumy)) / variance; - valOut = (sumy - (slope * sumx)) / N; } + valOut = (sumy - (slope * sumx)) / N; } return mcsv1_UDAF::SUCCESS; } diff --git a/utils/regr/regr_r2.cpp b/utils/regr/regr_r2.cpp index f8c923ee8..34e8888e8 100644 --- a/utils/regr/regr_r2.cpp +++ b/utils/regr/regr_r2.cpp @@ -61,8 +61,8 @@ mcsv1_UDAF::ReturnCode regr_r2::init(mcsv1Context* context, context->setUserDataSize(sizeof(regr_r2_data)); context->setResultType(CalpontSystemCatalog::DOUBLE); context->setColWidth(8); - context->setScale(colTypes[0].scale + 8); - context->setPrecision(19); + context->setScale(DECIMAL_NOT_SPECIFIED); + context->setPrecision(0); context->setRunFlag(mcsv1sdk::UDAF_IGNORE_NULLS); return mcsv1_UDAF::SUCCESS; diff --git a/utils/regr/regr_slope.cpp b/utils/regr/regr_slope.cpp index 721ab6a22..da178673a 100644 --- a/utils/regr/regr_slope.cpp +++ b/utils/regr/regr_slope.cpp @@ -60,8 +60,8 @@ mcsv1_UDAF::ReturnCode regr_slope::init(mcsv1Context* context, context->setUserDataSize(sizeof(regr_slope_data)); context->setResultType(CalpontSystemCatalog::DOUBLE); context->setColWidth(8); - context->setScale(colTypes[0].scale + 8); - context->setPrecision(19); + context->setScale(DECIMAL_NOT_SPECIFIED); + context->setPrecision(0); context->setRunFlag(mcsv1sdk::UDAF_IGNORE_NULLS); return mcsv1_UDAF::SUCCESS; @@ -141,14 +141,16 @@ mcsv1_UDAF::ReturnCode regr_slope::evaluate(mcsv1Context* context, static_any::a double N = data->cnt; if (N > 0) { + // COVAR_POP(y, x) / VAR_POP(x) double sumx = data->sumx; double sumy = data->sumy; double sumx2 = data->sumx2; double sumxy = data->sumxy; - double variance = (N * sumx2) - (sumx * sumx); - if (variance != 0) + double covar_pop = N * sumxy - sumx * sumy; + double var_pop = N * sumx2 - sumx * sumx; + if (var_pop != 0) { - double slope = ((N * sumxy) - (sumx * sumy)) / variance; + double slope = covar_pop / var_pop; valOut = slope; } } diff --git a/utils/regr/regr_sxx.cpp b/utils/regr/regr_sxx.cpp index a11b06a7d..3f06af61b 100644 --- a/utils/regr/regr_sxx.cpp +++ b/utils/regr/regr_sxx.cpp @@ -58,8 +58,8 @@ mcsv1_UDAF::ReturnCode regr_sxx::init(mcsv1Context* context, context->setUserDataSize(sizeof(regr_sxx_data)); context->setResultType(CalpontSystemCatalog::DOUBLE); context->setColWidth(8); - context->setScale(colTypes[0].scale + 8); - context->setPrecision(19); + context->setScale(DECIMAL_NOT_SPECIFIED); + context->setPrecision(0); context->setRunFlag(mcsv1sdk::UDAF_IGNORE_NULLS); return mcsv1_UDAF::SUCCESS; diff --git a/utils/regr/regr_sxy.cpp b/utils/regr/regr_sxy.cpp index e3df580b6..e6d005597 100644 --- a/utils/regr/regr_sxy.cpp +++ b/utils/regr/regr_sxy.cpp @@ -59,8 +59,8 @@ mcsv1_UDAF::ReturnCode regr_sxy::init(mcsv1Context* context, context->setUserDataSize(sizeof(regr_sxy_data)); context->setResultType(CalpontSystemCatalog::DOUBLE); context->setColWidth(8); - context->setScale(colTypes[0].scale + 8); - context->setPrecision(19); + context->setScale(DECIMAL_NOT_SPECIFIED); + context->setPrecision(0); context->setRunFlag(mcsv1sdk::UDAF_IGNORE_NULLS); return mcsv1_UDAF::SUCCESS; diff --git a/utils/regr/regr_syy.cpp b/utils/regr/regr_syy.cpp index 3b0ec7c8d..d0841f723 100644 --- a/utils/regr/regr_syy.cpp +++ b/utils/regr/regr_syy.cpp @@ -58,8 +58,8 @@ mcsv1_UDAF::ReturnCode regr_syy::init(mcsv1Context* context, context->setUserDataSize(sizeof(regr_syy_data)); context->setResultType(CalpontSystemCatalog::DOUBLE); context->setColWidth(8); - context->setScale(colTypes[0].scale + 8); - context->setPrecision(19); + context->setScale(DECIMAL_NOT_SPECIFIED); + context->setPrecision(0); context->setRunFlag(mcsv1sdk::UDAF_IGNORE_NULLS); return mcsv1_UDAF::SUCCESS; diff --git a/utils/regr/regrmysql.cpp b/utils/regr/regrmysql.cpp index 822d05ca6..08f46bb11 100644 --- a/utils/regr/regrmysql.cpp +++ b/utils/regr/regrmysql.cpp @@ -581,12 +581,13 @@ extern "C" double sumy = data->sumy; double sumx2 = data->sumx2; double sumxy = data->sumxy; + double slope = 0; double variance = (N * sumx2) - (sumx * sumx); if (variance) { - double slope = ((N * sumxy) - (sumx * sumy)) / variance; - return (sumy - (slope * sumx)) / N; + slope = ((N * sumxy) - (sumx * sumy)) / variance; } + return (sumy - (slope * sumx)) / N; } *is_null = 1; return 0; diff --git a/utils/rowgroup/rowaggregation.cpp b/utils/rowgroup/rowaggregation.cpp index d08781c07..8a80ca683 100644 --- a/utils/rowgroup/rowaggregation.cpp +++ b/utils/rowgroup/rowaggregation.cpp @@ -1673,11 +1673,10 @@ void RowAggregation::updateEntry(const Row& rowIn) { for (uint64_t i = 0; i < fFunctionCols.size(); i++) { - SP_ROWAGG_FUNC_t pFunctionCol = fFunctionCols[i]; - int64_t colIn = pFunctionCol->fInputColumnIndex; - int64_t colOut = pFunctionCol->fOutputColumnIndex; + int64_t colIn = fFunctionCols[i]->fInputColumnIndex; + int64_t colOut = fFunctionCols[i]->fOutputColumnIndex; - switch (pFunctionCol->fAggFunction) + switch (fFunctionCols[i]->fAggFunction) { case ROWAGG_COUNT_COL_NAME: @@ -1691,7 +1690,7 @@ void RowAggregation::updateEntry(const Row& rowIn) case ROWAGG_MIN: case ROWAGG_MAX: case ROWAGG_SUM: - doMinMaxSum(rowIn, colIn, colOut, pFunctionCol->fAggFunction); + doMinMaxSum(rowIn, colIn, colOut, fFunctionCols[i]->fAggFunction); break; case ROWAGG_AVG: @@ -1708,7 +1707,7 @@ void RowAggregation::updateEntry(const Row& rowIn) case ROWAGG_BIT_OR: case ROWAGG_BIT_XOR: { - doBitOp(rowIn, colIn, colOut, pFunctionCol->fAggFunction); + doBitOp(rowIn, colIn, colOut, fFunctionCols[i]->fAggFunction); break; } @@ -1731,7 +1730,7 @@ void RowAggregation::updateEntry(const Row& rowIn) { std::ostringstream errmsg; errmsg << "RowAggregation: function (id = " << - (uint64_t) pFunctionCol->fAggFunction << ") is not supported."; + (uint64_t) fFunctionCols[i]->fAggFunction << ") is not supported."; cerr << errmsg.str() << endl; throw logging::QueryDataExcept(errmsg.str(), logging::aggregateFuncErr); break; @@ -2015,7 +2014,6 @@ void RowAggregation::doUDAF(const Row& rowIn, int64_t colIn, int64_t colOut, for (uint32_t i = 0; i < paramCount; ++i) { - SP_ROWAGG_FUNC_t pFunctionCol = fFunctionCols[funcColsIdx]; mcsv1sdk::ColumnDatum& datum = valsIn[i]; // Turn on NULL flags based on the data dataFlags[i] = 0; @@ -2024,9 +2022,9 @@ void RowAggregation::doUDAF(const Row& rowIn, int64_t colIn, int64_t colOut, // to acces the constant value rather than a row value. cc = NULL; - if (pFunctionCol->fpConstCol) + if (fFunctionCols[funcColsIdx]->fpConstCol) { - cc = dynamic_cast(pFunctionCol->fpConstCol.get()); + cc = dynamic_cast(fFunctionCols[funcColsIdx]->fpConstCol.get()); } if ((cc && cc->type() == ConstantColumn::NULLDATA) @@ -2243,9 +2241,8 @@ void RowAggregation::doUDAF(const Row& rowIn, int64_t colIn, int64_t colOut, && fFunctionCols[funcColsIdx + 1]->fAggFunction == ROWAGG_MULTI_PARM) { ++funcColsIdx; - SP_ROWAGG_FUNC_t pFunctionCol = fFunctionCols[funcColsIdx]; - colIn = pFunctionCol->fInputColumnIndex; - colOut = pFunctionCol->fOutputColumnIndex; + colIn = fFunctionCols[funcColsIdx]->fInputColumnIndex; + colOut = fFunctionCols[funcColsIdx]->fOutputColumnIndex; } else { From bb096737c3d216b2f4a39b425745d29a8465e1f2 Mon Sep 17 00:00:00 2001 From: David Hill Date: Tue, 20 Nov 2018 16:47:22 -0600 Subject: [PATCH 21/34] MCOL-1944 - move cplogger from post-install to syslogSetup --- oam/install_scripts/syslogSetup.sh | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/oam/install_scripts/syslogSetup.sh b/oam/install_scripts/syslogSetup.sh index 31f457505..da8bd7684 100755 --- a/oam/install_scripts/syslogSetup.sh +++ b/oam/install_scripts/syslogSetup.sh @@ -232,6 +232,11 @@ if [ ! -z "$syslog_conf" ] ; then chmod 644 /etc/logrotate.d/columnstore restartSyslog + + #log install message + test -f $installdir/post/functions && . $installdir/post/functions + LD_LIBRARY_PATH=$LD_LIBRARY_PATH:$installdir/lib $installdir/bin/cplogger -i 19 "***** MariaDB Columnstore Installed *****" + fi } From f3ce33d9ea585fcaee41ddeb37014227d3ba5f0e Mon Sep 17 00:00:00 2001 From: Andrew Hutchings Date: Wed, 21 Nov 2018 11:23:12 +0000 Subject: [PATCH 22/34] MCOL-1790 Switch to MariaDB's case type detection MariaDB added a generic way to detect case type so remove our hack and switch to that. --- dbcon/mysql/ha_calpont_execplan.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/dbcon/mysql/ha_calpont_execplan.cpp b/dbcon/mysql/ha_calpont_execplan.cpp index a872ace4f..e048d5f34 100644 --- a/dbcon/mysql/ha_calpont_execplan.cpp +++ b/dbcon/mysql/ha_calpont_execplan.cpp @@ -3813,7 +3813,7 @@ FunctionColumn* buildCaseFunction(Item_func* item, gp_walk_info& gwi, bool& nonS FuncExp* funcexp = FuncExp::instance(); string funcName = "case_simple"; - if (strcasecmp(((Item_func_case*)item)->case_type(), "searched") == 0) + if (item->functype() == Item_func::CASE_SEARCHED_FUNC) { funcName = "case_searched"; } @@ -3857,7 +3857,7 @@ FunctionColumn* buildCaseFunction(Item_func* item, gp_walk_info& gwi, bool& nonS // some cpu cycles trying to build a ReturnedColumn as below. // Every even numbered arg is a WHEN. In between are the THEN. // An odd number of args indicates an ELSE residing in the last spot. - if (funcName == "case_searched" && + if ((item->functype() == Item_func::CASE_SEARCHED_FUNC) && (i < arg_offset)) { // MCOL-1472 Nested CASE with an ISNULL predicate. We don't want the predicate From 430f7c494cd69482b5ca8069486e39e8b87caddc Mon Sep 17 00:00:00 2001 From: David Hill Date: Wed, 21 Nov 2018 10:25:19 -0600 Subject: [PATCH 23/34] MCOL-1944 - move cplogger to syslogSetup.sh --- oam/install_scripts/post-install | 8 -------- oam/install_scripts/syslogSetup.sh | 32 ++++++++++++++++++++++-------- 2 files changed, 24 insertions(+), 16 deletions(-) diff --git a/oam/install_scripts/post-install b/oam/install_scripts/post-install index 6fac1eaae..d82bbb953 100755 --- a/oam/install_scripts/post-install +++ b/oam/install_scripts/post-install @@ -315,14 +315,6 @@ if [ -z "aws" ]; then $installdir/bin/MCSgetCredentials.sh >/dev/null 2>&1 fi -#log install message -test -f $installdir/post/functions && . $installdir/post/functions -if [ $user = "root" ]; then - $installdir/bin/cplogger -i 19 "***** MariaDB Columnstore Installed *****" -else - LD_LIBRARY_PATH=$LD_LIBRARY_PATH:$installdir/lib $installdir/bin/cplogger -i 19 "***** MariaDB Columnstore Installed *****" -fi - #setup hadoop hadoop=`which hadoop 2>/dev/null` if [ -z "$hadoop" ]; then diff --git a/oam/install_scripts/syslogSetup.sh b/oam/install_scripts/syslogSetup.sh index da8bd7684..22cf99054 100755 --- a/oam/install_scripts/syslogSetup.sh +++ b/oam/install_scripts/syslogSetup.sh @@ -227,16 +227,32 @@ if [ ! -z "$syslog_conf" ] ; then fi fi - # install Columnstore Log Rotate File - cp $installdir/bin/columnstoreLogRotate /etc/logrotate.d/columnstore > /dev/null 2>&1 - chmod 644 /etc/logrotate.d/columnstore - restartSyslog - - #log install message - test -f $installdir/post/functions && . $installdir/post/functions - LD_LIBRARY_PATH=$LD_LIBRARY_PATH:$installdir/lib $installdir/bin/cplogger -i 19 "***** MariaDB Columnstore Installed *****" + # install Columnstore Log Rotate File + if [ -d /etc/logrotate.d ]; then + cp $installdir/bin/columnstoreLogRotate /etc/logrotate.d/columnstore > /dev/null 2>&1 + chmod 644 /etc/logrotate.d/columnstore + + #do the logrotate to start with a fresh log file during install + logrotate -f /etc/logrotate.d/columnstore > /dev/null 2>&1 + fi + + #log install message and find the least permission that allows logging to work + CHMOD_LIST=("750" "770" "775" "777") + for CHMOD in "${CHMOD_LIST[@]}"; do + chmod $CHMOD /var/log/mariadb + chmod $CHMOD /var/log/mariadb/columnstore + + test -f $installdir/post/functions && . $installdir/post/functions + LD_LIBRARY_PATH=$LD_LIBRARY_PATH:$installdir/lib $installdir/bin/cplogger -i 19 "***** MariaDB Columnstore Installed *****" + + if [ -f /var/log/mariadb/columnstore/info.log ]; then + if [ ! -s /var/log/mariadb/columnstore/info.log ]; then + break + fi + fi + done fi } From 6563f48e32d26df2e5df6bd152cb300eb32a44cd Mon Sep 17 00:00:00 2001 From: Roman Nozdrin Date: Tue, 20 Nov 2018 20:47:11 +0300 Subject: [PATCH 24/34] MCOL-1786 Reduce the performance degradation caused by iequals. --- writeengine/bulk/we_bulkloadbuffer.cpp | 114 +++++++++++-------------- writeengine/bulk/we_bulkloadbuffer.h | 9 ++ 2 files changed, 59 insertions(+), 64 deletions(-) diff --git a/writeengine/bulk/we_bulkloadbuffer.cpp b/writeengine/bulk/we_bulkloadbuffer.cpp index 877f0b033..da568f9a6 100644 --- a/writeengine/bulk/we_bulkloadbuffer.cpp +++ b/writeengine/bulk/we_bulkloadbuffer.cpp @@ -31,8 +31,6 @@ #include #include -#include - #include "we_bulkload.h" #include "we_bulkloadbuffer.h" #include "we_brm.h" @@ -370,18 +368,11 @@ void BulkLoadBuffer::convert(char* field, int fieldLength, { errno = 0; - if (iequals(field, "true")) - { - fVal = 1; - } - else - { #ifdef _MSC_VER - fVal = (float)strtod( field, 0 ); + fVal = (float)strtod( field, 0 ); #else - fVal = strtof( field, 0 ); + fVal = strtof( field, 0 ); #endif - } if (errno == ERANGE) { @@ -414,6 +405,11 @@ void BulkLoadBuffer::convert(char* field, int fieldLength, fVal = minFltSat; bufStats.satCount++; } + if ( fVal == 0 + && isTrueWord(const_cast(field), fieldLength) ) + { + fVal = 1; + } } } @@ -474,14 +470,7 @@ void BulkLoadBuffer::convert(char* field, int fieldLength, { errno = 0; - if (iequals(field, "true")) - { - dVal = 1; - } - else - { - dVal = strtod(field, 0); - } + dVal = strtod(field, 0); if (errno == ERANGE) { @@ -514,6 +503,11 @@ void BulkLoadBuffer::convert(char* field, int fieldLength, dVal = column.fMinDblSat; bufStats.satCount++; } + else if (dVal == 0 + && isTrueWord(const_cast(field), fieldLength)) + { + dVal = 1; + } } } @@ -629,12 +623,6 @@ void BulkLoadBuffer::convert(char* field, int fieldLength, } else { - if (iequals(field, "true")) - { - strcpy(field, "1"); - fieldLength = 1; - } - if ( (column.dataType == CalpontSystemCatalog::DECIMAL ) || (column.dataType == CalpontSystemCatalog::UDECIMAL) ) { @@ -664,6 +652,11 @@ void BulkLoadBuffer::convert(char* field, int fieldLength, origVal = static_cast(column.fMaxIntSat); bSatVal = true; } + else if ( origVal == 0 + && isTrueWord(const_cast(field), fieldLength) ) + { + origVal = 1; + } if (bSatVal) bufStats.satCount++; @@ -722,15 +715,8 @@ void BulkLoadBuffer::convert(char* field, int fieldLength, { errno = 0; - if (iequals(field, "true")) - { - origVal = 1; - } - else - { - origVal = strtoll(field, 0, 10); - } - + origVal = strtoll(field, 0, 10); + if (errno == ERANGE) bSatVal = true; } @@ -747,6 +733,11 @@ void BulkLoadBuffer::convert(char* field, int fieldLength, origVal = static_cast(column.fMaxIntSat); bSatVal = true; } + else if ( origVal == 0 + && isTrueWord(const_cast(field), fieldLength) ) + { + origVal = 1; + } if (bSatVal) bufStats.satCount++; @@ -805,12 +796,12 @@ void BulkLoadBuffer::convert(char* field, int fieldLength, } else { - if (iequals(field, "true")) + if (isTrueWord(const_cast(field), fieldLength)) { strcpy(field, "1"); fieldLength = 1; } - + if ( (column.dataType == CalpontSystemCatalog::DECIMAL ) || (column.dataType == CalpontSystemCatalog::UDECIMAL)) { @@ -898,14 +889,7 @@ void BulkLoadBuffer::convert(char* field, int fieldLength, { errno = 0; - if (iequals(field, "true")) - { - origVal = 1; - } - else - { - origVal = strtoll(field, 0, 10); - } + origVal = strtoll(field, 0, 10); if (errno == ERANGE) bSatVal = true; @@ -923,6 +907,11 @@ void BulkLoadBuffer::convert(char* field, int fieldLength, origVal = static_cast(column.fMaxIntSat); bSatVal = true; } + else if ( origVal == 0 + && isTrueWord(const_cast(field), fieldLength) ) + { + origVal = 1; + } if (bSatVal) bufStats.satCount++; @@ -981,12 +970,12 @@ void BulkLoadBuffer::convert(char* field, int fieldLength, } else { - if (iequals(field, "true")) + if (isTrueWord(const_cast(field), fieldLength)) { strcpy(field, "1"); fieldLength = 1; } - + if ( (column.dataType == CalpontSystemCatalog::DECIMAL) || (column.dataType == CalpontSystemCatalog::UDECIMAL)) { @@ -1018,6 +1007,7 @@ void BulkLoadBuffer::convert(char* field, int fieldLength, bSatVal = true; } + if (bSatVal) bufStats.satCount++; @@ -1199,14 +1189,7 @@ void BulkLoadBuffer::convert(char* field, int fieldLength, { errno = 0; - if (iequals(field, "true")) - { - ullVal = 1; - } - else - { - ullVal = strtoull(field, 0, 10); - } + ullVal = strtoull(field, 0, 10); if (errno == ERANGE) bSatVal = true; @@ -1220,6 +1203,11 @@ void BulkLoadBuffer::convert(char* field, int fieldLength, ullVal = column.fMaxIntSat; bSatVal = true; } + else if ( ullVal == 0 + && isTrueWord(const_cast(field), fieldLength) ) + { + ullVal = 1; + } if (bSatVal) bufStats.satCount++; @@ -1276,14 +1264,7 @@ void BulkLoadBuffer::convert(char* field, int fieldLength, { errno = 0; - if (iequals(field, "true")) - { - origVal = 1; - } - else - { - origVal = strtoll(field, 0, 10); - } + origVal = strtoll(field, 0, 10); if (errno == ERANGE) bSatVal = true; @@ -1301,6 +1282,11 @@ void BulkLoadBuffer::convert(char* field, int fieldLength, origVal = static_cast(column.fMaxIntSat); bSatVal = true; } + else if ( origVal == 0 + && isTrueWord(const_cast(field), fieldLength) ) + { + origVal = 1; + } if (bSatVal) bufStats.satCount++; @@ -1361,12 +1347,12 @@ void BulkLoadBuffer::convert(char* field, int fieldLength, } else { - if (iequals(field, "true")) + if (isTrueWord(const_cast(field), fieldLength)) { strcpy(field, "1"); fieldLength = 1; } - + if ( (column.dataType == CalpontSystemCatalog::DECIMAL) || (column.dataType == CalpontSystemCatalog::UDECIMAL)) { diff --git a/writeengine/bulk/we_bulkloadbuffer.h b/writeengine/bulk/we_bulkloadbuffer.h index 85d60980c..649e9847e 100644 --- a/writeengine/bulk/we_bulkloadbuffer.h +++ b/writeengine/bulk/we_bulkloadbuffer.h @@ -383,5 +383,14 @@ public: } }; +inline bool isTrueWord(const char *field, int fieldLength) +{ + //return false; + return fieldLength == 4 && ( field[0] == 'T' || field[0] == 't' ) + && ( field[1] == 'R' || field[1] == 'r' ) + && ( field[2] == 'U' || field[2] == 'u' ) + && ( field[3] == 'E' || field[3] == 'e' ); +} + } #endif From 7c1853feab6f55874e8e2306f9870b2c44b10e5a Mon Sep 17 00:00:00 2001 From: David Hill Date: Wed, 21 Nov 2018 10:52:48 -0600 Subject: [PATCH 25/34] MCOL-1944 - move cplogger to syslogSetup.sh --- oam/install_scripts/syslogSetup.sh | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/oam/install_scripts/syslogSetup.sh b/oam/install_scripts/syslogSetup.sh index 22cf99054..7b088f522 100755 --- a/oam/install_scripts/syslogSetup.sh +++ b/oam/install_scripts/syslogSetup.sh @@ -183,12 +183,6 @@ makeDir() { test -d /var/log/mariadb/columnstore/corefiles || mkdir /var/log/mariadb/columnstore/corefiles >/dev/null 2>&1 test -d /var/log/mariadb/columnstore/trace || mkdir /var/log/mariadb/columnstore/trace >/dev/null 2>&1 fi - - if [ $non_root_user == "yes" ]; then - chmod 777 /var/log/mariadb - chmod 777 /var/log/mariadb/columnstore - fi - } install() { @@ -243,6 +237,9 @@ if [ ! -z "$syslog_conf" ] ; then for CHMOD in "${CHMOD_LIST[@]}"; do chmod $CHMOD /var/log/mariadb chmod $CHMOD /var/log/mariadb/columnstore + chmod $CHMOD /var/log/mariadb/columnstore/archive + chmod $CHMOD /var/log/mariadb/columnstore/corefiles + chmod $CHMOD /var/log/mariadb/columnstore/trace test -f $installdir/post/functions && . $installdir/post/functions LD_LIBRARY_PATH=$LD_LIBRARY_PATH:$installdir/lib $installdir/bin/cplogger -i 19 "***** MariaDB Columnstore Installed *****" From bb23798e7413b026828065b8c5d0dba9a35496a3 Mon Sep 17 00:00:00 2001 From: David Hill Date: Wed, 21 Nov 2018 11:10:27 -0600 Subject: [PATCH 26/34] MCOL-1944 - move cplogger to syslogSetup.sh --- oam/install_scripts/syslogSetup.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/oam/install_scripts/syslogSetup.sh b/oam/install_scripts/syslogSetup.sh index 7b088f522..b937b7ab2 100755 --- a/oam/install_scripts/syslogSetup.sh +++ b/oam/install_scripts/syslogSetup.sh @@ -172,7 +172,7 @@ fi } makeDir() { - if [ ! -d /var/log/mariadb/columnstore ];then + if [ ! -d /var/log/mariadb/columnstore ]; then mkdir -p /var/log/mariadb/columnstore >/dev/null 2>&1 test -d /var/log/mariadb/columnstore/archive || mkdir /var/log/mariadb/columnstore/archive >/dev/null 2>&1 test -d /var/log/mariadb/columnstore/corefiles || mkdir /var/log/mariadb/columnstore/corefiles >/dev/null 2>&1 @@ -246,7 +246,7 @@ if [ ! -z "$syslog_conf" ] ; then if [ -f /var/log/mariadb/columnstore/info.log ]; then if [ ! -s /var/log/mariadb/columnstore/info.log ]; then - break + exit 0 fi fi done From b422396371f2ef431b077b9f305c23a40482cbac Mon Sep 17 00:00:00 2001 From: David Hill Date: Wed, 21 Nov 2018 11:43:05 -0600 Subject: [PATCH 27/34] MCOL-1944 - move cplogger to syslogSetup.sh --- oam/install_scripts/syslogSetup.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/oam/install_scripts/syslogSetup.sh b/oam/install_scripts/syslogSetup.sh index b937b7ab2..babbd1c88 100755 --- a/oam/install_scripts/syslogSetup.sh +++ b/oam/install_scripts/syslogSetup.sh @@ -245,7 +245,7 @@ if [ ! -z "$syslog_conf" ] ; then LD_LIBRARY_PATH=$LD_LIBRARY_PATH:$installdir/lib $installdir/bin/cplogger -i 19 "***** MariaDB Columnstore Installed *****" if [ -f /var/log/mariadb/columnstore/info.log ]; then - if [ ! -s /var/log/mariadb/columnstore/info.log ]; then + if [ -s /var/log/mariadb/columnstore/info.log ]; then exit 0 fi fi From 00ce77b742ce284d41cdce3b5a8a7ce2247bea52 Mon Sep 17 00:00:00 2001 From: Andrew Hutchings Date: Thu, 22 Nov 2018 21:58:03 +0000 Subject: [PATCH 28/34] MCOL-1624 mcssystemready() does more testing Now checks if system catalog exists and if we can query the system catalogue in FE mode (and therefore tests ExeMgr and PrimProc). --- dbcon/mysql/ha_calpont_impl.cpp | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/dbcon/mysql/ha_calpont_impl.cpp b/dbcon/mysql/ha_calpont_impl.cpp index 3be44d2b8..9536ff24f 100644 --- a/dbcon/mysql/ha_calpont_impl.cpp +++ b/dbcon/mysql/ha_calpont_impl.cpp @@ -2015,6 +2015,7 @@ extern "C" Oam oam; DBRM dbrm(true); SystemStatus systemstatus; + WriteEngine::FileOp fileOp; try { @@ -2022,8 +2023,15 @@ extern "C" if (systemstatus.SystemOpState == ACTIVE && dbrm.getSystemReady() - && dbrm.getSystemQueryReady()) + && dbrm.getSystemQueryReady() + && fileOp.existsOIDDir(1001)) { + // Test getting system catalogue data from ExeMgr + boost::shared_ptr systemCatalogPtr = + execplan::CalpontSystemCatalog::makeCalpontSystemCatalog(0); + systemCatalogPtr->identity(execplan::CalpontSystemCatalog::FE); + systemCatalogPtr->getTableCount(); + return 1; } } From 176ef2f2c10fcc264a0045078e99d8d379b9628b Mon Sep 17 00:00:00 2001 From: David Hall Date: Fri, 23 Nov 2018 12:42:29 -0600 Subject: [PATCH 29/34] MCOL-1793 Add udafContext to the copy constructor of WindowFunctionColumn. --- dbcon/execplan/windowfunctioncolumn.cpp | 3 +- dbcon/execplan/windowfunctioncolumn.h | 4 + genii.vpw | 3 - oam/oamcpp/oamcpp.vpj | 444 ++++++------ oamapps/hardwareMonitor/HardwareMonitor.vpj | 430 ++++++------ oamapps/postConfigure/postConfigure.vpj | 440 ++++++------ .../ReplayTransactionLog.vpj | 426 ++++++------ oamapps/resourceMonitor/resourceMonitor.vpj | 426 ++++++------ oamapps/serverMonitor/ServerMonitor.vpj | 468 ++++++------- primitives/blockcache/blockcache.vpj | 466 ++++++------- procmgr/procmgr.vpj | 436 ++++++------ procmon/procmon.vpj | 434 ++++++------ tools/configMgt/autoConfigure.vpj | 636 +++++++++--------- utils/batchloader/batchloader.vpj | 430 ++++++------ utils/cacheutils/cacheutils.vpj | 430 ++++++------ utils/compress/compress.vpj | 456 ++++++------- utils/configcpp/configcpp.vpj | 452 ++++++------- utils/dataconvert/dataconvert.vpj | 448 ++++++------ utils/dataconvert/tdriver.vpj | 436 ++++++------ utils/ddlcleanup/ddlcleanup.vpj | 430 ++++++------ utils/idbdatafile/idbdatafile.vpj | 484 ++++++------- utils/idbhdfs/idbhdfs.vpj | 454 ++++++------- utils/multicast/multicast.vpj | 456 ++++++------- utils/regr/regr_intercept.cpp | 9 +- utils/rwlock/rwlock.vpj | 438 ++++++------ utils/startup/startup.vpj | 438 ++++++------ versioning/BRM/brm.vpj | 604 ++++++++--------- writeengine/bulk/bulk.vpj | 596 ++++++++-------- writeengine/client/writeengineclient.vpj | 576 ++++++++-------- writeengine/dictionary/dictionary.vpj | 438 ++++++------ writeengine/splitter/splitter.vpj | 506 +++++++------- writeengine/wrapper/wrapper.vpj | 452 ++++++------- 32 files changed, 6575 insertions(+), 6574 deletions(-) diff --git a/dbcon/execplan/windowfunctioncolumn.cpp b/dbcon/execplan/windowfunctioncolumn.cpp index 5c84ff2d1..b1377880f 100644 --- a/dbcon/execplan/windowfunctioncolumn.cpp +++ b/dbcon/execplan/windowfunctioncolumn.cpp @@ -233,7 +233,8 @@ WindowFunctionColumn::WindowFunctionColumn( const WindowFunctionColumn& rhs, con fFunctionName(rhs.functionName()), fFunctionParms(rhs.functionParms()), fPartitions (rhs.partitions()), - fOrderBy (rhs.orderBy()) + fOrderBy (rhs.orderBy()), + udafContext(rhs.getUDAFContext()) {} const string WindowFunctionColumn::toString() const diff --git a/dbcon/execplan/windowfunctioncolumn.h b/dbcon/execplan/windowfunctioncolumn.h index 3fc983e55..47c82b805 100644 --- a/dbcon/execplan/windowfunctioncolumn.h +++ b/dbcon/execplan/windowfunctioncolumn.h @@ -140,6 +140,10 @@ public: { return udafContext; } + const mcsv1sdk::mcsv1Context& getUDAFContext() const + { + return udafContext; + } private: /** diff --git a/genii.vpw b/genii.vpw index 686b01ed7..417da26ce 100644 --- a/genii.vpw +++ b/genii.vpw @@ -4,12 +4,9 @@ - - - diff --git a/oam/oamcpp/oamcpp.vpj b/oam/oamcpp/oamcpp.vpj index ce008c9d5..45ed51878 100644 --- a/oam/oamcpp/oamcpp.vpj +++ b/oam/oamcpp/oamcpp.vpj @@ -1,225 +1,225 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + Version="10.0" + VendorName="SlickEdit" + TemplateName="GNU C/C++" + WorkingDir="."> + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/oamapps/hardwareMonitor/HardwareMonitor.vpj b/oamapps/hardwareMonitor/HardwareMonitor.vpj index 6e48fd6cf..a376e6628 100644 --- a/oamapps/hardwareMonitor/HardwareMonitor.vpj +++ b/oamapps/hardwareMonitor/HardwareMonitor.vpj @@ -1,218 +1,218 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + Version="10.0" + VendorName="SlickEdit" + TemplateName="GNU C/C++" + WorkingDir="."> + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/oamapps/postConfigure/postConfigure.vpj b/oamapps/postConfigure/postConfigure.vpj index 81d372dee..cc89393af 100644 --- a/oamapps/postConfigure/postConfigure.vpj +++ b/oamapps/postConfigure/postConfigure.vpj @@ -1,223 +1,223 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + Version="10.0" + VendorName="SlickEdit" + TemplateName="GNU C/C++" + WorkingDir="."> + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/oamapps/replayTransactionLog/ReplayTransactionLog.vpj b/oamapps/replayTransactionLog/ReplayTransactionLog.vpj index 034df4c88..ce3154c21 100644 --- a/oamapps/replayTransactionLog/ReplayTransactionLog.vpj +++ b/oamapps/replayTransactionLog/ReplayTransactionLog.vpj @@ -1,216 +1,216 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + Version="10.0" + VendorName="SlickEdit" + TemplateName="GNU C/C++" + WorkingDir="."> + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/oamapps/resourceMonitor/resourceMonitor.vpj b/oamapps/resourceMonitor/resourceMonitor.vpj index b8c69a721..c88870832 100644 --- a/oamapps/resourceMonitor/resourceMonitor.vpj +++ b/oamapps/resourceMonitor/resourceMonitor.vpj @@ -1,216 +1,216 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + Version="10.0" + VendorName="SlickEdit" + TemplateName="GNU C/C++" + WorkingDir="."> + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/oamapps/serverMonitor/ServerMonitor.vpj b/oamapps/serverMonitor/ServerMonitor.vpj index a5e5909e2..212a1cf81 100644 --- a/oamapps/serverMonitor/ServerMonitor.vpj +++ b/oamapps/serverMonitor/ServerMonitor.vpj @@ -1,237 +1,237 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + Version="10.0" + VendorName="SlickEdit" + TemplateName="GNU C/C++" + WorkingDir="."> + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/primitives/blockcache/blockcache.vpj b/primitives/blockcache/blockcache.vpj index 672753473..da41e8eb5 100644 --- a/primitives/blockcache/blockcache.vpj +++ b/primitives/blockcache/blockcache.vpj @@ -1,236 +1,236 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + Version="10.0" + VendorName="SlickEdit" + TemplateName="GNU C/C++" + WorkingDir="."> + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/procmgr/procmgr.vpj b/procmgr/procmgr.vpj index ab587c6a7..443588442 100644 --- a/procmgr/procmgr.vpj +++ b/procmgr/procmgr.vpj @@ -1,221 +1,221 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + Version="10.0" + VendorName="SlickEdit" + TemplateName="GNU C/C++" + WorkingDir="."> + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/procmon/procmon.vpj b/procmon/procmon.vpj index 0c8c399a8..bb0c6c9fa 100644 --- a/procmon/procmon.vpj +++ b/procmon/procmon.vpj @@ -1,220 +1,220 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + Version="10.0" + VendorName="SlickEdit" + TemplateName="GNU C/C++" + WorkingDir="."> + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/tools/configMgt/autoConfigure.vpj b/tools/configMgt/autoConfigure.vpj index 5d788fe40..edd1497f5 100644 --- a/tools/configMgt/autoConfigure.vpj +++ b/tools/configMgt/autoConfigure.vpj @@ -1,321 +1,321 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + Version="10.0" + VendorName="SlickEdit" + TemplateName="GNU C/C++" + WorkingDir="."> + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/utils/batchloader/batchloader.vpj b/utils/batchloader/batchloader.vpj index 16d7753ee..c48a19007 100644 --- a/utils/batchloader/batchloader.vpj +++ b/utils/batchloader/batchloader.vpj @@ -1,218 +1,218 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + Version="10.0" + VendorName="SlickEdit" + TemplateName="GNU C/C++" + WorkingDir="."> + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/utils/cacheutils/cacheutils.vpj b/utils/cacheutils/cacheutils.vpj index eaefca217..2a8bec873 100644 --- a/utils/cacheutils/cacheutils.vpj +++ b/utils/cacheutils/cacheutils.vpj @@ -1,218 +1,218 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + Version="10.0" + VendorName="SlickEdit" + TemplateName="GNU C/C++" + WorkingDir="."> + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/utils/compress/compress.vpj b/utils/compress/compress.vpj index 860ce8693..02ecfff14 100644 --- a/utils/compress/compress.vpj +++ b/utils/compress/compress.vpj @@ -1,231 +1,231 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + Version="10.0" + VendorName="SlickEdit" + TemplateName="GNU C/C++" + WorkingDir="."> + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/utils/configcpp/configcpp.vpj b/utils/configcpp/configcpp.vpj index d8dff4d13..73b33cfb3 100644 --- a/utils/configcpp/configcpp.vpj +++ b/utils/configcpp/configcpp.vpj @@ -1,229 +1,229 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + Version="10.0" + VendorName="SlickEdit" + TemplateName="GNU C/C++" + WorkingDir="."> + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/utils/dataconvert/dataconvert.vpj b/utils/dataconvert/dataconvert.vpj index d38ab71c0..e2b2b9763 100644 --- a/utils/dataconvert/dataconvert.vpj +++ b/utils/dataconvert/dataconvert.vpj @@ -1,227 +1,227 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + Version="10.0" + VendorName="SlickEdit" + TemplateName="GNU C/C++" + WorkingDir="."> + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/utils/dataconvert/tdriver.vpj b/utils/dataconvert/tdriver.vpj index 2e665c68c..4b6a8630a 100644 --- a/utils/dataconvert/tdriver.vpj +++ b/utils/dataconvert/tdriver.vpj @@ -1,221 +1,221 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + Version="10.0" + VendorName="SlickEdit" + TemplateName="GNU C/C++" + WorkingDir="."> + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/utils/ddlcleanup/ddlcleanup.vpj b/utils/ddlcleanup/ddlcleanup.vpj index 777a12f05..b207dd826 100644 --- a/utils/ddlcleanup/ddlcleanup.vpj +++ b/utils/ddlcleanup/ddlcleanup.vpj @@ -1,218 +1,218 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + Version="10.0" + VendorName="SlickEdit" + TemplateName="GNU C/C++" + WorkingDir="."> + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/utils/idbdatafile/idbdatafile.vpj b/utils/idbdatafile/idbdatafile.vpj index cacaa56bc..cd2d7e77c 100644 --- a/utils/idbdatafile/idbdatafile.vpj +++ b/utils/idbdatafile/idbdatafile.vpj @@ -1,245 +1,245 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + Version="10.0" + VendorName="SlickEdit" + TemplateName="GNU C/C++" + WorkingDir="."> + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/utils/idbhdfs/idbhdfs.vpj b/utils/idbhdfs/idbhdfs.vpj index 7bac0bf8e..a3704675d 100644 --- a/utils/idbhdfs/idbhdfs.vpj +++ b/utils/idbhdfs/idbhdfs.vpj @@ -1,230 +1,230 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + Version="10.0" + VendorName="SlickEdit" + TemplateName="GNU C/C++" + WorkingDir="."> + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/utils/multicast/multicast.vpj b/utils/multicast/multicast.vpj index 963327b70..42a806ce0 100644 --- a/utils/multicast/multicast.vpj +++ b/utils/multicast/multicast.vpj @@ -1,231 +1,231 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + Version="10.0" + VendorName="SlickEdit" + TemplateName="GNU C/C++" + WorkingDir="."> + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/utils/regr/regr_intercept.cpp b/utils/regr/regr_intercept.cpp index 3b1d78e18..6d4c35a47 100644 --- a/utils/regr/regr_intercept.cpp +++ b/utils/regr/regr_intercept.cpp @@ -145,13 +145,12 @@ mcsv1_UDAF::ReturnCode regr_intercept::evaluate(mcsv1Context* context, static_an double sumy = data->sumy; double sumx2 = data->sumx2; double sumxy = data->sumxy; - double slope = 0; - double variance = (N * sumx2) - (sumx * sumx); - if (variance != 0) + double numerator = sumy * sumx2 - sumx * sumxy; + double var_pop = (N * sumx2) - (sumx * sumx); + if (var_pop != 0) { - slope = ((N * sumxy) - (sumx * sumy)) / variance; + valOut = numerator / var_pop; } - valOut = (sumy - (slope * sumx)) / N; } return mcsv1_UDAF::SUCCESS; } diff --git a/utils/rwlock/rwlock.vpj b/utils/rwlock/rwlock.vpj index cb1007232..3d2e45619 100644 --- a/utils/rwlock/rwlock.vpj +++ b/utils/rwlock/rwlock.vpj @@ -1,222 +1,222 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + Version="10.0" + VendorName="SlickEdit" + TemplateName="GNU C/C++" + WorkingDir="."> + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/utils/startup/startup.vpj b/utils/startup/startup.vpj index 3fc1d2480..8f2340702 100644 --- a/utils/startup/startup.vpj +++ b/utils/startup/startup.vpj @@ -1,222 +1,222 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + Version="10.0" + VendorName="SlickEdit" + TemplateName="GNU C/C++" + WorkingDir="."> + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/versioning/BRM/brm.vpj b/versioning/BRM/brm.vpj index 5f28016da..07d034a19 100644 --- a/versioning/BRM/brm.vpj +++ b/versioning/BRM/brm.vpj @@ -1,305 +1,305 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + Version="10.0" + VendorName="SlickEdit" + TemplateName="GNU C/C++" + WorkingDir="."> + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/writeengine/bulk/bulk.vpj b/writeengine/bulk/bulk.vpj index 402a35a27..de86cdd04 100644 --- a/writeengine/bulk/bulk.vpj +++ b/writeengine/bulk/bulk.vpj @@ -1,301 +1,301 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + Version="10.0" + VendorName="SlickEdit" + TemplateName="GNU C/C++" + WorkingDir="."> + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/writeengine/client/writeengineclient.vpj b/writeengine/client/writeengineclient.vpj index 65093428e..4afe9c836 100644 --- a/writeengine/client/writeengineclient.vpj +++ b/writeengine/client/writeengineclient.vpj @@ -1,291 +1,291 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + Version="10.0" + VendorName="SlickEdit" + TemplateName="GNU C/C++" + WorkingDir="."> + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/writeengine/dictionary/dictionary.vpj b/writeengine/dictionary/dictionary.vpj index c30613ee2..9aa57303f 100644 --- a/writeengine/dictionary/dictionary.vpj +++ b/writeengine/dictionary/dictionary.vpj @@ -1,222 +1,222 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + Version="10.0" + VendorName="SlickEdit" + TemplateName="GNU C/C++" + WorkingDir="."> + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/writeengine/splitter/splitter.vpj b/writeengine/splitter/splitter.vpj index e7754a250..be853818e 100644 --- a/writeengine/splitter/splitter.vpj +++ b/writeengine/splitter/splitter.vpj @@ -1,256 +1,256 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + Version="10.0" + VendorName="SlickEdit" + TemplateName="GNU C/C++" + WorkingDir="." + VCSProject="Subversion:"> + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/writeengine/wrapper/wrapper.vpj b/writeengine/wrapper/wrapper.vpj index b26d790a4..48fa6614c 100644 --- a/writeengine/wrapper/wrapper.vpj +++ b/writeengine/wrapper/wrapper.vpj @@ -1,229 +1,229 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + Version="10.0" + VendorName="SlickEdit" + TemplateName="GNU C/C++" + WorkingDir="."> + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + From 61579c71778fb9a323bf6e737935c0deae87779b Mon Sep 17 00:00:00 2001 From: Andrew Hutchings Date: Sat, 24 Nov 2018 21:52:33 +0000 Subject: [PATCH 30/34] Revert "MCOL-1624 mcssystemready() does more testing" This reverts commit 00ce77b742ce284d41cdce3b5a8a7ce2247bea52. --- dbcon/mysql/ha_calpont_impl.cpp | 10 +--------- 1 file changed, 1 insertion(+), 9 deletions(-) diff --git a/dbcon/mysql/ha_calpont_impl.cpp b/dbcon/mysql/ha_calpont_impl.cpp index 9536ff24f..3be44d2b8 100644 --- a/dbcon/mysql/ha_calpont_impl.cpp +++ b/dbcon/mysql/ha_calpont_impl.cpp @@ -2015,7 +2015,6 @@ extern "C" Oam oam; DBRM dbrm(true); SystemStatus systemstatus; - WriteEngine::FileOp fileOp; try { @@ -2023,15 +2022,8 @@ extern "C" if (systemstatus.SystemOpState == ACTIVE && dbrm.getSystemReady() - && dbrm.getSystemQueryReady() - && fileOp.existsOIDDir(1001)) + && dbrm.getSystemQueryReady()) { - // Test getting system catalogue data from ExeMgr - boost::shared_ptr systemCatalogPtr = - execplan::CalpontSystemCatalog::makeCalpontSystemCatalog(0); - systemCatalogPtr->identity(execplan::CalpontSystemCatalog::FE); - systemCatalogPtr->getTableCount(); - return 1; } } From ffb3a68a86568c4aec4d6745a6ead4d9cb85f87c Mon Sep 17 00:00:00 2001 From: Roman Nozdrin Date: Sun, 25 Nov 2018 12:24:38 +0300 Subject: [PATCH 31/34] MCOL-1716 Disable GROUP BY handler for queries with NOT IN and correlated subquery. --- dbcon/mysql/ha_calpont.cpp | 36 +++++++++++++++++++++++++++++++----- 1 file changed, 31 insertions(+), 5 deletions(-) diff --git a/dbcon/mysql/ha_calpont.cpp b/dbcon/mysql/ha_calpont.cpp index e16a8ba83..6ea5f85a6 100644 --- a/dbcon/mysql/ha_calpont.cpp +++ b/dbcon/mysql/ha_calpont.cpp @@ -1161,9 +1161,9 @@ void check_walk(const Item* item, void* arg) { case Item::FUNC_ITEM: { - Item_func* ifp = (Item_func*)item; + const Item_func* ifp = static_cast(item); - if ( ifp->functype() != Item_func::EQ_FUNC ) + if ( ifp->functype() != Item_func::EQ_FUNC ) // NON-equi JOIN { if ( ifp->argument_count() == 2 && ifp->arguments()[0]->type() == Item::FIELD_ITEM && @@ -1178,11 +1178,36 @@ void check_walk(const Item* item, void* arg) return; } } + else // IN + correlated subquery + { + if ( ifp->functype() == Item_func::NOT_FUNC + && ifp->arguments()[0]->type() == Item::EXPR_CACHE_ITEM ) + { + check_walk(ifp->arguments()[0], arg); + } + } + } + break; + } + + case Item::EXPR_CACHE_ITEM: // IN + correlated subquery + { + const Item_cache_wrapper* icw = static_cast(item); + if ( icw->get_orig_item()->type() == Item::FUNC_ITEM ) + { + const Item_func *ifp = static_cast(icw->get_orig_item()); + if ( ifp->argument_count() == 2 && + ( ifp->arguments()[0]->type() == Item::Item::SUBSELECT_ITEM + || ifp->arguments()[1]->type() == Item::Item::SUBSELECT_ITEM )) + { + *unsupported_feature = true; + return; + } } break; } - case Item::COND_ITEM: + case Item::COND_ITEM: // OR in cods is unsupported yet { Item_cond* icp = (Item_cond*)item; if ( is_cond_or(icp) ) @@ -1219,6 +1244,7 @@ static group_by_handler* create_calpont_group_by_handler(THD* thd, Query* query) { ha_calpont_group_by_handler* handler = NULL; + // same as thd->lex->current_select SELECT_LEX *select_lex = query->from->select_lex; // Create a handler if query is valid. See comments for details. @@ -1236,7 +1262,7 @@ create_calpont_group_by_handler(THD* thd, Query* query) unsupported_feature = true; } - // Unsupported JOIN conditions check. + // Unsupported conditions check. if ( !unsupported_feature ) { JOIN *join = select_lex->join; @@ -1246,7 +1272,7 @@ create_calpont_group_by_handler(THD* thd, Query* query) icp = reinterpret_cast(join->conds); if ( unsupported_feature == false - && join->table_count > 1 && icp ) + && icp ) { icp->traverse_cond(check_walk, &unsupported_feature, Item::POSTFIX); } From f9922a37dc907e9b1c242e719a68453eb03d5a87 Mon Sep 17 00:00:00 2001 From: Patrick LeBlanc Date: Mon, 26 Nov 2018 10:44:11 -0600 Subject: [PATCH 32/34] MCOL-1558. Changed one other place the _current file is used. --- procmgr/processmanager.cpp | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/procmgr/processmanager.cpp b/procmgr/processmanager.cpp index 3203c67b2..f21459f22 100644 --- a/procmgr/processmanager.cpp +++ b/procmgr/processmanager.cpp @@ -9176,7 +9176,11 @@ int ProcessManager::getDBRMData(messageqcpp::IOSocket fIos, std::string moduleNa char line[200]; oldFile.getline(line, 200); - currentDbrmFile = line; + // MCOL-1558. Handle absolute and relative paths. + if (line[0] == '/') + currentDbrmFile = line; + else + currentDbrmFile = DBRMroot.substr(0, DBRMroot.find_last_of('/') + 1) + line; } else { From 07d4ffecc846dacee7f5a93447a20d0d2ba1dbd5 Mon Sep 17 00:00:00 2001 From: david hill Date: Mon, 26 Nov 2018 15:05:39 -0600 Subject: [PATCH 33/34] Update Readme change to GA --- README | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README b/README index 541e8872f..bc8efde2c 100644 --- a/README +++ b/README @@ -1,9 +1,9 @@ This is MariaDB ColumnStore 1.2 -MariaDB ColumnStore 1.2 is the development version of MariaDB ColumnStore. +MariaDB ColumnStore 1.2 is the GA version of MariaDB ColumnStore. It is built by porting InfiniDB 4.6.7 on MariaDB 10.2 and adding entirely new features not found anywhere else. -MariaDB ColumnStore 1.2 is a pre-release. +MariaDB ColumnStore 1.2 is a GA. Additional features will be pushed in future releases. A few things to notice: From 8be6cc4e6ea63932838b6465cb355e79e3d1f48f Mon Sep 17 00:00:00 2001 From: david hill Date: Mon, 26 Nov 2018 15:07:01 -0600 Subject: [PATCH 34/34] Update Readme.md --- README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index d9e21c977..5e4c33b6d 100644 --- a/README.md +++ b/README.md @@ -1,10 +1,10 @@ # MariaDB ColumnStore Storage/Execution engine 1.2 -MariaDB ColumnStore 1.2 is the development version of MariaDB ColumnStore. +MariaDB ColumnStore 1.2 is the GA version of MariaDB ColumnStore. It is built by porting InfiniDB 4.6.7 on MariaDB 10.2 and adding entirely new features not found anywhere else. -# MariaDB ColumnStore 1.2 is a pre-release. +# MariaDB ColumnStore 1.2 is a GA release. - Do not use pre-releases on production systems.