database/mariadb-columnstore-engine
1
0
Fork 0
mirror of https://github.com/mariadb-corporation/mariadb-columnstore-engine.git synced 2025-04-18 21:44:02 +03:00
Code

MCOL-5464: Fixes of bugs from ASAN warnings, part one (#2792)

Browse Source 
* Fixes of bugs from ASAN warnings, part one

* MQC as static library, with nifty counter for global map and mutex

* Switch clang to 16

* link messageqcpp to execplan
This commit is contained in:
Leonid Fedorov 2023-04-04 02:33:23 +03:00 committed by GitHub
parent ac8881091b
commit 2e1394149b
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
76 changed files with 630 additions and 2050 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
.drone.jsonnet
View File

@ -31,7 +31,7 @@ local cmakeflags = '-DCMAKE_BUILD_TYPE=RelWithDebInfo -DBUILD_CONFIG=mysql_relea
'-DPLUGIN_GSSAPI=NO -DPLUGIN_SPIDER=NO -DPLUGIN_OQGRAPH=NO -DPLUGIN_SPHINX=NO ' +
'-DWITH_EMBEDDED_SERVER=NO -DWITH_WSREP=NO -DWITH_COREDUMPS=ON';
local clang_version = '14';
local clang_version = '16';
local gcc_version = '11';
local clang_update_alternatives = 'update-alternatives --install /usr/bin/clang clang /usr/bin/clang-' + clang_version + ' 100 --slave /usr/bin/clang++ clang++ /usr/bin/clang++-' + clang_version + ' && update-alternatives --install /usr/bin/cc cc /usr/bin/clang 100 && update-alternatives --install /usr/bin/c++ c++ /usr/bin/clang++ 100 ';

9
CMakeLists.txt
View File

@ -205,7 +205,7 @@ IF (NOT INSTALL_LAYOUT)
ENDIF()
OPTION(SECURITY_HARDENED "Use security-enhancing compiler features (stack protector, relro, etc)" ${security_default})
OPTION(SECURITY_HARDENED_NEW "Use new security-enhancing compilier features" OFF)
IF(SECURITY_HARDENED)
IF(SECURITY_HARDENED AND NOT WITH_ASAN AND NOT WITH_UBSAN AND NOT WITH_TSAN AND NOT WITH_GPROF)
# security-enhancing flags
MY_CHECK_AND_SET_COMPILER_FLAG("-pie -fPIC")
MY_CHECK_AND_SET_COMPILER_FLAG("-Wl,-z,relro,-z,now")
@ -237,10 +237,15 @@ elseif (CMAKE_CXX_COMPILER_ID STREQUAL "GNU")
MY_CHECK_AND_SET_COMPILER_FLAG("-Wno-register")
endif()
IF (WITH_COLUMNSTORE_ASAN)
MY_CHECK_AND_SET_COMPILER_FLAG("-U_FORTIFY_SOURCE" DEBUG RELWITHDEBINFO)
MY_CHECK_AND_SET_COMPILER_FLAG("-fsanitize=address -fsanitize-address-use-after-scope -fPIC")
ENDIF()
MY_CHECK_AND_SET_COMPILER_FLAG("-Wno-deprecated-copy" DEBUG RELEASE RELWITHDEBINFO MINSIZEREL)
MY_CHECK_AND_SET_COMPILER_FLAG("-Wno-deprecated-declarations" DEBUG RELEASE RELWITHDEBINFO MINSIZEREL)
MY_CHECK_AND_SET_COMPILER_FLAG("-Werror -Wall")
MY_CHECK_AND_SET_COMPILER_FLAG("-Werror -Wall -Wextra")
SET (ENGINE_LDFLAGS "-Wl,--no-as-needed -Wl,--add-needed")
SET (ENGINE_DT_LIB datatypes)
SET (ENGINE_COMMON_LIBS messageqcpp loggingcpp configcpp idbboot boost_thread xml2 pthread rt ${ENGINE_DT_LIB})

132
build/bootstrap_mcs.sh
View File

@ -18,17 +18,25 @@ message "Building Mariadb Server from $color_yellow$MDB_SOURCE_PATH$color_normal
BUILD_TYPE_OPTIONS=("Debug" "RelWithDebInfo")
DISTRO_OPTIONS=("Ubuntu" "CentOS" "Debian" "Rocky")
cd $SCRIPT_LOCATION
CURRENT_BRANCH=$(git rev-parse --abbrev-ref HEAD)
BRANCHES=($(git branch --list --no-color| grep "[^* ]+" -Eo))
cd -
optparse.define short=t long=build-type desc="Build Type: ${BUILD_TYPE_OPTIONS[*]}" variable=MCS_BUILD_TYPE
optparse.define short=d long=distro desc="Choouse your OS: ${DISTRO_OPTIONS[*]}" variable=OS
optparse.define short=s long=skip-deps desc="Skip install dependences" variable=SKIP_DEPS default=false value=true
optparse.define short=D long=install-deps desc="Install dependences" variable=INSTALL_DEPS default=false value=true
optparse.define short=C long=force-cmake-reconfig desc="Force cmake reconfigure" variable=FORCE_CMAKE_CONFIG default=false value=true
optparse.define short=S long=skip-columnstore-submodules desc="Skip columnstore submodules initialization" variable=SKIP_SUBMODULES default=false value=true
optparse.define short=u long=skip-unit-tests desc="Skip UnitTests" variable=SKIP_UNIT_TESTS default=false value=true
optparse.define short=B long=run-microbench="Compile and run microbenchmarks " variable=RUN_BENCHMARKS default=false value=true
optparse.define short=b long=branch desc="Choouse git branch ('none' for menu)" variable=BRANCH
optparse.define short=b long=branch desc="Choose git branch. For menu use -b \"\"" variable=BRANCH default=$CURRENT_BRANCH
optparse.define short=D long=without-core-dumps desc="Do not produce core dumps" variable=WITHOUT_COREDUMPS default=false value=true
optparse.define short=A long=asan desc="Build with ASAN" variable=ASAN default=false value=true
optparse.define short=v long=verbose desc="Verbose makefile commands" variable=MAKEFILE_VERBOSE default=false value=true
optparse.define short=P long=report-path desc="Path for storing reports and profiles" variable=REPORT_PATH default="/core"
source $( optparse.build )
@ -45,18 +53,27 @@ INSTALL_PREFIX="/usr/"
DATA_DIR="/var/lib/mysql/data"
CMAKE_BIN_NAME=cmake
CTEST_BIN_NAME=ctest
CONFIG_DIR="/etc/my.cnf.d"
if [[ $OS = 'Ubuntu' || $OS = 'Debian' ]]; then
CONFIG_DIR="/etc/mysql/mariadb.conf.d"
fi
select_branch()
{
cd $SCRIPT_LOCATION
CURRENT_BRANCH=$(git rev-parse --abbrev-ref HEAD)
if [[ ! " ${BRANCHES[*]} " =~ " ${BRANCH} " ]]; then
if [[ $BRANCH = 'none' ]]; then
if [[ $BRANCH = "" ]]; then
getChoice -q "Select your branch" -o BRANCHES
BRANCH=$selectedChoice
fi
cd $SCRIPT_LOCATION
message "Selecting $BRANCH branch for Columnstore"
git checkout $BRANCH
cd -
if [[ $BRANCH != $CURRENT_BRANCH ]]; then
message "Selecting $BRANCH branch for Columnstore"
git checkout $BRANCH
fi
message "Turning off Columnstore submodule auto update via gitconfig"
cd $MDB_SOURCE_PATH
@ -64,8 +81,6 @@ select_branch()
cd -
fi
cd $SCRIPT_LOCATION
CURRENT_BRANCH=$(git rev-parse --abbrev-ref HEAD)
cd -
message "Columnstore will be built from $color_yellow$CURRENT_BRANCH$color_normal branch"
}
@ -136,7 +151,9 @@ clean_old_installation()
rm -rf /var/lib/columnstore/local/
rm -f /var/lib/columnstore/storagemanager/storagemanager-lock
rm -f /var/lib/columnstore/storagemanager/cs-initialized
rm -rf /var/log/mariadb/columnstore/*
rm -rf /tmp/*
rm -rf $REPORT_PATH
rm -rf /var/lib/mysql
rm -rf /var/run/mysqld
rm -rf $DATA_DIR
@ -161,7 +178,8 @@ build()
-DWITH_WSREP=OFF
-DWITH_SSL=system
-DCMAKE_INSTALL_PREFIX:PATH=$INSTALL_PREFIX
-DCMAKE_EXPORT_COMPILE_COMMANDS=1"
-DCMAKE_EXPORT_COMPILE_COMMANDS=1
"
if [[ $SKIP_UNIT_TESTS = true ]] ; then
@ -172,11 +190,21 @@ build()
message "Buiding with unittests"
fi
if [[ $ASAN = true ]] ; then
warn "Building with ASAN"
MDB_CMAKE_FLAGS="${MDB_CMAKE_FLAGS} -DWITH_ASAN=ON -DWITH_COLUMNSTORE_ASAN=ON -DWITH_COLUMNSTORE_REPORT_PATH=${REPORT_PATH}"
fi
if [[ $WITHOUT_COREDUMPS = true ]] ; then
warn "Cores are not dumped"
else
MDB_CMAKE_FLAGS="${MDB_CMAKE_FLAGS} -DWITH_COREDUMPS=ON"
echo "${REPORT_PATH}/core_%e.%p" | sudo tee /proc/sys/kernel/core_pattern
fi
if [[ $MAKEFILE_VERBOSE = true ]] ; then
warn "Verbosing Makefile Commands"
MDB_CMAKE_FLAGS="${MDB_CMAKE_FLAGS} -DCMAKE_VERBOSE_MAKEFILE:BOOL=ON"
fi
if [[ $RUN_BENCHMARKS = true ]] ; then
@ -197,7 +225,6 @@ build()
message "Buiding without microbenchmarks"
fi
cd $MDB_SOURCE_PATH
if [[ $SKIP_SUBMODULES = true ]] ; then
@ -228,11 +255,13 @@ build()
message "building with flags $MDB_CMAKE_FLAGS"
local CPUS=$(getconf _NPROCESSORS_ONLN)
${CMAKE_BIN_NAME} . -DCMAKE_BUILD_TYPE=$MCS_BUILD_TYPE $MDB_CMAKE_FLAGS && \
make -j $CPUS install
${CMAKE_BIN_NAME} -DCMAKE_BUILD_TYPE=$MCS_BUILD_TYPE $MDB_CMAKE_FLAGS && \ |
make -j $CPUS
message "Installing silently"
make -j $CPUS install > /dev/null
if [ $? -ne 0 ]; then
error "!!!! BUILD FAILED"
error "!!!! BUILD FAILED !!!!"
exit 1
fi
cd -
@ -259,7 +288,7 @@ run_unit_tests()
else
message "Running unittests"
cd $MDB_SOURCE_PATH
${CTEST_BIN_NAME} . -R columnstore: -j $(nproc)
${CTEST_BIN_NAME} . -R columnstore: -j $(nproc) --progress
cd -
fi
}
@ -271,14 +300,56 @@ run_microbenchmarks_tests()
else
message "Runnning microbenchmarks"
cd $MDB_SOURCE_PATH
${CTEST_BIN_NAME} . -V -R columnstore_microbenchmarks: -j $(nproc)
${CTEST_BIN_NAME} . -V -R columnstore_microbenchmarks: -j $(nproc) --progress
cd -
fi
}
disable_plugins_for_bootstrap()
{
find /etc -type f -exec sed -i 's/plugin-load-add=auth_gssapi.so//g' {} +
find /etc -type f -exec sed -i 's/plugin-load-add=ha_columnstore.so//g' {} +
}
enable_columnstore_back()
{
echo plugin-load-add=ha_columnstore.so >> $CONFIG_DIR/columnstore.cnf
}
fix_config_files()
{
message Fixing config files
THREAD_STACK_SIZE="20M"
SYSTEMD_SERVICE_DIR="/usr/lib/systemd/system"
if [[ $ASAN = true ]] ; then
COLUMNSTORE_CONFIG=$CONFIG_DIR/columnstore.cnf
if grep -q thread_stack $COLUMNSTORE_CONFIG; then
warn "MDB Server has thread_stack settings on $COLUMNSTORE_CONFIG check it's compatibility with ASAN"
else
echo "thread_stack = ${THREAD_STACK_SIZE}" >> $COLUMNSTORE_CONFIG
message "thread_stack was set to ${THREAD_STACK_SIZE} in $COLUMNSTORE_CONFIG"
fi
MDB_SERVICE_FILE=$SYSTEMD_SERVICE_DIR/mariadb.service
if grep -q ASAN $MDB_SERVICE_FILE; then
warn "MDB Server has ASAN options in $MDB_SERVICE_FILE, check it's compatibility"
else
echo Environment="'ASAN_OPTIONS=abort_on_error=1:disable_coredump=0,print_stats=false,detect_odr_violation=0,check_initialization_order=1,detect_stack_use_after_return=1,atexit=false,log_path=${ASAN_PATH}'" >> $MDB_SERVICE_FILE
message "ASAN options were added to $MDB_SERVICE_FILE"
fi
fi
systemctl daemon-reload
}
install()
{
message "Installing MariaDB"
disable_plugins_for_bootstrap
mkdir -p $REPORT_PATH
chmod 777 $REPORT_PATH
check_user_and_group
@ -288,10 +359,15 @@ install()
socket=/run/mysqld/mysqld.sock" > /etc/my.cnf.d/socket.cnf'
mv $INSTALL_PREFIX/lib/mysql/plugin/ha_columnstore.so /tmp/ha_columnstore_1.so || mv $INSTALL_PREFIX/lib64/mysql/plugin/ha_columnstore.so /tmp/ha_columnstore_2.so
mkdir -p /var/lib/mysql
chown mysql:mysql /var/lib/mysql
message "Running mysql_install_db"
mysql_install_db --rpm --user=mysql
sudo -u mysql mysql_install_db --rpm --user=mysql > /dev/null
mv /tmp/ha_columnstore_1.so $INSTALL_PREFIX/lib/mysql/plugin/ha_columnstore.so || mv /tmp/ha_columnstore_2.so $INSTALL_PREFIX/lib64/mysql/plugin/ha_columnstore.so
enable_columnstore_back
mkdir -p /etc/columnstore
cp $MDB_SOURCE_PATH/storage/columnstore/columnstore/oam/etc/Columnstore.xml /etc/columnstore/Columnstore.xml
@ -308,7 +384,7 @@ socket=/run/mysqld/mysqld.sock" > /etc/my.cnf.d/socket.cnf'
> /etc/mysql/debian.cnf
fi
systemctl daemon-reload
fix_config_files
if [ -d "/etc/mysql/mariadb.conf.d/" ]; then
message "Copying configs from /etc/mysql/mariadb.conf.d/ to /etc/my.cnf.d"
@ -345,9 +421,23 @@ socket=/run/mysqld/mysqld.sock" > /etc/my.cnf.d/socket.cnf'
chmod 777 /var/log/mariadb/columnstore
}
smoke()
{
message "Creating test database"
mariadb -e "create database if not exists test;"
message "Selecting magic numbers"
MAGIC=`mysql -N test < $MDB_SOURCE_PATH/storage/columnstore/columnstore/tests/scripts/smoke.sql`
if [[ $MAGIC == '42' ]] ; then
message "Great answer correct"
else
warn "Smoke failed, answer is '$MAGIC'"
fi
}
select_branch
if [[ $SKIP_DEPS = false ]] ; then
if [[ $INSTALL_DEPS = true ]] ; then
install_deps
fi
@ -358,4 +448,6 @@ run_unit_tests
run_microbenchmarks_tests
install
start_service
smoke
message "$color_green FINISHED $color_normal"

21
datatypes/mcs_decimal.h
View File

@ -165,6 +165,27 @@ const int128_t mcs_pow_10_128[20] = {
100000000000000000000000000000000000000_xxl,
};
const long long columnstore_precision[19] = {0,
9,
99,
999,
9999,
99999,
999999,
9999999,
99999999,
999999999,
9999999999LL,
99999999999LL,
999999999999LL,
9999999999999LL,
99999999999999LL,
999999999999999LL,
9999999999999999LL,
99999999999999999LL,
999999999999999999LL};
const int128_t ConversionRangeMaxValue[20] = {9999999999999999999_xxl,
99999999999999999999_xxl,
999999999999999999999_xxl,

102
dbcon/ddlpackageproc/libddlpackageproc.rc
View File

@ -1,102 +0,0 @@
// Microsoft Visual C++ generated resource script.
//
#include "resource.h"
#define APSTUDIO_READONLY_SYMBOLS
/////////////////////////////////////////////////////////////////////////////
//
// Generated from the TEXTINCLUDE 2 resource.
//
#include "afxres.h"
/////////////////////////////////////////////////////////////////////////////
#undef APSTUDIO_READONLY_SYMBOLS
/////////////////////////////////////////////////////////////////////////////
// English (U.S.) resources
#if !defined(AFX_RESOURCE_DLL) || defined(AFX_TARG_ENU)
#ifdef _WIN32
LANGUAGE LANG_ENGLISH, SUBLANG_ENGLISH_US
#pragma code_page(1252)
#endif //_WIN32
#ifdef APSTUDIO_INVOKED
/////////////////////////////////////////////////////////////////////////////
//
// TEXTINCLUDE
//
1 TEXTINCLUDE
BEGIN
"resource.h\0"
END
2 TEXTINCLUDE
BEGIN
"#include ""afxres.h""\r\n"
"\0"
END
3 TEXTINCLUDE
BEGIN
"\r\n"
"\0"
END
#endif // APSTUDIO_INVOKED
/////////////////////////////////////////////////////////////////////////////
//
// Version
//
VS_VERSION_INFO VERSIONINFO
FILEVERSION 4,6,0,0
PRODUCTVERSION 4,6,0,0
FILEFLAGSMASK 0x17L
#ifdef _DEBUG
FILEFLAGS 0x1L
#else
FILEFLAGS 0x0L
#endif
FILEOS 0x4L
FILETYPE 0x2L
FILESUBTYPE 0x0L
BEGIN
BLOCK "StringFileInfo"
BEGIN
BLOCK "040904b0"
BEGIN
VALUE "CompanyName", "InfiniDB, Inc."
VALUE "FileDescription", "InfiniDB DDL API"
VALUE "FileVersion", "4.6.0-0"
VALUE "InternalName", "libddlpackageproc"
VALUE "LegalCopyright", "Copyright (C) 2014"
VALUE "OriginalFilename", "libddlpackageproc.dll"
VALUE "ProductName", "InfiniDB"
VALUE "ProductVersion", "4.6.0.0 Beta"
END
END
BLOCK "VarFileInfo"
BEGIN
VALUE "Translation", 0x409, 1200
END
END
#endif // English (U.S.) resources
/////////////////////////////////////////////////////////////////////////////
#ifndef APSTUDIO_INVOKED
/////////////////////////////////////////////////////////////////////////////
//
// Generated from the TEXTINCLUDE 3 resource.
//
/////////////////////////////////////////////////////////////////////////////
#endif // not APSTUDIO_INVOKED

102
dbcon/dmlpackageproc/libdmlpackageproc.rc
View File

@ -1,102 +0,0 @@
// Microsoft Visual C++ generated resource script.
//
#include "resource.h"
#define APSTUDIO_READONLY_SYMBOLS
/////////////////////////////////////////////////////////////////////////////
//
// Generated from the TEXTINCLUDE 2 resource.
//
#include "afxres.h"
/////////////////////////////////////////////////////////////////////////////
#undef APSTUDIO_READONLY_SYMBOLS
/////////////////////////////////////////////////////////////////////////////
// English (U.S.) resources
#if !defined(AFX_RESOURCE_DLL) || defined(AFX_TARG_ENU)
#ifdef _WIN32
LANGUAGE LANG_ENGLISH, SUBLANG_ENGLISH_US
#pragma code_page(1252)
#endif //_WIN32
#ifdef APSTUDIO_INVOKED
/////////////////////////////////////////////////////////////////////////////
//
// TEXTINCLUDE
//
1 TEXTINCLUDE
BEGIN
"resource.h\0"
END
2 TEXTINCLUDE
BEGIN
"#include ""afxres.h""\r\n"
"\0"
END
3 TEXTINCLUDE
BEGIN
"\r\n"
"\0"
END
#endif // APSTUDIO_INVOKED
/////////////////////////////////////////////////////////////////////////////
//
// Version
//
VS_VERSION_INFO VERSIONINFO
FILEVERSION 4,6,0,0
PRODUCTVERSION 4,6,0,0
FILEFLAGSMASK 0x17L
#ifdef _DEBUG
FILEFLAGS 0x1L
#else
FILEFLAGS 0x0L
#endif
FILEOS 0x4L
FILETYPE 0x2L
FILESUBTYPE 0x0L
BEGIN
BLOCK "StringFileInfo"
BEGIN
BLOCK "040904b0"
BEGIN
VALUE "CompanyName", "InfiniDB, Inc."
VALUE "FileDescription", "InfiniDB DML API"
VALUE "FileVersion", "4.6.0-0"
VALUE "InternalName", "libdmlpackageproc"
VALUE "LegalCopyright", "Copyright (C) 2014"
VALUE "OriginalFilename", "libdmlpackageproc.dll"
VALUE "ProductName", "InfiniDB"
VALUE "ProductVersion", "4.6.0.0 Beta"
END
END
BLOCK "VarFileInfo"
BEGIN
VALUE "Translation", 0x409, 1200
END
END
#endif // English (U.S.) resources
/////////////////////////////////////////////////////////////////////////////
#ifndef APSTUDIO_INVOKED
/////////////////////////////////////////////////////////////////////////////
//
// Generated from the TEXTINCLUDE 3 resource.
//
/////////////////////////////////////////////////////////////////////////////
#endif // not APSTUDIO_INVOKED

2
dbcon/execplan/CMakeLists.txt
View File

@ -51,6 +51,6 @@ add_library(execplan SHARED ${execplan_LIB_SRCS})
add_dependencies(execplan loggingcpp)
target_link_libraries(execplan ${NETSNMP_LIBRARIES} ${MARIADB_STRING_LIBS} ${ENGINE_DT_LIB})
target_link_libraries(execplan messageqcpp ${NETSNMP_LIBRARIES} ${MARIADB_STRING_LIBS} ${ENGINE_DT_LIB})
install(TARGETS execplan DESTINATION ${ENGINE_LIBDIR} COMPONENT columnstore-engine)

1
dbcon/execplan/calpontsystemcatalog.cpp
View File

@ -54,6 +54,7 @@ using namespace joblist;
#include "bytestream.h"
#include "messagequeue.h"
#include "messagequeuepool.h"
using namespace messageqcpp;
#include "configcpp.h"

102
dbcon/joblist/libjoblist.rc
View File

@ -1,102 +0,0 @@
// Microsoft Visual C++ generated resource script.
//
#include "resource.h"
#define APSTUDIO_READONLY_SYMBOLS
/////////////////////////////////////////////////////////////////////////////
//
// Generated from the TEXTINCLUDE 2 resource.
//
#include "afxres.h"
/////////////////////////////////////////////////////////////////////////////
#undef APSTUDIO_READONLY_SYMBOLS
/////////////////////////////////////////////////////////////////////////////
// English (U.S.) resources
#if !defined(AFX_RESOURCE_DLL) || defined(AFX_TARG_ENU)
#ifdef _WIN32
LANGUAGE LANG_ENGLISH, SUBLANG_ENGLISH_US
#pragma code_page(1252)
#endif //_WIN32
#ifdef APSTUDIO_INVOKED
/////////////////////////////////////////////////////////////////////////////
//
// TEXTINCLUDE
//
1 TEXTINCLUDE
BEGIN
"resource.h\0"
END
2 TEXTINCLUDE
BEGIN
"#include ""afxres.h""\r\n"
"\0"
END
3 TEXTINCLUDE
BEGIN
"\r\n"
"\0"
END
#endif // APSTUDIO_INVOKED
/////////////////////////////////////////////////////////////////////////////
//
// Version
//
VS_VERSION_INFO VERSIONINFO
FILEVERSION 4,6,0,0
PRODUCTVERSION 4,6,0,0
FILEFLAGSMASK 0x17L
#ifdef _DEBUG
FILEFLAGS 0x1L
#else
FILEFLAGS 0x0L
#endif
FILEOS 0x4L
FILETYPE 0x2L
FILESUBTYPE 0x0L
BEGIN
BLOCK "StringFileInfo"
BEGIN
BLOCK "040904b0"
BEGIN
VALUE "CompanyName", "InfiniDB, Inc."
VALUE "FileDescription", "InfiniDB Job List API"
VALUE "FileVersion", "4.6.0-0"
VALUE "InternalName", "libjoblist.dll"
VALUE "LegalCopyright", "Copyright (C) 2014"
VALUE "OriginalFilename", "libjoblist.dll"
VALUE "ProductName", "InfiniDB"
VALUE "ProductVersion", "4.6.0.0 Beta"
END
END
BLOCK "VarFileInfo"
BEGIN
VALUE "Translation", 0x409, 1200
END
END
#endif // English (U.S.) resources
/////////////////////////////////////////////////////////////////////////////
#ifndef APSTUDIO_INVOKED
/////////////////////////////////////////////////////////////////////////////
//
// Generated from the TEXTINCLUDE 3 resource.
//
/////////////////////////////////////////////////////////////////////////////
#endif // not APSTUDIO_INVOKED

3
dbcon/joblist/primitivemsg.h
View File

@ -29,6 +29,7 @@
#include "blocksize.h"
#include "calpontsystemcatalog.h"
#include "joblisttypes.h"
#include <columnwidth.h>
#include <vector>
@ -711,7 +712,7 @@ const uint8_t ROUND_POS = 0x01; // actual value larger/longer than the stored v
const uint8_t ROUND_NEG = 0x80; // actual value less than the stored value
// Tied to ColByScanRequestHeader and ColByScanRangeRequestHeader. Check other headers if modifying.
struct NewColRequestHeader
struct /*alignas(utils::MAXCOLUMNWIDTH)*/ NewColRequestHeader
{
ISMPacketHeader ism;
PrimitiveHeader hdr;

13
dbcon/joblist/tupleannexstep.cpp
View File

@ -298,7 +298,6 @@ void TupleAnnexStep::join()
uint32_t TupleAnnexStep::nextBand(messageqcpp::ByteStream& bs)
{
RGData rgDataOut;
bool more = false;
uint32_t rowCount = 0;
@ -306,18 +305,18 @@ uint32_t TupleAnnexStep::nextBand(messageqcpp::ByteStream& bs)
{
bs.restart();
more = fOutputDL->next(fOutputIterator, &rgDataOut);
more = fOutputDL->next(fOutputIterator, &fRgDataOut);
if (more && !cancelled())
{
fRowGroupDeliver.setData(&rgDataOut);
fRowGroupDeliver.setData(&fRgDataOut);
fRowGroupDeliver.serializeRGData(bs);
rowCount = fRowGroupDeliver.getRowCount();
}
else
{
while (more)
more = fOutputDL->next(fOutputIterator, &rgDataOut);
more = fOutputDL->next(fOutputIterator, &fRgDataOut);
fEndOfResult = true;
}
@ -327,7 +326,7 @@ uint32_t TupleAnnexStep::nextBand(messageqcpp::ByteStream& bs)
handleException(std::current_exception(), logging::ERR_IN_DELIVERY, logging::ERR_ALWAYS_CRITICAL,
"TupleAnnexStep::nextBand()");
while (more)
more = fOutputDL->next(fOutputIterator, &rgDataOut);
more = fOutputDL->next(fOutputIterator, &fRgDataOut);
fEndOfResult = true;
}
@ -335,8 +334,8 @@ uint32_t TupleAnnexStep::nextBand(messageqcpp::ByteStream& bs)
if (fEndOfResult)
{
// send an empty / error band
rgDataOut.reinit(fRowGroupDeliver, 0);
fRowGroupDeliver.setData(&rgDataOut);
fRgDataOut.reinit(fRowGroupDeliver, 0);
fRowGroupDeliver.setData(&fRgDataOut);
fRowGroupDeliver.resetRowGroup(0);
fRowGroupDeliver.setStatus(status());
fRowGroupDeliver.serializeRGData(bs);

1
dbcon/joblist/tupleannexstep.h
View File

@ -118,6 +118,7 @@ class TupleAnnexStep : public JobStep, public TupleDeliveryStep
rowgroup::RowGroup fRowGroupIn;
rowgroup::RowGroup fRowGroupOut;
rowgroup::RowGroup fRowGroupDeliver;
rowgroup::RGData fRgDataOut;
rowgroup::Row fRowIn;
rowgroup::Row fRowOut;

2
dbcon/joblist/windowfunctionstep.cpp
View File

@ -1054,7 +1054,7 @@ void WindowFunctionStep::doPostProcessForSelect()
for (int64_t i = begin; i < end; i++)
{
if (rgData.rowData.get() == NULL)
if (!rgData.hasRowData())
{
rgCapacity = ((rowsLeft > 8192) ? 8192 : rowsLeft);
rowsLeft -= rgCapacity;

2
dbcon/mysql/ha_mcs.cpp
View File

@ -1941,7 +1941,7 @@ int ha_mcs_cache::flush_insert_cache()
int error, error2;
ha_maria* from = cache_handler;
uchar* record = table->record[0];
ulonglong copied_rows = 0;
[[maybe_unused]] ulonglong copied_rows = 0;
DBUG_ENTER("flush_insert_cache");
DBUG_ASSERT(from->file->state->records == share->cached_rows);

2
dbcon/mysql/ha_mcs_execplan.cpp
View File

@ -2962,7 +2962,7 @@ SimpleColumn* getSmallestColumn(boost::shared_ptr<CalpontSystemCatalog> csc,
CalpontSelectExecutionPlan::ReturnedColumnList::const_iterator iter;
ReturnedColumn* rc;
ReturnedColumn* rc = nullptr;
for (iter = cols.begin(); iter != cols.end(); iter++)
{

4
dbcon/mysql/ha_mcs_partition.cpp
View File

@ -569,7 +569,7 @@ extern "C"
void calshowpartitions_deinit(UDF_INIT* initid)
{
delete initid->ptr;
delete[] initid->ptr;
}
const char* calshowpartitions(UDF_INIT* initid, UDF_ARGS* args, char* result, unsigned long* length,
@ -1164,7 +1164,7 @@ extern "C"
void calshowpartitionsbyvalue_deinit(UDF_INIT* initid)
{
delete initid->ptr;
delete[] initid->ptr;
}
const char* calshowpartitionsbyvalue(UDF_INIT* initid, UDF_ARGS* args, char* result,

102
ddlproc/DDLProc.rc
View File

@ -1,102 +0,0 @@
// Microsoft Visual C++ generated resource script.
//
#include "resource.h"
#define APSTUDIO_READONLY_SYMBOLS
/////////////////////////////////////////////////////////////////////////////
//
// Generated from the TEXTINCLUDE 2 resource.
//
#include "afxres.h"
/////////////////////////////////////////////////////////////////////////////
#undef APSTUDIO_READONLY_SYMBOLS
/////////////////////////////////////////////////////////////////////////////
// English (U.S.) resources
#if !defined(AFX_RESOURCE_DLL) || defined(AFX_TARG_ENU)
#ifdef _WIN32
LANGUAGE LANG_ENGLISH, SUBLANG_ENGLISH_US
#pragma code_page(1252)
#endif //_WIN32
#ifdef APSTUDIO_INVOKED
/////////////////////////////////////////////////////////////////////////////
//
// TEXTINCLUDE
//
1 TEXTINCLUDE
BEGIN
"resource.h\0"
END
2 TEXTINCLUDE
BEGIN
"#include ""afxres.h""\r\n"
"\0"
END
3 TEXTINCLUDE
BEGIN
"\r\n"
"\0"
END
#endif // APSTUDIO_INVOKED
/////////////////////////////////////////////////////////////////////////////
//
// Version
//
VS_VERSION_INFO VERSIONINFO
FILEVERSION 4,6,0,0
PRODUCTVERSION 4,6,0,0
FILEFLAGSMASK 0x17L
#ifdef _DEBUG
FILEFLAGS 0x1L
#else
FILEFLAGS 0x0L
#endif
FILEOS 0x4L
FILETYPE 0x1L
FILESUBTYPE 0x0L
BEGIN
BLOCK "StringFileInfo"
BEGIN
BLOCK "040904b0"
BEGIN
VALUE "CompanyName", "InfiniDB, Inc."
VALUE "FileDescription", "InfiniDB DDL Processor"
VALUE "FileVersion", "4.6.0-0"
VALUE "InternalName", "DDLProc"
VALUE "LegalCopyright", "Copyright (C) 2014"
VALUE "OriginalFilename", "DDLProc.exe"
VALUE "ProductName", "InfiniDB"
VALUE "ProductVersion", "4.6.0.0 Beta"
END
END
BLOCK "VarFileInfo"
BEGIN
VALUE "Translation", 0x409, 1200
END
END
#endif // English (U.S.) resources
/////////////////////////////////////////////////////////////////////////////
#ifndef APSTUDIO_INVOKED
/////////////////////////////////////////////////////////////////////////////
//
// Generated from the TEXTINCLUDE 3 resource.
//
/////////////////////////////////////////////////////////////////////////////
#endif // not APSTUDIO_INVOKED

1
debian/mariadb-plugin-columnstore.install vendored
View File

@ -70,7 +70,6 @@ usr/lib/*/libjoiner.so
usr/lib/*/liblibmysql_client.so
usr/lib/*/libloggingcpp.so
usr/lib/*/libmarias3.so
usr/lib/*/libmessageqcpp.so
usr/lib/*/liboamcpp.so
usr/lib/*/libquerystats.so
usr/lib/*/libquerytele.so

102
dmlproc/DMLProc.rc
View File

@ -1,102 +0,0 @@
// Microsoft Visual C++ generated resource script.
//
#include "resource.h"
#define APSTUDIO_READONLY_SYMBOLS
/////////////////////////////////////////////////////////////////////////////
//
// Generated from the TEXTINCLUDE 2 resource.
//
#include "afxres.h"
/////////////////////////////////////////////////////////////////////////////
#undef APSTUDIO_READONLY_SYMBOLS
/////////////////////////////////////////////////////////////////////////////
// English (U.S.) resources
#if !defined(AFX_RESOURCE_DLL) || defined(AFX_TARG_ENU)
#ifdef _WIN32
LANGUAGE LANG_ENGLISH, SUBLANG_ENGLISH_US
#pragma code_page(1252)
#endif //_WIN32
#ifdef APSTUDIO_INVOKED
/////////////////////////////////////////////////////////////////////////////
//
// TEXTINCLUDE
//
1 TEXTINCLUDE
BEGIN
"resource.h\0"
END
2 TEXTINCLUDE
BEGIN
"#include ""afxres.h""\r\n"
"\0"
END
3 TEXTINCLUDE
BEGIN
"\r\n"
"\0"
END
#endif // APSTUDIO_INVOKED
/////////////////////////////////////////////////////////////////////////////
//
// Version
//
VS_VERSION_INFO VERSIONINFO
FILEVERSION 4,6,0,0
PRODUCTVERSION 4,6,0,0
FILEFLAGSMASK 0x17L
#ifdef _DEBUG
FILEFLAGS 0x1L
#else
FILEFLAGS 0x0L
#endif
FILEOS 0x4L
FILETYPE 0x1L
FILESUBTYPE 0x0L
BEGIN
BLOCK "StringFileInfo"
BEGIN
BLOCK "040904b0"
BEGIN
VALUE "CompanyName", "InfiniDB, Inc."
VALUE "FileDescription", "InfiniDB DML Processor"
VALUE "FileVersion", "4.6.0-0"
VALUE "InternalName", "DMLProc"
VALUE "LegalCopyright", "Copyright (C) 2014"
VALUE "OriginalFilename", "DMLProc.exe"
VALUE "ProductName", "InfiniDB"
VALUE "ProductVersion", "4.6.0.0 Beta"
END
END
BLOCK "VarFileInfo"
BEGIN
VALUE "Translation", 0x409, 1200
END
END
#endif // English (U.S.) resources
/////////////////////////////////////////////////////////////////////////////
#ifndef APSTUDIO_INVOKED
/////////////////////////////////////////////////////////////////////////////
//
// Generated from the TEXTINCLUDE 3 resource.
//
/////////////////////////////////////////////////////////////////////////////
#endif // not APSTUDIO_INVOKED

28
oam/install_scripts/CMakeLists.txt
View File

@ -8,6 +8,34 @@ if (WITH_COREDUMPS)
endif (WITH_COREDUMPS)
SET(LD_PRELOAD_STRING "LD_PRELOAD=$(ldconfig -p | grep -m1 libjemalloc | awk '{print $1}')")
SET(ALLOC_CONFIG "MALLOC_CONF=''")
SET(PRIMPROC_ALLOC_CONFIG ${ALLOC_CONFIG})
SET(DMLPROC_ALLOC_CONFIG ${ALLOC_CONFIG})
SET(DDLPROC_ALLOC_CONFIG ${ALLOC_CONFIG})
SET(WRITEENGINE_ALLOC_CONFIG ${ALLOC_CONFIG})
SET(CONTROLLERNODE_ALLOC_CONFIG ${ALLOC_CONFIG})
SET(WORKERNODE_ALLOC_CONFIG ${ALLOC_CONFIG})
SET(STORAGEMANAGER_ALLOC_CONFIG ${ALLOC_CONFIG})
if (WITH_COLUMNSTORE_ASAN)
SET(ASAN_PATH "/tmp/asan")
if (WITH_COLUMNSTORE_REPORT_PATH)
SET(ASAN_PATH "${WITH_COLUMNSTORE_REPORT_PATH}/asan")
endif (WITH_COLUMNSTORE_REPORT_PATH)
SET(LD_PRELOAD_STRING "")
SET(ALLOC_CONFIG "ASAN_OPTIONS=abort_on_error=1:disable_coredump=0,print_stats=false,detect_odr_violation=0,check_initialization_order=1,detect_stack_use_after_return=1,atexit=false,log_path=${ASAN_PATH}")
SET(PRIMPROC_ALLOC_CONFIG ${ALLOC_CONFIG})
SET(DMLPROC_ALLOC_CONFIG ${ALLOC_CONFIG})
SET(DDLPROC_ALLOC_CONFIG ${ALLOC_CONFIG})
SET(WRITEENGINE_ALLOC_CONFIG ${ALLOC_CONFIG})
SET(CONTROLLERNODE_ALLOC_CONFIG ${ALLOC_CONFIG})
SET(WORKERNODE_ALLOC_CONFIG ${ALLOC_CONFIG})
SET(STORAGEMANAGER_ALLOC_CONFIG ${ALLOC_CONFIG})
endif()
configure_file("${CMAKE_CURRENT_SOURCE_DIR}/columnstoreSyslogSetup.sh.in" "${CMAKE_CURRENT_SOURCE_DIR}/columnstoreSyslogSetup.sh" @ONLY)
configure_file("${CMAKE_CURRENT_SOURCE_DIR}/columnstore-post-install.in" "${CMAKE_CURRENT_SOURCE_DIR}/columnstore-post-install" @ONLY)

2
oam/install_scripts/mcs-controllernode.service.in
View File

@ -13,7 +13,7 @@ Group=@DEFAULT_GROUP@
LimitNOFILE=65536
LimitNPROC=65536
LimitCORE=@CORE_DUMPS@
Environment="@CONTROLLERNODE_ALLOC_CONFIG@"
ExecStart=@ENGINE_BINDIR@/controllernode
ExecStop=@ENGINE_BINDIR@/mcs-stop-controllernode.sh $MAINPID

1
oam/install_scripts/mcs-ddlproc.service.in
View File

@ -14,6 +14,7 @@ LimitNOFILE=65536
LimitNPROC=65536
LimitCORE=@CORE_DUMPS@
Environment="@DDLPROC_ALLOC_CONFIG@"
ExecStart=@ENGINE_BINDIR@/DDLProc
Restart=on-failure

1
oam/install_scripts/mcs-dmlproc.service.in
View File

@ -14,6 +14,7 @@ LimitNOFILE=65536
LimitNPROC=65536
LimitCORE=@CORE_DUMPS@
Environment="@DMLPROC_ALLOC_CONFIG@"
ExecStart=@ENGINE_BINDIR@/DMLProc
Restart=on-failure

2
oam/install_scripts/mcs-primproc.service.in
View File

@ -16,7 +16,7 @@ LimitNPROC=65536
LimitCORE=@CORE_DUMPS@
ExecStartPre=/usr/bin/env bash -c "ldconfig -p | grep -m1 libjemalloc > /dev/null || echo 'Please install jemalloc to avoid ColumnStore performance degradation and unexpected service interruptions.'"
ExecStart=/usr/bin/env bash -c "LD_PRELOAD=$(ldconfig -p | grep -m1 libjemalloc | awk '{print $1}') exec @ENGINE_BINDIR@/PrimProc"
ExecStart=/usr/bin/env bash -c "@PRIMPROC_ALLOC_CONFIG@ @LD_PRELOAD_STRING@ exec @ENGINE_BINDIR@/PrimProc"
Restart=on-failure
TimeoutStopSec=2

2
oam/install_scripts/mcs-storagemanager.service.in
View File

@ -10,5 +10,5 @@ LimitNOFILE=65536
LimitNPROC=65536
LimitCORE=@CORE_DUMPS@
ExecStart=/usr/bin/env bash -c "LD_PRELOAD=$(ldconfig -p | grep -m1 libjemalloc | awk '{print $1}') exec @ENGINE_BINDIR@/StorageManager"
ExecStart=/usr/bin/env bash -c "@STORAGEMANAGER_ALLOC_CONFIG@ @LD_PRELOAD_STRING@ exec @ENGINE_BINDIR@/StorageManager"
SuccessExitStatus=255

1
oam/install_scripts/mcs-workernode.service.in
View File

@ -11,6 +11,7 @@ LimitNOFILE=65536
LimitNPROC=65536
LimitCORE=@CORE_DUMPS@
Environment="@WORKERNODE_ALLOC_CONFIG@"
ExecStart=@ENGINE_BINDIR@/workernode DBRM_Worker%i
ExecStopPost=@ENGINE_BINDIR@/mcs-savebrm.py
ExecStopPost=/usr/bin/env bash -c "clearShm > /dev/null 2>&1"

2
oam/install_scripts/mcs-writeengineserver.service.in
View File

@ -14,7 +14,7 @@ LimitNOFILE=65536
LimitNPROC=65536
LimitCORE=@CORE_DUMPS@
ExecStart=/usr/bin/env bash -c "LD_PRELOAD=$(ldconfig -p | grep -m1 libjemalloc | awk '{print $1}') exec @ENGINE_BINDIR@/WriteEngineServer"
ExecStart=/usr/bin/env bash -c "@WRITEENGINE_ALLOC_CONFIG@ @LD_PRELOAD_STRING@ exec @ENGINE_BINDIR@/WriteEngineServer"
Restart=on-failure
TimeoutStopSec=20

102
oam/oamcpp/liboamcpp.rc
View File

@ -1,102 +0,0 @@
// Microsoft Visual C++ generated resource script.
//
#include "resource.h"
#define APSTUDIO_READONLY_SYMBOLS
/////////////////////////////////////////////////////////////////////////////
//
// Generated from the TEXTINCLUDE 2 resource.
//
#include "afxres.h"
/////////////////////////////////////////////////////////////////////////////
#undef APSTUDIO_READONLY_SYMBOLS
/////////////////////////////////////////////////////////////////////////////
// English (U.S.) resources
#if !defined(AFX_RESOURCE_DLL) || defined(AFX_TARG_ENU)
#ifdef _WIN32
LANGUAGE LANG_ENGLISH, SUBLANG_ENGLISH_US
#pragma code_page(1252)
#endif //_WIN32
#ifdef APSTUDIO_INVOKED
/////////////////////////////////////////////////////////////////////////////
//
// TEXTINCLUDE
//
1 TEXTINCLUDE
BEGIN
"resource.h\0"
END
2 TEXTINCLUDE
BEGIN
"#include ""afxres.h""\r\n"
"\0"
END
3 TEXTINCLUDE
BEGIN
"\r\n"
"\0"
END
#endif // APSTUDIO_INVOKED
/////////////////////////////////////////////////////////////////////////////
//
// Version
//
VS_VERSION_INFO VERSIONINFO
FILEVERSION 4,6,0,0
PRODUCTVERSION 4,6,0,0
FILEFLAGSMASK 0x17L
#ifdef _DEBUG
FILEFLAGS 0x1L
#else
FILEFLAGS 0x0L
#endif
FILEOS 0x4L
FILETYPE 0x2L
FILESUBTYPE 0x0L
BEGIN
BLOCK "StringFileInfo"
BEGIN
BLOCK "040904b0"
BEGIN
VALUE "CompanyName", "InfiniDB, Inc."
VALUE "FileDescription", "InfiniDB OAM API"
VALUE "FileVersion", "4.6.0-0"
VALUE "InternalName", "liboamcpp"
VALUE "LegalCopyright", "Copyright (C) 2014"
VALUE "OriginalFilename", "liboamcpp.dll"
VALUE "ProductName", "InfiniDB"
VALUE "ProductVersion", "4.6.0.0 Beta"
END
END
BLOCK "VarFileInfo"
BEGIN
VALUE "Translation", 0x409, 1200
END
END
#endif // English (U.S.) resources
/////////////////////////////////////////////////////////////////////////////
#ifndef APSTUDIO_INVOKED
/////////////////////////////////////////////////////////////////////////////
//
// Generated from the TEXTINCLUDE 3 resource.
//
/////////////////////////////////////////////////////////////////////////////
#endif // not APSTUDIO_INVOKED

102
primitives/primproc/PrimProc.rc
View File

@ -1,102 +0,0 @@
// Microsoft Visual C++ generated resource script.
//
#include "resource.h"
#define APSTUDIO_READONLY_SYMBOLS
/////////////////////////////////////////////////////////////////////////////
//
// Generated from the TEXTINCLUDE 2 resource.
//
#include "afxres.h"
/////////////////////////////////////////////////////////////////////////////
#undef APSTUDIO_READONLY_SYMBOLS
/////////////////////////////////////////////////////////////////////////////
// English (U.S.) resources
#if !defined(AFX_RESOURCE_DLL) || defined(AFX_TARG_ENU)
#ifdef _WIN32
LANGUAGE LANG_ENGLISH, SUBLANG_ENGLISH_US
#pragma code_page(1252)
#endif //_WIN32
#ifdef APSTUDIO_INVOKED
/////////////////////////////////////////////////////////////////////////////
//
// TEXTINCLUDE
//
1 TEXTINCLUDE
BEGIN
"resource.h\0"
END
2 TEXTINCLUDE
BEGIN
"#include ""afxres.h""\r\n"
"\0"
END
3 TEXTINCLUDE
BEGIN
"\r\n"
"\0"
END
#endif // APSTUDIO_INVOKED
/////////////////////////////////////////////////////////////////////////////
//
// Version
//
VS_VERSION_INFO VERSIONINFO
FILEVERSION 4,6,0,0
PRODUCTVERSION 4,6,0,0
FILEFLAGSMASK 0x17L
#ifdef _DEBUG
FILEFLAGS 0x1L
#else
FILEFLAGS 0x0L
#endif
FILEOS 0x4L
FILETYPE 0x1L
FILESUBTYPE 0x0L
BEGIN
BLOCK "StringFileInfo"
BEGIN
BLOCK "040904b0"
BEGIN
VALUE "CompanyName", "InfiniDB, Inc."
VALUE "FileDescription", "InfiniDB Primitive Processor"
VALUE "FileVersion", "4.6.0-0"
VALUE "InternalName", "PrimProc"
VALUE "LegalCopyright", "Copyright (C) 2014"
VALUE "OriginalFilename", "PrimProc.exe"
VALUE "ProductName", "InfiniDB"
VALUE "ProductVersion", "4.6.0.0 Beta"
END
END
BLOCK "VarFileInfo"
BEGIN
VALUE "Translation", 0x409, 1200
END
END
#endif // English (U.S.) resources
/////////////////////////////////////////////////////////////////////////////
#ifndef APSTUDIO_INVOKED
/////////////////////////////////////////////////////////////////////////////
//
// Generated from the TEXTINCLUDE 3 resource.
//
/////////////////////////////////////////////////////////////////////////////
#endif // not APSTUDIO_INVOKED

2
primitives/primproc/batchprimitiveprocessor.cpp
View File

@ -931,7 +931,7 @@ void BatchPrimitiveProcessor::initProcessor()
strValues.reset(new utils::NullString[LOGICAL_BLOCK_RIDS]);
outMsgSize = defaultBufferSize;
outputMsg.reset(reinterpret_cast<uint8_t*>(aligned_alloc(utils::MAXCOLUMNWIDTH, outMsgSize)));
outputMsg.reset(new(std::align_val_t(MAXCOLUMNWIDTH)) uint8_t[outMsgSize]);
if (ot == ROW_GROUP)
{

2
primitives/primproc/batchprimitiveprocessor.h
View File

@ -232,7 +232,7 @@ class BatchPrimitiveProcessor
/* Common space for primitive data */
alignas(utils::MAXCOLUMNWIDTH) uint8_t blockData[BLOCK_SIZE * utils::MAXCOLUMNWIDTH];
uint8_t blockDataAux[BLOCK_SIZE * execplan::AUX_COL_WIDTH];
boost::scoped_array<uint8_t> outputMsg;
std::unique_ptr<uint8_t[], utils::AlignedDeleter> outputMsg;
uint32_t outMsgSize;
std::vector<SCommand> filterSteps;

3
primitives/primproc/columncommand.cpp
View File

@ -631,7 +631,8 @@ void ColumnCommand::fillInPrimitiveMessageHeader(const int8_t outputType, const
size_t inputMsgBufSize = baseMsgLength + (LOGICAL_BLOCK_RIDS * sizeof(primitives::RIDType));
if (!inputMsg)
inputMsg.reset(reinterpret_cast<uint8_t*>(aligned_alloc(utils::MAXCOLUMNWIDTH, inputMsgBufSize)));
inputMsg.reset(new(std::align_val_t(utils::MAXCOLUMNWIDTH)) uint8_t[inputMsgBufSize]);
primMsg = (NewColRequestHeader*)inputMsg.get();
outMsg = (ColResultHeader*)bpp->outputMsg.get();

4
primitives/primproc/columncommand.h
View File

@ -30,6 +30,8 @@
#pragma once
#include <memory>
#include "columnwidth.h"
#include "command.h"
#include "calpontsystemcatalog.h"
@ -164,7 +166,7 @@ class ColumnCommand : public Command
bool _isScan;
boost::scoped_array<uint8_t> inputMsg;
std::unique_ptr<uint8_t[], utils::AlignedDeleter> inputMsg;
NewColRequestHeader* primMsg;
ColResultHeader* outMsg;

6
primitives/primproc/serviceexemgr.h
View File

@ -370,9 +370,9 @@ namespace exemgr
// shared by multiple threads per session.
ThreadCntPerSessionMap_t threadCntPerSessionMap_;
std::mutex threadCntPerSessionMapMutex_;
ActiveStatementCounter* statementsRunningCount_;
joblist::DistributedEngineComm* dec_;
joblist::ResourceManager* rm_;
ActiveStatementCounter* statementsRunningCount_ = nullptr;
joblist::DistributedEngineComm* dec_ = nullptr;
joblist::ResourceManager* rm_ = nullptr;
// Its attributes are set in Child()
querytele::QueryTeleServerParms teleServerParms_;
std::vector<struct in_addr> localNetIfaceSins_;

4
primitives/primproc/sqlfrontsessionthread.cpp
View File

@ -248,7 +248,7 @@ void SQLFrontSessionThread::analyzeTableExecute(messageqcpp::ByteStream& bs, job
auto rowCount = jl->projectTable(dummyTableOid, bs);
while (rowCount)
{
auto outRG = (static_cast<joblist::TupleJobList*>(jl.get()))->getOutputRowGroup();
auto const& outRG = (static_cast<joblist::TupleJobList*>(jl.get()))->getOutputRowGroup();
statisticsManager->collectSample(outRG);
rowCount = jl->projectTable(dummyTableOid, bs);
}
@ -894,7 +894,7 @@ void SQLFrontSessionThread::operator()()
std::unique_lock<std::mutex> scoped(jlMutex);
destructing++;
std::thread bgdtor(
[jl, &jlMutex, &jlCleanupDone, stmtID, &li, &destructing, &msgLog]
[jl, &jlMutex, &jlCleanupDone, stmtID, li, &destructing, &msgLog]
{
std::unique_lock<std::mutex> scoped(jlMutex);
const_cast<joblist::SJLP&>(jl).reset(); // this happens second; does real destruction

10
storage-manager/src/Config.h
View File

@ -17,7 +17,17 @@
#pragma once
#ifndef __clang__
#pragma GCC diagnostic push
#pragma GCC diagnostic ignored "-Wmaybe-uninitialized"
#endif
#include <boost/property_tree/ptree.hpp>
#ifndef __clang__
#pragma GCC diagnostic pop
#endif
#include <boost/thread.hpp>
#include <sys/types.h>
#include <functional>

9
storage-manager/src/MetadataFile.cpp
View File

@ -22,7 +22,16 @@
#include <set>
#include <boost/filesystem.hpp>
#define BOOST_SPIRIT_THREADSAFE
#ifndef __clang__
#pragma GCC diagnostic push
#pragma GCC diagnostic ignored "-Wmaybe-uninitialized"
#endif
#include <boost/property_tree/ptree.hpp>
#ifndef __clang__
#pragma GCC diagnostic pop
#endif
#include <boost/property_tree/json_parser.hpp>
#include <boost/foreach.hpp>
#include <boost/uuid/uuid.hpp>

9
storage-manager/src/S3Storage.cpp
View File

@ -27,7 +27,16 @@
#include <boost/uuid/uuid_io.hpp>
#include <boost/uuid/random_generator.hpp>
#define BOOST_SPIRIT_THREADSAFE
#ifndef __clang__
#pragma GCC diagnostic push
#pragma GCC diagnostic ignored "-Wmaybe-uninitialized"
#endif
#include <boost/property_tree/ptree.hpp>
#ifndef __clang__
#pragma GCC diagnostic pop
#endif
#include <boost/property_tree/json_parser.hpp>
#include "Utilities.h"

16
tests/scripts/fullmtr.sh
View File

@ -3,14 +3,14 @@ mysql -e "create database if not exists test;"
SOCKET=`mysql -e "show variables like 'socket';" | grep socket | cut -f2`
cd /usr/share/mysql/mysql-test
./mtr --force --max-test-fail=0 --testcase-timeout=60 --extern socket=$SOCKET --suite=columnstore/setup
./mtr --force --max-test-fail=0 --testcase-timeout=60 --extern socket=$SOCKET --suite=columnstore/basic | tee $CURRENT_DIR/mtr.basic.log 2>&1
./mtr --force --max-test-fail=0 --testcase-timeout=60 --extern socket=$SOCKET --suite=columnstore/bugfixes | tee $CURRENT_DIR/mtr.bugfixes.log 2>&1
./mtr --force --max-test-fail=0 --testcase-timeout=60 --extern socket=$SOCKET --suite=columnstore/devregression | tee $CURRENT_DIR/mtr.devregression.log 2>&1
./mtr --force --max-test-fail=0 --testcase-timeout=60 --extern socket=$SOCKET --suite=columnstore/autopilot | tee $CURRENT_DIR/mtr.autopilot.log 2>&1
./mtr --force --max-test-fail=0 --testcase-timeout=60 --extern socket=$SOCKET --suite=columnstore/extended | tee $CURRENT_DIR/mtr.extended.log 2>&1
./mtr --force --max-test-fail=0 --testcase-timeout=60 --extern socket=$SOCKET --suite=columnstore/multinode | tee $CURRENT_DIR/mtr.multinode.log 2>&1
./mtr --force --max-test-fail=0 --testcase-timeout=60 --extern socket=$SOCKET --suite=columnstore/oracle | tee $CURRENT_DIR/mtr.multinode.log 2>&1
./mtr --force --max-test-fail=0 --testcase-timeout=60 --extern socket=$SOCKET --suite=columnstore/1pmonly | tee $CURRENT_DIR/mtr.1pmonly.log 2>&1
./mtr --force --max-test-fail=0 --testcase-timeout=60 --extern socket=$SOCKET --suite=columnstore/setup | tee $CURRENT_DIR/mtr.setup.log 2>&1
./mtr --force --max-test-fail=0 --testcase-timeout=60 --extern socket=$SOCKET --suite=columnstore/bugfixes | tee $CURRENT_DIR/mtr.bugfixes.log 2>&1
./mtr --force --max-test-fail=0 --testcase-timeout=60 --extern socket=$SOCKET --suite=columnstore/devregression | tee $CURRENT_DIR/mtr.devregression.log 2>&1
./mtr --force --max-test-fail=0 --testcase-timeout=60 --extern socket=$SOCKET --suite=columnstore/autopilot | tee $CURRENT_DIR/mtr.autopilot.log 2>&1
./mtr --force --max-test-fail=0 --testcase-timeout=60 --extern socket=$SOCKET --suite=columnstore/extended | tee $CURRENT_DIR/mtr.extended.log 2>&1
./mtr --force --max-test-fail=0 --testcase-timeout=60 --extern socket=$SOCKET --suite=columnstore/multinode | tee $CURRENT_DIR/mtr.multinode.log 2>&1
./mtr --force --max-test-fail=0 --testcase-timeout=60 --extern socket=$SOCKET --suite=columnstore/oracle | tee $CURRENT_DIR/mtr.oracle.log 2>&1
./mtr --force --max-test-fail=0 --testcase-timeout=60 --extern socket=$SOCKET --suite=columnstore/1pmonly | tee $CURRENT_DIR/mtr.1pmonly.log 2>&1
cd -

5
tests/scripts/smoke.sql Normal file
View File

@ -0,0 +1,5 @@
DROP TABLE IF EXISTS smoke_table;
CREATE TABLE smoke_table(a int) ENGINE COLUMNSTORE;
INSERT INTO smoke_table VALUES(42);
SELECT * FROM smoke_table;
DROP TABLE smoke_table;

102
tools/dbloadxml/colxml.rc
View File

@ -1,102 +0,0 @@
// Microsoft Visual C++ generated resource script.
//
#include "resource.h"
#define APSTUDIO_READONLY_SYMBOLS
/////////////////////////////////////////////////////////////////////////////
//
// Generated from the TEXTINCLUDE 2 resource.
//
#include "afxres.h"
/////////////////////////////////////////////////////////////////////////////
#undef APSTUDIO_READONLY_SYMBOLS
/////////////////////////////////////////////////////////////////////////////
// English (U.S.) resources
#if !defined(AFX_RESOURCE_DLL) || defined(AFX_TARG_ENU)
#ifdef _WIN32
LANGUAGE LANG_ENGLISH, SUBLANG_ENGLISH_US
#pragma code_page(1252)
#endif //_WIN32
#ifdef APSTUDIO_INVOKED
/////////////////////////////////////////////////////////////////////////////
//
// TEXTINCLUDE
//
1 TEXTINCLUDE
BEGIN
"resource.h\0"
END
2 TEXTINCLUDE
BEGIN
"#include ""afxres.h""\r\n"
"\0"
END
3 TEXTINCLUDE
BEGIN
"\r\n"
"\0"
END
#endif // APSTUDIO_INVOKED
/////////////////////////////////////////////////////////////////////////////
//
// Version
//
VS_VERSION_INFO VERSIONINFO
FILEVERSION 4,6,0,0
PRODUCTVERSION 4,6,0,0
FILEFLAGSMASK 0x17L
#ifdef _DEBUG
FILEFLAGS 0x1L
#else
FILEFLAGS 0x0L
#endif
FILEOS 0x4L
FILETYPE 0x1L
FILESUBTYPE 0x0L
BEGIN
BLOCK "StringFileInfo"
BEGIN
BLOCK "040904b0"
BEGIN
VALUE "CompanyName", "InfiniDB, Inc."
VALUE "FileDescription", "InfiniDB Bulk Load Utility"
VALUE "FileVersion", "4.6.0-0"
VALUE "InternalName", "colxml"
VALUE "LegalCopyright", "Copyright (C) 2014"
VALUE "OriginalFilename", "colxml.exe"
VALUE "ProductName", "InfiniDB"
VALUE "ProductVersion", "4.6.0.0 Beta"
END
END
BLOCK "VarFileInfo"
BEGIN
VALUE "Translation", 0x409, 1200
END
END
#endif // English (U.S.) resources
/////////////////////////////////////////////////////////////////////////////
#ifndef APSTUDIO_INVOKED
/////////////////////////////////////////////////////////////////////////////
//
// Generated from the TEXTINCLUDE 3 resource.
//
/////////////////////////////////////////////////////////////////////////////
#endif // not APSTUDIO_INVOKED

12
utils/common/columnwidth.h
View File

@ -21,8 +21,16 @@
namespace utils
{
const uint8_t MAXLEGACYWIDTH = 8ULL;
const uint8_t MAXCOLUMNWIDTH = 16ULL;
constexpr uint8_t MAXLEGACYWIDTH = 8ULL;
constexpr uint8_t MAXCOLUMNWIDTH = 16ULL;
struct AlignedDeleter
{
void operator()(uint8_t* ptr)
{
operator delete[](ptr, std::align_val_t(utils::MAXCOLUMNWIDTH));
};
};
inline bool isWide(uint8_t width)
{

43
utils/configcpp/configcpp.cpp
View File

@ -69,15 +69,17 @@ const fs::path defaultConfigFilePath(configDefaultFileName);
namespace config
{
Config* globConfigInstancePtr = nullptr;
Config::configMap_t Config::fInstanceMap;
boost::mutex Config::fInstanceMapMutex;
Config::configMap_t Config::fInstanceMap;
// duplicate to that in the Config class
boost::mutex Config::fXmlLock;
// duplicate to that in the Config class
boost::mutex Config::fWriteXmlLock;
std::atomic_bool globHasConfig;
ConfigUniqPtr globConfigInstancePtr;
void Config::checkAndReloadConfig()
{
struct stat statbuf;
@ -105,20 +107,20 @@ Config* Config::makeConfig(const string& cf)
if (globConfigInstancePtr)
{
globConfigInstancePtr->checkAndReloadConfig();
return globConfigInstancePtr;
return globConfigInstancePtr.get();
}
// Make this configurable at least at compile-time.
std::string configFilePath =
std::string(MCSSYSCONFDIR) + std::string("/columnstore/") + configDefaultFileName;
globConfigInstancePtr = new Config(configFilePath);
globConfigInstancePtr.reset(new Config(configFilePath));
globHasConfig.store(true, std::memory_order_relaxed);
return globConfigInstancePtr;
return globConfigInstancePtr.get();
}
boost::mutex::scoped_lock lk(fInstanceMapMutex);
globConfigInstancePtr->checkAndReloadConfig();
return globConfigInstancePtr;
return globConfigInstancePtr.get();
}
boost::mutex::scoped_lock lk(fInstanceMapMutex);
@ -526,21 +528,6 @@ void Config::writeConfigFile(messageqcpp::ByteStream msg) const
/* static */
void Config::deleteInstanceMap()
{
boost::mutex::scoped_lock lk(fInstanceMapMutex);
for (Config::configMap_t::iterator iter = fInstanceMap.begin(); iter != fInstanceMap.end(); ++iter)
{
Config* instance = iter->second;
delete instance;
}
fInstanceMap.clear();
if (globConfigInstancePtr)
{
delete globConfigInstancePtr;
globConfigInstancePtr = nullptr;
}
}
/* static */
@ -643,4 +630,18 @@ std::string Config::getTempFileDir(Config::TempDirPurpose what)
return {};
}
void Config::ConfigDeleter::operator()(Config* config)
{
boost::mutex::scoped_lock lk(fInstanceMapMutex);
for (Config::configMap_t::iterator iter = fInstanceMap.begin(); iter != fInstanceMap.end(); ++iter)
{
Config* instance = iter->second;
delete instance;
}
fInstanceMap.clear();
delete config;
}
} // namespace config

11
utils/configcpp/configcpp.h
View File

@ -56,6 +56,11 @@ namespace config
class Config
{
public:
struct ConfigDeleter
{
void operator()(Config* config);
};
/** @brief Config factory method
*
* Creates a singleton Config object
@ -249,8 +254,14 @@ class Config
*
*/
void checkAndReloadConfig();
};
using ConfigUniqPtr = std::unique_ptr<Config, Config::ConfigDeleter>;
} // namespace config
#undef EXPORT

102
utils/configcpp/libconfigcpp.rc
View File

@ -1,102 +0,0 @@
// Microsoft Visual C++ generated resource script.
//
#include "resource.h"
#define APSTUDIO_READONLY_SYMBOLS
/////////////////////////////////////////////////////////////////////////////
//
// Generated from the TEXTINCLUDE 2 resource.
//
#include "afxres.h"
/////////////////////////////////////////////////////////////////////////////
#undef APSTUDIO_READONLY_SYMBOLS
/////////////////////////////////////////////////////////////////////////////
// English (U.S.) resources
#if !defined(AFX_RESOURCE_DLL) || defined(AFX_TARG_ENU)
#ifdef _WIN32
LANGUAGE LANG_ENGLISH, SUBLANG_ENGLISH_US
#pragma code_page(1252)
#endif //_WIN32
#ifdef APSTUDIO_INVOKED
/////////////////////////////////////////////////////////////////////////////
//
// TEXTINCLUDE
//
1 TEXTINCLUDE
BEGIN
"resource.h\0"
END
2 TEXTINCLUDE
BEGIN
"#include ""afxres.h""\r\n"
"\0"
END
3 TEXTINCLUDE
BEGIN
"\r\n"
"\0"
END
#endif // APSTUDIO_INVOKED
/////////////////////////////////////////////////////////////////////////////
//
// Version
//
VS_VERSION_INFO VERSIONINFO
FILEVERSION 4,6,0,0
PRODUCTVERSION 4,6,0,0
FILEFLAGSMASK 0x17L
#ifdef _DEBUG
FILEFLAGS 0x1L
#else
FILEFLAGS 0x0L
#endif
FILEOS 0x4L
FILETYPE 0x2L
FILESUBTYPE 0x0L
BEGIN
BLOCK "StringFileInfo"
BEGIN
BLOCK "040904b0"
BEGIN
VALUE "CompanyName", "InfiniDB, Inc."
VALUE "FileDescription", "InfiniDB Config API"
VALUE "FileVersion", "4.6.0-0"
VALUE "InternalName", "libconfigcpp"
VALUE "LegalCopyright", "Copyright (C) 2014"
VALUE "OriginalFilename", "libconfigcpp.dll"
VALUE "ProductName", "InfiniDB"
VALUE "ProductVersion", "4.6.0.0 Beta"
END
END
BLOCK "VarFileInfo"
BEGIN
VALUE "Translation", 0x409, 1200
END
END
#endif // English (U.S.) resources
/////////////////////////////////////////////////////////////////////////////
#ifndef APSTUDIO_INVOKED
/////////////////////////////////////////////////////////////////////////////
//
// Generated from the TEXTINCLUDE 3 resource.
//
/////////////////////////////////////////////////////////////////////////////
#endif // not APSTUDIO_INVOKED

1
utils/configcpp/xmlparser.cpp
View File

@ -150,6 +150,7 @@ void XMLParser::setConfig(xmlDocPtr doc, const string& section, const string& na
{
xmlAddChild(cur2, xmlNewText((const xmlChar*)"\t"));
cur3 = cur2->xmlChildrenNode;
xmlFree(cur3->content);
}
else
{

46
utils/dataconvert/dataconvert.cpp
View File

@ -48,26 +48,6 @@ using namespace logging;
namespace
{
const int64_t columnstore_precision[19] = {0,
9,
99,
999,
9999,
99999,
999999,
9999999,
99999999,
999999999,
9999999999LL,
99999999999LL,
999999999999LL,
9999999999999LL,
99999999999999LL,
999999999999999LL,
9999999999999999LL,
99999999999999999LL,
999999999999999999LL};
template <class T>
bool from_string(T& t, const std::string& s, std::ios_base& (*f)(std::ios_base&))
{
@ -475,25 +455,16 @@ void number_int_value(const string& data, cscDataType typeCode,
if ((typeCode == datatypes::SystemCatalog::DECIMAL) || (typeCode == datatypes::SystemCatalog::UDECIMAL) ||
(ct.scale > 0))
{
T rangeUp, rangeLow;
if (ct.precision < 19)
auto precision =
ct.precision == rowgroup::MagicPrecisionForCountAgg ? datatypes::INT128MAXPRECISION : ct.precision;
if (precision > datatypes::INT128MAXPRECISION || precision < 0)
{
rangeUp = (T)columnstore_precision[ct.precision];
}
else
{
auto precision =
ct.precision == rowgroup::MagicPrecisionForCountAgg ? datatypes::INT128MAXPRECISION : ct.precision;
if (precision > datatypes::INT128MAXPRECISION || precision < 0)
{
throw QueryDataExcept("Unsupported precision " + std::to_string(precision) + " converting DECIMAL ",
dataTypeErr);
}
rangeUp = datatypes::ConversionRangeMaxValue[ct.precision - 19];
throw QueryDataExcept("Unsupported precision " + std::to_string(precision) + " converting DECIMAL ",
dataTypeErr);
}
rangeLow = -rangeUp;
T rangeUp = dataconvert::decimalRangeUp<T>(precision);
T rangeLow = -rangeUp;
if (intVal > rangeUp)
{
@ -2849,7 +2820,8 @@ int64_t DataConvert::stringToTime(const string& data)
{
if (!hasDate)
{
day = strtol(data.substr(0, pos).c_str(), &end, 10);
std::string tmpDataSegment = data.substr(0, pos);
day = strtol(tmpDataSegment.c_str(), &end, 10);
if (*end != '\0')
return -1;

15
utils/dataconvert/dataconvert.h
View File

@ -104,6 +104,7 @@ const int64_t IDB_pow[19] = {1,
100000000000000000LL,
1000000000000000000LL};
const int32_t SECS_PER_MIN = 60;
const int32_t MINS_PER_HOUR = 60;
const int32_t HOURS_PER_DAY = 24;
@ -1553,6 +1554,20 @@ inline int128_t strtoll128(const char* data, bool& saturate, char** ep)
return res;
}
template <class T>
T decimalRangeUp(int32_t precision)
{
if (precision < 19)
{
return (T)datatypes::columnstore_precision[precision];
}
else
{
return datatypes::ConversionRangeMaxValue[precision - 19];
}
}
template <>
inline int128_t string_to_ll<int128_t>(const std::string& data, bool& bSaturate)
{

1
utils/funcexp/func_cast.cpp
View File

@ -1072,7 +1072,6 @@ IDB_Decimal Func_cast_decimal::getDecimalVal(Row& row, FunctionParm& parm, bool&
if (decimal.isTSInt128ByPrecision())
{
int128_t max_number_decimal = datatypes::ConversionRangeMaxValue[max_length - 19];
uint128_t uval = parm[0]->data()->getUintVal(row, isNull);
if (uval > (uint128_t)datatypes::Decimal::maxInt128)

23
utils/funcexp/func_date_add.cpp
View File

@ -591,6 +591,12 @@ uint64_t dateAdd(uint64_t time, const string& expr, IntervalColumn::interval_typ
if (-day < month_length[monthSave])
{
if (monthSave == 0)
{
monthSave = 12;
tmpYear--;
}
month--;
monthSave--;
@ -613,6 +619,12 @@ uint64_t dateAdd(uint64_t time, const string& expr, IntervalColumn::interval_typ
// BUG 5448 - changed from '==' to '<='
if (day <= 0)
{
if (monthSave == 0)
{
monthSave = 12;
tmpYear--;
}
month--;
monthSave--;
@ -635,6 +647,17 @@ uint64_t dateAdd(uint64_t time, const string& expr, IntervalColumn::interval_typ
break;
}
if (monthSave == 0)
{
monthSave = 12;
tmpYear--;
if (isLeapYear(tmpYear))
month_length[2] = 29;
else
month_length[2] = 28;
}
month--;
monthSave--;

2
utils/funcexp/func_json_unquote.cpp
View File

@ -37,7 +37,7 @@ std::string Func_json_unquote::getStrVal(rowgroup::Row& row, FunctionParm& fp, b
if (unlikely(jsEg.s.error) || jsEg.value_type != JSON_VALUE_STRING)
return js.safeString();
char* buf = (char*)alloca(jsEg.value_len);
char* buf = (char*)alloca(jsEg.value_len + 1);
if ((strLen = json_unescape(cs, jsEg.value, jsEg.value + jsEg.value_len, &my_charset_utf8mb3_general_ci,
(uchar*)buf, (uchar*)(buf + jsEg.value_len))) >= 0)
{

26
utils/idbdatafile/IDBFactory.cpp
View File

@ -47,15 +47,10 @@ bool IDBFactory::installDefaultPlugins()
// protect these methods since we are changing our static data structure
boost::mutex::scoped_lock lock(fac_guard);
s_plugins[IDBDataFile::BUFFERED] =
FileFactoryEnt(IDBDataFile::BUFFERED, "buffered", new BufferedFileFactory(), new PosixFileSystem());
s_plugins[IDBDataFile::UNBUFFERED] = FileFactoryEnt(IDBDataFile::UNBUFFERED, "unbuffered",
new UnbufferedFileFactory(), new PosixFileSystem());
// TODO: use the installPlugin fcn below instead of declaring this statically, then remove the dependency
// IDBDatafile -> cloudio
// s_plugins[IDBDataFile::CLOUD] = FileFactoryEnt(IDBDataFile::CLOUD, "cloud", new SMFileFactory(), new
// SMFileSystem());
s_plugins.emplace(IDBDataFile::BUFFERED, FileFactoryEnt(IDBDataFile::BUFFERED, "buffered", new BufferedFileFactory(),
new PosixFileSystem()));
s_plugins.emplace(IDBDataFile::UNBUFFERED, FileFactoryEnt(IDBDataFile::UNBUFFERED, "unbuffered",
new UnbufferedFileFactory(), new PosixFileSystem()));
return false;
}
@ -86,7 +81,7 @@ bool IDBFactory::installPlugin(const std::string& plugin)
}
FileFactoryEnt ent = (*(FileFactoryEntryFunc)functor)();
s_plugins[ent.type] = ent;
s_plugins.emplace(ent.type, std::move(ent));
std::ostringstream oss;
oss << "IDBFactory::installPlugin: installed filesystem plugin " << plugin;
@ -112,7 +107,7 @@ IDBDataFile* IDBFactory::open(IDBDataFile::Types type, const char* fname, const
throw std::runtime_error(oss.str());
}
return s_plugins[type].factory->open(fname, mode, opts, colWidth);
return s_plugins.at(type).factory->open(fname, mode, opts, colWidth);
}
IDBFileSystem& IDBFactory::getFs(IDBDataFile::Types type)
@ -124,7 +119,14 @@ IDBFileSystem& IDBFactory::getFs(IDBDataFile::Types type)
throw std::runtime_error(oss.str());
}
return *(s_plugins[type].filesystem);
return *(s_plugins.at(type).filesystem);
}
FileFactoryEnt::~FileFactoryEnt()
{
delete filesystem;
delete factory;
}
} // namespace idbdatafile

34
utils/idbdatafile/IDBFactory.h
View File

@ -31,21 +31,31 @@ class IDBFileSystem;
struct FileFactoryEnt
{
FileFactoryEnt() : type(IDBDataFile::UNKNOWN), name("unknown"), factory(0), filesystem(0)
{
;
}
FileFactoryEnt(IDBDataFile::Types t, const std::string& n, FileFactoryBase* f, IDBFileSystem* fs)
: type(t), name(n), factory(f), filesystem(fs)
{
;
}
IDBDataFile::Types type;
std::string name;
FileFactoryBase* factory;
IDBFileSystem* filesystem;
FileFactoryEnt(const FileFactoryEnt&) = delete;
FileFactoryEnt& operator=(const FileFactoryEnt&) = delete;
FileFactoryEnt& operator=(FileFactoryEnt&&) = delete;
FileFactoryEnt(FileFactoryEnt&& temporary)
: factory(temporary.factory)
, filesystem(temporary.filesystem)
{
temporary.factory = nullptr;
temporary.filesystem = nullptr;
}
~FileFactoryEnt();
IDBDataFile::Types type = IDBDataFile::UNKNOWN;
std::string name = "unknown";
FileFactoryBase* factory = nullptr;
IDBFileSystem* filesystem = nullptr;
};
typedef FileFactoryEnt (*FileFactoryEntryFunc)();
@ -101,8 +111,8 @@ class IDBFactory
static FactoryMap s_plugins;
IDBFactory();
virtual ~IDBFactory();
IDBFactory() = delete;
~IDBFactory() = delete;
};
inline const std::string& IDBFactory::name(IDBDataFile::Types type)
@ -112,7 +122,7 @@ inline const std::string& IDBFactory::name(IDBDataFile::Types type)
throw std::runtime_error("unknown plugin type in IDBFactory::name");
}
return s_plugins[type].name;
return s_plugins.at(type).name;
}
} // namespace idbdatafile

2
utils/messageqcpp/CMakeLists.txt
View File

@ -14,7 +14,7 @@ set(messageqcpp_LIB_SRCS
bytestreampool.cpp
)
add_library(messageqcpp SHARED ${messageqcpp_LIB_SRCS})
add_library(messageqcpp STATIC ${messageqcpp_LIB_SRCS})
add_dependencies(messageqcpp loggingcpp)

104
utils/messageqcpp/messagequeuepool.cpp
View File

@ -15,23 +15,51 @@
Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
MA 02110-1301, USA. */
#include <memory>
#include <sstream>
#include <map>
#include <time.h>
#include <mutex>
#include "messagequeuepool.h"
#include "messagequeue.h"
#include <new>
#include <type_traits>
namespace messageqcpp
{
std::mutex& getQueueMutex()
using ClientMapType = std::multimap<std::string, std::unique_ptr<ClientObject>>;
struct LockedClientMap
{
static std::mutex queueMutex;
return queueMutex;
LockedClientMap()
{
}
~LockedClientMap()
{
}
ClientMapType clientMap;
std::mutex queueMutex;
};
static int clientMapNiftyCounter;
static typename std::aligned_storage<sizeof(LockedClientMap), alignof(LockedClientMap)>::type clientMapBuf;
auto& lockedMap = reinterpret_cast<LockedClientMap&>(clientMapBuf);
LockedClientMapInitilizer::LockedClientMapInitilizer ()
{
if (clientMapNiftyCounter++ == 0) new (&lockedMap) LockedClientMap (); // placement new
}
LockedClientMapInitilizer::~LockedClientMapInitilizer ()
{
if (--clientMapNiftyCounter == 0) (&lockedMap)->~LockedClientMap();
}
// Make linker happy
std::multimap<std::string, ClientObject*> MessageQueueClientPool::clientMap;
// 300 seconds idle until cleanup
#define MAX_IDLE_TIME 300
@ -43,7 +71,7 @@ static uint64_t TimeSpecToSeconds(struct timespec* ts)
MessageQueueClient* MessageQueueClientPool::getInstance(const std::string& dnOrIp, uint64_t port)
{
std::scoped_lock lock(getQueueMutex());
auto lock = std::scoped_lock(lockedMap.queueMutex);
std::ostringstream oss;
oss << dnOrIp << "_" << port;
@ -63,16 +91,17 @@ MessageQueueClient* MessageQueueClientPool::getInstance(const std::string& dnOrI
clock_gettime(CLOCK_MONOTONIC, &now);
uint64_t nowSeconds = TimeSpecToSeconds(&now);
newClientObject->client = new MessageQueueClient(dnOrIp, port);
newClientObject->client.reset(new MessageQueueClient(dnOrIp, port));
newClientObject->inUse = true;
newClientObject->lastUsed = nowSeconds;
clientMap.insert(std::pair<std::string, ClientObject*>(searchString, newClientObject));
return newClientObject->client;
lockedMap.clientMap.emplace(std::move(searchString), std::move(newClientObject));
return newClientObject->client.get();
}
MessageQueueClient* MessageQueueClientPool::getInstance(const std::string& module)
{
std::scoped_lock lock(getQueueMutex());
auto lock = std::scoped_lock(lockedMap.queueMutex);
MessageQueueClient* returnClient = MessageQueueClientPool::findInPool(module);
@ -83,16 +112,19 @@ MessageQueueClient* MessageQueueClientPool::getInstance(const std::string& modul
}
// We didn't find one, create new one
ClientObject* newClientObject = new ClientObject();
auto newClientObject = std::make_unique<ClientObject>();
struct timespec now;
clock_gettime(CLOCK_MONOTONIC, &now);
uint64_t nowSeconds = TimeSpecToSeconds(&now);
newClientObject->client = new MessageQueueClient(module);
newClientObject->client.reset(new MessageQueueClient(module));
newClientObject->inUse = true;
newClientObject->lastUsed = nowSeconds;
clientMap.insert(std::pair<std::string, ClientObject*>(module, newClientObject));
return newClientObject->client;
auto result = newClientObject->client.get();
lockedMap.clientMap.emplace(std::move(module), std::move(newClientObject));
return result;
}
MessageQueueClient* MessageQueueClientPool::findInPool(const std::string& search)
@ -102,40 +134,37 @@ MessageQueueClient* MessageQueueClientPool::findInPool(const std::string& search
uint64_t nowSeconds = TimeSpecToSeconds(&now);
MessageQueueClient* returnClient = NULL;
std::multimap<std::string, ClientObject*>::iterator it = clientMap.begin();
auto it = lockedMap.clientMap.begin();
// Scan pool
while (it != clientMap.end())
while (it != lockedMap.clientMap.end())
{
ClientObject* clientObject = it->second;
ClientObject* clientObject = it->second.get();
uint64_t elapsedTime = nowSeconds - clientObject->lastUsed;
// If connection hasn't been used for MAX_IDLE_TIME we probably don't need it so drop it
// Don't drop in use connections that have been in use a long time
if ((elapsedTime >= MAX_IDLE_TIME) && (!clientObject->inUse))
{
delete clientObject->client;
delete clientObject;
// Do this so we don't invalidate current interator
std::multimap<std::string, ClientObject*>::iterator toDelete = it;
auto toDelete = it;
it++;
clientMap.erase(toDelete);
lockedMap.clientMap.erase(toDelete);
continue;
}
if (!clientObject->inUse)
{
MessageQueueClient* client = clientObject->client;
MessageQueueClient* client = clientObject->client.get();
// If the unused socket isn't connected or has data pending read, destroy it
if (!client->isConnected() || client->hasData())
{
delete client;
delete clientObject;
// Do this so we don't invalidate current interator
std::multimap<std::string, ClientObject*>::iterator toDelete = it;
auto toDelete = it;
it++;
clientMap.erase(toDelete);
lockedMap.clientMap.erase(toDelete);
continue;
}
}
@ -145,7 +174,7 @@ MessageQueueClient* MessageQueueClientPool::findInPool(const std::string& search
{
if ((returnClient == NULL) && (!clientObject->inUse))
{
returnClient = clientObject->client;
returnClient = clientObject->client.get();
clientObject->inUse = true;
return returnClient;
}
@ -165,12 +194,12 @@ void MessageQueueClientPool::releaseInstance(MessageQueueClient* client)
if (client == NULL)
return;
std::scoped_lock lock(getQueueMutex());
std::multimap<std::string, ClientObject*>::iterator it = clientMap.begin();
auto lock = std::scoped_lock(lockedMap.queueMutex);
auto it = lockedMap.clientMap.begin();
while (it != clientMap.end())
while (it != lockedMap.clientMap.end())
{
if (it->second->client == client)
if (it->second->client.get() == client)
{
struct timespec now;
clock_gettime(CLOCK_MONOTONIC, &now);
@ -193,16 +222,15 @@ void MessageQueueClientPool::deleteInstance(MessageQueueClient* client)
if (client == NULL)
return;
std::scoped_lock lock(getQueueMutex());
std::multimap<std::string, ClientObject*>::iterator it = clientMap.begin();
while (it != clientMap.end())
auto lock = std::scoped_lock(lockedMap.queueMutex);
auto it = lockedMap.clientMap.begin();
while (it != lockedMap.clientMap.end())
{
if (it->second->client == client)
if (it->second->client.get() == client)
{
delete it->second->client;
delete it->second;
clientMap.erase(it);
lockedMap.clientMap.erase(it);
return;
}

22
utils/messageqcpp/messagequeuepool.h
View File

@ -19,18 +19,24 @@
#include <map>
#include "messagequeue.h"
#include <memory>
#include <boost/stacktrace.hpp>
namespace messageqcpp
{
static struct LockedClientMapInitilizer {
LockedClientMapInitilizer ();
~LockedClientMapInitilizer ();
} clientMapInitilizer; // static initializer for every translation unit
struct ClientObject
{
MessageQueueClient* client;
uint64_t lastUsed;
bool inUse;
ClientObject() : client(NULL), lastUsed(0), inUse(false)
{
}
std::unique_ptr<MessageQueueClient> client;
uint64_t lastUsed = 0;
bool inUse = false;
};
class MessageQueueClientPool
@ -45,8 +51,6 @@ class MessageQueueClientPool
private:
MessageQueueClientPool(){};
~MessageQueueClientPool(){};
static std::multimap<std::string, ClientObject*> clientMap;
};
} // namespace messageqcpp

106
utils/rowgroup/rowgroup.cpp
View File

@ -50,19 +50,7 @@ namespace rowgroup
{
using cscType = execplan::CalpontSystemCatalog::ColDataType;
StringStore::StringStore() : empty(true), fUseStoreStringMutex(false)
{
}
StringStore::StringStore(const StringStore&)
{
throw logic_error("Don't call StringStore copy ctor");
}
StringStore& StringStore::operator=(const StringStore&)
{
throw logic_error("Don't call StringStore operator=");
}
StringStore::~StringStore()
{
@ -86,10 +74,10 @@ StringStore::~StringStore()
uint64_t StringStore::storeString(const uint8_t* data, uint32_t len)
{
MemChunk* lastMC = NULL;
MemChunk* lastMC = nullptr;
uint64_t ret = 0;
empty = false; // At least a NULL is being stored.
empty = false; // At least a nullptr is being stored.
// Sometimes the caller actually wants "" to be returned....... argggghhhh......
// if (len == 0)
@ -121,7 +109,7 @@ uint64_t StringStore::storeString(const uint8_t* data, uint32_t len)
}
else
{
if ((lastMC == NULL) || (lastMC->capacity - lastMC->currentSize < (len + 4)))
if ((lastMC == nullptr) || (lastMC->capacity - lastMC->currentSize < (len + 4)))
{
// mem usage debugging
// if (lastMC)
@ -215,20 +203,12 @@ void StringStore::clear()
empty = true;
}
UserDataStore::UserDataStore() : fUseUserDataMutex(false)
{
}
UserDataStore::~UserDataStore()
{
}
uint32_t UserDataStore::storeUserData(mcsv1sdk::mcsv1Context& context,
boost::shared_ptr<mcsv1sdk::UserData> data, uint32_t len)
{
uint32_t ret = 0;
if (len == 0 || data == NULL)
if (len == 0 || data == nullptr)
{
return numeric_limits<uint32_t>::max();
}
@ -305,7 +285,7 @@ void UserDataStore::deserialize(ByteStream& bs)
}
mcsv1sdk::mcsv1_UDAF::ReturnCode rc;
mcsv1sdk::UserData* userData = NULL;
mcsv1sdk::UserData* userData = nullptr;
rc = funcIter->second->createUserData(userData, vStoreData[i].length);
if (rc != mcsv1sdk::mcsv1_UDAF::SUCCESS)
@ -323,10 +303,6 @@ void UserDataStore::deserialize(ByteStream& bs)
return;
}
RGData::RGData()
{
// cout << "rgdata++ = " << __sync_add_and_fetch(&rgDataCount, 1) << endl;
}
RGData::RGData(const RowGroup& rg, uint32_t rowCount)
{
@ -336,6 +312,9 @@ RGData::RGData(const RowGroup& rg, uint32_t rowCount)
if (rg.usesStringTable() && rowCount > 0)
strings.reset(new StringStore());
userDataStore.reset();
#ifdef VALGRIND
/* In a PM-join, we can serialize entire tables; not every value has been
* filled in yet. Need to look into that. Valgrind complains that
@ -354,6 +333,8 @@ RGData::RGData(const RowGroup& rg)
if (rg.usesStringTable())
strings.reset(new StringStore());
userDataStore.reset();
#ifdef VALGRIND
/* In a PM-join, we can serialize entire tables; not every value has been
* filled in yet. Need to look into that. Valgrind complains that
@ -366,6 +347,7 @@ RGData::RGData(const RowGroup& rg)
void RGData::reinit(const RowGroup& rg, uint32_t rowCount)
{
rowData.reset(new uint8_t[rg.getDataSize(rowCount)]);
userDataStore.reset();
if (rg.usesStringTable())
strings.reset(new StringStore());
@ -386,16 +368,6 @@ void RGData::reinit(const RowGroup& rg)
reinit(rg, 8192);
}
RGData::RGData(const RGData& r) : rowData(r.rowData), strings(r.strings), userDataStore(r.userDataStore)
{
// cout << "rgdata++ = " << __sync_add_and_fetch(&rgDataCount, 1) << endl;
}
RGData::~RGData()
{
// cout << "rgdata-- = " << __sync_sub_and_fetch(&rgDataCount, 1) << endl;
}
void RGData::serialize(ByteStream& bs, uint32_t amount) const
{
// cout << "serializing!\n";
@ -464,6 +436,7 @@ void RGData::clear()
{
rowData.reset();
strings.reset();
userDataStore.reset();
}
// UserDataStore is only used for UDAF.
@ -478,10 +451,6 @@ UserDataStore* RGData::getUserDataStore()
return userDataStore.get();
}
Row::Row() : data(NULL), strings(NULL), userDataStore(NULL)
{
}
Row::Row(const Row& r)
: columnCount(r.columnCount)
, baseRid(r.baseRid)
@ -501,11 +470,7 @@ Row::Row(const Row& r)
, hasLongStringField(r.hasLongStringField)
, sTableThreshold(r.sTableThreshold)
, forceInline(r.forceInline)
, userDataStore(NULL)
{
}
Row::~Row()
, userDataStore(nullptr)
{
}
@ -1023,7 +988,7 @@ bool Row::equals(const Row& r2, uint32_t lastCol) const
const CHARSET_INFO* Row::getCharset(uint32_t col) const
{
if (charsets[col] == NULL)
if (charsets[col] == nullptr)
{
const_cast<CHARSET_INFO**>(charsets)[col] = &datatypes::Charset(charsetNumbers[col]).getCharset();
}
@ -1031,14 +996,6 @@ const CHARSET_INFO* Row::getCharset(uint32_t col) const
}
RowGroup::RowGroup()
: columnCount(0)
, data(NULL)
, rgData(NULL)
, strings(NULL)
, useStringTable(true)
, hasCollation(false)
, hasLongStringField(false)
, sTableThreshold(20)
{
// 1024 is too generous to waste.
oldOffsets.reserve(10);
@ -1057,7 +1014,7 @@ RowGroup::RowGroup(uint32_t colCount, const vector<uint32_t>& positions, const v
const vector<uint32_t>& cprecision, uint32_t stringTableThreshold, bool stringTable,
const vector<bool>& forceInlineData)
: columnCount(colCount)
, data(NULL)
, data(nullptr)
, oldOffsets(positions)
, oids(roids)
, keys(tkeys)
@ -1065,8 +1022,8 @@ RowGroup::RowGroup(uint32_t colCount, const vector<uint32_t>& positions, const v
, charsetNumbers(csNumbers)
, scale(cscale)
, precision(cprecision)
, rgData(NULL)
, strings(NULL)
, rgData(nullptr)
, strings(nullptr)
, sTableThreshold(stringTableThreshold)
{
uint32_t i;
@ -1107,8 +1064,8 @@ RowGroup::RowGroup(uint32_t colCount, const vector<uint32_t>& positions, const v
useStringTable = (stringTable && hasLongStringField);
offsets = (useStringTable ? &stOffsets[0] : &oldOffsets[0]);
// Set all the charsets to NULL for jit initialization.
charsets.insert(charsets.begin(), charsetNumbers.size(), NULL);
// Set all the charsets to nullptr for jit initialization.
charsets.insert(charsets.begin(), charsetNumbers.size(), nullptr);
}
RowGroup::RowGroup(const RowGroup& r)
@ -1176,14 +1133,6 @@ RowGroup& RowGroup::operator=(const RowGroup& r)
}
RowGroup::RowGroup(ByteStream& bs)
: columnCount(0)
, data(nullptr)
, rgData(nullptr)
, strings(nullptr)
, useStringTable(true)
, hasCollation(false)
, hasLongStringField(false)
, sTableThreshold(20)
{
this->deserialize(bs);
}
@ -1254,22 +1203,13 @@ void RowGroup::deserialize(ByteStream& bs)
else if (!useStringTable && !oldOffsets.empty())
offsets = &oldOffsets[0];
// Set all the charsets to NULL for jit initialization.
charsets.insert(charsets.begin(), charsetNumbers.size(), NULL);
// Set all the charsets to nullptr for jit initialization.
charsets.insert(charsets.begin(), charsetNumbers.size(), nullptr);
}
void RowGroup::serializeRGData(ByteStream& bs) const
{
// cout << "****** serializing\n" << toString() << en
// if (useStringTable || !hasLongStringField)
rgData->serialize(bs, getDataSize());
// else {
// uint64_t size;
// RGData *compressed = convertToStringTable(&size);
// compressed->serialize(bs, size);
// if (compressed != rgData)
// delete compressed;
// }
}
uint32_t RowGroup::getDataSize() const
@ -1354,7 +1294,7 @@ string RowGroup::toString(const std::vector<uint64_t>& used) const
// os << "strings = " << hex << (int64_t) strings << "\n";
// os << "data = " << (int64_t) data << "\n" << dec;
if (data != NULL)
if (data != nullptr)
{
Row r;
initRow(&r);
@ -1580,7 +1520,7 @@ void RowGroup::addToSysDataList(execplan::CalpontSystemCatalog::NJLSysDataList&
const CHARSET_INFO* RowGroup::getCharset(uint32_t col)
{
if (charsets[col] == NULL)
if (charsets[col] == nullptr)
{
charsets[col] = &datatypes::Charset(charsetNumbers[col]).getCharset();
}

160
utils/rowgroup/rowgroup.h
View File

@ -132,7 +132,11 @@ inline T derefFromTwoVectorPtrs(const std::vector<T>* outer, const std::vector<T
class StringStore
{
public:
StringStore();
StringStore() = default;
StringStore(const StringStore&) = delete;
StringStore(StringStore&&) = delete;
StringStore& operator=(const StringStore&) = delete;
StringStore& operator=(StringStore&&) = delete;
virtual ~StringStore();
inline utils::NullString getString(uint64_t offset) const;
@ -177,17 +181,14 @@ class StringStore
private:
std::string empty_str;
StringStore(const StringStore&);
StringStore& operator=(const StringStore&);
static constexpr const uint32_t CHUNK_SIZE = 64 * 1024; // allocators like powers of 2
std::vector<boost::shared_array<uint8_t>> mem;
// To store strings > 64KB (BLOB/TEXT)
std::vector<boost::shared_array<uint8_t>> longStrings;
bool empty;
bool fUseStoreStringMutex; //@bug6065, make StringStore::storeString() thread safe
bool empty = true;
bool fUseStoreStringMutex = false; //@bug6065, make StringStore::storeString() thread safe
boost::mutex fMutex;
};
@ -214,8 +215,13 @@ class UserDataStore
};
public:
UserDataStore();
virtual ~UserDataStore();
UserDataStore() = default;
virtual ~UserDataStore() = default;
UserDataStore(const UserDataStore&) = delete;
UserDataStore(UserDataStore&&) = delete;
UserDataStore& operator=(const UserDataStore&) = delete;
UserDataStore& operator=(UserDataStore&&) = delete;
void serialize(messageqcpp::ByteStream&) const;
void deserialize(messageqcpp::ByteStream&);
@ -237,12 +243,10 @@ class UserDataStore
boost::shared_ptr<mcsv1sdk::UserData> getUserData(uint32_t offset) const;
private:
UserDataStore(const UserDataStore&);
UserDataStore& operator=(const UserDataStore&);
std::vector<StoreData> vStoreData;
bool fUseUserDataMutex;
bool fUseUserDataMutex = false;
boost::mutex fMutex;
};
@ -254,13 +258,16 @@ class Row;
class RGData
{
public:
RGData(); // useless unless followed by an = or a deserialize operation
RGData() = default; // useless unless followed by an = or a deserialize operation
RGData(const RowGroup& rg, uint32_t rowCount); // allocates memory for rowData
explicit RGData(const RowGroup& rg);
RGData(const RGData&);
virtual ~RGData();
RGData& operator=(const RGData&) = default;
RGData& operator=(RGData&&) = default;
RGData(const RGData&) = default;
RGData(RGData&&) = default;
virtual ~RGData() = default;
inline RGData& operator=(const RGData&);
// amount should be the # returned by RowGroup::getDataSize()
void serialize(messageqcpp::ByteStream&, uint32_t amount) const;
@ -274,7 +281,7 @@ class RGData
void clear();
void reinit(const RowGroup& rg);
void reinit(const RowGroup& rg, uint32_t rowCount);
inline void setStringStore(boost::shared_ptr<StringStore>& ss)
inline void setStringStore(std::shared_ptr<StringStore>& ss)
{
strings = ss;
}
@ -307,18 +314,21 @@ class RGData
return (userDataStore ? (userDataStore->useUserDataMutex()) : false);
}
boost::shared_array<uint8_t> rowData;
boost::shared_ptr<StringStore> strings;
boost::shared_ptr<UserDataStore> userDataStore;
bool hasRowData() const
{
return !!rowData;
}
private:
// boost::shared_array<uint8_t> rowData;
// boost::shared_ptr<StringStore> strings;
std::shared_ptr<uint8_t[]> rowData;
std::shared_ptr<StringStore> strings;
std::shared_ptr<UserDataStore> userDataStore;
// Need sig to support backward compat. RGData can deserialize both forms.
static const uint32_t RGDATA_SIG = 0xffffffff; // won't happen for 'old' Rowgroup data
friend class RowGroup;
friend class RowGroupStorage;
};
class Row
@ -326,28 +336,26 @@ class Row
public:
struct Pointer
{
inline Pointer() : data(NULL), strings(NULL), userDataStore(NULL)
{
}
inline Pointer() = default;
// Pointer(uint8_t*) implicitly makes old code compatible with the string table impl;
inline Pointer(uint8_t* d) : data(d), strings(NULL), userDataStore(NULL)
inline Pointer(uint8_t* d) : data(d)
{
}
inline Pointer(uint8_t* d, StringStore* s) : data(d), strings(s), userDataStore(NULL)
inline Pointer(uint8_t* d, StringStore* s) : data(d), strings(s)
{
}
inline Pointer(uint8_t* d, StringStore* s, UserDataStore* u) : data(d), strings(s), userDataStore(u)
{
}
uint8_t* data;
StringStore* strings;
UserDataStore* userDataStore;
uint8_t* data = nullptr;
StringStore* strings = nullptr;
UserDataStore* userDataStore = nullptr;
};
Row();
Row() = default;
Row(const Row&);
~Row();
~Row() = default;
Row& operator=(const Row&);
bool operator==(const Row&) const;
@ -500,7 +508,7 @@ class Row
template <typename T>
inline void setBinaryField_offset(const T* value, uint32_t width, uint32_t colIndex);
// support VARBINARY
// Add 2-byte length at the CHARSET_INFO*beginning of the field. NULL and zero length field are
// Add 2-byte length at the CHARSET_INFO*beginning of the field. nullptr and zero length field are
// treated the same, could use one of the length bit to distinguish these two cases.
inline void setVarBinaryField(const utils::NullString& val, uint32_t colIndex);
// No string construction is necessary for better performance.
@ -598,31 +606,32 @@ class Row
const CHARSET_INFO* getCharset(uint32_t col) const;
private:
uint32_t columnCount;
uint64_t baseRid;
private:
inline bool inStringTable(uint32_t col) const;
private:
uint32_t columnCount = 0;
uint64_t baseRid = 0;
// Note, the mem behind these pointer fields is owned by RowGroup not Row
uint32_t* oldOffsets;
uint32_t* stOffsets;
uint32_t* offsets;
uint32_t* colWidths;
execplan::CalpontSystemCatalog::ColDataType* types;
uint32_t* charsetNumbers;
CHARSET_INFO** charsets;
uint8_t* data;
uint32_t* scale;
uint32_t* precision;
uint32_t* oldOffsets = nullptr;
uint32_t* stOffsets = nullptr;
uint32_t* offsets = nullptr;
uint32_t* colWidths = nullptr;
execplan::CalpontSystemCatalog::ColDataType* types = nullptr;
uint32_t* charsetNumbers = nullptr;
CHARSET_INFO** charsets = nullptr;
uint8_t* data = nullptr;
uint32_t* scale = nullptr;
uint32_t* precision = nullptr;
StringStore* strings;
bool useStringTable;
bool hasCollation;
bool hasLongStringField;
uint32_t sTableThreshold;
StringStore* strings = nullptr;
bool useStringTable = true;
bool hasCollation = false;
bool hasLongStringField = false;
uint32_t sTableThreshold = 20;
boost::shared_array<bool> forceInline;
inline bool inStringTable(uint32_t col) const;
UserDataStore* userDataStore; // For UDAF
UserDataStore* userDataStore = nullptr; // For UDAF
friend class RowGroup;
};
@ -1538,9 +1547,6 @@ class RowGroup : public messageqcpp::Serializeable
inline bool usesStringTable() const;
inline void setUseStringTable(bool);
// RGData *convertToInlineData(uint64_t *size = NULL) const; // caller manages the memory returned by
// this void convertToInlineDataInPlace(); RGData *convertToStringTable(uint64_t *size = NULL)
// const; void convertToStringTableInPlace();
void serializeRGData(messageqcpp::ByteStream&) const;
inline uint32_t getStringTableThreshold() const;
@ -1576,17 +1582,17 @@ class RowGroup : public messageqcpp::Serializeable
const uint16_t& blockNum);
inline void getLocation(uint32_t* partNum, uint16_t* segNum, uint8_t* extentNum, uint16_t* blockNum);
inline void setStringStore(boost::shared_ptr<StringStore>);
inline void setStringStore(std::shared_ptr<StringStore>);
const CHARSET_INFO* getCharset(uint32_t col);
private:
uint32_t columnCount;
uint8_t* data;
uint32_t columnCount = 0;
uint8_t* data = nullptr;
std::vector<uint32_t> oldOffsets; // inline data offsets
std::vector<uint32_t> stOffsets; // string table offsets
uint32_t* offsets; // offsets either points to oldOffsets or stOffsets
uint32_t* offsets = nullptr; // offsets either points to oldOffsets or stOffsets
std::vector<uint32_t> colWidths;
// oids: the real oid of the column, may have duplicates with alias.
// This oid is necessary for front-end to decide the real column width.
@ -1604,12 +1610,12 @@ class RowGroup : public messageqcpp::Serializeable
std::vector<uint32_t> precision;
// string table impl
RGData* rgData;
StringStore* strings; // note, strings and data belong to rgData
bool useStringTable;
bool hasCollation;
bool hasLongStringField;
uint32_t sTableThreshold;
RGData* rgData = nullptr;
StringStore* strings = nullptr; // note, strings and data belong to rgData
bool useStringTable = true;
bool hasCollation = false;
bool hasLongStringField = false;
uint32_t sTableThreshold = 20;
boost::shared_array<bool> forceInline;
static const uint32_t headerSize = 18;
@ -1646,7 +1652,7 @@ every row, they're a measurable performance penalty */
inline uint32_t RowGroup::getRowCount() const
{
// idbassert(data);
// if (!data) throw std::logic_error("RowGroup::getRowCount(): data is NULL!");
// if (!data) throw std::logic_error("RowGroup::getRowCount(): data is nullptr!");
return *((uint32_t*)&data[rowCountOffset]);
}
@ -1677,8 +1683,8 @@ inline void RowGroup::getRow(uint32_t rowNum, Row* r) const
inline void RowGroup::setData(uint8_t* d)
{
data = d;
strings = NULL;
rgData = NULL;
strings = nullptr;
rgData = nullptr;
setUseStringTable(false);
}
@ -1712,7 +1718,7 @@ inline void RowGroup::setUseStringTable(bool b)
offsets = &oldOffsets[0];
if (!useStringTable)
strings = NULL;
strings = nullptr;
}
inline uint64_t RowGroup::getBaseRid() const
@ -1772,7 +1778,7 @@ inline uint32_t RowGroup::getRowSizeWithStrings() const
inline uint64_t RowGroup::getSizeWithStrings(uint64_t n) const
{
if (strings == NULL)
if (strings == nullptr)
return getDataSize(n);
else
return getDataSize(n) + strings->getSize();
@ -1896,7 +1902,7 @@ inline uint32_t RowGroup::getStringTableThreshold() const
return sTableThreshold;
}
inline void RowGroup::setStringStore(boost::shared_ptr<StringStore> ss)
inline void RowGroup::setStringStore(std::shared_ptr<StringStore> ss)
{
if (useStringTable)
{
@ -2123,7 +2129,7 @@ inline const uint8_t* StringStore::getPointer(uint64_t off) const
inline bool StringStore::isNullValue(uint64_t off) const
{
if (off == std::numeric_limits<uint64_t>::max())
if (off == std::numeric_limits<uint64_t>::max())
return true;
return false;
}
@ -2190,14 +2196,6 @@ inline uint64_t StringStore::getSize() const
return ret;
}
inline RGData& RGData::operator=(const RGData& r)
{
rowData = r.rowData;
strings = r.strings;
userDataStore = r.userDataStore;
return *this;
}
inline void RGData::getRow(uint32_t num, Row* row)
{
uint32_t size = row->getSize();

102
utils/udfsdk/libudf_mysql.rc
View File

@ -1,102 +0,0 @@
// Microsoft Visual C++ generated resource script.
//
#include "resource.h"
#define APSTUDIO_READONLY_SYMBOLS
/////////////////////////////////////////////////////////////////////////////
//
// Generated from the TEXTINCLUDE 2 resource.
//
#include "afxres.h"
/////////////////////////////////////////////////////////////////////////////
#undef APSTUDIO_READONLY_SYMBOLS
/////////////////////////////////////////////////////////////////////////////
// English (U.S.) resources
#if !defined(AFX_RESOURCE_DLL) || defined(AFX_TARG_ENU)
#ifdef _WIN32
LANGUAGE LANG_ENGLISH, SUBLANG_ENGLISH_US
#pragma code_page(1252)
#endif //_WIN32
#ifdef APSTUDIO_INVOKED
/////////////////////////////////////////////////////////////////////////////
//
// TEXTINCLUDE
//
1 TEXTINCLUDE
BEGIN
"resource.h\0"
END
2 TEXTINCLUDE
BEGIN
"#include ""afxres.h""\r\n"
"\0"
END
3 TEXTINCLUDE
BEGIN
"\r\n"
"\0"
END
#endif // APSTUDIO_INVOKED
/////////////////////////////////////////////////////////////////////////////
//
// Version
//
VS_VERSION_INFO VERSIONINFO
FILEVERSION 4,6,0,0
PRODUCTVERSION 4,6,0,0
FILEFLAGSMASK 0x17L
#ifdef _DEBUG
FILEFLAGS 0x1L
#else
FILEFLAGS 0x0L
#endif
FILEOS 0x4L
FILETYPE 0x1L
FILESUBTYPE 0x0L
BEGIN
BLOCK "StringFileInfo"
BEGIN
BLOCK "040904b0"
BEGIN
VALUE "CompanyName", "InfiniDB, Inc."
VALUE "FileDescription", "InfiniDB UDF API"
VALUE "FileVersion", "4.6.0-0"
VALUE "InternalName", "libudfsdk"
VALUE "LegalCopyright", "Copyright (C) 2014"
VALUE "OriginalFilename", "libudfsdk.dll"
VALUE "ProductName", "InfiniDB"
VALUE "ProductVersion", "4.6.0.0 Beta"
END
END
BLOCK "VarFileInfo"
BEGIN
VALUE "Translation", 0x409, 1200
END
END
#endif // English (U.S.) resources
/////////////////////////////////////////////////////////////////////////////
#ifndef APSTUDIO_INVOKED
/////////////////////////////////////////////////////////////////////////////
//
// Generated from the TEXTINCLUDE 3 resource.
//
/////////////////////////////////////////////////////////////////////////////
#endif // not APSTUDIO_INVOKED

102
utils/udfsdk/libudfsdk.rc
View File

@ -1,102 +0,0 @@
// Microsoft Visual C++ generated resource script.
//
#include "resource.h"
#define APSTUDIO_READONLY_SYMBOLS
/////////////////////////////////////////////////////////////////////////////
//
// Generated from the TEXTINCLUDE 2 resource.
//
#include "afxres.h"
/////////////////////////////////////////////////////////////////////////////
#undef APSTUDIO_READONLY_SYMBOLS
/////////////////////////////////////////////////////////////////////////////
// English (U.S.) resources
#if !defined(AFX_RESOURCE_DLL) || defined(AFX_TARG_ENU)
#ifdef _WIN32
LANGUAGE LANG_ENGLISH, SUBLANG_ENGLISH_US
#pragma code_page(1252)
#endif //_WIN32
#ifdef APSTUDIO_INVOKED
/////////////////////////////////////////////////////////////////////////////
//
// TEXTINCLUDE
//
1 TEXTINCLUDE
BEGIN
"resource.h\0"
END
2 TEXTINCLUDE
BEGIN
"#include ""afxres.h""\r\n"
"\0"
END
3 TEXTINCLUDE
BEGIN
"\r\n"
"\0"
END
#endif // APSTUDIO_INVOKED
/////////////////////////////////////////////////////////////////////////////
//
// Version
//
VS_VERSION_INFO VERSIONINFO
FILEVERSION 4,6,0,0
PRODUCTVERSION 4,6,0,0
FILEFLAGSMASK 0x17L
#ifdef _DEBUG
FILEFLAGS 0x1L
#else
FILEFLAGS 0x0L
#endif
FILEOS 0x4L
FILETYPE 0x1L
FILESUBTYPE 0x0L
BEGIN
BLOCK "StringFileInfo"
BEGIN
BLOCK "040904b0"
BEGIN
VALUE "CompanyName", "InfiniDB, Inc."
VALUE "FileDescription", "InfiniDB UDF API"
VALUE "FileVersion", "4.6.0-0"
VALUE "InternalName", "libudfsdk"
VALUE "LegalCopyright", "Copyright (C) 2014"
VALUE "OriginalFilename", "libudfsdk.dll"
VALUE "ProductName", "InfiniDB"
VALUE "ProductVersion", "4.6.0.0 Beta"
END
END
BLOCK "VarFileInfo"
BEGIN
VALUE "Translation", 0x409, 1200
END
END
#endif // English (U.S.) resources
/////////////////////////////////////////////////////////////////////////////
#ifndef APSTUDIO_INVOKED
/////////////////////////////////////////////////////////////////////////////
//
// Generated from the TEXTINCLUDE 3 resource.
//
/////////////////////////////////////////////////////////////////////////////
#endif // not APSTUDIO_INVOKED

49
versioning/BRM/slavecomm.cpp
View File

@ -27,6 +27,7 @@
#include <fcntl.h>
#include <cstdio>
#include <ctime>
#include <memory>
#include "messagequeue.h"
#include "bytestream.h"
@ -57,21 +58,18 @@ void timespec_sub(const struct timespec& tv1, const struct timespec& tv2, double
namespace BRM
{
SlaveComm::SlaveComm(string hostname, SlaveDBRMNode* s)
: slave(s)
, currentSaveFile(NULL)
, journalh(NULL)
SlaveComm::SlaveComm(string hostname)
{
config::Config* config = config::Config::makeConfig();
string tmp;
slave = std::make_unique<SlaveDBRMNode>();
bool tellUser = true;
for (;;)
{
try
{
server = new MessageQueueServer(hostname);
server = std::make_unique<MessageQueueServer>(hostname);
break;
}
catch (runtime_error& re)
@ -133,8 +131,8 @@ SlaveComm::SlaveComm(string hostname, SlaveDBRMNode* s)
journalName = savefile + "_journal";
const char* filename = journalName.c_str();
journalh = IDBDataFile::open(IDBPolicy::getType(filename, IDBPolicy::WRITEENG), filename, "a", 0);
if (journalh == NULL)
journalh.reset(IDBDataFile::open(IDBPolicy::getType(filename, IDBPolicy::WRITEENG), filename, "a", 0));
if (journalh == nullptr)
throw runtime_error("Could not open the BRM journal for writing!");
}
else
@ -162,8 +160,6 @@ SlaveComm::SlaveComm(string hostname, SlaveDBRMNode* s)
}
SlaveComm::SlaveComm()
: currentSaveFile(NULL)
, journalh(NULL)
{
config::Config* config = config::Config::makeConfig();
@ -192,23 +188,9 @@ SlaveComm::SlaveComm()
server = NULL;
standalone = true;
printOnly = false;
slave = new SlaveDBRMNode();
slave = std::make_unique<SlaveDBRMNode>();
}
SlaveComm::~SlaveComm()
{
delete server;
server = NULL;
if (firstSlave)
{
delete currentSaveFile;
currentSaveFile = NULL;
}
delete journalh;
journalh = NULL;
}
void SlaveComm::stop()
{
@ -1946,8 +1928,7 @@ void SlaveComm::do_confirm()
{
if (!currentSaveFile)
{
currentSaveFile =
IDBDataFile::open(IDBPolicy::getType(tmp.c_str(), IDBPolicy::WRITEENG), tmp.c_str(), "wb", 0);
currentSaveFile.reset(IDBDataFile::open(IDBPolicy::getType(tmp.c_str(), IDBPolicy::WRITEENG), tmp.c_str(), "wb", 0));
}
if (currentSaveFile == NULL)
@ -1970,7 +1951,7 @@ void SlaveComm::do_confirm()
if (err < (int)relative.length())
{
ostringstream os;
os << "WorkerComm: currentfile write() returned " << err << " file pointer is " << currentSaveFile;
os << "WorkerComm: currentfile write() returned " << err << " file pointer is " << currentSaveFile.get();
if (err < 0)
os << " errno: " << strerror(errno);
@ -1979,13 +1960,13 @@ void SlaveComm::do_confirm()
}
currentSaveFile->flush();
delete currentSaveFile;
currentSaveFile = NULL;
currentSaveFile = nullptr;
saveFileToggle = !saveFileToggle;
delete journalh;
journalh = IDBDataFile::open(IDBPolicy::getType(journalName.c_str(), IDBPolicy::WRITEENG),
journalName.c_str(), "w+b", 0);
;
journalh.reset(IDBDataFile::open(IDBPolicy::getType(journalName.c_str(), IDBPolicy::WRITEENG),
journalName.c_str(), "w+b", 0));
if (!journalh)
throw runtime_error("Could not open the BRM journal for writing!");
@ -2245,7 +2226,6 @@ void SlaveComm::do_ownerCheck(ByteStream& msg)
master.write(reply);
}
// FIXME: needs to be refactored along with SessionManagerServer::lookupProcessStatus()
bool SlaveComm::processExists(const uint32_t pid, const string& pname)
{
string stat;
@ -2278,7 +2258,6 @@ bool SlaveComm::processExists(const uint32_t pid, const string& pname)
return true;
}
void SlaveComm::do_dmlLockLBIDRanges(ByteStream& msg)
{
ByteStream reply;

14
versioning/BRM/slavecomm.h
View File

@ -54,8 +54,10 @@ class SlaveComm
EXPORT SlaveComm();
/** Use this ctor to have it connected to the rest of the DBRM system */
EXPORT SlaveComm(std::string hostname, SlaveDBRMNode* s); // hostname = 'DBRM_WorkerN'
EXPORT ~SlaveComm();
EXPORT SlaveComm(std::string hostname); // hostname = 'DBRM_WorkerN'
EXPORT ~SlaveComm() {};
SlaveDBRMNode& getSlaveNode() { return *slave; }
EXPORT void run();
EXPORT void stop();
@ -112,15 +114,15 @@ class SlaveComm
void saveDelta();
bool processExists(const uint32_t pid, const std::string& pname);
messageqcpp::MessageQueueServer* server;
std::unique_ptr<messageqcpp::MessageQueueServer> server;
messageqcpp::IOSocket master;
SlaveDBRMNode* slave;
std::unique_ptr<SlaveDBRMNode> slave;
std::string savefile;
bool release, die, firstSlave, saveFileToggle, takeSnapshot, doSaveDelta, standalone, printOnly;
messageqcpp::ByteStream delta;
idbdatafile::IDBDataFile* currentSaveFile;
std::unique_ptr<idbdatafile::IDBDataFile> currentSaveFile;
std::string journalName;
idbdatafile::IDBDataFile* journalh;
std::unique_ptr<idbdatafile::IDBDataFile> journalh;
int64_t snapshotInterval, journalCount;
struct timespec MSG_TIMEOUT;
};

16
versioning/BRM/slavenode.cpp
View File

@ -22,6 +22,7 @@
#include <iostream>
#include <signal.h>
#include <memory>
#include <string>
#include <clocale>
#include "slavedbrmnode.h"
@ -39,7 +40,7 @@
using namespace BRM;
using namespace std;
SlaveComm* comm;
std::unique_ptr<SlaveComm> comm;
bool die = false;
boost::thread_group monitorThreads;
@ -120,12 +121,11 @@ int ServiceWorkerNode::Child()
{
setupChildSignalHandlers();
SlaveDBRMNode slave;
ShmKeys keys;
try
{
comm = new SlaveComm(std::string(m_nodename), &slave);
comm = std::make_unique<SlaveComm>(std::string(m_nodename));
NotifyServiceStarted();
}
catch (exception& e)
@ -139,12 +139,12 @@ int ServiceWorkerNode::Child()
}
/* Start 4 threads to monitor write lock state */
monitorThreads.create_thread(RWLockMonitor(&die, slave.getEMFLLockStatus(), keys.KEYRANGE_EMFREELIST_BASE));
monitorThreads.create_thread(RWLockMonitor(&die, slave.getEMLockStatus(), keys.KEYRANGE_EXTENTMAP_BASE));
monitorThreads.create_thread(RWLockMonitor(&die, slave.getVBBMLockStatus(), keys.KEYRANGE_VBBM_BASE));
monitorThreads.create_thread(RWLockMonitor(&die, slave.getVSSLockStatus(), keys.KEYRANGE_VSS_BASE));
monitorThreads.create_thread(RWLockMonitor(&die, comm->getSlaveNode().getEMFLLockStatus(), keys.KEYRANGE_EMFREELIST_BASE));
monitorThreads.create_thread(RWLockMonitor(&die, comm->getSlaveNode().getEMLockStatus(), keys.KEYRANGE_EXTENTMAP_BASE));
monitorThreads.create_thread(RWLockMonitor(&die, comm->getSlaveNode().getVBBMLockStatus(), keys.KEYRANGE_VBBM_BASE));
monitorThreads.create_thread(RWLockMonitor(&die, comm->getSlaveNode().getVSSLockStatus(), keys.KEYRANGE_VSS_BASE));
monitorThreads.create_thread(
RWLockMonitor(&die, slave.getEMIndexLockStatus(), keys.KEYRANGE_EXTENTMAP_INDEX_BASE));
RWLockMonitor(&die, comm->getSlaveNode().getEMIndexLockStatus(), keys.KEYRANGE_EXTENTMAP_INDEX_BASE));
try
{

2
writeengine/build/startdbrm
View File

@ -1,2 +0,0 @@
nohup controlernode &
nohup workernode DBRM_Worker1 &

48
writeengine/build/we_rules.mak
View File

@ -1,48 +0,0 @@
IPCS_CLEANUP=$(EXPORT_ROOT)/bin/dbrm stop; ipcs-pat -d > /dev/null; sleep 2; $(EXPORT_ROOT)/bin/dbrm start;
CPPFLAGS=-I. -I$(EXPORT_ROOT)/include -I/usr/include/libxml2
CXXFLAGS+=$(DEBUG_FLAGS) -D_FILE_OFFSET_BITS=64 -Wall -fpic
LLIBS=-L. -L$(EXPORT_ROOT)/lib -lxml2 -lrwlock -lconfigcpp -lmessageqcpp -lbrm -lboost_thread \
-lloggingcpp -lboost_date_time -lboost_filesystem
TLIBS=$(LLIBS) -lcppunit -ldl
GLIBS=$(TLIBS:-lcppunit=)
LIBDIR=../obj
LOBJS_SHARED= \
$(LIBDIR)/we_blockop.o \
$(LIBDIR)/we_brm.o \
$(LIBDIR)/we_bulkrollbackmgr.o \
$(LIBDIR)/we_bulkrollbackfile.o \
$(LIBDIR)/we_bulkrollbackfilecompressed.o \
$(LIBDIR)/we_bulkrollbackfilecompressedhdfs.o \
$(LIBDIR)/we_cache.o \
$(LIBDIR)/we_chunkmanager.o \
$(LIBDIR)/we_config.o \
$(LIBDIR)/we_confirmhdfsdbfile.o \
$(LIBDIR)/we_convertor.o \
$(LIBDIR)/we_dbfileop.o \
$(LIBDIR)/we_dbrootextenttracker.o \
$(LIBDIR)/we_define.o \
$(LIBDIR)/we_fileop.o \
$(LIBDIR)/we_log.o \
$(LIBDIR)/we_rbmetawriter.o \
$(LIBDIR)/we_simplesyslog.o \
$(LIBDIR)/we_stats.o
LOBJS_INDEX= \
$(LIBDIR)/we_freemgr.o \
$(LIBDIR)/we_indexlist.o \
$(LIBDIR)/we_indexlist_common.o \
$(LIBDIR)/we_indexlist_find_delete.o \
$(LIBDIR)/we_indexlist_multiple_narray.o \
$(LIBDIR)/we_indexlist_narray.o \
$(LIBDIR)/we_indexlist_update_hdr_sub.o \
$(LIBDIR)/we_indextree.o
LOBJS_DCTNRY= $(LIBDIR)/we_dctnry.o
LOBJS_XML= $(LIBDIR)/we_xmlop.o \
$(LIBDIR)/we_xmljob.o \
$(LIBDIR)/we_xmlgendata.o \
$(LIBDIR)/we_xmlgenproc.o

100
writeengine/bulk/cpimport.rc
View File

@ -1,100 +0,0 @@
// Microsoft Visual C++ generated resource script.
//
#include "resource.h"
#define APSTUDIO_READONLY_SYMBOLS
/////////////////////////////////////////////////////////////////////////////
//
// Generated from the TEXTINCLUDE 2 resource.
//
#include "afxres.h"
/////////////////////////////////////////////////////////////////////////////
#undef APSTUDIO_READONLY_SYMBOLS
/////////////////////////////////////////////////////////////////////////////
// English (United States) resources
#if !defined(AFX_RESOURCE_DLL) || defined(AFX_TARG_ENU)
LANGUAGE LANG_ENGLISH, SUBLANG_ENGLISH_US
#pragma code_page(1252)
#ifdef APSTUDIO_INVOKED
/////////////////////////////////////////////////////////////////////////////
//
// TEXTINCLUDE
//
1 TEXTINCLUDE
BEGIN
"resource.h\0"
END
2 TEXTINCLUDE
BEGIN
"#include ""afxres.h""\r\n"
"\0"
END
3 TEXTINCLUDE
BEGIN
"\r\n"
"\0"
END
#endif // APSTUDIO_INVOKED
/////////////////////////////////////////////////////////////////////////////
//
// Version
//
VS_VERSION_INFO VERSIONINFO
FILEVERSION 4,6,0,0
PRODUCTVERSION 4,6,0,0
FILEFLAGSMASK 0x17L
#ifdef _DEBUG
FILEFLAGS 0x1L
#else
FILEFLAGS 0x0L
#endif
FILEOS 0x4L
FILETYPE 0x1L
FILESUBTYPE 0x0L
BEGIN
BLOCK "StringFileInfo"
BEGIN
BLOCK "040904b0"
BEGIN
VALUE "CompanyName", "InfiniDB"
VALUE "FileDescription", "InfiniDB Bulk Loader"
VALUE "FileVersion", "4.6.0-0"
VALUE "InternalName", "cpimport"
VALUE "LegalCopyright", "Copyright (C) 2014"
VALUE "OriginalFilename", "cpimport.exe"
VALUE "ProductName", "InfiniDB"
VALUE "ProductVersion", "4.6.0.0 Beta"
END
END
BLOCK "VarFileInfo"
BEGIN
VALUE "Translation", 0x409, 1200
END
END
#endif // English (United States) resources
/////////////////////////////////////////////////////////////////////////////
#ifndef APSTUDIO_INVOKED
/////////////////////////////////////////////////////////////////////////////
//
// Generated from the TEXTINCLUDE 3 resource.
//
/////////////////////////////////////////////////////////////////////////////
#endif // not APSTUDIO_INVOKED

10
writeengine/bulk/we_bulkloadbuffer.cpp
View File

@ -2415,12 +2415,10 @@ void BulkLoadBuffer::tokenize(const boost::ptr_vector<ColumnInfo>& columnsInfo,
//------------------------------------------------------------------
case FLD_PARSE_ENCLOSED_STATE:
{
char next = *(p + 1);
if ((p + 1 < pEndOfData) &&
(((c == ESCAPE_CHAR) && ((next == STRING_ENCLOSED_CHAR) || (next == ESCAPE_CHAR) ||
(next == LINE_FEED) || (next == CARRIAGE_RETURN))) ||
((c == STRING_ENCLOSED_CHAR) && (next == STRING_ENCLOSED_CHAR))))
(((c == ESCAPE_CHAR) && ((*(p + 1) == STRING_ENCLOSED_CHAR) || (*(p + 1) == ESCAPE_CHAR) ||
(*(p + 1) == LINE_FEED) || (*(p + 1) == CARRIAGE_RETURN))) ||
((c == STRING_ENCLOSED_CHAR) && (*(p + 1) == STRING_ENCLOSED_CHAR))))
{
// Create/save original data before stripping out bytes
if (rawDataRowLength == 0)
@ -2515,7 +2513,7 @@ void BulkLoadBuffer::tokenize(const boost::ptr_vector<ColumnInfo>& columnsInfo,
// cout << "triming ... " << endl;
char* tmp = p;
while (*(--tmp) == ' ')
while (tmp != lastRowHead && *(--tmp) == ' ')
{
// cout << "offset is " << offset <<endl;
offset--;

102
writeengine/libwriteengine.rc
View File

@ -1,102 +0,0 @@
// Microsoft Visual C++ generated resource script.
//
#include "resource.h"
#define APSTUDIO_READONLY_SYMBOLS
/////////////////////////////////////////////////////////////////////////////
//
// Generated from the TEXTINCLUDE 2 resource.
//
#include "afxres.h"
/////////////////////////////////////////////////////////////////////////////
#undef APSTUDIO_READONLY_SYMBOLS
/////////////////////////////////////////////////////////////////////////////
// English (U.S.) resources
#if !defined(AFX_RESOURCE_DLL) || defined(AFX_TARG_ENU)
#ifdef _WIN32
LANGUAGE LANG_ENGLISH, SUBLANG_ENGLISH_US
#pragma code_page(1252)
#endif //_WIN32
#ifdef APSTUDIO_INVOKED
/////////////////////////////////////////////////////////////////////////////
//
// TEXTINCLUDE
//
1 TEXTINCLUDE
BEGIN
"resource.h\0"
END
2 TEXTINCLUDE
BEGIN
"#include ""afxres.h""\r\n"
"\0"
END
3 TEXTINCLUDE
BEGIN
"\r\n"
"\0"
END
#endif // APSTUDIO_INVOKED
/////////////////////////////////////////////////////////////////////////////
//
// Version
//
VS_VERSION_INFO VERSIONINFO
FILEVERSION 4,6,0,0
PRODUCTVERSION 4,6,0,0
FILEFLAGSMASK 0x17L
#ifdef _DEBUG
FILEFLAGS 0x1L
#else
FILEFLAGS 0x0L
#endif
FILEOS 0x4L
FILETYPE 0x2L
FILESUBTYPE 0x0L
BEGIN
BLOCK "StringFileInfo"
BEGIN
BLOCK "040904b0"
BEGIN
VALUE "CompanyName", "InfiniDB, Inc."
VALUE "FileDescription", "InfiniDB Write Engine API"
VALUE "FileVersion", "4.6.0-0"
VALUE "InternalName", "libwriteengine"
VALUE "LegalCopyright", "Copyright (C) 2014"
VALUE "OriginalFilename", "libwriteengine.dll"
VALUE "ProductName", "InfiniDB"
VALUE "ProductVersion", "4.6.0.0 Beta"
END
END
BLOCK "VarFileInfo"
BEGIN
VALUE "Translation", 0x409, 1200
END
END
#endif // English (U.S.) resources
/////////////////////////////////////////////////////////////////////////////
#ifndef APSTUDIO_INVOKED
/////////////////////////////////////////////////////////////////////////////
//
// Generated from the TEXTINCLUDE 3 resource.
//
/////////////////////////////////////////////////////////////////////////////
#endif // not APSTUDIO_INVOKED

102
writeengine/server/WriteEngineServer.rc
View File

@ -1,102 +0,0 @@
// Microsoft Visual C++ generated resource script.
//
#include "resource.h"
#define APSTUDIO_READONLY_SYMBOLS
/////////////////////////////////////////////////////////////////////////////
//
// Generated from the TEXTINCLUDE 2 resource.
//
#include "afxres.h"
/////////////////////////////////////////////////////////////////////////////
#undef APSTUDIO_READONLY_SYMBOLS
/////////////////////////////////////////////////////////////////////////////
// English (U.S.) resources
#if !defined(AFX_RESOURCE_DLL) || defined(AFX_TARG_ENU)
#ifdef _WIN32
LANGUAGE LANG_ENGLISH, SUBLANG_ENGLISH_US
#pragma code_page(1252)
#endif //_WIN32
#ifdef APSTUDIO_INVOKED
/////////////////////////////////////////////////////////////////////////////
//
// TEXTINCLUDE
//
1 TEXTINCLUDE
BEGIN
"resource.h\0"
END
2 TEXTINCLUDE
BEGIN
"#include ""afxres.h""\r\n"
"\0"
END
3 TEXTINCLUDE
BEGIN
"\r\n"
"\0"
END
#endif // APSTUDIO_INVOKED
/////////////////////////////////////////////////////////////////////////////
//
// Version
//
VS_VERSION_INFO VERSIONINFO
FILEVERSION 4,6,0,0
PRODUCTVERSION 4,6,0,0
FILEFLAGSMASK 0x17L
#ifdef _DEBUG
FILEFLAGS 0x1L
#else
FILEFLAGS 0x0L
#endif
FILEOS 0x4L
FILETYPE 0x1L
FILESUBTYPE 0x0L
BEGIN
BLOCK "StringFileInfo"
BEGIN
BLOCK "040904b0"
BEGIN
VALUE "CompanyName", "InfiniDB, Inc."
VALUE "FileDescription", "InfiniDB Write Engine Server"
VALUE "FileVersion", "4.6.0-0"
VALUE "InternalName", "WriteEngineServer"
VALUE "LegalCopyright", "Copyright (C) 2014"
VALUE "OriginalFilename", "WriteEngineServer.exe"
VALUE "ProductName", "InfiniDB"
VALUE "ProductVersion", "4.6.0.0 Beta"
END
END
BLOCK "VarFileInfo"
BEGIN
VALUE "Translation", 0x409, 1200
END
END
#endif // English (U.S.) resources
/////////////////////////////////////////////////////////////////////////////
#ifndef APSTUDIO_INVOKED
/////////////////////////////////////////////////////////////////////////////
//
// Generated from the TEXTINCLUDE 3 resource.
//
/////////////////////////////////////////////////////////////////////////////
#endif // not APSTUDIO_INVOKED

32
writeengine/shared/we_fileop.cpp
View File

@ -60,7 +60,7 @@ namespace WriteEngine
{
/*static*/ boost::mutex FileOp::m_createDbRootMutexes;
/*static*/ boost::mutex FileOp::m_mkdirMutex;
/*static*/ std::map<int, boost::mutex*> FileOp::m_DbRootAddExtentMutexes;
/*static*/ std::map<int, boost::mutex> FileOp::m_DbRootAddExtentMutexes;
// in 1 call to fwrite(), during initialization
// StopWatch timer;
@ -993,7 +993,7 @@ int FileOp::initColumnExtent(IDBDataFile* pFile, uint16_t dbRoot, int nBlocks, c
Stats::startParseEvent(WE_STATS_WAIT_TO_CREATE_COL_EXTENT);
#endif
boost::mutex::scoped_lock lk(*m_DbRootAddExtentMutexes[dbRoot]);
boost::mutex::scoped_lock lk(m_DbRootAddExtentMutexes[dbRoot]);
#ifdef PROFILE
if (bExpandExtent)
@ -1714,7 +1714,7 @@ int FileOp::initDctnryExtent(IDBDataFile* pFile, uint16_t dbRoot, int nBlocks, u
Stats::startParseEvent(WE_STATS_WAIT_TO_CREATE_DCT_EXTENT);
#endif
boost::mutex::scoped_lock lk(*m_DbRootAddExtentMutexes[dbRoot]);
boost::mutex::scoped_lock lk(m_DbRootAddExtentMutexes[dbRoot]);
#ifdef PROFILE
if (bExpandExtent)
@ -1809,33 +1809,13 @@ void FileOp::initDbRootExtentMutexes()
for (size_t i = 0; i < rootIds.size(); i++)
{
boost::mutex* pM = new boost::mutex;
m_DbRootAddExtentMutexes[rootIds[i]] = pM;
m_DbRootAddExtentMutexes.emplace(std::piecewise_construct,
std::forward_as_tuple(rootIds[i]),
std::forward_as_tuple());
}
}
}
/***********************************************************
* DESCRIPTION:
* Cleans up memory allocated to the DBRoot extent mutexes. Calling
* this function is not necessary, but it is provided for completeness,
* to complement initDbRootExtentMutexes(), and to provide a way to
* free up memory at the end of program execution.
***********************************************************/
/* static */
void FileOp::removeDbRootExtentMutexes()
{
boost::mutex::scoped_lock lk(m_createDbRootMutexes);
std::map<int, boost::mutex*>::iterator k = m_DbRootAddExtentMutexes.begin();
while (k != m_DbRootAddExtentMutexes.end())
{
delete k->second;
++k;
}
}
/***********************************************************
* DESCRIPTION:
* Write out (reinitialize) a partial extent in a column file.

3
writeengine/shared/we_fileop.h
View File

@ -443,7 +443,6 @@ class FileOp : public BlockOp, public WeUIDGID
execplan::CalpontSystemCatalog::ColDataType colDataType);
static void initDbRootExtentMutexes();
static void removeDbRootExtentMutexes();
int writeInitialCompColumnChunk(IDBDataFile* pFile, int nBlocksAllocated, int nRows,
const uint8_t* emptyVal, int width, BRM::LBID_t lbid,
@ -457,7 +456,7 @@ class FileOp : public BlockOp, public WeUIDGID
static boost::mutex m_createDbRootMutexes;
// Mutexes used to serialize extent creation within each DBRoot
static std::map<int, boost::mutex*> m_DbRootAddExtentMutexes;
static std::map<int, boost::mutex> m_DbRootAddExtentMutexes;
// protect race condition in creating directories
static boost::mutex m_mkdirMutex;

4
writeengine/shared/we_type.h
View File

@ -400,8 +400,8 @@ struct JobColumn /** @brief Job Column Structure */
int compressionType; /** @brief compression type */
bool autoIncFlag; /** @brief auto increment flag */
DctnryStruct dctnry; /** @brief dictionary structure */
int64_t fMinIntSat; /** @brief For integer type, the min saturation value */
uint64_t fMaxIntSat; /** @brief For integer type, the max saturation value */
int128_t fMinIntSat; /** @brief For integer type, the min saturation value */
uint128_t fMaxIntSat; /** @brief For integer type, the max saturation value */
double fMinDblSat; /** @brief for float/double, the min saturation value */
double fMaxDblSat; /** @brief for float/double, the max saturation value */
bool fWithDefault; /** @brief With default */

101
writeengine/splitter/splitter.rc
View File

@ -1,101 +0,0 @@
// Microsoft Visual C++ generated resource script.
//
#include "resource.h"
#define APSTUDIO_READONLY_SYMBOLS
/////////////////////////////////////////////////////////////////////////////
//
// Generated from the TEXTINCLUDE 2 resource.
//
#include "afxres.h"
/////////////////////////////////////////////////////////////////////////////
#undef APSTUDIO_READONLY_SYMBOLS
/////////////////////////////////////////////////////////////////////////////
// English (U.S.) resources
#if !defined(AFX_RESOURCE_DLL) || defined(AFX_TARG_ENU)
#ifdef _WIN32
LANGUAGE LANG_ENGLISH, SUBLANG_ENGLISH_US
#pragma code_page(1252)
#endif //_WIN32
#ifdef APSTUDIO_INVOKED
/////////////////////////////////////////////////////////////////////////////
//
// TEXTINCLUDE
//
1 TEXTINCLUDE
BEGIN
"resource.h\0"
END
2 TEXTINCLUDE
BEGIN
"#include ""afxres.h""\r\n"
"\0"
END
3 TEXTINCLUDE
BEGIN
"\r\n"
"\0"
END
#endif // APSTUDIO_INVOKED
/////////////////////////////////////////////////////////////////////////////
//
// Version
//
VS_VERSION_INFO VERSIONINFO
FILEVERSION 4,6,0,0
PRODUCTVERSION 4,6,0,0
FILEFLAGSMASK 0x17L
#ifdef _DEBUG
FILEFLAGS 0x1L
#else
FILEFLAGS 0x0L
#endif
FILEOS 0x4L
FILETYPE 0x1L
FILESUBTYPE 0x0L
BEGIN
BLOCK "StringFileInfo"
BEGIN
BLOCK "040904b0"
BEGIN
VALUE "FileDescription", "InfiniDB Bulk Loader"
VALUE "FileVersion", "4.6.0-0"
VALUE "InternalName", "splitter"
VALUE "LegalCopyright", "Copyright (C) 2014"
VALUE "OriginalFilename", "cpimport.exe"
VALUE "ProductName", "InfiniDB"
VALUE "ProductVersion", "4.6.0.0 Beta"
END
END
BLOCK "VarFileInfo"
BEGIN
VALUE "Translation", 0x409, 1200
END
END
#endif // English (U.S.) resources
/////////////////////////////////////////////////////////////////////////////
#ifndef APSTUDIO_INVOKED
/////////////////////////////////////////////////////////////////////////////
//
// Generated from the TEXTINCLUDE 3 resource.
//
/////////////////////////////////////////////////////////////////////////////
#endif // not APSTUDIO_INVOKED

2
writeengine/splitter/we_cmdargs.cpp
View File

@ -464,7 +464,7 @@ bool WECmdArgs::str2PmList(std::string& PmList, VecInts& V)
if (aLen > 0)
{
strncpy(aBuff, PmList.c_str(), BUFLEN);
strncpy(aBuff, PmList.c_str(), BUFLEN - 1);
aBuff[BUFLEN - 1] = 0;
}
else

31
writeengine/wrapper/writeengine.cpp
View File

@ -65,6 +65,7 @@ using namespace idbdatafile;
#include "dataconvert.h"
#include "string_prefixes.h"
#include "mcs_decimal.h"
namespace WriteEngine
//#define PROFILE 1
@ -919,7 +920,7 @@ int WriteEngineWrapper::fillColumn(const TxnID& txnid, const OID& dataOid,
Column refCol;
ColType newColType;
ColType refColType;
boost::scoped_array<char> defVal(new char[MAX_COLUMN_BOUNDARY]);
boost::scoped_array<char> defVal(new char[datatypes::MAXDECIMALWIDTH]);
ColumnOp* colOpNewCol = m_colOp[op(compressionType)];
ColumnOp* refColOp = m_colOp[op(refCompressionType)];
Dctnry* dctnry = m_dctnry[op(compressionType)];
@ -994,6 +995,29 @@ int WriteEngineWrapper::fillColumn(const TxnID& txnid, const OID& dataOid,
return rc;
}
// TODO: Get rid of this
void emptyValueToAny(boost::any* any, const uint8_t* emptyValue, int colWidth)
{
switch (colWidth)
{
case 16:
*any = *(uint128_t*)emptyValue;
break;
case 8:
*any = *(uint64_t*)emptyValue;
break;
case 4:
*any = *(uint32_t*)emptyValue;
break;
case 2:
*any = *(uint16_t*)emptyValue;
break;
default:
*any = *emptyValue;
}
}
int WriteEngineWrapper::deleteRow(const TxnID& txnid, const vector<CSCTypesList>& colExtentsColType,
vector<ColStructList>& colExtentsStruct, vector<void*>& colOldValueList,
vector<RIDList>& ridLists, const int32_t tableOid, bool hasAUXCol)
@ -1034,10 +1058,7 @@ int WriteEngineWrapper::deleteRow(const TxnID& txnid, const vector<CSCTypesList>
const uint8_t* emptyVal = m_colOp[op(curColStruct.fCompressionType)]->getEmptyRowValue(
curColStruct.colDataType, curColStruct.colWidth);
if (curColStruct.colWidth == datatypes::MAXDECIMALWIDTH)
curTuple.data = *(int128_t*)emptyVal;
else
curTuple.data = *(int64_t*)emptyVal;
emptyValueToAny(&curTuple.data, emptyVal, curColStruct.colWidth);
curTupleList.push_back(curTuple);
colValueList.push_back(curTupleList);

30
writeengine/xml/we_xmljob.cpp
View File

@ -21,6 +21,7 @@
*******************************************************************************/
/** @file */
#include "mcs_basic_types.h"
#define WRITEENGINEXMLJOB_DLLEXPORT
#include "we_xmljob.h"
#undef WRITEENGINEXMLJOB_DLLEXPORT
@ -46,28 +47,6 @@ using namespace execplan;
namespace WriteEngine
{
// Maximum saturation value for DECIMAL types based on precision
// TODO MCOL-641 add support here. see dataconvert.cpp
const long long columnstore_precision[19] = {0,
9,
99,
999,
9999,
99999,
999999,
9999999,
99999999,
999999999,
9999999999LL,
99999999999LL,
999999999999LL,
9999999999999LL,
99999999999999LL,
999999999999999LL,
9999999999999999LL,
99999999999999999LL,
999999999999999999LL};
//------------------------------------------------------------------------------
// Constructor
//------------------------------------------------------------------------------
@ -695,13 +674,14 @@ void XMLJob::initSatLimits(JobColumn& curColumn) const
}
else if (curColumn.typeName == ColDataTypeStr[CalpontSystemCatalog::DECIMAL])
{
curColumn.fMinIntSat = -columnstore_precision[curColumn.precision];
curColumn.fMaxIntSat = columnstore_precision[curColumn.precision];
curColumn.fMaxIntSat = dataconvert::decimalRangeUp<int128_t>(curColumn.precision);
curColumn.fMinIntSat = -curColumn.fMaxIntSat;
}
else if (curColumn.typeName == ColDataTypeStr[CalpontSystemCatalog::UDECIMAL])
{
curColumn.fMinIntSat = 0;
curColumn.fMaxIntSat = columnstore_precision[curColumn.precision];
curColumn.fMaxIntSat = dataconvert::decimalRangeUp<int128_t>(curColumn.precision);
}
else if (curColumn.typeName == ColDataTypeStr[CalpontSystemCatalog::FLOAT])
{