diff --git a/.gitignore b/.gitignore index aba662bbf..93be85e28 100644 --- a/.gitignore +++ b/.gitignore @@ -106,3 +106,5 @@ install_manifest_storage-engine.txt _CPack_Packages columnstoreversion.h .idea/ +.build +/.vs diff --git a/CMakeLists.txt b/CMakeLists.txt index 94fdde894..662d43d2a 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -34,10 +34,25 @@ IF(NOT CMAKE_BUILD_TYPE) SET(CMAKE_BUILD_TYPE RELWITHDEBINFO CACHE STRING "Choose the type of build, options are: None(CMAKE_CXX_FLAGS or CMAKE_C_FLAGS used) Debug Release RelWithDebInfo MinSizeRel" FORCE) ENDIF(NOT CMAKE_BUILD_TYPE) +SET_PROPERTY(CACHE CMAKE_BUILD_TYPE PROPERTY STRINGS "Debug" "Release" "MinSizeRel" "RelWithDebInfo") -#set( CMAKE_VERBOSE_MAKEFILE on ) +INCLUDE(ExternalProject) -INCLUDE(columnstore_version.cmake) +SET(CMAKE_CXX_STANDARD 11) +SET(CMAKE_CXX_STANDARD_REQUIRED TRUE) +SET(CMAKE_CXX_EXTENSIONS FALSE) +SET(CMAKE_EXPORT_COMPILE_COMMANDS TRUE) +SET(CMAKE_POSITION_INDEPENDENT_CODE TRUE) +SET(CMAKE_ARCHIVE_OUTPUT_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/obj) +SET(CMAKE_RUNTIME_OUTPUT_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/bin) +SET(CMAKE_LIBRARY_OUTPUT_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/lib) + +SET_PROPERTY(DIRECTORY PROPERTY EP_BASE ${CMAKE_CURRENT_BINARY_DIR}/external) +LIST(APPEND CMAKE_MODULE_PATH ${CMAKE_CURRENT_LIST_DIR}/cmake) + +find_package(Boost 1.53.0 REQUIRED COMPONENTS system filesystem thread regex date_time) +find_package(BISON REQUIRED) +INCLUDE(columnstore_version) SET (PACKAGE columnstore) SET (PACKAGE_NAME columnstore) @@ -75,13 +90,13 @@ IF("${isSystemDir}" STREQUAL "-1") SET(CMAKE_INSTALL_RPATH "${CMAKE_INSTALL_RPATH};${INSTALL_ENGINE}/mysql/lib") ENDIF("${isSystemDir}" STREQUAL "-1") -INCLUDE (configureEngine.cmake) +INCLUDE (configureEngine) # releasenum is used by external scripts for various tasks. Leave it alone. CONFIGURE_FILE(${CMAKE_CURRENT_SOURCE_DIR}/build/releasenum.in ${CMAKE_CURRENT_BINARY_DIR}/build/releasenum IMMEDIATE) INSTALL(FILES ${CMAKE_CURRENT_BINARY_DIR}/build/releasenum DESTINATION ${INSTALL_ENGINE} COMPONENT platform) CONFIGURE_FILE(${CMAKE_CURRENT_SOURCE_DIR}/columnstoreversion.h.in ${CMAKE_CURRENT_SOURCE_DIR}/columnstoreversion.h) -CONFIGURE_FILE(${CMAKE_CURRENT_SOURCE_DIR}/config.h.cmake ${CMAKE_CURRENT_BINARY_DIR}/config.h) +CONFIGURE_FILE(${CMAKE_CURRENT_SOURCE_DIR}/config.h.in ${CMAKE_CURRENT_BINARY_DIR}/config.h) exec_program("git" ${CMAKE_CURRENT_SOURCE_DIR} @@ -91,8 +106,6 @@ exec_program("git" CONFIGURE_FILE(${CMAKE_CURRENT_SOURCE_DIR}/gitversionEngine.in ${CMAKE_CURRENT_BINARY_DIR}/gitversionEngine IMMEDIATE) INSTALL(FILES ${CMAKE_CURRENT_BINARY_DIR}/gitversionEngine DESTINATION ${INSTALL_ENGINE} COMPONENT platform) -INCLUDE(bison.cmake) - FIND_PROGRAM(LEX_EXECUTABLE flex DOC "path to the flex executable") if(NOT LEX_EXECUTABLE) FIND_PROGRAM(LEX_EXECUTABLE lex DOC "path to the lex executable") @@ -101,13 +114,13 @@ if(NOT LEX_EXECUTABLE) endif() endif() -INCLUDE (FindLibXml2) +FIND_PACKAGE(LibXml2) if (NOT LIBXML2_FOUND) MESSAGE(FATAL_ERROR "Could not find a usable libxml2 development environment!") endif() -INCLUDE (FindSnappy.cmake) +INCLUDE (FindSnappy) if (NOT SNAPPY_FOUND) MESSAGE(FATAL_ERROR "Snappy not found please install snappy-devel for CentOS/RedHat or libsnappy-dev for Ubuntu/Debian") endif() @@ -116,7 +129,7 @@ endif() IF (EXISTS "/etc/SuSE-release") SET(JEMALLOC_LIBRARIES "") ELSE () - INCLUDE (FindJeMalloc.cmake) + INCLUDE (FindJeMalloc) if (NOT JEMALLOC_FOUND) message(FATAL_ERROR "jemalloc not found!") SET(JEMALLOC_LIBRARIES "") @@ -128,7 +141,7 @@ if(NOT AWK_EXECUTABLE) message(FATAL_ERROR "awk not found!") endif() -INCLUDE(check_compiler_flag.cmake) +INCLUDE(check_compiler_flag) MY_CHECK_AND_SET_COMPILER_FLAG("-g -O3 -fno-omit-frame-pointer -fno-strict-aliasing -Wall -fno-tree-vectorize -D_GLIBCXX_ASSERTIONS -DDBUG_OFF -DHAVE_CONFIG_H" RELEASE RELWITHDEBINFO MINSIZEREL) MY_CHECK_AND_SET_COMPILER_FLAG("-ggdb3 -fno-omit-frame-pointer -fno-tree-vectorize -D_GLIBCXX_ASSERTIONS -DSAFE_MUTEX -DSAFEMALLOC -DENABLED_DEBUG_SYNC -O0 -Wall -D_DEBUG -DHAVE_CONFIG_H" DEBUG) @@ -157,10 +170,6 @@ IF(SECURITY_HARDENED) ENDIF() SET (ENGINE_LDFLAGS "-Wl,--no-as-needed -Wl,--add-needed") - - -FIND_PACKAGE(Boost 1.53.0 REQUIRED COMPONENTS system filesystem thread regex date_time) - SET (ENGINE_LIBDIR "${INSTALL_ENGINE}/lib") SET (ENGINE_BINDIR "${INSTALL_ENGINE}/bin") SET (ENGINE_INCDIR "${INSTALL_ENGINE}/include") @@ -276,6 +285,15 @@ ADD_SUBDIRECTORY(writeengine/server) ADD_SUBDIRECTORY(writeengine/bulk) ADD_SUBDIRECTORY(writeengine/splitter) -INCLUDE(cpackEngineRPM.cmake) -INCLUDE(cpackEngineDEB.cmake) +# WriteEngine component tests +IF( WITH_SHARED_COMP_TESTS ) + # search for cppunit + INCLUDE (findcppunit.cmake) + if (NOT CPPUNIT_FOUND) + MESSAGE(FATAL_ERROR "CPPUnit not found please install cppunit-devel for CentOS/RedHat or libcppunit-dev for Ubuntu/Debian") + endif() + ADD_SUBDIRECTORY(writeengine/shared) +ENDIF( WITH_SHARED_COMP_TESTS ) +INCLUDE(cpackEngineRPM) +INCLUDE(cpackEngineDEB) diff --git a/VERSION b/VERSION index f5d54e457..e0328ce70 100644 --- a/VERSION +++ b/VERSION @@ -1,4 +1,4 @@ COLUMNSTORE_VERSION_MAJOR=1 -COLUMNSTORE_VERSION_MINOR=3 +COLUMNSTORE_VERSION_MINOR=4 COLUMNSTORE_VERSION_PATCH=0 COLUMNSTORE_VERSION_RELEASE=1 diff --git a/bison.cmake b/bison.cmake deleted file mode 100644 index d5c725fbb..000000000 --- a/bison.cmake +++ /dev/null @@ -1,81 +0,0 @@ -# Copyright (c) 2009 Sun Microsystems, Inc. -# Use is subject to license terms. -# -# This program is free software; you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation; version 2 of the License. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program; if not, write to the Free Software -# Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA - -IF(CMAKE_SYSTEM_NAME MATCHES "SunOS") - # On Solaris, /opt/csw often contains a newer bison - IF(NOT BISON_EXECUTABLE AND EXISTS /opt/csw/bin/bison) - SET(BISON_EXECUTABLE /opt/csw/bin/bison) - ENDIF() -ENDIF() -FIND_PROGRAM(BISON_EXECUTABLE bison DOC "path to the bison executable") -MARK_AS_ADVANCED(BISON_EXECUTABLE "") -IF(NOT BISON_EXECUTABLE) - MESSAGE("Warning: Bison executable not found in PATH") -ELSEIF(BISON_EXECUTABLE AND NOT BISON_USABLE) - # Check version as well - EXEC_PROGRAM(${BISON_EXECUTABLE} ARGS --version OUTPUT_VARIABLE BISON_VERSION_STR) - # Get first line in case it's multiline - STRING(REGEX REPLACE "([^\n]+).*" "\\1" FIRST_LINE "${BISON_VERSION_STR}") - # get version information - STRING(REGEX REPLACE ".* ([0-9]+)\\.([0-9]+)" "\\1" BISON_VERSION_MAJOR "${FIRST_LINE}") - STRING(REGEX REPLACE ".* ([0-9]+)\\.([0-9]+)" "\\2" BISON_VERSION_MINOR "${FIRST_LINE}") - IF (BISON_VERSION_MAJOR LESS 2) - MESSAGE("Warning: bison version is old. please update to version 2") - ELSE() - SET(BISON_USABLE 1 CACHE INTERNAL "Bison version 2 or higher") - ENDIF() -ENDIF() - -# Use bison to generate C++ and header file -MACRO (RUN_BISON input_yy output_cc output_h) - IF(BISON_TOO_OLD) - IF(EXISTS ${output_cc} AND EXISTS ${output_h}) - SET(BISON_USABLE FALSE) - ENDIF() - ENDIF() - IF(BISON_USABLE) - ADD_CUSTOM_COMMAND( - OUTPUT ${output_cc} - ${output_h} - COMMAND ${BISON_EXECUTABLE} -y -p MYSQL - --output=${output_cc} - --defines=${output_h} - ${input_yy} - DEPENDS ${input_yy} - ) - ELSE() - # Bison is missing or not usable, e.g too old - IF(EXISTS ${output_cc} AND EXISTS ${output_h}) - IF(${input_yy} IS_NEWER_THAN ${output_cc} OR ${input_yy} IS_NEWER_THAN ${output_h}) - # Possibly timestamps are messed up in source distribution. - MESSAGE("Warning: no usable bison found, ${input_yy} will not be rebuilt.") - ENDIF() - ELSE() - # Output files are missing, bail out. - SET(ERRMSG - "Bison (GNU parser generator) is required to build MySQL." - "Please install bison." - ) - IF(WIN32) - SET(ERRMSG ${ERRMSG} - "You can download bison from http://gnuwin32.sourceforge.net/packages/bison.htm " - "Choose 'Complete package, except sources' installation. We recommend to " - "install bison into a directory without spaces, e.g C:\\GnuWin32.") - ENDIF() - MESSAGE(FATAL_ERROR ${ERRMSG}) - ENDIF() - ENDIF() -ENDMACRO() diff --git a/FindJeMalloc.cmake b/cmake/FindJeMalloc.cmake similarity index 100% rename from FindJeMalloc.cmake rename to cmake/FindJeMalloc.cmake diff --git a/FindSnappy.cmake b/cmake/FindSnappy.cmake similarity index 100% rename from FindSnappy.cmake rename to cmake/FindSnappy.cmake diff --git a/cmake/boost.CMakeLists.txt.in b/cmake/boost.CMakeLists.txt.in new file mode 100644 index 000000000..5df75c405 --- /dev/null +++ b/cmake/boost.CMakeLists.txt.in @@ -0,0 +1,33 @@ +cmake_minimum_required(VERSION @CMAKE_VERSION@) + +include(ExternalProject) + +if(CMAKE_CXX_COMPILER_ID MATCHES "GNU") + set(_toolset "gcc") +elseif(CMAKE_CXX_COMPILER_ID MATCHES ".*Clang") + set(_toolset "clang") +elseif(CMAKE_CXX_COMPILER_ID MATCHES "Intel") + set(_toolset "intel-linux") +endif() + +set(_b2args link=shared;threading=multi;variant=release;toolset=${_toolset};--with-system;--with-filesystem;--with-thread;--with-regex;--with-date_time) + +ExternalProject_Add(boost + PREFIX build + URL https://sourceforge.net/projects/boost/files/boost/1.55.0/boost_1_55_0.zip + URL_HASH SHA256=ae85620e810b87a03e1acf8bbf0d4ad87c0cf7040cf6a4e1d8958488ebe42e7e + DOWNLOAD_NO_PROGRESS TRUE + UPDATE_COMMAND "" + CONFIGURE_COMMAND /bootstrap.sh + --with-toolset=${_toolset} + --prefix=${CMAKE_CURRENT_SOURCE_DIR}/../boost + --with-libraries=system,filesystem,thread,regex,date_time + BUILD_COMMAND /b2 -q ${_b2args} + LOG_BUILD TRUE + BUILD_IN_SOURCE TRUE + INSTALL_COMMAND /b2 -q install ${_b2args} + LOG_INSTALL TRUE +) + +unset(_b2args) +unset(_toolset) \ No newline at end of file diff --git a/cmake/boost.cmake b/cmake/boost.cmake new file mode 100644 index 000000000..ae129287d --- /dev/null +++ b/cmake/boost.cmake @@ -0,0 +1,34 @@ + +# boost super build (see superbuild.md) + + +configure_file(${CMAKE_CURRENT_LIST_DIR}/boost.CMakeLists.txt.in ${CMAKE_CURRENT_BINARY_DIR}/.boost/CMakeLists.txt @ONLY) + +execute_process( + COMMAND ${CMAKE_COMMAND} . + -G "${CMAKE_GENERATOR}" + -DCMAKE_C_COMPILER=${CMAKE_C_COMPILER} + -DCMAKE_CXX_COMPILER=${CMAKE_CXX_COMPILER} + WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/.boost + RESULT_VARIABLE _exec_ret +) + +if(${_exec_ret}) + message(FATAL_ERROR "Error ${_exec_ret} configuring boost dependency.") +endif() + +execute_process( + COMMAND ${CMAKE_COMMAND} --build . + WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/.boost + RESULT_VARIABLE _exec_ret +) + +if(${_exec_ret}) + message(FATAL_ERROR "Error ${_exec_ret} building boost dependency: ${_exec_ret}") +endif() + +unset(_exec_ret) + +set(BOOST_ROOT ${CMAKE_CURRENT_BINARY_DIR}/boost) + +find_package(Boost 1.55.0 REQUIRED COMPONENTS system filesystem thread regex date_time) diff --git a/check_compiler_flag.cmake b/cmake/check_compiler_flag.cmake similarity index 100% rename from check_compiler_flag.cmake rename to cmake/check_compiler_flag.cmake diff --git a/columnstore_version.cmake b/cmake/columnstore_version.cmake similarity index 100% rename from columnstore_version.cmake rename to cmake/columnstore_version.cmake diff --git a/configureEngine.cmake b/cmake/configureEngine.cmake similarity index 100% rename from configureEngine.cmake rename to cmake/configureEngine.cmake diff --git a/cpackEngineDEB.cmake b/cmake/cpackEngineDEB.cmake similarity index 100% rename from cpackEngineDEB.cmake rename to cmake/cpackEngineDEB.cmake diff --git a/cpackEngineRPM.cmake b/cmake/cpackEngineRPM.cmake similarity index 100% rename from cpackEngineRPM.cmake rename to cmake/cpackEngineRPM.cmake diff --git a/cmake/superbuild.md b/cmake/superbuild.md new file mode 100644 index 000000000..6bedaeabc --- /dev/null +++ b/cmake/superbuild.md @@ -0,0 +1,24 @@ +CMake Super Build +================= +A super build is a process to download, build and install dependencies with cmake at configure time. This ensure dependencies are available during the initial configure stage. Its accomplished by executing separate cmake configure and build processes inline with the main project cmake which builds and installs the missing dependency. + +Rationale: +---------- +It maybe observed that ExternalProject accomplishes a similar task, however, the target of an ExternalProject is not available until after the build stage. Any scripting logic which requires the dependency during the configure stage will fail. The super build solves this by ensuring the dependency is built independent of the main projects configuration which uses it. + +Example: +-------- +# In the context of the main projects cmake scripts, subshells of cmake are executed to configure and build the dependency +configure_file(some_dependency.CMakeLists.txt.in some_dep_dir\CMakeLists.txt @ONLY) # drop a top-level CMakeLists.txt in a folder for the dependency +execute_process(COMMAND ${CMAKE_COMMAND} . WORKING_DIRECTORY some_dep_dir) # execute configure stage of dependency against newly created CMakeLists.txt from above step +execute_process(COMMAND ${CMAKE_COMMAND} --build . WORKING_DIRECTORY some_dep_dir) # install the dependency +find_package(some_dependency) # the dependency should be installed and can be 'found' or used as appropriate + +NOTES +----- + o The bulk of the work is performed in the generated/copied CMakeLists.txt to download (optional), build and install the dependency. It typically contains the full set of ExternalProject statements and error handling. + o CMake scripts executed in a sub-process with execute_process are independent and share no state whatsoever with the calling process. There are two ways to share state with the sub-shell + - Wrap appropriate @VARIABLE@ decorations in the CMakeLists.in template which get substituted with values when configure_file is executed + - Pass them on the command line of the execute_process statement. e.g.: execute_process(COMMAND ${CMAKE_COMMAND} -DSOME_VAR=${SOME_VAL} -DANOTHER_VAR=${ANOTHER_VAL} ... + +x \ No newline at end of file diff --git a/config.h.cmake b/config.h.cmake deleted file mode 100644 index 74d707b11..000000000 --- a/config.h.cmake +++ /dev/null @@ -1,390 +0,0 @@ -/* config.h.cmake */ -#ifndef TEST_CONFIG_H -#define TEST_CONFIG_H - -/* Define to 1 to let the system come up without using OAM */ -#cmakedefine SKIP_OAM_INIT 1 - -/* Define to 1 if you have the `alarm' function. */ -#cmakedefine HAVE_ALARM 1 - -/* Define to 1 if you have `alloca', as a function or macro. */ -#cmakedefine HAVE_ALLOCA 1 - -/* Define to 1 if you have and it should be used (not on Ultrix). - */ -#cmakedefine HAVE_ALLOCA_H 1 - -/* Define to 1 if you have the header file. */ -#cmakedefine HAVE_ARPA_INET_H 1 - -/* Define to 1 if you have the `btowc' function. */ -#cmakedefine HAVE_BTOWC 1 - -/* Define to 1 if you have the declaration of `getenv', and to 0 if you don't. - */ -#cmakedefine HAVE_DECL_GETENV 1 - -/* Define to 1 if you have the declaration of `strerror_r', and to 0 if you - don't. */ -#cmakedefine HAVE_DECL_STRERROR_R 1 - -/* Define to 1 if you have the header file. */ -#cmakedefine HAVE_DLFCN_H 1 - -/* Define to 1 if you have the `dup2' function. */ -#cmakedefine HAVE_DUP2 1 - -/* Define to 1 if you have the header file. */ -#cmakedefine HAVE_FCNTL_H 1 - -/* Define to 1 if you have the `floor' function. */ -#cmakedefine HAVE_FLOOR 1 - -/* Define to 1 if you have the `fork' function. */ -#cmakedefine HAVE_FORK 1 - -/* Define to 1 if you have the `ftime' function. */ -#cmakedefine HAVE_FTIME 1 - -/* Define to 1 if you have the `ftruncate' function. */ -#cmakedefine HAVE_FTRUNCATE 1 - -/* Define to 1 if you have the `gethostbyname' function. */ -#cmakedefine HAVE_GETHOSTBYNAME 1 - -/* Define to 1 if you have the `getpagesize' function. */ -#cmakedefine HAVE_GETPAGESIZE 1 - -/* Define to 1 if you have the `gettimeofday' function. */ -#cmakedefine HAVE_GETTIMEOFDAY 1 - -/* Define to 1 if you have the `inet_ntoa' function. */ -#cmakedefine HAVE_INET_NTOA 1 - -/* Define to 1 if you have the header file. */ -#cmakedefine HAVE_INTTYPES_H 1 - -/* Define to 1 if you have the `isascii' function. */ -#cmakedefine HAVE_ISASCII 1 - -/* Define to 1 if you have the header file. */ -#cmakedefine HAVE_LIMITS_H 1 - -/* Define to 1 if you have the `localtime_r' function. */ -#cmakedefine HAVE_LOCALTIME_R 1 - -/* Define to 1 if your system has a GNU libc compatible `malloc' function, and - to 0 otherwise. */ -#cmakedefine HAVE_MALLOC 1 - -/* Define to 1 if you have the header file. */ -#cmakedefine HAVE_MALLOC_H 1 - -/* Define to 1 if you have the `mbsrtowcs' function. */ -#cmakedefine HAVE_MBSRTOWCS 1 - -/* Define to 1 if declares mbstate_t. */ -#cmakedefine HAVE_MBSTATE_T 1 - -/* Define to 1 if you have the `memchr' function. */ -#cmakedefine HAVE_MEMCHR 1 - -/* Define to 1 if you have the `memmove' function. */ -#cmakedefine HAVE_MEMMOVE 1 - -/* Define to 1 if you have the header file. */ -#cmakedefine HAVE_MEMORY_H 1 - -/* Define to 1 if you have the `mempcpy' function. */ -#cmakedefine HAVE_MEMPCPY 1 - -/* Define to 1 if you have the `memset' function. */ -#cmakedefine HAVE_MEMSET 1 - -/* Define to 1 if you have the `mkdir' function. */ -#cmakedefine HAVE_MKDIR 1 - -/* Define to 1 if you have the header file. */ -#cmakedefine HAVE_NCURSES_H 1 - -/* Define to 1 if you have the header file. */ -#cmakedefine HAVE_NETDB_H 1 - -/* Define to 1 if you have the header file. */ -#cmakedefine HAVE_NETINET_IN_H 1 - -/* Define to 1 if you have the `pow' function. */ -#cmakedefine HAVE_POW 1 - -/* Define to 1 if the system has the type `ptrdiff_t'. */ -#cmakedefine HAVE_PTRDIFF_T 1 - -/* Define to 1 if you have the header file. */ -#cmakedefine HAVE_READLINE_READLINE_H 1 - -/* Define to 1 if you have the `regcomp' function. */ -#cmakedefine HAVE_REGCOMP 1 - -/* Define to 1 if you have the `rmdir' function. */ -#cmakedefine HAVE_RMDIR 1 - -/* Define to 1 if you have the `select' function. */ -#cmakedefine HAVE_SELECT 1 - -/* Define to 1 if you have the `setenv' function. */ -#cmakedefine HAVE_SETENV 1 - -/* Define to 1 if you have the `setlocale' function. */ -#cmakedefine HAVE_SETLOCALE 1 - -/* Define to 1 if you have the `socket' function. */ -#cmakedefine HAVE_SOCKET 1 - -/* Define to 1 if `stat' has the bug that it succeeds when given the - zero-length file name argument. */ -#cmakedefine HAVE_STAT_EMPTY_STRING_BUG 1 - -/* Define to 1 if stdbool.h conforms to C99. */ -#cmakedefine HAVE_STDBOOL_H 1 - -/* Define to 1 if you have the header file. */ -#cmakedefine HAVE_STDDEF_H 1 - -/* Define to 1 if you have the header file. */ -#cmakedefine HAVE_STDINT_H 1 - -/* Define to 1 if you have the header file. */ -#cmakedefine HAVE_STDLIB_H 1 - -/* Define to 1 if you have the `strcasecmp' function. */ -#cmakedefine HAVE_STRCASECMP 1 - -/* Define to 1 if you have the `strchr' function. */ -#cmakedefine HAVE_STRCHR 1 - -/* Define to 1 if you have the `strcspn' function. */ -#cmakedefine HAVE_STRCSPN 1 - -/* Define to 1 if you have the `strdup' function. */ -#cmakedefine HAVE_STRDUP 1 - -/* Define to 1 if you have the `strerror' function. */ -#cmakedefine HAVE_STRERROR 1 - -/* Define to 1 if you have the `strerror_r' function. */ -#cmakedefine HAVE_STRERROR_R 1 - -/* Define to 1 if you have the `strftime' function. */ -#cmakedefine HAVE_STRFTIME 1 - -/* Define to 1 if you have the header file. */ -#cmakedefine HAVE_STRINGS_H 1 - -/* Define to 1 if you have the header file. */ -#cmakedefine HAVE_STRING_H 1 - -/* Define to 1 if you have the `strrchr' function. */ -#cmakedefine HAVE_STRRCHR 1 - -/* Define to 1 if you have the `strspn' function. */ -#cmakedefine HAVE_STRSPN 1 - -/* Define to 1 if you have the `strstr' function. */ -#cmakedefine HAVE_STRSTR 1 - -/* Define to 1 if you have the `strtol' function. */ -#cmakedefine HAVE_STRTOL 1 - -/* Define to 1 if you have the `strtoul' function. */ -#cmakedefine HAVE_STRTOUL 1 - -/* Define to 1 if you have the `strtoull' function. */ -#cmakedefine HAVE_STRTOULL 1 - -/* Define to 1 if you have the header file. */ -#cmakedefine HAVE_SYSLOG_H 1 - -/* Define to 1 if you have the header file. */ -#cmakedefine HAVE_SYS_FILE_H 1 - -/* Define to 1 if you have the header file. */ -#cmakedefine HAVE_SYS_MOUNT_H 1 - -/* Define to 1 if you have the header file. */ -#cmakedefine HAVE_SYS_SELECT_H 1 - -/* Define to 1 if you have the header file. */ -#cmakedefine HAVE_SYS_SOCKET_H 1 - -/* Define to 1 if you have the header file. */ -#cmakedefine HAVE_SYS_STATFS_H 1 - -/* Define to 1 if you have the header file. */ -#cmakedefine HAVE_SYS_STAT_H 1 - -/* Define to 1 if you have the header file. */ -#cmakedefine HAVE_SYS_TIMEB_H 1 - -/* Define to 1 if you have the header file. */ -#cmakedefine HAVE_SYS_TIME_H 1 - -/* Define to 1 if you have the header file. */ -#cmakedefine HAVE_SYS_TYPES_H 1 - -/* Define to 1 if you have that is POSIX.1 compatible. */ -#cmakedefine HAVE_SYS_WAIT_H 1 - -/* Define to 1 if you have the header file. */ -#cmakedefine HAVE_UNISTD_H 1 - -/* Define to 1 if you have the `utime' function. */ -#cmakedefine HAVE_UTIME 1 - -/* Define to 1 if you have the header file. */ -#cmakedefine HAVE_UTIME_H 1 - -/* Define to 1 if `utime(file, NULL)' sets file's timestamp to the present. */ -#cmakedefine HAVE_UTIME_NULL 1 - -/* Define to 1 if you have the header file. */ -#cmakedefine HAVE_VALUES_H 1 - -/* Define to 1 if you have the `vfork' function. */ -#cmakedefine HAVE_VFORK 1 - -/* Define to 1 if you have the header file. */ -#cmakedefine HAVE_VFORK_H 1 - -/* Define to 1 if you have the header file. */ -#cmakedefine HAVE_WCHAR_H 1 - -/* Define to 1 if you have the header file. */ -#cmakedefine HAVE_WCTYPE_H 1 - -/* Define to 1 if you have the `wmempcpy' function. */ -#cmakedefine HAVE_WMEMPCPY 1 - -/* Define to 1 if `fork' works. */ -#cmakedefine HAVE_WORKING_FORK 1 - -/* Define to 1 if `vfork' works. */ -#cmakedefine HAVE_WORKING_VFORK 1 - -/* Define to 1 if you have the header file. */ -#cmakedefine HAVE_ZLIB_H 1 - -/* Define to 1 if the system has the type `_Bool'. */ -#cmakedefine HAVE__BOOL 1 - -/* Define to 1 if `lstat' dereferences a symlink specified with a trailing - slash. */ -#cmakedefine LSTAT_FOLLOWS_SLASHED_SYMLINK 1 - -/* Name of package */ -#cmakedefine PACKAGE "${PACKAGE}" - -/* Define to the address where bug reports for this package should be sent. */ -#cmakedefine PACKAGE_BUGREPORT "${PACKAGE_BUGREPORT}" - -/* Define to the full name of this package. */ -#cmakedefine PACKAGE_NAME "${PACKAGE_NAME}" - -/* Define to the full name and version of this package. */ -#cmakedefine PACKAGE_STRING "${PACKAGE_STRING}" - -/* Define to the one symbol short name of this package. */ -#cmakedefine PACKAGE_TARNAME "${PACKAGE_TARNAME}" - -/* Define to the home page for this package. */ -#cmakedefine PACKAGE_URL "${PACKAGE_URL}" - -/* Define to the version of this package. */ -#cmakedefine PACKAGE_VERSION "${PACKAGE_VERSION}" - -/* Define as the return type of signal handlers (`int' or `void'). */ -#cmakedefine RETSIGTYPE ${RETSIGTYPE} - -/* Define to the type of arg 1 for `select'. */ -#cmakedefine SELECT_TYPE_ARG1 ${SELECT_TYPE_ARG1} - -/* Define to the type of args 2, 3 and 4 for `select'. */ -#cmakedefine SELECT_TYPE_ARG234 (${SELECT_TYPE_ARG234}) - -/* Define to the type of arg 5 for `select'. */ -#cmakedefine SELECT_TYPE_ARG5 (${SELECT_TYPE_ARG5}) - -/* Define to 1 if the `S_IS*' macros in do not work properly. */ -#cmakedefine STAT_MACROS_BROKEN 1 - -/* Define to 1 if you have the ANSI C header files. */ -#cmakedefine STDC_HEADERS 1 - -/* Define to 1 if strerror_r returns char *. */ -#cmakedefine STRERROR_R_CHAR_P 1 - -/* Define to 1 if you can safely include both and . */ -#cmakedefine TIME_WITH_SYS_TIME 1 - -/* Define to 1 if your declares `struct tm'. */ -#cmakedefine TM_IN_SYS_TIME 1 - -/* Version number of package */ -#cmakedefine VERSION "${VERSION}" - -/* Define to 1 if `lex' declares `yytext' as a `char *' by default, not a - `char[]'. */ -#cmakedefine YYTEXT_POINTER 1 - -/* Define to empty if `const' does not conform to ANSI C. */ -#cmakedefine const - -/* Define to rpl_fnmatch if the replacement function should be used. */ -#cmakedefine fnmatch - -/* Define to `__inline__' or `__inline' if that's what the C compiler - calls it, or to nothing if 'inline' is not supported under any name. */ -#ifndef __cplusplus -#cmakedefine inline ${inline} -#endif - -/* Define to rpl_malloc if the replacement function should be used. */ -#cmakedefine malloc - -/* Define to a type if does not define. */ -#cmakedefine mbstate_t - -/* Define to `int' if does not define. */ -#cmakedefine mode_t ${mode_t} - -/* Define to `long int' if does not define. */ -#cmakedefine off_t ${off_t} - -/* Define to `int' if does not define. */ -#cmakedefine pid_t ${pid_t} - -/* Define to the equivalent of the C99 'restrict' keyword, or to - nothing if this is not supported. Do not define if restrict is - supported directly. */ -#cmakedefine restrict ${restrict} -/* Work around a bug in Sun C++: it does not support _Restrict or - __restrict__, even though the corresponding Sun C compiler ends up with - "#define restrict _Restrict" or "#define restrict __restrict__" in the - previous line. Perhaps some future version of Sun C++ will work with - restrict; if so, hopefully it defines __RESTRICT like Sun C does. */ -#if defined __SUNPRO_CC && !defined __RESTRICT -# define _Restrict -# define __restrict__ -#endif - -/* Define to `unsigned int' if does not define. */ -#cmakedefine size_t ${size_t} - -/* Define as `fork' if `vfork' does not work. */ -#cmakedefine vfork ${VFORK} - -/* Define to empty if the keyword `volatile' does not work. Warning: valid - code using `volatile' can become incorrect without. Disable with care. */ -#cmakedefine volatile - -#endif diff --git a/config.h.in b/config.h.in index 6a451addb..74d707b11 100644 --- a/config.h.in +++ b/config.h.in @@ -1,386 +1,372 @@ -/* config.h.in. Generated from configure.ac by autoheader. */ +/* config.h.cmake */ +#ifndef TEST_CONFIG_H +#define TEST_CONFIG_H -/* Define to one of `_getb67', `GETB67', `getb67' for Cray-2 and Cray-YMP - systems. This function is required for `alloca.c' support on those systems. - */ -#undef CRAY_STACKSEG_END - -/* Define to 1 if using `alloca.c'. */ -#undef C_ALLOCA +/* Define to 1 to let the system come up without using OAM */ +#cmakedefine SKIP_OAM_INIT 1 /* Define to 1 if you have the `alarm' function. */ -#undef HAVE_ALARM +#cmakedefine HAVE_ALARM 1 /* Define to 1 if you have `alloca', as a function or macro. */ -#undef HAVE_ALLOCA +#cmakedefine HAVE_ALLOCA 1 /* Define to 1 if you have and it should be used (not on Ultrix). */ -#undef HAVE_ALLOCA_H +#cmakedefine HAVE_ALLOCA_H 1 /* Define to 1 if you have the header file. */ -#undef HAVE_ARPA_INET_H +#cmakedefine HAVE_ARPA_INET_H 1 /* Define to 1 if you have the `btowc' function. */ -#undef HAVE_BTOWC +#cmakedefine HAVE_BTOWC 1 /* Define to 1 if you have the declaration of `getenv', and to 0 if you don't. */ -#undef HAVE_DECL_GETENV +#cmakedefine HAVE_DECL_GETENV 1 /* Define to 1 if you have the declaration of `strerror_r', and to 0 if you don't. */ -#undef HAVE_DECL_STRERROR_R +#cmakedefine HAVE_DECL_STRERROR_R 1 /* Define to 1 if you have the header file. */ -#undef HAVE_DLFCN_H +#cmakedefine HAVE_DLFCN_H 1 /* Define to 1 if you have the `dup2' function. */ -#undef HAVE_DUP2 +#cmakedefine HAVE_DUP2 1 /* Define to 1 if you have the header file. */ -#undef HAVE_FCNTL_H +#cmakedefine HAVE_FCNTL_H 1 /* Define to 1 if you have the `floor' function. */ -#undef HAVE_FLOOR +#cmakedefine HAVE_FLOOR 1 /* Define to 1 if you have the `fork' function. */ -#undef HAVE_FORK +#cmakedefine HAVE_FORK 1 /* Define to 1 if you have the `ftime' function. */ -#undef HAVE_FTIME +#cmakedefine HAVE_FTIME 1 /* Define to 1 if you have the `ftruncate' function. */ -#undef HAVE_FTRUNCATE +#cmakedefine HAVE_FTRUNCATE 1 /* Define to 1 if you have the `gethostbyname' function. */ -#undef HAVE_GETHOSTBYNAME +#cmakedefine HAVE_GETHOSTBYNAME 1 /* Define to 1 if you have the `getpagesize' function. */ -#undef HAVE_GETPAGESIZE +#cmakedefine HAVE_GETPAGESIZE 1 /* Define to 1 if you have the `gettimeofday' function. */ -#undef HAVE_GETTIMEOFDAY +#cmakedefine HAVE_GETTIMEOFDAY 1 /* Define to 1 if you have the `inet_ntoa' function. */ -#undef HAVE_INET_NTOA +#cmakedefine HAVE_INET_NTOA 1 /* Define to 1 if you have the header file. */ -#undef HAVE_INTTYPES_H +#cmakedefine HAVE_INTTYPES_H 1 /* Define to 1 if you have the `isascii' function. */ -#undef HAVE_ISASCII +#cmakedefine HAVE_ISASCII 1 /* Define to 1 if you have the header file. */ -#undef HAVE_LIMITS_H +#cmakedefine HAVE_LIMITS_H 1 /* Define to 1 if you have the `localtime_r' function. */ -#undef HAVE_LOCALTIME_R +#cmakedefine HAVE_LOCALTIME_R 1 /* Define to 1 if your system has a GNU libc compatible `malloc' function, and to 0 otherwise. */ -#undef HAVE_MALLOC +#cmakedefine HAVE_MALLOC 1 /* Define to 1 if you have the header file. */ -#undef HAVE_MALLOC_H +#cmakedefine HAVE_MALLOC_H 1 /* Define to 1 if you have the `mbsrtowcs' function. */ -#undef HAVE_MBSRTOWCS +#cmakedefine HAVE_MBSRTOWCS 1 /* Define to 1 if declares mbstate_t. */ -#undef HAVE_MBSTATE_T +#cmakedefine HAVE_MBSTATE_T 1 /* Define to 1 if you have the `memchr' function. */ -#undef HAVE_MEMCHR +#cmakedefine HAVE_MEMCHR 1 /* Define to 1 if you have the `memmove' function. */ -#undef HAVE_MEMMOVE +#cmakedefine HAVE_MEMMOVE 1 /* Define to 1 if you have the header file. */ -#undef HAVE_MEMORY_H +#cmakedefine HAVE_MEMORY_H 1 /* Define to 1 if you have the `mempcpy' function. */ -#undef HAVE_MEMPCPY +#cmakedefine HAVE_MEMPCPY 1 /* Define to 1 if you have the `memset' function. */ -#undef HAVE_MEMSET +#cmakedefine HAVE_MEMSET 1 /* Define to 1 if you have the `mkdir' function. */ -#undef HAVE_MKDIR +#cmakedefine HAVE_MKDIR 1 /* Define to 1 if you have the header file. */ -#undef HAVE_NCURSES_H +#cmakedefine HAVE_NCURSES_H 1 /* Define to 1 if you have the header file. */ -#undef HAVE_NETDB_H +#cmakedefine HAVE_NETDB_H 1 /* Define to 1 if you have the header file. */ -#undef HAVE_NETINET_IN_H +#cmakedefine HAVE_NETINET_IN_H 1 /* Define to 1 if you have the `pow' function. */ -#undef HAVE_POW +#cmakedefine HAVE_POW 1 /* Define to 1 if the system has the type `ptrdiff_t'. */ -#undef HAVE_PTRDIFF_T +#cmakedefine HAVE_PTRDIFF_T 1 /* Define to 1 if you have the header file. */ -#undef HAVE_READLINE_READLINE_H +#cmakedefine HAVE_READLINE_READLINE_H 1 /* Define to 1 if you have the `regcomp' function. */ -#undef HAVE_REGCOMP +#cmakedefine HAVE_REGCOMP 1 /* Define to 1 if you have the `rmdir' function. */ -#undef HAVE_RMDIR +#cmakedefine HAVE_RMDIR 1 /* Define to 1 if you have the `select' function. */ -#undef HAVE_SELECT +#cmakedefine HAVE_SELECT 1 /* Define to 1 if you have the `setenv' function. */ -#undef HAVE_SETENV +#cmakedefine HAVE_SETENV 1 /* Define to 1 if you have the `setlocale' function. */ -#undef HAVE_SETLOCALE +#cmakedefine HAVE_SETLOCALE 1 /* Define to 1 if you have the `socket' function. */ -#undef HAVE_SOCKET +#cmakedefine HAVE_SOCKET 1 /* Define to 1 if `stat' has the bug that it succeeds when given the zero-length file name argument. */ -#undef HAVE_STAT_EMPTY_STRING_BUG +#cmakedefine HAVE_STAT_EMPTY_STRING_BUG 1 /* Define to 1 if stdbool.h conforms to C99. */ -#undef HAVE_STDBOOL_H +#cmakedefine HAVE_STDBOOL_H 1 /* Define to 1 if you have the header file. */ -#undef HAVE_STDDEF_H +#cmakedefine HAVE_STDDEF_H 1 /* Define to 1 if you have the header file. */ -#undef HAVE_STDINT_H +#cmakedefine HAVE_STDINT_H 1 /* Define to 1 if you have the header file. */ -#undef HAVE_STDLIB_H +#cmakedefine HAVE_STDLIB_H 1 /* Define to 1 if you have the `strcasecmp' function. */ -#undef HAVE_STRCASECMP +#cmakedefine HAVE_STRCASECMP 1 /* Define to 1 if you have the `strchr' function. */ -#undef HAVE_STRCHR +#cmakedefine HAVE_STRCHR 1 /* Define to 1 if you have the `strcspn' function. */ -#undef HAVE_STRCSPN +#cmakedefine HAVE_STRCSPN 1 /* Define to 1 if you have the `strdup' function. */ -#undef HAVE_STRDUP +#cmakedefine HAVE_STRDUP 1 /* Define to 1 if you have the `strerror' function. */ -#undef HAVE_STRERROR +#cmakedefine HAVE_STRERROR 1 /* Define to 1 if you have the `strerror_r' function. */ -#undef HAVE_STRERROR_R +#cmakedefine HAVE_STRERROR_R 1 /* Define to 1 if you have the `strftime' function. */ -#undef HAVE_STRFTIME +#cmakedefine HAVE_STRFTIME 1 /* Define to 1 if you have the header file. */ -#undef HAVE_STRINGS_H +#cmakedefine HAVE_STRINGS_H 1 /* Define to 1 if you have the header file. */ -#undef HAVE_STRING_H +#cmakedefine HAVE_STRING_H 1 /* Define to 1 if you have the `strrchr' function. */ -#undef HAVE_STRRCHR +#cmakedefine HAVE_STRRCHR 1 /* Define to 1 if you have the `strspn' function. */ -#undef HAVE_STRSPN +#cmakedefine HAVE_STRSPN 1 /* Define to 1 if you have the `strstr' function. */ -#undef HAVE_STRSTR +#cmakedefine HAVE_STRSTR 1 /* Define to 1 if you have the `strtol' function. */ -#undef HAVE_STRTOL +#cmakedefine HAVE_STRTOL 1 /* Define to 1 if you have the `strtoul' function. */ -#undef HAVE_STRTOUL +#cmakedefine HAVE_STRTOUL 1 /* Define to 1 if you have the `strtoull' function. */ -#undef HAVE_STRTOULL +#cmakedefine HAVE_STRTOULL 1 /* Define to 1 if you have the header file. */ -#undef HAVE_SYSLOG_H +#cmakedefine HAVE_SYSLOG_H 1 /* Define to 1 if you have the header file. */ -#undef HAVE_SYS_FILE_H +#cmakedefine HAVE_SYS_FILE_H 1 /* Define to 1 if you have the header file. */ -#undef HAVE_SYS_MOUNT_H +#cmakedefine HAVE_SYS_MOUNT_H 1 /* Define to 1 if you have the header file. */ -#undef HAVE_SYS_SELECT_H +#cmakedefine HAVE_SYS_SELECT_H 1 /* Define to 1 if you have the header file. */ -#undef HAVE_SYS_SOCKET_H +#cmakedefine HAVE_SYS_SOCKET_H 1 /* Define to 1 if you have the header file. */ -#undef HAVE_SYS_STATFS_H +#cmakedefine HAVE_SYS_STATFS_H 1 /* Define to 1 if you have the header file. */ -#undef HAVE_SYS_STAT_H +#cmakedefine HAVE_SYS_STAT_H 1 /* Define to 1 if you have the header file. */ -#undef HAVE_SYS_TIMEB_H +#cmakedefine HAVE_SYS_TIMEB_H 1 /* Define to 1 if you have the header file. */ -#undef HAVE_SYS_TIME_H +#cmakedefine HAVE_SYS_TIME_H 1 /* Define to 1 if you have the header file. */ -#undef HAVE_SYS_TYPES_H +#cmakedefine HAVE_SYS_TYPES_H 1 /* Define to 1 if you have that is POSIX.1 compatible. */ -#undef HAVE_SYS_WAIT_H +#cmakedefine HAVE_SYS_WAIT_H 1 /* Define to 1 if you have the header file. */ -#undef HAVE_UNISTD_H +#cmakedefine HAVE_UNISTD_H 1 /* Define to 1 if you have the `utime' function. */ -#undef HAVE_UTIME +#cmakedefine HAVE_UTIME 1 /* Define to 1 if you have the header file. */ -#undef HAVE_UTIME_H +#cmakedefine HAVE_UTIME_H 1 /* Define to 1 if `utime(file, NULL)' sets file's timestamp to the present. */ -#undef HAVE_UTIME_NULL +#cmakedefine HAVE_UTIME_NULL 1 /* Define to 1 if you have the header file. */ -#undef HAVE_VALUES_H +#cmakedefine HAVE_VALUES_H 1 /* Define to 1 if you have the `vfork' function. */ -#undef HAVE_VFORK +#cmakedefine HAVE_VFORK 1 /* Define to 1 if you have the header file. */ -#undef HAVE_VFORK_H +#cmakedefine HAVE_VFORK_H 1 /* Define to 1 if you have the header file. */ -#undef HAVE_WCHAR_H +#cmakedefine HAVE_WCHAR_H 1 /* Define to 1 if you have the header file. */ -#undef HAVE_WCTYPE_H +#cmakedefine HAVE_WCTYPE_H 1 /* Define to 1 if you have the `wmempcpy' function. */ -#undef HAVE_WMEMPCPY +#cmakedefine HAVE_WMEMPCPY 1 /* Define to 1 if `fork' works. */ -#undef HAVE_WORKING_FORK +#cmakedefine HAVE_WORKING_FORK 1 /* Define to 1 if `vfork' works. */ -#undef HAVE_WORKING_VFORK +#cmakedefine HAVE_WORKING_VFORK 1 /* Define to 1 if you have the header file. */ -#undef HAVE_ZLIB_H +#cmakedefine HAVE_ZLIB_H 1 /* Define to 1 if the system has the type `_Bool'. */ -#undef HAVE__BOOL +#cmakedefine HAVE__BOOL 1 /* Define to 1 if `lstat' dereferences a symlink specified with a trailing slash. */ -#undef LSTAT_FOLLOWS_SLASHED_SYMLINK - -/* Define to the sub-directory where libtool stores uninstalled libraries. */ -#undef LT_OBJDIR +#cmakedefine LSTAT_FOLLOWS_SLASHED_SYMLINK 1 /* Name of package */ -#undef PACKAGE +#cmakedefine PACKAGE "${PACKAGE}" /* Define to the address where bug reports for this package should be sent. */ -#undef PACKAGE_BUGREPORT +#cmakedefine PACKAGE_BUGREPORT "${PACKAGE_BUGREPORT}" /* Define to the full name of this package. */ -#undef PACKAGE_NAME +#cmakedefine PACKAGE_NAME "${PACKAGE_NAME}" /* Define to the full name and version of this package. */ -#undef PACKAGE_STRING +#cmakedefine PACKAGE_STRING "${PACKAGE_STRING}" /* Define to the one symbol short name of this package. */ -#undef PACKAGE_TARNAME +#cmakedefine PACKAGE_TARNAME "${PACKAGE_TARNAME}" /* Define to the home page for this package. */ -#undef PACKAGE_URL +#cmakedefine PACKAGE_URL "${PACKAGE_URL}" /* Define to the version of this package. */ -#undef PACKAGE_VERSION +#cmakedefine PACKAGE_VERSION "${PACKAGE_VERSION}" /* Define as the return type of signal handlers (`int' or `void'). */ -#undef RETSIGTYPE +#cmakedefine RETSIGTYPE ${RETSIGTYPE} /* Define to the type of arg 1 for `select'. */ -#undef SELECT_TYPE_ARG1 +#cmakedefine SELECT_TYPE_ARG1 ${SELECT_TYPE_ARG1} /* Define to the type of args 2, 3 and 4 for `select'. */ -#undef SELECT_TYPE_ARG234 +#cmakedefine SELECT_TYPE_ARG234 (${SELECT_TYPE_ARG234}) /* Define to the type of arg 5 for `select'. */ -#undef SELECT_TYPE_ARG5 - -/* If using the C implementation of alloca, define if you know the - direction of stack growth for your system; otherwise it will be - automatically deduced at runtime. - STACK_DIRECTION > 0 => grows toward higher addresses - STACK_DIRECTION < 0 => grows toward lower addresses - STACK_DIRECTION = 0 => direction of growth unknown */ -#undef STACK_DIRECTION +#cmakedefine SELECT_TYPE_ARG5 (${SELECT_TYPE_ARG5}) /* Define to 1 if the `S_IS*' macros in do not work properly. */ -#undef STAT_MACROS_BROKEN +#cmakedefine STAT_MACROS_BROKEN 1 /* Define to 1 if you have the ANSI C header files. */ -#undef STDC_HEADERS +#cmakedefine STDC_HEADERS 1 /* Define to 1 if strerror_r returns char *. */ -#undef STRERROR_R_CHAR_P +#cmakedefine STRERROR_R_CHAR_P 1 /* Define to 1 if you can safely include both and . */ -#undef TIME_WITH_SYS_TIME +#cmakedefine TIME_WITH_SYS_TIME 1 /* Define to 1 if your declares `struct tm'. */ -#undef TM_IN_SYS_TIME +#cmakedefine TM_IN_SYS_TIME 1 /* Version number of package */ -#undef VERSION +#cmakedefine VERSION "${VERSION}" /* Define to 1 if `lex' declares `yytext' as a `char *' by default, not a `char[]'. */ -#undef YYTEXT_POINTER +#cmakedefine YYTEXT_POINTER 1 /* Define to empty if `const' does not conform to ANSI C. */ -#undef const +#cmakedefine const /* Define to rpl_fnmatch if the replacement function should be used. */ -#undef fnmatch +#cmakedefine fnmatch /* Define to `__inline__' or `__inline' if that's what the C compiler calls it, or to nothing if 'inline' is not supported under any name. */ #ifndef __cplusplus -#undef inline +#cmakedefine inline ${inline} #endif /* Define to rpl_malloc if the replacement function should be used. */ -#undef malloc +#cmakedefine malloc /* Define to a type if does not define. */ -#undef mbstate_t +#cmakedefine mbstate_t /* Define to `int' if does not define. */ -#undef mode_t +#cmakedefine mode_t ${mode_t} /* Define to `long int' if does not define. */ -#undef off_t +#cmakedefine off_t ${off_t} /* Define to `int' if does not define. */ -#undef pid_t +#cmakedefine pid_t ${pid_t} /* Define to the equivalent of the C99 'restrict' keyword, or to nothing if this is not supported. Do not define if restrict is supported directly. */ -#undef restrict +#cmakedefine restrict ${restrict} /* Work around a bug in Sun C++: it does not support _Restrict or __restrict__, even though the corresponding Sun C compiler ends up with "#define restrict _Restrict" or "#define restrict __restrict__" in the @@ -392,11 +378,13 @@ #endif /* Define to `unsigned int' if does not define. */ -#undef size_t +#cmakedefine size_t ${size_t} /* Define as `fork' if `vfork' does not work. */ -#undef vfork +#cmakedefine vfork ${VFORK} /* Define to empty if the keyword `volatile' does not work. Warning: valid code using `volatile' can become incorrect without. Disable with care. */ -#undef volatile +#cmakedefine volatile + +#endif diff --git a/dbcon/ddlpackage/CMakeLists.txt b/dbcon/ddlpackage/CMakeLists.txt index 30bc97124..3b3e231da 100644 --- a/dbcon/ddlpackage/CMakeLists.txt +++ b/dbcon/ddlpackage/CMakeLists.txt @@ -11,7 +11,6 @@ ADD_CUSTOM_COMMAND( ADD_CUSTOM_TARGET(ddl-lexer DEPENDS ${CMAKE_CURRENT_SOURCE_DIR}/ddl-scan.cpp) ADD_CUSTOM_TARGET(ddl-parser DEPENDS ${CMAKE_CURRENT_SOURCE_DIR}/ddl-gram.cpp) # Parser puts extra info to stderr. -INCLUDE(../../check_compiler_flag.cmake) MY_CHECK_AND_SET_COMPILER_FLAG("-DYYDEBUG=1" DEBUG) ########### next target ############### diff --git a/dbcon/ddlpackageproc/altertableprocessor.cpp b/dbcon/ddlpackageproc/altertableprocessor.cpp index cc14424e2..989dffce2 100644 --- a/dbcon/ddlpackageproc/altertableprocessor.cpp +++ b/dbcon/ddlpackageproc/altertableprocessor.cpp @@ -2069,7 +2069,8 @@ void AlterTableProcessor::tableComment(uint32_t sessionID, execplan::CalpontSyst } else { - throw std::runtime_error("Invalid table comment"); + // Generic table comment, we don't need to do anything + return; } // Get the OID for autoinc (if exists) diff --git a/dbcon/execplan/predicateoperator.h b/dbcon/execplan/predicateoperator.h index b83931394..08f0c40cf 100644 --- a/dbcon/execplan/predicateoperator.h +++ b/dbcon/execplan/predicateoperator.h @@ -36,6 +36,7 @@ #endif #include #include +#include #include "expressionparser.h" #include "returnedcolumn.h" @@ -456,16 +457,20 @@ inline bool PredicateOperator::getBoolVal(rowgroup::Row& row, bool& isNull, Retu return !ret; } + // MCOL-1559 + std::string val1 = lop->getStrVal(row, isNull); if (isNull) return false; - const std::string& val1 = lop->getStrVal(row, isNull); - + std::string val2 = rop->getStrVal(row, isNull); if (isNull) return false; - return strCompare(val1, rop->getStrVal(row, isNull)) && !isNull; - } + boost::trim_right_if(val1, boost::is_any_of(" ")); + boost::trim_right_if(val2, boost::is_any_of(" ")); + + return strCompare(val1, val2); + } //FIXME: ??? case execplan::CalpontSystemCatalog::VARBINARY: diff --git a/dbcon/execplan/treenode.h b/dbcon/execplan/treenode.h index 91ceb2a11..106b40d37 100644 --- a/dbcon/execplan/treenode.h +++ b/dbcon/execplan/treenode.h @@ -608,7 +608,7 @@ inline const std::string& TreeNode::getStrVal() int exponent = (int)floor(log10( fabs(fResult.floatVal))); // This will round down the exponent double base = fResult.floatVal * pow(10, -1.0 * exponent); - if (isnan(exponent) || isnan(base)) + if (isnan(exponent) || std::isnan(base)) { snprintf(tmp, 312, "%f", fResult.floatVal); fResult.strVal = removeTrailing0(tmp, 312); @@ -643,7 +643,7 @@ inline const std::string& TreeNode::getStrVal() int exponent = (int)floor(log10( fabs(fResult.doubleVal))); // This will round down the exponent double base = fResult.doubleVal * pow(10, -1.0 * exponent); - if (isnan(exponent) || isnan(base)) + if (isnan(exponent) || std::isnan(base)) { snprintf(tmp, 312, "%f", fResult.doubleVal); fResult.strVal = removeTrailing0(tmp, 312); diff --git a/dbcon/joblist/jlf_execplantojoblist.cpp b/dbcon/joblist/jlf_execplantojoblist.cpp index f3782c9d5..fff1b12fb 100644 --- a/dbcon/joblist/jlf_execplantojoblist.cpp +++ b/dbcon/joblist/jlf_execplantojoblist.cpp @@ -1636,7 +1636,6 @@ const JobStepVector doSimpleFilter(SimpleFilter* sf, JobInfo& jobInfo) string constval(cc->constval()); - CalpontSystemCatalog::OID dictOid = 0; CalpontSystemCatalog::ColType ct = sc->colType(); const PseudoColumn* pc = dynamic_cast(sc); diff --git a/dbcon/joblist/lbidlist.cpp b/dbcon/joblist/lbidlist.cpp index c317defc9..7852562ef 100644 --- a/dbcon/joblist/lbidlist.cpp +++ b/dbcon/joblist/lbidlist.cpp @@ -749,6 +749,7 @@ bool LBIDList::CasualPartitionPredicate(const int64_t Min, int64_t tMax = Max; dataconvert::DataConvert::trimWhitespace(tMin); dataconvert::DataConvert::trimWhitespace(tMax); + dataconvert::DataConvert::trimWhitespace(value); scan = compareVal(order_swap(tMin), order_swap(tMax), order_swap(value), op, lcf); diff --git a/dbcon/mysql/CMakeLists.txt b/dbcon/mysql/CMakeLists.txt index fa2df6eaa..c74e19343 100644 --- a/dbcon/mysql/CMakeLists.txt +++ b/dbcon/mysql/CMakeLists.txt @@ -29,7 +29,6 @@ add_library(calmysql SHARED ${libcalmysql_SRCS}) target_link_libraries(calmysql ${ENGINE_LDFLAGS} ${ENGINE_WRITE_LIBS} ${NETSNMP_LIBRARIES} ${SERVER_BUILD_INCLUDE_DIR}/../libservices/libmysqlservices.a threadpool) -SET_TARGET_PROPERTIES(calmysql PROPERTIES LINK_FLAGS "${calmysql_link_flags} -Wl,-E") set_target_properties(calmysql PROPERTIES VERSION 1.0.0 SOVERSION 1) SET ( is_columnstore_tables_SRCS diff --git a/dbcon/mysql/columnstore_info.sql b/dbcon/mysql/columnstore_info.sql index d0433a0d9..7655b5f16 100644 --- a/dbcon/mysql/columnstore_info.sql +++ b/dbcon/mysql/columnstore_info.sql @@ -98,4 +98,23 @@ BEGIN SELECT CONCAT((SELECT SUM(data_size) FROM information_schema.columnstore_extents ce left join information_schema.columnstore_columns cc on ce.object_id = cc.object_id where compression_type='Snappy') / (SELECT SUM(compressed_data_size) FROM information_schema.columnstore_files WHERE compressed_data_size IS NOT NULL), ':1') COMPRESSION_RATIO; END // +create procedure columnstore_upgrade() +`columnstore_upgrade`: BEGIN + DECLARE done INTEGER DEFAULT 0; + DECLARE schema_table VARCHAR(100) DEFAULT ""; + DECLARE table_list CURSOR FOR select concat('`', table_schema,'`.`',table_name,'`') from information_schema.tables where engine='columnstore'; + DECLARE CONTINUE HANDLER FOR NOT FOUND SET done = 1; + OPEN table_list; + tlist: LOOP + FETCH table_list INTO schema_table; + IF done = 1 THEN LEAVE tlist; + END IF; + SET @sql_query = concat('ALTER TABLE ', schema_table, ' COMMENT=\'\''); + PREPARE stmt FROM @sql_query; + EXECUTE stmt; + DEALLOCATE PREPARE stmt; + END LOOP; +END // +delimiter ; + DELIMITER ; diff --git a/dbcon/mysql/ha_calpont.cpp b/dbcon/mysql/ha_calpont.cpp index d4ff366bd..552a895d6 100644 --- a/dbcon/mysql/ha_calpont.cpp +++ b/dbcon/mysql/ha_calpont.cpp @@ -181,87 +181,6 @@ static int infinidb_done_func(void* p) DBUG_RETURN(0); } -#if 0 -/** - @brief - Example of simple lock controls. The "share" it creates is a - structure we will pass to each example handler. Do you have to have - one of these? Well, you have pieces that are used for locking, and - they are needed to function. -*/ - -static CALPONT_SHARE* get_share(const char* table_name, TABLE* table) -{ - CALPONT_SHARE* share; - uint32_t length; - char* tmp_name; -#ifndef _MSC_VER - pthread_mutex_lock(&calpont_mutex); -#endif - length = (uint32_t) strlen(table_name); - - if (!(share = (CALPONT_SHARE*) hash_search(&calpont_open_tables, - (uchar*) table_name, - length))) - { - if (!(share = (CALPONT_SHARE*) - my_multi_malloc(MYF(MY_WME | MY_ZEROFILL), - &share, sizeof(*share), - &tmp_name, length + 1, - NullS))) - { - pthread_mutex_unlock(&calpont_mutex); - return NULL; - } - - share->use_count = 0; - share->table_name_length = length; - share->table_name = tmp_name; - strmov(share->table_name, table_name); - - if (my_hash_insert(&calpont_open_tables, (uchar*) share)) - goto error; - - thr_lock_init(&share->lock); - pthread_mutex_init(&share->mutex, MY_MUTEX_INIT_FAST); - } - - share->use_count++; - pthread_mutex_unlock(&calpont_mutex); - - return share; - -error: - pthread_mutex_destroy(&share->mutex); - my_free(share, MYF(0)); - - return NULL; -} - -/** - @brief - Free lock controls. We call this whenever we close a table. If the table had - the last reference to the share, then we free memory associated with it. -*/ - -static int free_share(CALPONT_SHARE* share) -{ - pthread_mutex_lock(&calpont_mutex); - - if (!--share->use_count) - { - hash_delete(&calpont_open_tables, (uchar*) share); - thr_lock_delete(&share->lock); - pthread_mutex_destroy(&share->mutex); - my_free(share, MYF(0)); - } - - pthread_mutex_unlock(&calpont_mutex); - - return 0; -} -#endif - static handler* calpont_create_handler(handlerton* hton, TABLE_SHARE* table, MEM_ROOT* mem_root) diff --git a/dbcon/mysql/ha_calpont_ddl.cpp b/dbcon/mysql/ha_calpont_ddl.cpp index a59701e9b..191ff610a 100644 --- a/dbcon/mysql/ha_calpont_ddl.cpp +++ b/dbcon/mysql/ha_calpont_ddl.cpp @@ -2095,8 +2095,7 @@ int ha_calpont_impl_create_(const char* name, TABLE* table_arg, HA_CREATE_INFO* if ( schemaSyncOnly && isCreate) return rc; - //this is replcated DDL, treat it just like SSO - if (thd->slave_thread) + if (thd->slave_thread && !ci.replicationEnabled) return rc; //@bug 5660. Error out REAL DDL/DML on slave node. @@ -2294,8 +2293,7 @@ int ha_calpont_impl_delete_table_(const char* db, const char* name, cal_connecti return 0; } - //this is replcated DDL, treat it just like SSO - if (thd->slave_thread) + if (thd->slave_thread && !ci.replicationEnabled) return 0; //@bug 5660. Error out REAL DDL/DML on slave node. @@ -2434,8 +2432,7 @@ int ha_calpont_impl_rename_table_(const char* from, const char* to, cal_connecti pair toPair; string stmt; - //this is replicated DDL, treat it just like SSO - if (thd->slave_thread) + if (thd->slave_thread && !ci.replicationEnabled) return 0; //@bug 5660. Error out REAL DDL/DML on slave node. diff --git a/dbcon/mysql/ha_calpont_dml.cpp b/dbcon/mysql/ha_calpont_dml.cpp index dd2b09135..66edb13fa 100644 --- a/dbcon/mysql/ha_calpont_dml.cpp +++ b/dbcon/mysql/ha_calpont_dml.cpp @@ -2080,7 +2080,8 @@ int ha_calpont_impl_commit_ (handlerton* hton, THD* thd, bool all, cal_connectio thd->infinidb_vtable.vtable_state == THD::INFINIDB_SELECT_VTABLE ) return rc; - if (thd->slave_thread) return 0; + if (thd->slave_thread && !ci.replicationEnabled) + return 0; std::string command("COMMIT"); #ifdef INFINIDB_DEBUG diff --git a/dbcon/mysql/ha_calpont_execplan.cpp b/dbcon/mysql/ha_calpont_execplan.cpp index e553254e3..e617e5959 100644 --- a/dbcon/mysql/ha_calpont_execplan.cpp +++ b/dbcon/mysql/ha_calpont_execplan.cpp @@ -7140,7 +7140,9 @@ int getSelectPlan(gp_walk_info& gwi, SELECT_LEX& select_lex, SCSEP& csep, bool i // for subquery, order+limit by will be supported in infinidb. build order by columns // @todo union order by and limit support - if (gwi.hasWindowFunc || gwi.subSelectType != CalpontSelectExecutionPlan::MAIN_SELECT) + if (gwi.hasWindowFunc + || gwi.subSelectType != CalpontSelectExecutionPlan::MAIN_SELECT + || ( isUnion && ordercol )) { for (; ordercol; ordercol = ordercol->next) { diff --git a/dbcon/mysql/ha_calpont_impl.cpp b/dbcon/mysql/ha_calpont_impl.cpp index 02164e239..83442b3d3 100644 --- a/dbcon/mysql/ha_calpont_impl.cpp +++ b/dbcon/mysql/ha_calpont_impl.cpp @@ -179,6 +179,82 @@ inline uint32_t tid2sid(const uint32_t tid) return CalpontSystemCatalog::idb_tid2sid(tid); } + +/** + @brief + Wrapper around logging facility. + + @details + Reduces the boiler plate code. + + Called from number of places(mostly DML) in + ha_calpont_impl.cpp(). +*/ +void log_this(THD *thd, const char *message, + logging::LOG_TYPE log_type, unsigned sid) +{ + // corresponds with dbcon in SubsystemID vector + // in messagelog.cpp + unsigned int subSystemId = 24; + logging::LoggingID logid( subSystemId, sid, 0); + logging::Message::Args args1; + logging::Message msg(1); + args1.add(message); + msg.format( args1 ); + Logger logger(logid.fSubsysID); + logger.logMessage(log_type, msg, logid); +} + +/** + @brief + Forcely close a FEP connection. + + @details + Plugin code opens network connection with ExMgr to + get: + the result of meta-data queries + the result of DML or DQL query in any mode + statistics + This code allows to explicitly close the connection + if any error happens using a non-existing protocol + code 0. This causes ExeMgr loop to drop the + connection. + + Called from many places in ha_calpont_impl.cpp(). +*/ +void force_close_fep_conn(THD *thd, cal_connection_info* ci, bool check_prev_rc = false) +{ + if (!ci->cal_conn_hndl) + { + return; + } + + if(check_prev_rc && !ci->rc) + { + return; + } + + // send ExeMgr an unknown signal to force him to close + // the connection + ByteStream msg; + ByteStream::quadbyte qb = 0; + msg << qb; + + try + { + ci->cal_conn_hndl->exeMgr->write(msg); + } + catch (...) + { + // Add details into the message. + log_this(thd, "Exception in force_close_fep_conn().", + logging::LOG_TYPE_DEBUG, tid2sid(thd->thread_id)); + } + + sm::sm_cleanup(ci->cal_conn_hndl); + ci->cal_conn_hndl = 0; +} + void storeNumericField(Field** f, int64_t value, CalpontSystemCatalog::ColType& ct) { // unset null bit first @@ -936,7 +1012,6 @@ uint32_t doUpdateDelete(THD* thd) cal_connection_info* ci = reinterpret_cast(get_fe_conn_info_ptr()); - //@bug 5660. Error out DDL/DML on slave node, or on local query node if (ci->isSlaveNode && !thd->slave_thread) { string emsg = logging::IDBErrorInfo::instance()->errorMsg(ERR_DML_DDL_SLAVE); @@ -958,7 +1033,14 @@ uint32_t doUpdateDelete(THD* thd) // stats start ci->stats.reset(); ci->stats.setStartTime(); - ci->stats.fUser = thd->main_security_ctx.user; + if (thd->main_security_ctx.user) + { + ci->stats.fUser = thd->main_security_ctx.user; + } + else + { + ci->stats.fUser = ""; + } if (thd->main_security_ctx.host) ci->stats.fHost = thd->main_security_ctx.host; @@ -2053,8 +2135,9 @@ int ha_calpont_impl_rnd_init(TABLE* table) // prevent "create table as select" from running on slave thd->infinidb_vtable.hasInfiniDBTable = true; - /* If this node is the slave, ignore DML to IDB tables */ - if (thd->slave_thread && ( + cal_connection_info* ci = reinterpret_cast(get_fe_conn_info_ptr()); + + if (thd->slave_thread && !ci->replicationEnabled && ( thd->lex->sql_command == SQLCOM_INSERT || thd->lex->sql_command == SQLCOM_INSERT_SELECT || thd->lex->sql_command == SQLCOM_UPDATE || @@ -2107,8 +2190,6 @@ int ha_calpont_impl_rnd_init(TABLE* table) if (get_fe_conn_info_ptr() == NULL) set_fe_conn_info_ptr((void*)new cal_connection_info()); - cal_connection_info* ci = reinterpret_cast(get_fe_conn_info_ptr()); - idbassert(ci != 0); // MySQL sometimes calls rnd_init multiple times, plan should only be @@ -2119,26 +2200,7 @@ int ha_calpont_impl_rnd_init(TABLE* table) if (thd->killed == KILL_QUERY || thd->killed == KILL_QUERY_HARD) { - if (ci->cal_conn_hndl) - { - // send ExeMgr a signal before closing the connection - ByteStream msg; - ByteStream::quadbyte qb = 0; - msg << qb; - - try - { - ci->cal_conn_hndl->exeMgr->write(msg); - } - catch (...) - { - // canceling query. ignore connection failure. - } - - sm::sm_cleanup(ci->cal_conn_hndl); - ci->cal_conn_hndl = 0; - } - + force_close_fep_conn(thd, ci); return 0; } @@ -2239,7 +2301,14 @@ int ha_calpont_impl_rnd_init(TABLE* table) { ci->stats.reset(); // reset query stats ci->stats.setStartTime(); - ci->stats.fUser = thd->main_security_ctx.user; + if (thd->main_security_ctx.user) + { + ci->stats.fUser = thd->main_security_ctx.user; + } + else + { + ci->stats.fUser = ""; + } if (thd->main_security_ctx.host) ci->stats.fHost = thd->main_security_ctx.host; @@ -2604,8 +2673,9 @@ int ha_calpont_impl_rnd_next(uchar* buf, TABLE* table) { THD* thd = current_thd; - /* If this node is the slave, ignore DML to IDB tables */ - if (thd->slave_thread && ( + cal_connection_info* ci = reinterpret_cast(get_fe_conn_info_ptr()); + + if (thd->slave_thread && !ci->replicationEnabled && ( thd->lex->sql_command == SQLCOM_INSERT || thd->lex->sql_command == SQLCOM_INSERT_SELECT || thd->lex->sql_command == SQLCOM_UPDATE || @@ -2616,7 +2686,6 @@ int ha_calpont_impl_rnd_next(uchar* buf, TABLE* table) thd->lex->sql_command == SQLCOM_LOAD)) return 0; - if (thd->infinidb_vtable.vtable_state == THD::INFINIDB_ERROR) return ER_INTERNAL_ERROR; @@ -2645,31 +2714,10 @@ int ha_calpont_impl_rnd_next(uchar* buf, TABLE* table) if (get_fe_conn_info_ptr() == NULL) set_fe_conn_info_ptr((void*)new cal_connection_info()); - cal_connection_info* ci = reinterpret_cast(get_fe_conn_info_ptr()); - // @bug 3078 if (thd->killed == KILL_QUERY || thd->killed == KILL_QUERY_HARD) { - if (ci->cal_conn_hndl) - { - // send ExeMgr a signal before cloing the connection - ByteStream msg; - ByteStream::quadbyte qb = 0; - msg << qb; - - try - { - ci->cal_conn_hndl->exeMgr->write(msg); - } - catch (...) - { - // cancel query. ignore connection failure. - } - - sm::sm_cleanup(ci->cal_conn_hndl); - ci->cal_conn_hndl = 0; - } - + force_close_fep_conn(thd, ci); return 0; } @@ -2729,8 +2777,17 @@ int ha_calpont_impl_rnd_end(TABLE* table) int rc = 0; THD* thd = current_thd; cal_connection_info* ci = NULL; + bool replicationEnabled = false; - if (thd->slave_thread && ( + if (thd->infinidb_vtable.cal_conn_info) + ci = reinterpret_cast(thd->infinidb_vtable.cal_conn_info); + + if (ci && ci->replicationEnabled) + { + replicationEnabled = true; + } + + if (thd->slave_thread && !replicationEnabled && ( thd->lex->sql_command == SQLCOM_INSERT || thd->lex->sql_command == SQLCOM_INSERT_SELECT || thd->lex->sql_command == SQLCOM_UPDATE || @@ -2745,7 +2802,6 @@ int ha_calpont_impl_rnd_end(TABLE* table) if (get_fe_conn_info_ptr() != NULL) ci = reinterpret_cast(get_fe_conn_info_ptr()); - if (thd->infinidb_vtable.vtable_state == THD::INFINIDB_ORDER_BY ) { thd->infinidb_vtable.vtable_state = THD::INFINIDB_SELECT_VTABLE; // flip back to normal state @@ -2767,27 +2823,7 @@ int ha_calpont_impl_rnd_end(TABLE* table) if (((thd->lex)->sql_command == SQLCOM_INSERT) || ((thd->lex)->sql_command == SQLCOM_INSERT_SELECT) ) { - // @bug 4022. error handling for select part of dml - if (ci->cal_conn_hndl && ci->rc) - { - // send ExeMgr a signal before closing the connection - ByteStream msg; - ByteStream::quadbyte qb = 0; - msg << qb; - - try - { - ci->cal_conn_hndl->exeMgr->write(msg); - } - catch (...) - { - // this is error handling, so ignore connection failure. - } - - sm::sm_cleanup(ci->cal_conn_hndl); - ci->cal_conn_hndl = 0; - return rc; - } + force_close_fep_conn(thd, ci, true); // checking prev command rc } if (!ci) @@ -2802,29 +2838,9 @@ int ha_calpont_impl_rnd_end(TABLE* table) (thd->lex)->sql_command != SQLCOM_INSERT_SELECT && thd->variables.select_limit != (uint64_t) - 1)) { - if (ci->cal_conn_hndl) - { - // send ExeMgr a signal before closing the connection - ByteStream msg; - ByteStream::quadbyte qb = 0; - msg << qb; - - try - { - ci->cal_conn_hndl->exeMgr->write(msg); - } - catch (...) - { - // this is the end of query. Ignore the exception if exemgr connection failed - // for whatever reason. - } - - sm::sm_cleanup(ci->cal_conn_hndl); - ci->cal_conn_hndl = 0; - // clear querystats because no query stats available for cancelled query - ci->queryStats = ""; - } - + force_close_fep_conn(thd, ci); + // clear querystats because no query stats available for cancelled query + ci->queryStats = ""; return 0; } @@ -2906,6 +2922,7 @@ int ha_calpont_impl_rnd_end(TABLE* table) ci->warningMsg.clear(); // reset expressionId just in case ci->expressionId = 0; + return rc; } @@ -3043,9 +3060,8 @@ int ha_calpont_impl_write_row(uchar* buf, TABLE* table) cal_connection_info* ci = reinterpret_cast(get_fe_conn_info_ptr()); - if (thd->slave_thread) return 0; - - + if (thd->slave_thread && !ci->replicationEnabled) + return 0; if (ci->alterTableState > 0) return 0; @@ -3130,7 +3146,8 @@ void ha_calpont_impl_start_bulk_insert(ha_rows rows, TABLE* table) if (thd->infinidb_vtable.vtable_state != THD::INFINIDB_ALTER_VTABLE) thd->infinidb_vtable.isInfiniDBDML = true; - if (thd->slave_thread) return; + if (thd->slave_thread && !ci->replicationEnabled) + return; //@bug 5660. Error out DDL/DML on slave node, or on local query node if (ci->isSlaveNode && thd->infinidb_vtable.vtable_state != THD::INFINIDB_ALTER_VTABLE) @@ -3243,16 +3260,11 @@ void ha_calpont_impl_start_bulk_insert(ha_rows rows, TABLE* table) } //Log the statement to debug.log - LoggingID logid( 24, tid2sid(thd->thread_id), 0); - logging::Message::Args args1; - logging::Message msg(1); - args1.add("Start SQL statement: "); - ostringstream oss; - oss << idb_mysql_query_str(thd) << "; |" << table->s->db.str << "|"; - args1.add(oss.str()); - msg.format( args1 ); - Logger logger(logid.fSubsysID); - logger.logMessage(LOG_TYPE_DEBUG, msg, logid); + { + ostringstream oss; + oss << "Start SQL statement: " << idb_mysql_query_str(thd) << "; |" << table->s->db.str << "|"; + log_this(thd, oss.str().c_str(), logging::LOG_TYPE_DEBUG, tid2sid(thd->thread_id)); + } //start process cpimport mode 1 ci->mysqld_pid = getpid(); @@ -3290,13 +3302,8 @@ void ha_calpont_impl_start_bulk_insert(ha_rows rows, TABLE* table) { setError(current_thd, ER_INTERNAL_ERROR, logging::IDBErrorInfo::instance()->errorMsg(ERR_LOCAL_QUERY_UM)); ci->singleInsert = true; - LoggingID logid( 24, tid2sid(thd->thread_id), 0); - logging::Message::Args args1; - logging::Message msg(1); - args1.add("End SQL statement"); - msg.format( args1 ); - Logger logger(logid.fSubsysID); - logger.logMessage(LOG_TYPE_DEBUG, msg, logid); + log_this(thd, "End SQL statement", logging::LOG_TYPE_DEBUG, + tid2sid(thd->thread_id)); return; } else @@ -3470,16 +3477,8 @@ void ha_calpont_impl_start_bulk_insert(ha_rows rows, TABLE* table) errnum << "); " << errmsg; setError(current_thd, ER_INTERNAL_ERROR, oss.str()); ci->singleInsert = true; - LoggingID logid( 24, tid2sid(thd->thread_id), 0); - logging::Message::Args args1, args2; - logging::Message emsg(1), msg(1); - args1.add(oss.str()); - emsg.format( args1 ); - Logger logger(logid.fSubsysID); - logger.logMessage(LOG_TYPE_ERROR, emsg, logid); - args2.add("End SQL statement"); - msg.format( args2 ); - logger.logMessage(LOG_TYPE_DEBUG, msg, logid); + log_this(thd, oss.str(), logging::LOG_TYPE_ERROR, tid2sid(thd->thread_id)); + log_this(thd, "End SQL statement", logging::LOG_TYPE_DEBUG, tid2sid(thd->thread_id)); return; } @@ -3500,13 +3499,7 @@ void ha_calpont_impl_start_bulk_insert(ha_rows rows, TABLE* table) errnum << "); " << strerror(errnum); setError(current_thd, ER_INTERNAL_ERROR, oss.str()); ci->singleInsert = true; - LoggingID logid( 24, tid2sid(thd->thread_id), 0); - logging::Message::Args args1; - logging::Message msg(1); - args1.add("End SQL statement"); - msg.format( args1 ); - Logger logger(logid.fSubsysID); - logger.logMessage(LOG_TYPE_DEBUG, msg, logid); + log_this(thd, "End SQL statement", logging::LOG_TYPE_DEBUG, tid2sid(thd->thread_id)); return; } @@ -3522,13 +3515,7 @@ void ha_calpont_impl_start_bulk_insert(ha_rows rows, TABLE* table) errnum << "); " << strerror(errnum); setError(current_thd, ER_INTERNAL_ERROR, oss.str()); ci->singleInsert = true; - LoggingID logid( 24, tid2sid(thd->thread_id), 0); - logging::Message::Args args1; - logging::Message msg(1); - args1.add("End SQL statement"); - msg.format( args1 ); - Logger logger(logid.fSubsysID); - logger.logMessage(LOG_TYPE_DEBUG, msg, logid); + log_this(thd, "End SQL statement", logging::LOG_TYPE_DEBUG, tid2sid(thd->thread_id)); return; } else if (aChPid == 0) // we are in child @@ -3565,13 +3552,8 @@ void ha_calpont_impl_start_bulk_insert(ha_rows rows, TABLE* table) setError(current_thd, ER_INTERNAL_ERROR, "Forking process cpimport failed."); ci->singleInsert = true; - LoggingID logid( 24, tid2sid(thd->thread_id), 0); - logging::Message::Args args1; - logging::Message msg(1); - args1.add("End SQL statement"); - msg.format( args1 ); - Logger logger(logid.fSubsysID); - logger.logMessage(LOG_TYPE_DEBUG, msg, logid); + log_this(thd, "End SQL statement", logging::LOG_TYPE_DEBUG, + tid2sid(thd->thread_id)); exit(1); } else // parent @@ -3604,7 +3586,14 @@ void ha_calpont_impl_start_bulk_insert(ha_rows rows, TABLE* table) // query stats. only collect execution time and rows inserted for insert/load_data_infile ci->stats.reset(); ci->stats.setStartTime(); - ci->stats.fUser = thd->main_security_ctx.user; + if (thd->main_security_ctx.user) + { + ci->stats.fUser = thd->main_security_ctx.user; + } + else + { + ci->stats.fUser = ""; + } if (thd->main_security_ctx.host) ci->stats.fHost = thd->main_security_ctx.host; @@ -3690,7 +3679,8 @@ int ha_calpont_impl_end_bulk_insert(bool abort, TABLE* table) cal_connection_info* ci = reinterpret_cast(get_fe_conn_info_ptr()); - if (thd->slave_thread) return 0; + if (thd->slave_thread && !ci->replicationEnabled) + return 0; int rc = 0; @@ -3736,13 +3726,7 @@ int ha_calpont_impl_end_bulk_insert(bool abort, TABLE* table) FormatMessage(FORMAT_MESSAGE_FROM_SYSTEM, NULL, errnum, 0, errmsg, 512, NULL); ostringstream oss; oss << "GenerateConsoleCtrlEvent: (errno-" << errnum << "); " << errmsg; - LoggingID logid( 24, 0, 0); - logging::Message::Args args1; - logging::Message msg(1); - args1.add(oss.str()); - msg.format( args1 ); - Logger logger(logid.fSubsysID); - logger.logMessage(LOG_TYPE_DEBUG, msg, logid); + log_this(thd, oss.str(), logging::LOG_TYPE_DEBUG,0); } // Close handles to the cpimport process and its primary thread. @@ -3846,18 +3830,15 @@ int ha_calpont_impl_end_bulk_insert(bool abort, TABLE* table) } #endif - LoggingID logid( 24, tid2sid(thd->thread_id), 0); - logging::Message::Args args1; - logging::Message msg(1); - if ( rc == 0) - args1.add("End SQL statement"); + { + log_this(thd, "End SQL statement", logging::LOG_TYPE_DEBUG, tid2sid(thd->thread_id)); + } else - args1.add("End SQL statement with error"); + { + log_this(thd, "End SQL statement with error", logging::LOG_TYPE_DEBUG, tid2sid(thd->thread_id)); + } - msg.format( args1 ); - Logger logger(logid.fSubsysID); - logger.logMessage(LOG_TYPE_DEBUG, msg, logid); ci->columnTypes.clear(); //get extra warning count if any ifstream dmlFile; @@ -4014,10 +3995,16 @@ int ha_calpont_impl_close_connection (handlerton* hton, THD* thd) execplan::CalpontSystemCatalog::removeCalpontSystemCatalog(tid2sid(thd->thread_id)); - if (get_fe_conn_info_ptr() == NULL) - set_fe_conn_info_ptr((void*)new cal_connection_info()); - - cal_connection_info* ci = reinterpret_cast(get_fe_conn_info_ptr()); + // MCOL-3247 Use THD::ha_data as a per-plugin per-session + // storage. Filled in external_lock when we remove a lock + // from vtable(lock_type = 2) + // An ugly way. I will use ha_data w/o external_lock. + // This in MCOL-2178 + cal_connection_info* ci = NULL; + if(thd_get_ha_data(thd, hton)) + { + ci = reinterpret_cast(thd_get_ha_data(thd, hton)); + } if (!ci) return 0; @@ -4183,26 +4170,7 @@ int ha_calpont_impl_external_lock(THD* thd, TABLE* table, int lock_type) if (thd->killed == KILL_QUERY || thd->killed == KILL_QUERY_HARD) { - if (ci->cal_conn_hndl) - { - // send ExeMgr a signal before cloing the connection - ByteStream msg; - ByteStream::quadbyte qb = 0; - msg << qb; - - try - { - ci->cal_conn_hndl->exeMgr->write(msg); - } - catch (...) - { - // this is the end of the query. Ignore connetion failure. - } - - sm::sm_cleanup(ci->cal_conn_hndl); - ci->cal_conn_hndl = 0; - } - + force_close_fep_conn(thd, ci); return 0; } @@ -4259,6 +4227,9 @@ int ha_calpont_impl_external_lock(THD* thd, TABLE* table, int lock_type) ci->miniStats = ci->cal_conn_hndl->miniStats; ci->queryState = 0; thd->infinidb_vtable.override_largeside_estimate = false; + // MCOL-3247 Use THD::ha_data as a per-plugin per-session + // storage for cal_conn_hndl to use it later in close_connection + thd_set_ha_data(thd, calpont_hton, get_fe_conn_info_ptr()); } } @@ -4357,26 +4328,7 @@ int ha_calpont_impl_group_by_init(ha_calpont_group_by_handler* group_hand, TABLE if (thd->killed == KILL_QUERY || thd->killed == KILL_QUERY_HARD) { - if (ci->cal_conn_hndl) - { - // send ExeMgr a signal before closing the connection - ByteStream msg; - ByteStream::quadbyte qb = 0; - msg << qb; - - try - { - ci->cal_conn_hndl->exeMgr->write(msg); - } - catch (...) - { - // canceling query. ignore connection failure. - } - - sm::sm_cleanup(ci->cal_conn_hndl); - ci->cal_conn_hndl = 0; - } - + force_close_fep_conn(thd, ci); return 0; } @@ -4391,7 +4343,14 @@ int ha_calpont_impl_group_by_init(ha_calpont_group_by_handler* group_hand, TABLE { ci->stats.reset(); // reset query stats ci->stats.setStartTime(); - ci->stats.fUser = thd->main_security_ctx.user; + if (thd->main_security_ctx.user) + { + ci->stats.fUser = thd->main_security_ctx.user; + } + else + { + ci->stats.fUser = ""; + } if (thd->main_security_ctx.host) ci->stats.fHost = thd->main_security_ctx.host; @@ -4810,19 +4769,6 @@ int ha_calpont_impl_group_by_next(ha_calpont_group_by_handler* group_hand, TABLE { THD* thd = current_thd; - /* If this node is the slave, ignore DML to IDB tables */ - if (thd->slave_thread && ( - thd->lex->sql_command == SQLCOM_INSERT || - thd->lex->sql_command == SQLCOM_INSERT_SELECT || - thd->lex->sql_command == SQLCOM_UPDATE || - thd->lex->sql_command == SQLCOM_UPDATE_MULTI || - thd->lex->sql_command == SQLCOM_DELETE || - thd->lex->sql_command == SQLCOM_DELETE_MULTI || - thd->lex->sql_command == SQLCOM_TRUNCATE || - thd->lex->sql_command == SQLCOM_LOAD)) - return 0; - - if (thd->infinidb_vtable.vtable_state == THD::INFINIDB_ERROR) return ER_INTERNAL_ERROR; @@ -4848,29 +4794,21 @@ int ha_calpont_impl_group_by_next(ha_calpont_group_by_handler* group_hand, TABLE cal_connection_info* ci = reinterpret_cast(get_fe_conn_info_ptr()); + if (thd->slave_thread && !ci->replicationEnabled && ( + thd->lex->sql_command == SQLCOM_INSERT || + thd->lex->sql_command == SQLCOM_INSERT_SELECT || + thd->lex->sql_command == SQLCOM_UPDATE || + thd->lex->sql_command == SQLCOM_UPDATE_MULTI || + thd->lex->sql_command == SQLCOM_DELETE || + thd->lex->sql_command == SQLCOM_DELETE_MULTI || + thd->lex->sql_command == SQLCOM_TRUNCATE || + thd->lex->sql_command == SQLCOM_LOAD)) + return 0; + // @bug 3078 if (thd->killed == KILL_QUERY || thd->killed == KILL_QUERY_HARD) { - if (ci->cal_conn_hndl) - { - // send ExeMgr a signal before cloing the connection - ByteStream msg; - ByteStream::quadbyte qb = 0; - msg << qb; - - try - { - ci->cal_conn_hndl->exeMgr->write(msg); - } - catch (...) - { - // cancel query. ignore connection failure. - } - - sm::sm_cleanup(ci->cal_conn_hndl); - ci->cal_conn_hndl = 0; - } - + force_close_fep_conn(thd, ci); return 0; } @@ -4932,8 +4870,19 @@ int ha_calpont_impl_group_by_end(ha_calpont_group_by_handler* group_hand, TABLE* THD* thd = current_thd; cal_connection_info* ci = NULL; + thd->infinidb_vtable.isNewQuery = true; + thd->infinidb_vtable.isUnion = false; - if (thd->slave_thread && ( + if (get_fe_conn_info_ptr() != NULL) + ci = reinterpret_cast(get_fe_conn_info_ptr()); + + if (!ci) + { + thd->infinidb_vtable.cal_conn_info = (void*)(new cal_connection_info()); + ci = reinterpret_cast(thd->infinidb_vtable.cal_conn_info); + } + + if (thd->slave_thread && !ci->replicationEnabled && ( thd->lex->sql_command == SQLCOM_INSERT || thd->lex->sql_command == SQLCOM_INSERT_SELECT || thd->lex->sql_command == SQLCOM_UPDATE || @@ -4944,43 +4893,11 @@ int ha_calpont_impl_group_by_end(ha_calpont_group_by_handler* group_hand, TABLE* thd->lex->sql_command == SQLCOM_LOAD)) return 0; - thd->infinidb_vtable.isNewQuery = true; - thd->infinidb_vtable.isUnion = false; - - if (get_fe_conn_info_ptr() != NULL) - ci = reinterpret_cast(get_fe_conn_info_ptr()); - - // MCOL-1052 - //if (thd->infinidb_vtable.vtable_state == THD::INFINIDB_ORDER_BY ) - //{ - // thd->infinidb_vtable.vtable_state = THD::INFINIDB_SELECT_VTABLE;// flip back to normal state - // return rc; - //} - - if (((thd->lex)->sql_command == SQLCOM_INSERT) || + if (((thd->lex)->sql_command == SQLCOM_INSERT) || ((thd->lex)->sql_command == SQLCOM_INSERT_SELECT) ) { - // @bug 4022. error handling for select part of dml - if (ci->cal_conn_hndl && ci->rc) - { - // send ExeMgr a signal before closing the connection - ByteStream msg; - ByteStream::quadbyte qb = 0; - msg << qb; - - try - { - ci->cal_conn_hndl->exeMgr->write(msg); - } - catch (...) - { - // this is error handling, so ignore connection failure. - } - - sm::sm_cleanup(ci->cal_conn_hndl); - ci->cal_conn_hndl = 0; - return rc; - } + force_close_fep_conn(thd, ci, true); // with checking prev command rc + return rc; } if (!ci) @@ -4995,33 +4912,15 @@ int ha_calpont_impl_group_by_end(ha_calpont_group_by_handler* group_hand, TABLE* (thd->lex)->sql_command != SQLCOM_INSERT_SELECT && thd->variables.select_limit != (uint64_t) - 1)) { - if (ci->cal_conn_hndl) + force_close_fep_conn(thd, ci); + // clear querystats because no query stats available for cancelled query + ci->queryStats = ""; + // Poping next ExeMgr connection out of the stack + if ( ci->cal_conn_hndl_st.size() ) { - // send ExeMgr a signal before closing the connection - ByteStream msg; - ByteStream::quadbyte qb = 0; - msg << qb; - - try - { - ci->cal_conn_hndl->exeMgr->write(msg); - } - catch (...) - { - // this is the end of query. Ignore the exception if exemgr connection failed - // for whatever reason. - } - - sm::sm_cleanup(ci->cal_conn_hndl); - ci->cal_conn_hndl = 0; - // clear querystats because no query stats available for cancelled query - ci->queryStats = ""; + ci->cal_conn_hndl_st.pop(); if ( ci->cal_conn_hndl_st.size() ) - { - ci->cal_conn_hndl_st.pop(); - if ( ci->cal_conn_hndl_st.size() ) - ci->cal_conn_hndl = ci->cal_conn_hndl_st.top(); - } + ci->cal_conn_hndl = ci->cal_conn_hndl_st.top(); } return 0; diff --git a/dbcon/mysql/ha_calpont_impl_if.h b/dbcon/mysql/ha_calpont_impl_if.h index 4f0046745..d53d6e652 100644 --- a/dbcon/mysql/ha_calpont_impl_if.h +++ b/dbcon/mysql/ha_calpont_impl_if.h @@ -251,10 +251,18 @@ struct cal_connection_info useXbit(false), utf8(false), useCpimport(1), - delimiter('\7') + delimiter('\7'), + replicationEnabled(false) { // check if this is a slave mysql daemon isSlaveNode = checkSlave(); + + std::string option = config::Config::makeConfig()->getConfig("SystemConfig", "ReplicationEnabled"); + + if (!option.compare("Y")) + { + replicationEnabled = true; + } } static bool checkSlave() @@ -319,6 +327,7 @@ struct cal_connection_info char delimiter; char enclosed_by; std::vector columnTypes; + bool replicationEnabled; // MCOL-1101 remove compilation unit variable rmParms std::vector rmParms; }; diff --git a/primitives/primproc/dictstep.cpp b/primitives/primproc/dictstep.cpp index abd99ada3..47bfeac25 100644 --- a/primitives/primproc/dictstep.cpp +++ b/primitives/primproc/dictstep.cpp @@ -30,6 +30,7 @@ #include #include +#include #include "bpp.h" #include "primitiveserver.h" @@ -93,6 +94,7 @@ void DictStep::createCommand(ByteStream& bs) for (uint32_t i = 0; i < filterCount; i++) { bs >> strTmp; + boost::trim_right_if(strTmp, boost::is_any_of(" ")); //cout << " " << strTmp << endl; eqFilter->insert(strTmp); } diff --git a/primitives/primproc/primitiveserver.cpp b/primitives/primproc/primitiveserver.cpp index 76ff8ef3c..9359556e9 100644 --- a/primitives/primproc/primitiveserver.cpp +++ b/primitives/primproc/primitiveserver.cpp @@ -28,6 +28,7 @@ #include #include #include +#include //#define NDEBUG #include #include @@ -1804,6 +1805,7 @@ private: for (i = 0; i < count; i++) { *bs >> str; + boost::trim_right_if(str, boost::is_any_of(" ")); filter->insert(str); } diff --git a/utils/configcpp/CMakeLists.txt b/utils/configcpp/CMakeLists.txt index cae83c461..f52f2a84e 100644 --- a/utils/configcpp/CMakeLists.txt +++ b/utils/configcpp/CMakeLists.txt @@ -9,5 +9,7 @@ add_library(configcpp SHARED ${configcpp_LIB_SRCS}) set_target_properties(configcpp PROPERTIES VERSION 1.0.0 SOVERSION 1) +target_compile_definitions(configcpp PUBLIC BOOST_NO_CXX11_SCOPED_ENUMS) + install(TARGETS configcpp DESTINATION ${ENGINE_LIBDIR} COMPONENT libs) diff --git a/utils/idbdatafile/BufferedFile.cpp b/utils/idbdatafile/BufferedFile.cpp index ddc3e9c66..05973ed23 100644 --- a/utils/idbdatafile/BufferedFile.cpp +++ b/utils/idbdatafile/BufferedFile.cpp @@ -279,4 +279,27 @@ int BufferedFile::close() return ret; } +/** + @brief + The wrapper for fallocate function. + @see + This one is used in shared/we_fileop.cpp to skip expensive file preallocation. +*/ +int BufferedFile::fallocate(int mode, off64_t offset, off64_t length) +{ + int ret = 0; + int savedErrno = 0; + + ret = ::fallocate( fileno(m_fp), mode, offset, length ); + savedErrno = errno; + + if ( IDBLogger::isEnabled() ) + { + IDBLogger::logNoArg(m_fname, this, "fallocate", errno); + } + + errno = savedErrno; + return ret; +} + } diff --git a/utils/idbdatafile/BufferedFile.h b/utils/idbdatafile/BufferedFile.h index c411ca8de..3e6ae040e 100644 --- a/utils/idbdatafile/BufferedFile.h +++ b/utils/idbdatafile/BufferedFile.h @@ -50,6 +50,7 @@ public: /* virtual */ off64_t tell(); /* virtual */ int flush(); /* virtual */ time_t mtime(); + /* virtual */ int fallocate(int mode, off64_t offset, off64_t length); protected: /* virtual */ diff --git a/utils/idbdatafile/CMakeLists.txt b/utils/idbdatafile/CMakeLists.txt index 7a39665aa..c8f63ad37 100644 --- a/utils/idbdatafile/CMakeLists.txt +++ b/utils/idbdatafile/CMakeLists.txt @@ -15,7 +15,9 @@ set(idbdatafile_LIB_SRCS add_library(idbdatafile SHARED ${idbdatafile_LIB_SRCS}) -target_link_libraries(idbdatafile ${NETSNMP_LIBRARIES}) +target_link_libraries(idbdatafile ${NETSNMP_LIBRARIES} ${ENGINE_OAM_LIBS}) + +target_compile_definitions(idbdatafile PUBLIC BOOST_NO_CXX11_SCOPED_ENUMS) set_target_properties(idbdatafile PROPERTIES VERSION 1.0.0 SOVERSION 1) diff --git a/utils/idbdatafile/IDBDataFile.h b/utils/idbdatafile/IDBDataFile.h index 1106087d9..7559b26f6 100644 --- a/utils/idbdatafile/IDBDataFile.h +++ b/utils/idbdatafile/IDBDataFile.h @@ -187,6 +187,15 @@ public: */ virtual time_t mtime() = 0; + /** + * The fallocate() method preallocates disk space cheaper then + * sequential write. fallocate() is supported by a limited number + * of FSes.This method is implemented for Un-/BufferedFile classes + * only. + * Returns -1 on error. + */ + virtual int fallocate(int mode, off64_t offset, off64_t length) = 0; + int colWidth() { return m_fColWidth; diff --git a/utils/idbdatafile/IDBPolicy.cpp b/utils/idbdatafile/IDBPolicy.cpp index b9eb29814..7bcdde27f 100644 --- a/utils/idbdatafile/IDBPolicy.cpp +++ b/utils/idbdatafile/IDBPolicy.cpp @@ -20,10 +20,11 @@ #include #include #include -#include // to_upper #include #include "configcpp.h" // for Config +#include "oamcache.h" +#include "liboamcpp.h" #include "IDBPolicy.h" #include "PosixFileSystem.h" //#include "HdfsFileSystem.h" @@ -49,6 +50,7 @@ int64_t IDBPolicy::s_hdfsRdwrBufferMaxSize = 0; std::string IDBPolicy::s_hdfsRdwrScratch; bool IDBPolicy::s_configed = false; boost::mutex IDBPolicy::s_mutex; +std::vector IDBPolicy::s_PreallocSpace; void IDBPolicy::init( bool bEnableLogging, bool bUseRdwrMemBuffer, const string& hdfsRdwrScratch, int64_t hdfsRdwrBufferMaxSize ) { @@ -186,10 +188,12 @@ void IDBPolicy::configIDBPolicy() bool idblog = false; string idblogstr = cf->getConfig("SystemConfig", "DataFileLog"); - if ( idblogstr.length() != 0 ) + // Must be faster. + if ( idblogstr.size() == 2 + && ( idblogstr[0] == 'O' || idblogstr[0] == 'o' ) + && ( idblogstr[1] == 'N' || idblogstr[1] == 'n' )) { - boost::to_upper(idblogstr); - idblog = ( idblogstr == "ON" ); + idblog = true; } //-------------------------------------------------------------------------- @@ -229,9 +233,78 @@ void IDBPolicy::configIDBPolicy() string scratch = cf->getConfig("SystemConfig", "hdfsRdwrScratch"); string hdfsRdwrScratch = tmpDir + scratch; + // MCOL-498. Use DBRootX.PreallocSpace to disable + // dbroots file space preallocation. + // The feature is used in the FileOp code and enabled by default. + char configSectionPref[] = "DBRoot"; + int confSectionLen = sizeof(configSectionPref)+oam::MAX_MODULE_ID_SIZE; + char configSection[confSectionLen]; + IDBPolicy::init( idblog, bUseRdwrMemBuffer, hdfsRdwrScratch, hdfsRdwrBufferMaxSize ); - s_configed = true; + + oam::OamCache* oamcache = oam::OamCache::makeOamCache(); + int PMId = oamcache->getLocalPMId(); + + oam::OamCache::PMDbrootsMap_t pmDbrootsMap; + pmDbrootsMap.reset(new oam::OamCache::PMDbrootsMap_t::element_type()); + oam::systemStorageInfo_t t; + oam::Oam oamInst; + t = oamInst.getStorageConfig(); + + oam::DeviceDBRootList moduledbrootlist = boost::get<2>(t); + oam::DeviceDBRootList::iterator pt = moduledbrootlist.begin(); + + while ( pt != moduledbrootlist.end() && (*pt).DeviceID != PMId) + { + pt++; + continue; + } + + // CS could return here b/c it initialised this singleton and + // there is no DBRootX sections in XML. + if (pt == moduledbrootlist.end()) + { + return; + } + + oam::DBRootConfigList &dbRootVec = (*pt).dbrootConfigList; + s_PreallocSpace.reserve(dbRootVec.size()>>1); + { + int rc; + oam::DBRootConfigList::iterator dbRootIter = dbRootVec.begin(); + for(; dbRootIter != dbRootVec.end(); dbRootIter++) + { + ::memset(configSection + sizeof(configSectionPref), 0, oam::MAX_MODULE_ID_SIZE); + rc = snprintf(configSection, confSectionLen, "%s%d", configSectionPref, *dbRootIter); + // gcc 8.2 warnings + if ( rc < 0 || rc >= confSectionLen) + { + ostringstream oss; + oss << "IDBPolicy::configIDBPolicy: failed to parse DBRootX section."; + throw runtime_error(oss.str()); + } + string setting = cf->getConfig(configSection, "PreallocSpace"); + + if ( setting.length() != 0 ) + { + if ( setting.size() == 3 + && ( setting[0] == 'O' || setting[0] == 'o' ) + && ( setting[1] == 'F' || setting[1] == 'f' ) + && ( setting[2] == 'F' || setting[2] == 'f' ) + ) + { + // int into uint16_t implicit conversion + s_PreallocSpace.push_back(*dbRootIter); + } + } + } + } +} + +void IDBPolicy::setPreallocSpace(uint16_t dbRoot) +{ + s_PreallocSpace.push_back(dbRoot); } } diff --git a/utils/idbdatafile/IDBPolicy.h b/utils/idbdatafile/IDBPolicy.h index 08fecb0e3..d05415117 100644 --- a/utils/idbdatafile/IDBPolicy.h +++ b/utils/idbdatafile/IDBPolicy.h @@ -19,6 +19,7 @@ #define IDBPOLICY_H_ #include +#include #include #include @@ -84,7 +85,12 @@ public: * Accessor method that returns whether or not cloud IO is enabled */ static bool useCloud(); - + + /** + * Checks for disk space preallocation feature status for a dbroot + */ + static bool PreallocSpace(uint16_t dbRoot); + /** * Accessor method that returns whether to use HDFS memory buffers */ @@ -128,6 +134,10 @@ public: static int listDirectory(const char* pathname, std::list& contents); static bool isDir(const char* pathname); static int copyFile(const char* srcPath, const char* destPath); + /** + * This is used in WE shared components Unit Tests + */ + static void setPreallocSpace(uint16_t dbRoot); private: /** @@ -140,6 +150,7 @@ private: static bool s_usehdfs; static bool s_usecloud; + static std::vector s_PreallocSpace; static bool s_bUseRdwrMemBuffer; static std::string s_hdfsRdwrScratch; static int64_t s_hdfsRdwrBufferMaxSize; @@ -165,6 +176,15 @@ bool IDBPolicy::useCloud() return s_usecloud; } +// MCOL-498 Looking for dbRoot in the List set in configIDBPolicy. +inline +bool IDBPolicy::PreallocSpace(uint16_t dbRoot) +{ + std::vector::iterator dbRootIter = + find(s_PreallocSpace.begin(), s_PreallocSpace.end(), dbRoot); + return dbRootIter != s_PreallocSpace.end(); +} + inline bool IDBPolicy::useRdwrMemBuffer() { diff --git a/utils/idbdatafile/UnbufferedFile.cpp b/utils/idbdatafile/UnbufferedFile.cpp index 3b89e13cd..5286c69b4 100644 --- a/utils/idbdatafile/UnbufferedFile.cpp +++ b/utils/idbdatafile/UnbufferedFile.cpp @@ -329,4 +329,27 @@ int UnbufferedFile::close() return ret; } +/** + @brief + The wrapper for fallocate function. + @see + This one is used in shared/we_fileop.cpp to skip expensive file preallocation. +*/ +int UnbufferedFile::fallocate(int mode, off64_t offset, off64_t length) +{ + int ret = 0; + int savedErrno = 0; + + ret = ::fallocate( m_fd, mode, offset, length ); + savedErrno = errno; + + if ( IDBLogger::isEnabled() ) + { + IDBLogger::logNoArg(m_fname, this, "fallocate", errno); + } + + errno = savedErrno; + return ret; +} + } diff --git a/utils/idbdatafile/UnbufferedFile.h b/utils/idbdatafile/UnbufferedFile.h index c2a2771e8..6564c59d7 100644 --- a/utils/idbdatafile/UnbufferedFile.h +++ b/utils/idbdatafile/UnbufferedFile.h @@ -47,6 +47,7 @@ public: /* virtual */ off64_t tell(); /* virtual */ int flush(); /* virtual */ time_t mtime(); + /* virtual */ int fallocate(int mode, off64_t offset, off64_t length); protected: /* virtual */ diff --git a/utils/idbhdfs/hdfs-shared/HdfsFile.cpp b/utils/idbhdfs/hdfs-shared/HdfsFile.cpp index 08260c13f..fc1ec6a2b 100644 --- a/utils/idbhdfs/hdfs-shared/HdfsFile.cpp +++ b/utils/idbhdfs/hdfs-shared/HdfsFile.cpp @@ -348,4 +348,10 @@ int HdfsFile::close() return ret; } +int HdfsFile::fallocate(int mode, off64_t offset, off64_t length) +{ + return 0; +} + + } diff --git a/utils/idbhdfs/hdfs-shared/HdfsFile.h b/utils/idbhdfs/hdfs-shared/HdfsFile.h index 64c68d4e7..704643931 100644 --- a/utils/idbhdfs/hdfs-shared/HdfsFile.h +++ b/utils/idbhdfs/hdfs-shared/HdfsFile.h @@ -62,6 +62,7 @@ public: /* virtual */ off64_t tell(); /* virtual */ int flush(); /* virtual */ time_t mtime(); + /* virtual */ int fallocate(int mode, off64_t offset, off64_t length); protected: /* virtual */ diff --git a/utils/idbhdfs/hdfs-shared/HdfsRdwrFileBuffer.cpp b/utils/idbhdfs/hdfs-shared/HdfsRdwrFileBuffer.cpp index 7d934b3b9..cab6ba545 100644 --- a/utils/idbhdfs/hdfs-shared/HdfsRdwrFileBuffer.cpp +++ b/utils/idbhdfs/hdfs-shared/HdfsRdwrFileBuffer.cpp @@ -317,4 +317,17 @@ int HdfsRdwrFileBuffer::close() return 0; } +/** + @brief + The dummy wrapper for fallocate function. + This is an open question which code must this method return. + fallocate fails for HDFS b/c it doesn't use it. + @see + This one is used in shared/we_fileop.cpp to skip expensive file preallocation. +*/ +int HdfsRdwrFileBuffer::fallocate(int mode, off64_t offset, off64_t length) +{ + return -1; +} + } diff --git a/utils/idbhdfs/hdfs-shared/HdfsRdwrFileBuffer.h b/utils/idbhdfs/hdfs-shared/HdfsRdwrFileBuffer.h index 74fdf7020..8b922f97a 100644 --- a/utils/idbhdfs/hdfs-shared/HdfsRdwrFileBuffer.h +++ b/utils/idbhdfs/hdfs-shared/HdfsRdwrFileBuffer.h @@ -62,6 +62,7 @@ public: /* virtual */ off64_t tell(); /* virtual */ int flush(); /* virtual */ time_t mtime(); + /* virtual*/ int fallocate(int mode, off64_t offset, off64_t length); protected: /* virtual */ diff --git a/utils/idbhdfs/hdfs-shared/HdfsRdwrMemBuffer.cpp b/utils/idbhdfs/hdfs-shared/HdfsRdwrMemBuffer.cpp index 2e6ad3c13..44bea060d 100644 --- a/utils/idbhdfs/hdfs-shared/HdfsRdwrMemBuffer.cpp +++ b/utils/idbhdfs/hdfs-shared/HdfsRdwrMemBuffer.cpp @@ -507,5 +507,10 @@ int HdfsRdwrMemBuffer::close() return 0; } +int HdfsRdwrMemBuffer::fallocate(int mode, off64_t offset, off64_t length) +{ + return 0; +} + } diff --git a/utils/idbhdfs/hdfs-shared/HdfsRdwrMemBuffer.h b/utils/idbhdfs/hdfs-shared/HdfsRdwrMemBuffer.h index c998d13a2..6258d6f70 100644 --- a/utils/idbhdfs/hdfs-shared/HdfsRdwrMemBuffer.h +++ b/utils/idbhdfs/hdfs-shared/HdfsRdwrMemBuffer.h @@ -62,6 +62,7 @@ public: /* virtual */ off64_t tell(); /* virtual */ int flush(); /* virtual */ time_t mtime(); + /* virtual */ int fallocate(int mode, off64_t offset, off64_t length); // Returns the total size of all currently allocated HdfsRdwrMemBuffers static size_t getTotalBuff() diff --git a/utils/regr/corr.h b/utils/regr/corr.h index eba7597eb..d1b5f55ac 100644 --- a/utils/regr/corr.h +++ b/utils/regr/corr.h @@ -25,8 +25,7 @@ * Columnstore interface for for the corr function * * - * CREATE AGGREGATE FUNCTION corr returns REAL - * soname 'libregr_mysql.so'; + * CREATE AGGREGATE FUNCTION corr returns REAL soname 'libregr_mysql.so'; * */ #ifndef HEADER_corr diff --git a/utils/regr/covar_pop.h b/utils/regr/covar_pop.h index fc47d4497..dda396fb9 100644 --- a/utils/regr/covar_pop.h +++ b/utils/regr/covar_pop.h @@ -25,8 +25,7 @@ * Columnstore interface for for the covar_pop function * * - * CREATE AGGREGATE FUNCTION covar_pop returns REAL - * soname 'libregr_mysql.so'; + * CREATE AGGREGATE FUNCTION covar_pop returns REAL soname 'libregr_mysql.so'; * */ #ifndef HEADER_covar_pop diff --git a/utils/regr/covar_samp.h b/utils/regr/covar_samp.h index 6aba65054..a65625520 100644 --- a/utils/regr/covar_samp.h +++ b/utils/regr/covar_samp.h @@ -25,8 +25,7 @@ * Columnstore interface for for the covar_samp function * * - * CREATE AGGREGATE FUNCTION covar_samp returns REAL - * soname 'libregr_mysql.so'; + * CREATE AGGREGATE FUNCTION covar_samp returns REAL soname 'libregr_mysql.so'; * */ #ifndef HEADER_covar_samp diff --git a/utils/regr/regr_avgx.cpp b/utils/regr/regr_avgx.cpp index bf010e648..8e4314d01 100644 --- a/utils/regr/regr_avgx.cpp +++ b/utils/regr/regr_avgx.cpp @@ -63,13 +63,6 @@ mcsv1_UDAF::ReturnCode regr_avgx::init(mcsv1Context* context, context->setErrorMessage("regr_avgx() with a non-numeric x argument"); return mcsv1_UDAF::ERROR; } - if (!(isNumeric(colTypes[1].dataType))) - { - // The error message will be prepended with - // "The storage engine for the table doesn't support " - context->setErrorMessage("regr_avgx() with a non-numeric independant (second) argument"); - return mcsv1_UDAF::ERROR; - } context->setUserDataSize(sizeof(regr_avgx_data)); context->setResultType(CalpontSystemCatalog::DOUBLE); diff --git a/utils/regr/regr_avgx.h b/utils/regr/regr_avgx.h index 75791f769..960a6a892 100644 --- a/utils/regr/regr_avgx.h +++ b/utils/regr/regr_avgx.h @@ -25,8 +25,7 @@ * Columnstore interface for for the regr_avgx function * * - * CREATE AGGREGATE FUNCTION regr_avgx returns REAL soname - * 'libregr_mysql.so'; + * CREATE AGGREGATE FUNCTION regr_avgx returns REAL soname 'libregr_mysql.so'; * */ #ifndef HEADER_regr_avgx diff --git a/utils/regr/regr_avgy.cpp b/utils/regr/regr_avgy.cpp index 7325d991f..3d49e96b4 100644 --- a/utils/regr/regr_avgy.cpp +++ b/utils/regr/regr_avgy.cpp @@ -60,14 +60,7 @@ mcsv1_UDAF::ReturnCode regr_avgy::init(mcsv1Context* context, { // The error message will be prepended with // "The storage engine for the table doesn't support " - context->setErrorMessage("regr_avgy() with a non-numeric x argument"); - return mcsv1_UDAF::ERROR; - } - if (!(isNumeric(colTypes[0].dataType))) - { - // The error message will be prepended with - // "The storage engine for the table doesn't support " - context->setErrorMessage("regr_avgy() with a non-numeric dependant (first) argument"); + context->setErrorMessage("regr_avgy() with a non-numeric y argument"); return mcsv1_UDAF::ERROR; } diff --git a/utils/regr/regr_avgy.h b/utils/regr/regr_avgy.h index c99021f9f..c2a3020da 100644 --- a/utils/regr/regr_avgy.h +++ b/utils/regr/regr_avgy.h @@ -25,8 +25,7 @@ * Columnstore interface for for the regr_avgy function * * - * CREATE AGGREGATE FUNCTION regr_avgy returns REAL soname - * 'libregr_mysql.so'; + * CREATE AGGREGATE FUNCTION regr_avgy returns REAL soname 'libregr_mysql.so'; * */ #ifndef HEADER_regr_avgy diff --git a/utils/regr/regr_count.h b/utils/regr/regr_count.h index 4f4fc558e..25cde7898 100644 --- a/utils/regr/regr_count.h +++ b/utils/regr/regr_count.h @@ -25,8 +25,7 @@ * Columnstore interface for for the regr_count function * * - * CREATE AGGREGATE FUNCTION regr_count returns INTEGER - * soname 'libregr_mysql.so'; + * CREATE AGGREGATE FUNCTION regr_count returns INTEGER soname 'libregr_mysql.so'; * */ #ifndef HEADER_regr_count diff --git a/utils/regr/regr_intercept.h b/utils/regr/regr_intercept.h index ed82477cd..ef8dc6de5 100644 --- a/utils/regr/regr_intercept.h +++ b/utils/regr/regr_intercept.h @@ -25,8 +25,7 @@ * Columnstore interface for for the regr_intercept function * * - * CREATE AGGREGATE FUNCTION regr_intercept returns REAL - * soname 'libregr_mysql.so'; + * CREATE AGGREGATE FUNCTION regr_intercept returns REAL soname 'libregr_mysql.so'; * */ #ifndef HEADER_regr_intercept diff --git a/utils/regr/regr_r2.h b/utils/regr/regr_r2.h index d440ad5a1..968814067 100644 --- a/utils/regr/regr_r2.h +++ b/utils/regr/regr_r2.h @@ -25,8 +25,7 @@ * Columnstore interface for for the regr_r2 function * * - * CREATE AGGREGATE FUNCTION regr_r2 returns REAL - * soname 'libregr_mysql.so'; + * CREATE AGGREGATE FUNCTION regr_r2 returns REAL soname 'libregr_mysql.so'; * */ #ifndef HEADER_regr_r2 diff --git a/utils/regr/regr_slope.h b/utils/regr/regr_slope.h index 9c148d895..8a20494c1 100644 --- a/utils/regr/regr_slope.h +++ b/utils/regr/regr_slope.h @@ -25,8 +25,7 @@ * Columnstore interface for for the regr_slope function * * - * CREATE AGGREGATE FUNCTION regr_slope returns REAL - * soname 'libregr_mysql.so'; + * CREATE AGGREGATE FUNCTION regr_slope returns REAL soname 'libregr_mysql.so'; * */ #ifndef HEADER_regr_slope diff --git a/utils/regr/regr_sxx.cpp b/utils/regr/regr_sxx.cpp index 5769a227b..4d5fac370 100644 --- a/utils/regr/regr_sxx.cpp +++ b/utils/regr/regr_sxx.cpp @@ -132,7 +132,7 @@ mcsv1_UDAF::ReturnCode regr_sxx::evaluate(mcsv1Context* context, static_any::any long double sumx2 = data->sumx2; long double var_popx = (sumx2 - (sumx * sumx / N)) / N; - valOut = static_cast(data->cnt * var_popx); + valOut = static_cast(N * var_popx); } return mcsv1_UDAF::SUCCESS; } diff --git a/utils/regr/regr_sxx.h b/utils/regr/regr_sxx.h index 14d82bd55..53c771b6f 100644 --- a/utils/regr/regr_sxx.h +++ b/utils/regr/regr_sxx.h @@ -25,8 +25,7 @@ * Columnstore interface for for the regr_sxx function * * - * CREATE AGGREGATE FUNCTION regr_sxx returns REAL - * soname 'libregr_mysql.so'; + * CREATE AGGREGATE FUNCTION regr_sxx returns REAL soname 'libregr_mysql.so'; * */ #ifndef HEADER_regr_sxx diff --git a/utils/regr/regr_sxy.h b/utils/regr/regr_sxy.h index 25aa34145..6371c6fed 100644 --- a/utils/regr/regr_sxy.h +++ b/utils/regr/regr_sxy.h @@ -25,8 +25,7 @@ * Columnstore interface for for the regr_sxy function * * - * CREATE AGGREGATE FUNCTION regr_sxy returns REAL - * soname 'libregr_mysql.so'; + * CREATE AGGREGATE FUNCTION regr_sxy returns REAL soname 'libregr_mysql.so'; * */ #ifndef HEADER_regr_sxy diff --git a/utils/regr/regr_syy.cpp b/utils/regr/regr_syy.cpp index 014a28389..6febb9579 100644 --- a/utils/regr/regr_syy.cpp +++ b/utils/regr/regr_syy.cpp @@ -132,7 +132,7 @@ mcsv1_UDAF::ReturnCode regr_syy::evaluate(mcsv1Context* context, static_any::any long double sumy2 = data->sumy2; long double var_popy = (sumy2 - (sumy * sumy / N)) / N; - valOut = static_cast(data->cnt * var_popy); + valOut = static_cast(N * var_popy); } return mcsv1_UDAF::SUCCESS; } diff --git a/utils/regr/regr_syy.h b/utils/regr/regr_syy.h index a837fab13..d1a582f4d 100644 --- a/utils/regr/regr_syy.h +++ b/utils/regr/regr_syy.h @@ -25,8 +25,7 @@ * Columnstore interface for for the regr_syy function * * - * CREATE AGGREGATE FUNCTION regr_syy returns REAL - * soname 'libregr_mysql.so'; + * CREATE AGGREGATE FUNCTION regr_syy returns REAL soname 'libregr_mysql.so'; * */ #ifndef HEADER_regr_syy diff --git a/utils/regr/regrmysql.cpp b/utils/regr/regrmysql.cpp index 4980108e3..2570163f1 100644 --- a/utils/regr/regrmysql.cpp +++ b/utils/regr/regrmysql.cpp @@ -147,7 +147,7 @@ extern "C" */ struct regr_avgx_data { - double sumx; + long double sumx; int64_t cnt; }; @@ -159,21 +159,29 @@ extern "C" struct regr_avgx_data* data; if (args->arg_count != 2) { - strcpy(message,"regr_avgx() requires two arguments"); - return 1; + strcpy(message,"regr_avgx() requires two arguments"); + return 1; } if (!(isNumeric(args->arg_type[1], args->attributes[1]))) { strcpy(message,"regr_avgx() with a non-numeric independant (second) argument"); return 1; } + if (args->arg_type[1] == DECIMAL_RESULT && initid->decimals != DECIMAL_NOT_SPECIFIED) + { + initid->decimals += 4; + } + else + { + initid->decimals = DECIMAL_NOT_SPECIFIED; + } - if (!(data = (struct regr_avgx_data*) malloc(sizeof(struct regr_avgx_data)))) + if (!(data = (struct regr_avgx_data*) malloc(sizeof(struct regr_avgx_data)))) { - strmov(message,"Couldn't allocate memory"); - return 1; + strmov(message,"Couldn't allocate memory"); + return 1; } - data->sumx = 0; + data->sumx = 0; data->cnt = 0; initid->ptr = (char*)data; @@ -226,7 +234,17 @@ extern "C" char* is_null, char* error __attribute__((unused))) { struct regr_avgx_data* data = (struct regr_avgx_data*)initid->ptr; - return data->sumx / data->cnt; + double valOut = 0; + if (data->cnt > 0) + { + valOut = static_cast(data->sumx / data->cnt); + } + else + { + *is_null = 1; + } + + return valOut; } //======================================================================= @@ -236,8 +254,8 @@ extern "C" */ struct regr_avgy_data { - double sumy; - int64_t cnt; + long double sumy; + int64_t cnt; }; #ifdef _MSC_VER @@ -248,8 +266,8 @@ extern "C" struct regr_avgy_data* data; if (args->arg_count != 2) { - strcpy(message,"regr_avgy() requires two arguments"); - return 1; + strcpy(message,"regr_avgy() requires two arguments"); + return 1; } if (!(isNumeric(args->arg_type[0], args->attributes[0]))) { @@ -257,10 +275,19 @@ extern "C" return 1; } - if (!(data = (struct regr_avgy_data*) malloc(sizeof(struct regr_avgy_data)))) + if (args->arg_type[0] == DECIMAL_RESULT && initid->decimals != DECIMAL_NOT_SPECIFIED) + { + initid->decimals += 4; + } + else + { + initid->decimals = DECIMAL_NOT_SPECIFIED; + } + + if (!(data = (struct regr_avgy_data*) malloc(sizeof(struct regr_avgy_data)))) { - strmov(message,"Couldn't allocate memory"); - return 1; + strmov(message,"Couldn't allocate memory"); + return 1; } data->sumy = 0; data->cnt = 0; @@ -315,7 +342,16 @@ extern "C" char* is_null, char* error __attribute__((unused))) { struct regr_avgy_data* data = (struct regr_avgy_data*)initid->ptr; - return data->sumy / data->cnt; + double valOut = 0; + if (data->cnt > 0) + { + valOut = static_cast(data->sumy / data->cnt); + } + else + { + *is_null = 1; + } + return valOut; } //======================================================================= @@ -336,14 +372,14 @@ extern "C" struct regr_count_data* data; if (args->arg_count != 2) { - strcpy(message,"regr_count() requires two arguments"); - return 1; + strcpy(message,"regr_count() requires two arguments"); + return 1; } if (!(data = (struct regr_count_data*) malloc(sizeof(struct regr_count_data)))) { - strmov(message,"Couldn't allocate memory"); - return 1; + strmov(message,"Couldn't allocate memory"); + return 1; } data->cnt = 0; @@ -405,10 +441,10 @@ extern "C" struct regr_slope_data { int64_t cnt; - double sumx; - double sumx2; // sum of (x squared) - double sumy; - double sumxy; // sum of (x*y) + long double sumx; + long double sumx2; // sum of (x squared) + long double sumy; + long double sumxy; // sum of (x*y) }; #ifdef _MSC_VER @@ -419,8 +455,8 @@ extern "C" struct regr_slope_data* data; if (args->arg_count != 2) { - strcpy(message,"regr_slope() requires two arguments"); - return 1; + strcpy(message,"regr_slope() requires two arguments"); + return 1; } if (!(isNumeric(args->arg_type[0], args->attributes[0]) && isNumeric(args->arg_type[1], args->attributes[1]))) { @@ -428,10 +464,12 @@ extern "C" return 1; } + initid->decimals = DECIMAL_NOT_SPECIFIED; + if (!(data = (struct regr_slope_data*) malloc(sizeof(struct regr_slope_data)))) { - strmov(message,"Couldn't allocate memory"); - return 1; + strmov(message,"Couldn't allocate memory"); + return 1; } data->cnt = 0; data->sumx = 0.0; @@ -497,20 +535,23 @@ extern "C" { struct regr_slope_data* data = (struct regr_slope_data*)initid->ptr; double N = data->cnt; + double valOut = 0; + *is_null = 1; if (N > 0) { - double sumx = data->sumx; - double sumy = data->sumy; - double sumx2 = data->sumx2; - double sumxy = data->sumxy; - double variance = (N * sumx2) - (sumx * sumx); - if (variance) + long double sumx = data->sumx; + long double sumy = data->sumy; + long double sumx2 = data->sumx2; + long double sumxy = data->sumxy; + long double covar_pop = N * sumxy - sumx * sumy; + long double var_pop = N * sumx2 - sumx * sumx; + if (var_pop != 0) { - return ((N * sumxy) - (sumx * sumy)) / variance; + valOut = static_cast(covar_pop / var_pop); + *is_null = 0; } } - *is_null = 1; - return 0; + return valOut; } //======================================================================= @@ -521,10 +562,10 @@ extern "C" struct regr_intercept_data { int64_t cnt; - double sumx; - double sumx2; // sum of (x squared) - double sumy; - double sumxy; // sum of (x*y) + long double sumx; + long double sumx2; // sum of (x squared) + long double sumy; + long double sumxy; // sum of (x*y) }; #ifdef _MSC_VER @@ -535,8 +576,8 @@ extern "C" struct regr_intercept_data* data; if (args->arg_count != 2) { - strcpy(message,"regr_intercept() requires two arguments"); - return 1; + strcpy(message,"regr_intercept() requires two arguments"); + return 1; } if (!(isNumeric(args->arg_type[0], args->attributes[0]) && isNumeric(args->arg_type[1], args->attributes[1]))) { @@ -544,10 +585,11 @@ extern "C" return 1; } - if (!(data = (struct regr_intercept_data*) malloc(sizeof(struct regr_intercept_data)))) + initid->decimals = DECIMAL_NOT_SPECIFIED; + if (!(data = (struct regr_intercept_data*) malloc(sizeof(struct regr_intercept_data)))) { - strmov(message,"Couldn't allocate memory"); - return 1; + strmov(message,"Couldn't allocate memory"); + return 1; } data->cnt = 0; data->sumx = 0.0; @@ -613,22 +655,23 @@ extern "C" { struct regr_intercept_data* data = (struct regr_intercept_data*)initid->ptr; double N = data->cnt; + double valOut = 0; + *is_null = 1; if (N > 0) { - double sumx = data->sumx; - double sumy = data->sumy; - double sumx2 = data->sumx2; - double sumxy = data->sumxy; - double slope = 0; - double variance = (N * sumx2) - (sumx * sumx); - if (variance) + long double sumx = data->sumx; + long double sumy = data->sumy; + long double sumx2 = data->sumx2; + long double sumxy = data->sumxy; + long double numerator = sumy * sumx2 - sumx * sumxy; + long double var_pop = (N * sumx2) - (sumx * sumx); + if (var_pop != 0) { - slope = ((N * sumxy) - (sumx * sumy)) / variance; + valOut = static_cast(numerator / var_pop); + *is_null = 0; } - return (sumy - (slope * sumx)) / N; } - *is_null = 1; - return 0; + return valOut; } //======================================================================= @@ -639,11 +682,11 @@ extern "C" struct regr_r2_data { int64_t cnt; - double sumx; - double sumx2; // sum of (x squared) - double sumy; - double sumy2; // sum of (y squared) - double sumxy; // sum of (x*y) + long double sumx; + long double sumx2; // sum of (x squared) + long double sumy; + long double sumy2; // sum of (y squared) + long double sumxy; // sum of (x*y) }; #ifdef _MSC_VER @@ -654,8 +697,8 @@ extern "C" struct regr_r2_data* data; if (args->arg_count != 2) { - strcpy(message,"regr_r2() requires two arguments"); - return 1; + strcpy(message,"regr_r2() requires two arguments"); + return 1; } if (!(isNumeric(args->arg_type[0], args->attributes[0]) && isNumeric(args->arg_type[1], args->attributes[1]))) { @@ -663,10 +706,12 @@ extern "C" return 1; } - if (!(data = (struct regr_r2_data*) malloc(sizeof(struct regr_r2_data)))) + initid->decimals = DECIMAL_NOT_SPECIFIED; + + if (!(data = (struct regr_r2_data*) malloc(sizeof(struct regr_r2_data)))) { - strmov(message,"Couldn't allocate memory"); - return 1; + strmov(message,"Couldn't allocate memory"); + return 1; } data->cnt = 0; data->sumx = 0.0; @@ -735,34 +780,38 @@ extern "C" { struct regr_r2_data* data = (struct regr_r2_data*)initid->ptr; double N = data->cnt; + double valOut = 0; if (N > 0) { - double sumx = data->sumx; - double sumy = data->sumy; - double sumx2 = data->sumx2; - double sumy2 = data->sumy2; - double sumxy = data->sumxy; - double var_popx = (sumx2 - (sumx * sumx / N)) / N; + long double sumx = data->sumx; + long double sumy = data->sumy; + long double sumx2 = data->sumx2; + long double sumy2 = data->sumy2; + long double sumxy = data->sumxy; + long double var_popx = (sumx2 - (sumx * sumx / N)) / N; if (var_popx == 0) { // When var_popx is 0, NULL is the result. *is_null = 1; return 0; } - double var_popy = (sumy2 - (sumy * sumy / N)) / N; + long double var_popy = (sumy2 - (sumy * sumy / N)) / N; if (var_popy == 0) { // When var_popy is 0, 1 is the result return 1; } - double std_popx = sqrt(var_popx); - double std_popy = sqrt(var_popy); - double covar_pop = (sumxy - ((sumx * sumy) / N)) / N; - double corr = covar_pop / (std_popy * std_popx); - return corr * corr; + long double std_popx = sqrt(var_popx); + long double std_popy = sqrt(var_popy); + long double covar_pop = (sumxy - ((sumx * sumy) / N)) / N; + long double corr = covar_pop / (std_popy * std_popx); + valOut = static_cast(corr * corr); } - *is_null = 1; - return 0; + else + { + *is_null = 1; + } + return valOut; } //======================================================================= @@ -773,11 +822,11 @@ extern "C" struct corr_data { int64_t cnt; - double sumx; - double sumx2; // sum of (x squared) - double sumy; - double sumy2; // sum of (y squared) - double sumxy; // sum of (x*y) + long double sumx; + long double sumx2; // sum of (x squared) + long double sumy; + long double sumy2; // sum of (y squared) + long double sumxy; // sum of (x*y) }; #ifdef _MSC_VER @@ -788,8 +837,8 @@ extern "C" struct corr_data* data; if (args->arg_count != 2) { - strcpy(message,"corr() requires two arguments"); - return 1; + strcpy(message,"corr() requires two arguments"); + return 1; } if (!(isNumeric(args->arg_type[0], args->attributes[0]) && isNumeric(args->arg_type[1], args->attributes[1]))) { @@ -797,10 +846,12 @@ extern "C" return 1; } - if (!(data = (struct corr_data*) malloc(sizeof(struct corr_data)))) + initid->decimals = DECIMAL_NOT_SPECIFIED; + + if (!(data = (struct corr_data*) malloc(sizeof(struct corr_data)))) { - strmov(message,"Couldn't allocate memory"); - return 1; + strmov(message,"Couldn't allocate memory"); + return 1; } data->cnt = 0; data->sumx = 0.0; @@ -869,34 +920,38 @@ extern "C" { struct corr_data* data = (struct corr_data*)initid->ptr; double N = data->cnt; + double valOut = 0; if (N > 0) { - double sumx = data->sumx; - double sumy = data->sumy; - double sumx2 = data->sumx2; - double sumy2 = data->sumy2; - double sumxy = data->sumxy; - double var_popx = (sumx2 - (sumx * sumx / N)) / N; + long double sumx = data->sumx; + long double sumy = data->sumy; + long double sumx2 = data->sumx2; + long double sumy2 = data->sumy2; + long double sumxy = data->sumxy; + long double var_popx = (sumx2 - (sumx * sumx / N)) / N; if (var_popx == 0) { // When var_popx is 0, NULL is the result. *is_null = 1; return 0; } - double var_popy = (sumy2 - (sumy * sumy / N)) / N; + long double var_popy = (sumy2 - (sumy * sumy / N)) / N; if (var_popy == 0) { // When var_popy is 0, 1 is the result return 1; } - double std_popx = sqrt(var_popx); - double std_popy = sqrt(var_popy); - double covar_pop = (sumxy - ((sumx * sumy) / N)) / N; - double corr = covar_pop / (std_popy * std_popx); - return corr; + long double std_popx = sqrt(var_popx); + long double std_popy = sqrt(var_popy); + long double covar_pop = (sumxy - ((sumx * sumy) / N)) / N; + long double corr = covar_pop / (std_popy * std_popx); + return static_cast(corr); } - *is_null = 1; - return 0; + else + { + *is_null = 1; + } + return valOut; } //======================================================================= @@ -907,8 +962,8 @@ extern "C" struct regr_sxx_data { int64_t cnt; - double sumx; - double sumx2; // sum of (x squared) + long double sumx; + long double sumx2; // sum of (x squared) }; #ifdef _MSC_VER @@ -919,8 +974,8 @@ extern "C" struct regr_sxx_data* data; if (args->arg_count != 2) { - strcpy(message,"regr_sxx() requires two arguments"); - return 1; + strcpy(message,"regr_sxx() requires two arguments"); + return 1; } if (!(isNumeric(args->arg_type[1], args->attributes[1]))) { @@ -928,10 +983,12 @@ extern "C" return 1; } - if (!(data = (struct regr_sxx_data*) malloc(sizeof(struct regr_sxx_data)))) + initid->decimals = DECIMAL_NOT_SPECIFIED; + + if (!(data = (struct regr_sxx_data*) malloc(sizeof(struct regr_sxx_data)))) { - strmov(message,"Couldn't allocate memory"); - return 1; + strmov(message,"Couldn't allocate memory"); + return 1; } data->cnt = 0; data->sumx = 0.0; @@ -990,15 +1047,19 @@ extern "C" { struct regr_sxx_data* data = (struct regr_sxx_data*)initid->ptr; double N = data->cnt; + double valOut = 0; if (N > 0) { - double sumx = data->sumx; - double sumx2 = data->sumx2; - double var_popx = (sumx2 - (sumx * sumx / N)) / N; - return data->cnt * var_popx; + long double sumx = data->sumx; + long double sumx2 = data->sumx2; + long double var_popx = (sumx2 - (sumx * sumx / N)) / N; + valOut = static_cast(N * var_popx); } - *is_null = 1; - return 0; + else + { + *is_null = 1; + } + return valOut; } //======================================================================= @@ -1008,8 +1069,8 @@ extern "C" struct regr_syy_data { int64_t cnt; - double sumy; - double sumy2; // sum of (y squared) + long double sumy; + long double sumy2; // sum of (y squared) }; #ifdef _MSC_VER @@ -1020,8 +1081,8 @@ extern "C" struct regr_syy_data* data; if (args->arg_count != 2) { - strcpy(message,"regr_syy() requires two arguments"); - return 1; + strcpy(message,"regr_syy() requires two arguments"); + return 1; } if (!(isNumeric(args->arg_type[0], args->attributes[0]))) { @@ -1029,10 +1090,12 @@ extern "C" return 1; } - if (!(data = (struct regr_syy_data*) malloc(sizeof(struct regr_syy_data)))) + initid->decimals = DECIMAL_NOT_SPECIFIED; + + if (!(data = (struct regr_syy_data*) malloc(sizeof(struct regr_syy_data)))) { - strmov(message,"Couldn't allocate memory"); - return 1; + strmov(message,"Couldn't allocate memory"); + return 1; } data->cnt = 0; data->sumy = 0.0; @@ -1091,15 +1154,19 @@ extern "C" { struct regr_syy_data* data = (struct regr_syy_data*)initid->ptr; double N = data->cnt; + double valOut = 0; if (N > 0) { - double sumy = data->sumy; - double sumy2 = data->sumy2; - double var_popy = (sumy2 - (sumy * sumy / N)) / N; - return data->cnt * var_popy; + long double sumy = data->sumy; + long double sumy2 = data->sumy2; + long double var_popy = (sumy2 - (sumy * sumy / N)) / N; + valOut = static_cast(N * var_popy); } - *is_null = 1; - return 0; + else + { + *is_null = 1; + } + return valOut; } //======================================================================= @@ -1110,9 +1177,9 @@ extern "C" struct regr_sxy_data { int64_t cnt; - double sumx; - double sumy; - double sumxy; // sum of (x*y) + long double sumx; + long double sumy; + long double sumxy; // sum of (x*y) }; #ifdef _MSC_VER @@ -1123,8 +1190,8 @@ extern "C" struct regr_sxy_data* data; if (args->arg_count != 2) { - strcpy(message,"regr_sxy() requires two arguments"); - return 1; + strcpy(message,"regr_sxy() requires two arguments"); + return 1; } if (!(isNumeric(args->arg_type[0], args->attributes[0]) && isNumeric(args->arg_type[1], args->attributes[1]))) { @@ -1132,10 +1199,12 @@ extern "C" return 1; } - if (!(data = (struct regr_sxy_data*) malloc(sizeof(struct regr_sxy_data)))) + initid->decimals = DECIMAL_NOT_SPECIFIED; + + if (!(data = (struct regr_sxy_data*) malloc(sizeof(struct regr_sxy_data)))) { - strmov(message,"Couldn't allocate memory"); - return 1; + strmov(message,"Couldn't allocate memory"); + return 1; } data->cnt = 0; data->sumx = 0.0; @@ -1198,16 +1267,21 @@ extern "C" { struct regr_sxy_data* data = (struct regr_sxy_data*)initid->ptr; double N = data->cnt; + double valOut = 0; if (N > 0) { - double sumx = data->sumx; - double sumy = data->sumy; - double sumxy = data->sumxy; - double covar_pop = (sumxy - ((sumx * sumy) / N)) / N; - return data->cnt * covar_pop; + long double sumx = data->sumx; + long double sumy = data->sumy; + long double sumxy = data->sumxy; + long double covar_pop = (sumxy - ((sumx * sumy) / N)) / N; + long double regr_sxy = N * covar_pop; + valOut = static_cast(regr_sxy); } - *is_null = 1; - return 0; + else + { + *is_null = 1; + } + return valOut; } //======================================================================= @@ -1218,9 +1292,9 @@ extern "C" struct covar_pop_data { int64_t cnt; - double sumx; - double sumy; - double sumxy; // sum of (x*y) + long double sumx; + long double sumy; + long double sumxy; // sum of (x*y) }; #ifdef _MSC_VER @@ -1231,8 +1305,8 @@ extern "C" struct covar_pop_data* data; if (args->arg_count != 2) { - strcpy(message,"covar_pop() requires two arguments"); - return 1; + strcpy(message,"covar_pop() requires two arguments"); + return 1; } if (!(isNumeric(args->arg_type[0], args->attributes[0]) && isNumeric(args->arg_type[1], args->attributes[1]))) { @@ -1240,10 +1314,12 @@ extern "C" return 1; } - if (!(data = (struct covar_pop_data*) malloc(sizeof(struct covar_pop_data)))) + initid->decimals = DECIMAL_NOT_SPECIFIED; + + if (!(data = (struct covar_pop_data*) malloc(sizeof(struct covar_pop_data)))) { - strmov(message,"Couldn't allocate memory"); - return 1; + strmov(message,"Couldn't allocate memory"); + return 1; } data->cnt = 0; data->sumx = 0.0; @@ -1306,16 +1382,20 @@ extern "C" { struct covar_pop_data* data = (struct covar_pop_data*)initid->ptr; double N = data->cnt; + double valOut = 0; if (N > 0) { - double sumx = data->sumx; - double sumy = data->sumy; - double sumxy = data->sumxy; - double covar_pop = (sumxy - ((sumx * sumy) / N)) / N; - return covar_pop; + long double sumx = data->sumx; + long double sumy = data->sumy; + long double sumxy = data->sumxy; + long double covar_pop = (sumxy - ((sumx * sumy) / N)) / N ; + valOut = static_cast(covar_pop); } - *is_null = 1; - return 0; + else + { + *is_null = 1; + } + return valOut; } //======================================================================= @@ -1325,9 +1405,9 @@ extern "C" struct covar_samp_data { int64_t cnt; - double sumx; - double sumy; - double sumxy; // sum of (x*y) + long double sumx; + long double sumy; + long double sumxy; // sum of (x*y) }; #ifdef _MSC_VER @@ -1338,8 +1418,8 @@ extern "C" struct covar_samp_data* data; if (args->arg_count != 2) { - strcpy(message,"covar_samp() requires two arguments"); - return 1; + strcpy(message,"covar_samp() requires two arguments"); + return 1; } if (!(isNumeric(args->arg_type[0], args->attributes[0]) && isNumeric(args->arg_type[1], args->attributes[1]))) { @@ -1347,10 +1427,12 @@ extern "C" return 1; } - if (!(data = (struct covar_samp_data*) malloc(sizeof(struct covar_samp_data)))) + initid->decimals = DECIMAL_NOT_SPECIFIED; + + if (!(data = (struct covar_samp_data*) malloc(sizeof(struct covar_samp_data)))) { - strmov(message,"Couldn't allocate memory"); - return 1; + strmov(message,"Couldn't allocate memory"); + return 1; } data->cnt = 0; data->sumx = 0.0; @@ -1413,16 +1495,20 @@ extern "C" { struct covar_samp_data* data = (struct covar_samp_data*)initid->ptr; double N = data->cnt; + double valOut = 0; if (N > 0) { - double sumx = data->sumx; - double sumy = data->sumy; - double sumxy = data->sumxy; - double covar_samp = (sumxy - ((sumx * sumy) / N)) / (N-1); - return covar_samp; + long double sumx = data->sumx; + long double sumy = data->sumy; + long double sumxy = data->sumxy; + long double covar_samp = (sumxy - ((sumx * sumy) / N)) / (N - 1); + valOut = static_cast(covar_samp); } - *is_null = 1; - return 0; + else + { + *is_null = 1; + } + return valOut; } } // vim:ts=4 sw=4: diff --git a/utils/rowgroup/rowaggregation.cpp b/utils/rowgroup/rowaggregation.cpp index 4bfbf87b6..55be66459 100644 --- a/utils/rowgroup/rowaggregation.cpp +++ b/utils/rowgroup/rowaggregation.cpp @@ -218,7 +218,6 @@ inline string getStringNullValue() namespace rowgroup { - const std::string typeStr(""); const static_any::any& RowAggregation::charTypeId((char)1); const static_any::any& RowAggregation::scharTypeId((signed char)1); @@ -590,7 +589,8 @@ inline bool RowAggregation::isNull(const RowGroup* pRowGroup, const Row& row, in RowAggregation::RowAggregation() : fAggMapPtr(NULL), fRowGroupOut(NULL), fTotalRowCount(0), fMaxTotalRowCount(AGG_ROWGROUP_SIZE), - fSmallSideRGs(NULL), fLargeSideRG(NULL), fSmallSideCount(0) + fSmallSideRGs(NULL), fLargeSideRG(NULL), fSmallSideCount(0), + fOrigFunctionCols(NULL) { } @@ -599,7 +599,8 @@ RowAggregation::RowAggregation(const vector& rowAggGroupByCol const vector& rowAggFunctionCols) : fAggMapPtr(NULL), fRowGroupOut(NULL), fTotalRowCount(0), fMaxTotalRowCount(AGG_ROWGROUP_SIZE), - fSmallSideRGs(NULL), fLargeSideRG(NULL), fSmallSideCount(0) + fSmallSideRGs(NULL), fLargeSideRG(NULL), fSmallSideCount(0), + fOrigFunctionCols(NULL) { fGroupByCols.assign(rowAggGroupByCols.begin(), rowAggGroupByCols.end()); fFunctionCols.assign(rowAggFunctionCols.begin(), rowAggFunctionCols.end()); @@ -610,7 +611,7 @@ RowAggregation::RowAggregation(const RowAggregation& rhs): fAggMapPtr(NULL), fRowGroupOut(NULL), fTotalRowCount(0), fMaxTotalRowCount(AGG_ROWGROUP_SIZE), fSmallSideRGs(NULL), fLargeSideRG(NULL), fSmallSideCount(0), - fRGContext(rhs.fRGContext) + fRGContext(rhs.fRGContext), fOrigFunctionCols(NULL) { //fGroupByCols.clear(); //fFunctionCols.clear(); @@ -714,11 +715,8 @@ void RowAggregation::setJoinRowGroups(vector* pSmallSideRG, RowGroup* // threads on the PM and by multple threads on the UM. It must remain // thread safe. //------------------------------------------------------------------------------ -void RowAggregation::resetUDAF(uint64_t funcColID) +void RowAggregation::resetUDAF(RowUDAFFunctionCol* rowUDAF) { - // Get the UDAF class pointer and store in the row definition object. - RowUDAFFunctionCol* rowUDAF = dynamic_cast(fFunctionCols[funcColID].get()); - // RowAggregation and it's functions need to be re-entrant which means // each instance (thread) needs its own copy of the context object. // Note: operator=() doesn't copy userData. @@ -786,7 +784,7 @@ void RowAggregation::initialize() { if (fFunctionCols[i]->fAggFunction == ROWAGG_UDAF) { - resetUDAF(i); + resetUDAF(dynamic_cast(fFunctionCols[i].get())); } } } @@ -838,7 +836,7 @@ void RowAggregation::aggReset() { if (fFunctionCols[i]->fAggFunction == ROWAGG_UDAF) { - resetUDAF(i); + resetUDAF(dynamic_cast(fFunctionCols[i].get())); } } } @@ -885,14 +883,28 @@ void RowAggregationUM::aggregateRowWithRemap(Row& row) inserted.first->second = RowPosition(fResultDataVec.size() - 1, fRowGroupOut->getRowCount() - 1); // If there's UDAF involved, reset the user data. - for (uint64_t i = 0; i < fFunctionCols.size(); i++) + if (fOrigFunctionCols) { - if (fFunctionCols[i]->fAggFunction == ROWAGG_UDAF) + // This is a multi-distinct query and fFunctionCols may not + // contain all the UDAF we need to reset + for (uint64_t i = 0; i < fOrigFunctionCols->size(); i++) { - resetUDAF(i); + if ((*fOrigFunctionCols)[i]->fAggFunction == ROWAGG_UDAF) + { + resetUDAF(dynamic_cast((*fOrigFunctionCols)[i].get())); + } + } + } + else + { + for (uint64_t i = 0; i < fFunctionCols.size(); i++) + { + if (fFunctionCols[i]->fAggFunction == ROWAGG_UDAF) + { + resetUDAF(dynamic_cast(fFunctionCols[i].get())); + } } } - // replace the key value with an equivalent copy, yes this is OK const_cast((inserted.first->first)) = pos; } @@ -946,14 +958,28 @@ void RowAggregation::aggregateRow(Row& row) RowPosition(fResultDataVec.size() - 1, fRowGroupOut->getRowCount() - 1); // If there's UDAF involved, reset the user data. - for (uint64_t i = 0; i < fFunctionCols.size(); i++) + if (fOrigFunctionCols) { - if (fFunctionCols[i]->fAggFunction == ROWAGG_UDAF) + // This is a multi-distinct query and fFunctionCols may not + // contain all the UDAF we need to reset + for (uint64_t i = 0; i < fOrigFunctionCols->size(); i++) { - resetUDAF(i); + if ((*fOrigFunctionCols)[i]->fAggFunction == ROWAGG_UDAF) + { + resetUDAF(dynamic_cast((*fOrigFunctionCols)[i].get())); + } + } + } + else + { + for (uint64_t i = 0; i < fFunctionCols.size(); i++) + { + if (fFunctionCols[i]->fAggFunction == ROWAGG_UDAF) + { + resetUDAF(dynamic_cast(fFunctionCols[i].get())); + } } } - } else { @@ -4699,7 +4725,7 @@ void RowAggregationMultiDistinct::doDistinctAggregation() { // backup the function column vector for finalize(). vector origFunctionCols = fFunctionCols; - + fOrigFunctionCols = &origFunctionCols; // aggregate data from each sub-aggregator to distinct aggregator for (uint64_t i = 0; i < fSubAggregators.size(); ++i) { @@ -4727,6 +4753,7 @@ void RowAggregationMultiDistinct::doDistinctAggregation() // restore the function column vector fFunctionCols = origFunctionCols; + fOrigFunctionCols = NULL; } @@ -4734,7 +4761,8 @@ void RowAggregationMultiDistinct::doDistinctAggregation_rowVec(vector origFunctionCols = fFunctionCols; - + fOrigFunctionCols = &origFunctionCols; + // aggregate data from each sub-aggregator to distinct aggregator for (uint64_t i = 0; i < fSubAggregators.size(); ++i) { @@ -4751,9 +4779,9 @@ void RowAggregationMultiDistinct::doDistinctAggregation_rowVec(vectorclear(); } - void resetUDAF(uint64_t funcColID); + void resetUDAF(RowUDAFFunctionCol* rowUDAF); inline bool isNull(const RowGroup* pRowGroup, const Row& row, int64_t col); inline void makeAggFieldsNull(Row& row); @@ -710,6 +710,9 @@ protected: static const static_any::any& doubleTypeId; static const static_any::any& longdoubleTypeId; static const static_any::any& strTypeId; + + // For UDAF along with with multiple distinct columns + vector* fOrigFunctionCols; }; //------------------------------------------------------------------------------ diff --git a/utils/rowgroup/rowgroup.h b/utils/rowgroup/rowgroup.h index 27f0b98a4..b91e0f0ef 100644 --- a/utils/rowgroup/rowgroup.h +++ b/utils/rowgroup/rowgroup.h @@ -1176,7 +1176,12 @@ inline bool Row::equals(const Row& r2, const std::vector& keyCols) con if (!isLongString(col)) { - if (getUintField(col) != r2.getUintField(col)) + if (getColType(i) == execplan::CalpontSystemCatalog::LONGDOUBLE) + { + if (getLongDoubleField(i) != r2.getLongDoubleField(i)) + return false; + } + else if (getUintField(col) != r2.getUintField(col)) return false; } else @@ -1204,7 +1209,12 @@ inline bool Row::equals(const Row& r2, uint32_t lastCol) const for (uint32_t i = 0; i <= lastCol; i++) if (!isLongString(i)) { - if (getUintField(i) != r2.getUintField(i)) + if (getColType(i) == execplan::CalpontSystemCatalog::LONGDOUBLE) + { + if (getLongDoubleField(i) != r2.getLongDoubleField(i)) + return false; + } + else if (getUintField(i) != r2.getUintField(i)) return false; } else diff --git a/writeengine/bulk/we_bulkload.h b/writeengine/bulk/we_bulkload.h index c4aecdbc2..4c1adab09 100644 --- a/writeengine/bulk/we_bulkload.h +++ b/writeengine/bulk/we_bulkload.h @@ -64,7 +64,7 @@ class BulkLoad : public FileOp public: /** - * @brief BulkLoad onstructor + * @brief BulkLoad constructor */ EXPORT BulkLoad(); diff --git a/writeengine/bulk/we_bulkloadbuffer.cpp b/writeengine/bulk/we_bulkloadbuffer.cpp index f0336d79d..01d6ed575 100644 --- a/writeengine/bulk/we_bulkloadbuffer.cpp +++ b/writeengine/bulk/we_bulkloadbuffer.cpp @@ -443,7 +443,7 @@ void BulkLoadBuffer::convert(char* field, int fieldLength, { memcpy(&dVal, field, sizeof(dVal)); - if ( isnan(dVal) ) + if ( std::isnan(dVal) ) { if (signbit(dVal)) dVal = column.fMinDblSat; diff --git a/writeengine/bulk/we_colbuf.cpp b/writeengine/bulk/we_colbuf.cpp index 1d164858c..da25704af 100644 --- a/writeengine/bulk/we_colbuf.cpp +++ b/writeengine/bulk/we_colbuf.cpp @@ -29,6 +29,7 @@ #include "we_columninfo.h" #include "we_log.h" #include "we_stats.h" +#include "we_blockop.h" #include #include "IDBDataFile.h" using namespace idbdatafile; @@ -100,24 +101,47 @@ void ColumnBuffer::resizeAndCopy(int newSize, int startOffset, int endOffset) //------------------------------------------------------------------------------ // Write data stored up in the output buffer to the segment column file. +// fillUpWEmpties is set when CS finishes with writing to add extra empty +// magics to fill up the block to its boundary. //------------------------------------------------------------------------------ -int ColumnBuffer::writeToFile(int startOffset, int writeSize) +int ColumnBuffer::writeToFile(int startOffset, int writeSize, bool fillUpWEmpties) { if (writeSize == 0) // skip unnecessary write, if 0 bytes given return NO_ERROR; + unsigned char *newBuf = NULL; + + if ( fillUpWEmpties ) + { + BlockOp blockOp; + newBuf = new unsigned char[BYTE_PER_BLOCK]; + uint64_t EmptyValue = blockOp.getEmptyRowValue(fColInfo->column.dataType, + fColInfo->column.width); + ::memcpy(static_cast(newBuf), + static_cast(fBuffer + startOffset), writeSize); + blockOp.setEmptyBuf(newBuf + writeSize, BYTE_PER_BLOCK - writeSize, + EmptyValue, fColInfo->column.width); + } #ifdef PROFILE Stats::startParseEvent(WE_STATS_WRITE_COL); #endif - size_t nitems = fFile->write(fBuffer + startOffset, writeSize) / writeSize; + size_t nitems; + if ( fillUpWEmpties ) + nitems = fFile->write(newBuf, BYTE_PER_BLOCK) / BYTE_PER_BLOCK; + else + nitems = fFile->write(fBuffer + startOffset, writeSize) / writeSize; if (nitems != 1) + { + delete newBuf; return ERR_FILE_WRITE; + } #ifdef PROFILE Stats::stopParseEvent(WE_STATS_WRITE_COL); #endif + delete newBuf; return NO_ERROR; } diff --git a/writeengine/bulk/we_colbuf.h b/writeengine/bulk/we_colbuf.h index d5f3af570..5416b4373 100644 --- a/writeengine/bulk/we_colbuf.h +++ b/writeengine/bulk/we_colbuf.h @@ -107,8 +107,11 @@ public: * * @param startOffset The buffer offset from where the write should begin * @param writeSize The number of bytes to be written to the file + * @param fillUpWEmpties The flag to fill the buffer with empty magic values + * up to the block boundary. */ - virtual int writeToFile(int startOffset, int writeSize); + virtual int writeToFile(int startOffset, int writeSize, + bool fillUpWEmpties = false); protected: diff --git a/writeengine/bulk/we_colbufcompressed.cpp b/writeengine/bulk/we_colbufcompressed.cpp index 9abc0038a..43534dc75 100644 --- a/writeengine/bulk/we_colbufcompressed.cpp +++ b/writeengine/bulk/we_colbufcompressed.cpp @@ -167,11 +167,16 @@ int ColumnBufferCompressed::resetToBeCompressedColBuf( // file, and instead buffer up the data to be compressed in 4M chunks before // writing it out. //------------------------------------------------------------------------------ -int ColumnBufferCompressed::writeToFile(int startOffset, int writeSize) +int ColumnBufferCompressed::writeToFile(int startOffset, int writeSize, + bool fillUpWEmpties) { if (writeSize == 0) // skip unnecessary write, if 0 bytes given return NO_ERROR; + int fillUpWEmptiesWriteSize = 0; + if (fillUpWEmpties) + fillUpWEmptiesWriteSize = BYTE_PER_BLOCK - writeSize % BYTE_PER_BLOCK; + // If we are starting a new file, we need to reinit the buffer and // find out what our file offset should be set to. if (!fToBeCompressedCapacity) @@ -219,7 +224,7 @@ int ColumnBufferCompressed::writeToFile(int startOffset, int writeSize) // Expand the compression buffer size if working with an abbrev extent, and // the bytes we are about to add will overflow the abbreviated extent. if ((fToBeCompressedCapacity < IDBCompressInterface::UNCOMPRESSED_INBUF_LEN) && - ((fNumBytes + writeSize) > fToBeCompressedCapacity) ) + ((fNumBytes + writeSize + fillUpWEmptiesWriteSize) > fToBeCompressedCapacity) ) { std::ostringstream oss; oss << "Expanding abbrev to-be-compressed buffer for: OID-" << @@ -231,7 +236,7 @@ int ColumnBufferCompressed::writeToFile(int startOffset, int writeSize) fToBeCompressedCapacity = IDBCompressInterface::UNCOMPRESSED_INBUF_LEN; } - if ((fNumBytes + writeSize) <= fToBeCompressedCapacity) + if ((fNumBytes + writeSize + fillUpWEmptiesWriteSize) <= fToBeCompressedCapacity) { if (fLog->isDebug( DEBUG_2 )) { @@ -242,12 +247,14 @@ int ColumnBufferCompressed::writeToFile(int startOffset, int writeSize) "; part-" << fColInfo->curCol.dataFile.fPartition << "; seg-" << fColInfo->curCol.dataFile.fSegment << "; addBytes-" << writeSize << + "; extraBytes-" << fillUpWEmptiesWriteSize << "; totBytes-" << (fNumBytes + writeSize); fLog->logMsg( oss.str(), MSGLVL_INFO2 ); } memcpy(bufOffset, (fBuffer + startOffset), writeSize); fNumBytes += writeSize; + fNumBytes += fillUpWEmptiesWriteSize; } else // Not enough room to add all the data to the to-be-compressed buffer { @@ -338,6 +345,7 @@ int ColumnBufferCompressed::writeToFile(int startOffset, int writeSize) memcpy(bufOffset, (fBuffer + startOffsetX), writeSizeOut); fNumBytes += writeSizeOut; + fNumBytes += fillUpWEmptiesWriteSize; } startOffsetX += writeSizeOut; @@ -347,7 +355,6 @@ int ColumnBufferCompressed::writeToFile(int startOffset, int writeSize) return NO_ERROR; } - //------------------------------------------------------------------------------ // Compress and write out the data in the to-be-compressed buffer. // Also may write out the compression header. diff --git a/writeengine/bulk/we_colbufcompressed.h b/writeengine/bulk/we_colbufcompressed.h index 6ea70339c..5c4cccffc 100644 --- a/writeengine/bulk/we_colbufcompressed.h +++ b/writeengine/bulk/we_colbufcompressed.h @@ -83,8 +83,11 @@ public: * * @param startOffset The buffer offset from where the write should begin * @param writeSize The number of bytes to be written to the file + * @param fillUpWEmpties The flag to fill the buffer with empty magic + * values up to the block boundary. */ - virtual int writeToFile(int startOffset, int writeSize); + virtual int writeToFile(int startOffset, int writeSize, + bool fillUpWEmpties = false); private: diff --git a/writeengine/bulk/we_colbufmgr.cpp b/writeengine/bulk/we_colbufmgr.cpp index 123847260..36f3e5493 100644 --- a/writeengine/bulk/we_colbufmgr.cpp +++ b/writeengine/bulk/we_colbufmgr.cpp @@ -45,7 +45,7 @@ namespace { // Minimum time to wait for a condition, so as to periodically wake up and // check the global job status, to see if the job needs to terminate. -const int COND_WAIT_SECONDS = 3; +const int COND_WAIT_SECONDS = 1; } namespace WriteEngine @@ -521,7 +521,9 @@ int ColumnBufferManager::writeToFile(int endOffset) // and the remaining buffer data will be written to the next segment file in // the DBRoot, partition, segement number sequence. // This function also catches and handles the case where an abbreviated -// extent needs to be expanded to a full extent on disk. +// extent needs to be expanded to a full extent on disk. When fillUpWEmpties is +// set then CS finishes with writing and has to fill with magics this block +// up to its boundary. // // WARNING: This means this function may change the information in the // ColumnInfo struct that owns this ColumnBufferManager, if a @@ -529,7 +531,7 @@ int ColumnBufferManager::writeToFile(int endOffset) // internal buffer, or if an abbreviated extent is expanded. //------------------------------------------------------------------------------ int ColumnBufferManager::writeToFileExtentCheck( - uint32_t startOffset, uint32_t writeSize) + uint32_t startOffset, uint32_t writeSize, bool fillUpWEmpties) { if (fLog->isDebug( DEBUG_3 )) @@ -571,7 +573,7 @@ int ColumnBufferManager::writeToFileExtentCheck( if (availableFileSize >= writeSize) { - int rc = fCBuf->writeToFile(startOffset, writeSize); + int rc = fCBuf->writeToFile(startOffset, writeSize, fillUpWEmpties); if (rc != NO_ERROR) { @@ -583,6 +585,12 @@ int ColumnBufferManager::writeToFileExtentCheck( return rc; } + // MCOL-498 Fill this block up to its boundary. + if ( fillUpWEmpties ) + { + writeSize = BLOCK_SIZE; + } + fColInfo->updateBytesWrittenCounts( writeSize ); } else @@ -624,7 +632,7 @@ int ColumnBufferManager::writeToFileExtentCheck( } int writeSize2 = writeSize - writeSize1; - fCBuf->writeToFile(startOffset + writeSize1, writeSize2); + rc = fCBuf->writeToFile(startOffset + writeSize1, writeSize2, fillUpWEmpties); if (rc != NO_ERROR) { @@ -636,6 +644,12 @@ int ColumnBufferManager::writeToFileExtentCheck( return rc; } + // MCOL-498 Fill this block up to its boundary. + if ( fillUpWEmpties ) + { + writeSize2 = BLOCK_SIZE; + } + fColInfo->updateBytesWrittenCounts( writeSize2 ); } @@ -643,7 +657,8 @@ int ColumnBufferManager::writeToFileExtentCheck( } //------------------------------------------------------------------------------ -// Flush the contents of internal fCBuf (column buffer) to disk. +// Flush the contents of internal fCBuf (column buffer) to disk. If CS flushes +// less then BLOCK_SIZE bytes then it propagates this event down the stack. //------------------------------------------------------------------------------ int ColumnBufferManager::flush( ) { @@ -668,17 +683,22 @@ int ColumnBufferManager::flush( ) int bufferSize = fCBuf->getSize(); + // MCOL-498 There are less the BLOCK_SIZE bytes in the buffer left + // so propagate this info down the stack to fill the buffer up + // with empty magics. // Account for circular buffer by making 2 calls to write the data, // if we are wrapping around at the end of the buffer. if (fBufFreeOffset < fBufWriteOffset) { - RETURN_ON_ERROR( writeToFileExtentCheck( - fBufWriteOffset, bufferSize - fBufWriteOffset) ); + bool fillUpWEmpties = ( static_cast(bufferSize - fBufWriteOffset) >= BLOCK_SIZE ) + ? false : true; + RETURN_ON_ERROR( writeToFileExtentCheck( fBufWriteOffset, + bufferSize - fBufWriteOffset, fillUpWEmpties) ); fBufWriteOffset = 0; } - + // MCOL-498 fill the buffer up with empty magics. RETURN_ON_ERROR( writeToFileExtentCheck( - fBufWriteOffset, fBufFreeOffset - fBufWriteOffset) ); + fBufWriteOffset, fBufFreeOffset - fBufWriteOffset, true) ); fBufWriteOffset = fBufFreeOffset; return NO_ERROR; diff --git a/writeengine/bulk/we_colbufmgr.h b/writeengine/bulk/we_colbufmgr.h index 8ec86fabe..574c8facc 100644 --- a/writeengine/bulk/we_colbufmgr.h +++ b/writeengine/bulk/we_colbufmgr.h @@ -193,9 +193,12 @@ protected: * write out the buffer. * @param startOffset The buffer offset where the write should begin * @param writeSize The number of bytes to be written to the file + * @param fillUpWEmpties The flag to fill the buffer with NULLs up to + * the block boundary. * @return success or fail status */ - virtual int writeToFileExtentCheck(uint32_t startOffset, uint32_t writeSize); + virtual int writeToFileExtentCheck(uint32_t startOffset, uint32_t writeSize, + bool fillUpWEmpties = false); //------------------------------------------------------------------------- // Protected Data Members diff --git a/writeengine/dictionary/CMakeLists.txt b/writeengine/dictionary/CMakeLists.txt index fe934813b..37317d2f1 100644 --- a/writeengine/dictionary/CMakeLists.txt +++ b/writeengine/dictionary/CMakeLists.txt @@ -1,44 +1,3 @@ - -include_directories(${KDE4_INCLUDES} ${KDE4_INCLUDE_DIR} ${QT_INCLUDES} ) - - ########### install files ############### install(FILES we_dctnry.h DESTINATION include) - - - -#original Makefile.am contents follow: - -## Copyright (C) 2014 InfiniDB, Inc. -## -## This program is free software; you can redistribute it and/or -## modify it under the terms of the GNU General Public License -## as published by the Free Software Foundation; version 2 of -## the License. -## -## This program is distributed in the hope that it will be useful, -## but WITHOUT ANY WARRANTY; without even the implied warranty of -## MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -## GNU General Public License for more details. -## -## You should have received a copy of the GNU General Public License -## along with this program; if not, write to the Free Software -## Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, -## MA 02110-1301, USA. -# -## $Id: Makefile.am 864 2009-04-02 19:22:49Z rdempsey $ -### Process this file with automake to produce Makefile.in -# -#include_HEADERS = we_dctnry.h -# -#test: -# -#coverage: -# -#leakcheck: -# -#docs: -# -#bootstrap: install-data-am -# diff --git a/writeengine/dictionary/we_dctnry.cpp b/writeengine/dictionary/we_dctnry.cpp index 4e93ca60a..3b77dae7a 100644 --- a/writeengine/dictionary/we_dctnry.cpp +++ b/writeengine/dictionary/we_dctnry.cpp @@ -29,6 +29,7 @@ #include #include #include +#include #include #include #include @@ -108,7 +109,6 @@ Dctnry::Dctnry() : &m_endHeader, HDR_UNIT_SIZE); m_curFbo = INVALID_NUM; m_curLbid = INVALID_LBID; - memset(m_sigArray, 0, MAX_STRING_CACHE_SIZE * sizeof(Signature)); m_arraySize = 0; clear();//files @@ -130,14 +130,16 @@ Dctnry::~Dctnry() ******************************************************************************/ void Dctnry::freeStringCache( ) { - for (int i = 0; i < m_arraySize; i++) + std::set::iterator it; + for (it=m_sigArray.begin(); it!=m_sigArray.end(); it++) { - delete [] m_sigArray[i].signature; - m_sigArray[i].signature = 0; + Signature sig = *it; + delete [] sig.signature; + sig.signature = 0; } - memset(m_sigArray, 0, MAX_STRING_CACHE_SIZE * sizeof(Signature)); m_arraySize = 0; + m_sigArray.clear(); } /******************************************************************************* @@ -161,7 +163,6 @@ int Dctnry::init() m_curOp = 0; memset( m_curBlock.data, 0, sizeof(m_curBlock.data)); m_curBlock.lbid = INVALID_LBID; - memset(m_sigArray, 0, MAX_STRING_CACHE_SIZE * sizeof(Signature)); m_arraySize = 0; return NO_ERROR; @@ -259,12 +260,15 @@ int Dctnry::createDctnry( const OID& dctnryOID, int colWidth, if ( m_dFile != NULL ) { + // MCOL-498 CS optimizes abbreviated extent + // creation. rc = FileOp::initDctnryExtent( m_dFile, m_dbRoot, totalSize, m_dctnryHeader2, m_totalHdrBytes, - false ); + false, + true ); // explicitly optimize if (rc != NO_ERROR) { @@ -329,7 +333,8 @@ int Dctnry::expandDctnryExtent() blksToAdd, m_dctnryHeader2, m_totalHdrBytes, - true ); + true, + true ); // explicitly optimize if (rc != NO_ERROR) return rc; @@ -623,19 +628,17 @@ int Dctnry::openDctnry(const OID& dctnryOID, ******************************************************************************/ bool Dctnry::getTokenFromArray(Signature& sig) { - for (int i = 0; i < (int)m_arraySize ; i++ ) - { - if (sig.size == m_sigArray[i].size) - { - if (!memcmp(sig.signature, m_sigArray[i].signature, sig.size)) - { - sig.token = m_sigArray[i].token; - return true; - }//endif sig compare - }//endif size compare - } + std::set::iterator it; + it = m_sigArray.find(sig); + if ( it == m_sigArray.end()){ + return false; + }else{ + Signature sigfound = *it; + sig.token = sigfound.token; + return true; + } - return false; + return false; } /******************************************************************************* @@ -1329,7 +1332,7 @@ void Dctnry::preLoadStringCache( const DataBlock& fileBlock ) memcpy(aSig.signature, &fileBlock.data[offBeg], len); aSig.token.op = op; aSig.token.fbo = m_curLbid; - m_sigArray[op - 1] = aSig; + m_sigArray.insert(aSig); offEnd = offBeg; hdrOffsetBeg += HDR_UNIT_SIZE; @@ -1368,7 +1371,7 @@ void Dctnry::addToStringCache( const Signature& newSig ) memcpy(asig.signature, newSig.signature, newSig.size ); asig.size = newSig.size; asig.token = newSig.token; - m_sigArray[m_arraySize] = asig; + m_sigArray.insert(asig); m_arraySize++; } @@ -1461,7 +1464,7 @@ int Dctnry::updateDctnry(unsigned char* sigValue, int& sigSize, sig.signature = new unsigned char[sigSize]; memcpy (sig.signature, sigValue, sigSize); sig.token = token; - m_sigArray[m_arraySize] = sig; + m_sigArray.insert(sig); m_arraySize++; } diff --git a/writeengine/dictionary/we_dctnry.h b/writeengine/dictionary/we_dctnry.h index 58d429142..93135da4d 100644 --- a/writeengine/dictionary/we_dctnry.h +++ b/writeengine/dictionary/we_dctnry.h @@ -56,6 +56,17 @@ typedef struct Signature Token token; } Signature; +struct sig_compare { + bool operator() (const Signature& a, const Signature& b) const { + if (a.size == b.size){ + return memcmp(a.signature,b.signature,a.size)<0;} + else if (a.size& oids); virtual int numOfBlocksInFile(); - Signature m_sigArray[MAX_STRING_CACHE_SIZE]; // string cache + std::set m_sigArray; int m_arraySize; // num strings in m_sigArray // m_dctnryHeader used for hdr when readSubBlockEntry is used to read a blk diff --git a/writeengine/dictionary/we_dctnrystore.cpp b/writeengine/dictionary/we_dctnrystore.cpp index 8e2982b57..4a7fefe73 100644 --- a/writeengine/dictionary/we_dctnrystore.cpp +++ b/writeengine/dictionary/we_dctnrystore.cpp @@ -133,7 +133,7 @@ const int DctnryStore::updateDctnryStore(unsigned char* sigValue, sig.signature = new unsigned char[sigSize]; memcpy (sig.signature, sigValue, sigSize); sig.token = token; - m_dctnry.m_sigArray[m_dctnry.m_arraySize] = sig; + m_dctnry.m_sigArray.insert(sig) = sig; m_dctnry.m_arraySize++; } diff --git a/writeengine/redistribute/CMakeLists.txt b/writeengine/redistribute/CMakeLists.txt index d62cff1a1..22308ced0 100644 --- a/writeengine/redistribute/CMakeLists.txt +++ b/writeengine/redistribute/CMakeLists.txt @@ -15,5 +15,8 @@ target_link_libraries(writeengineredistribute ${NETSNMP_LIBRARIES}) set_target_properties(writeengineredistribute PROPERTIES VERSION 1.0.0 SOVERSION 1) +target_compile_definitions(writeengineredistribute PUBLIC BOOST_NO_CXX11_SCOPED_ENUMS) + + install(TARGETS writeengineredistribute DESTINATION ${ENGINE_LIBDIR} COMPONENT libs) diff --git a/writeengine/shared/CMakeLists.txt b/writeengine/shared/CMakeLists.txt index f49b76e9e..898da28a0 100644 --- a/writeengine/shared/CMakeLists.txt +++ b/writeengine/shared/CMakeLists.txt @@ -1,44 +1,11 @@ -include_directories(${KDE4_INCLUDES} ${KDE4_INCLUDE_DIR} ${QT_INCLUDES} ) +include_directories(${ENGINE_COMMON_INCLUDES} ../dictionary) +add_executable(we_shared_components_tests ./shared_components_tests.cpp) +target_link_libraries(we_shared_components_tests ${ENGINE_LDFLAGS} ${MARIADB_CLIENT_LIBS} ${ENGINE_WRITE_LIBS} ${CPPUNIT_LIBRARIES}) + +install(TARGETS we_shared_components_tests DESTINATION ${ENGINE_BINDIR} COMPONENT platform) ########### install files ############### install(FILES we_index.h we_define.h we_type.h we_fileop.h we_blockop.h we_dbfileop.h we_obj.h we_log.h we_simplesyslog.h we_convertor.h we_brm.h we_macro.h we_config.h we_cache.h we_stats.h we_bulkrollbackmgr.h we_typeext.h we_chunkmanager.h we_bulkrollbackfilecompressed.h we_bulkrollbackfilecompressedhdfs.h we_bulkrollbackfile.h we_rbmetawriter.h we_dbrootextenttracker.h we_confirmhdfsdbfile.h DESTINATION include) - - - -#original Makefile.am contents follow: - -## Copyright (C) 2014 InfiniDB, Inc. -## -## This program is free software; you can redistribute it and/or -## modify it under the terms of the GNU General Public License -## as published by the Free Software Foundation; version 2 of -## the License. -## -## This program is distributed in the hope that it will be useful, -## but WITHOUT ANY WARRANTY; without even the implied warranty of -## MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -## GNU General Public License for more details. -## -## You should have received a copy of the GNU General Public License -## along with this program; if not, write to the Free Software -## Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, -## MA 02110-1301, USA. -# -## $Id: Makefile.am 3720 2012-04-04 18:18:49Z rdempsey $ -### Process this file with automake to produce Makefile.in -# -#include_HEADERS = we_index.h we_define.h we_type.h we_fileop.h we_blockop.h we_dbfileop.h we_obj.h we_log.h we_simplesyslog.h we_convertor.h we_brm.h we_macro.h we_config.h we_cache.h we_stats.h we_bulkrollbackmgr.h we_typeext.h we_chunkmanager.h we_bulkrollbackfilecompressed.h we_bulkrollbackfilecompressedhdfs.h we_bulkrollbackfile.h we_rbmetawriter.h we_dbrootextenttracker.h we_confirmhdfsdbfile.h -# -#test: -# -#coverage: -# -#leakcheck: -# -#docs: -# -#bootstrap: install-data-am -# diff --git a/writeengine/shared/tshared.cpp b/writeengine/shared/shared_components_tests.cpp similarity index 77% rename from writeengine/shared/tshared.cpp rename to writeengine/shared/shared_components_tests.cpp index 22fa0723f..e569ecee6 100644 --- a/writeengine/shared/tshared.cpp +++ b/writeengine/shared/shared_components_tests.cpp @@ -29,18 +29,28 @@ using namespace boost; #include -#include -#include -#include -#include -#include -#include -#include +#include "we_dbfileop.h" +#include "we_type.h" +#include "we_log.h" +#include "we_convertor.h" +#include "we_brm.h" +#include "we_cache.h" +#include "we_colop.h" +#include "IDBDataFile.h" +#include "BufferedFile.h" +#include "IDBPolicy.h" +#include "IDBFileSystem.h" +#include "idbcompress.h" +#include "calpontsystemcatalog.h" +#include "we_colopcompress.h" +#include "we_dctnrycompress.h" +using namespace compress; +using namespace idbdatafile; using namespace WriteEngine; using namespace BRM; -int compare (const void* a, const void* b) +/*int compare (const void* a, const void* b) { return ( *(uint32_t*)a - * (uint32_t*)b ); } @@ -49,51 +59,55 @@ int compare1(const void* a, const void* b) { return ( (*(SortTuple*)a).key - (*(SortTuple*)b).key ); } - +*/ class SharedTest : public CppUnit::TestFixture { - CPPUNIT_TEST_SUITE( SharedTest ); + CPPUNIT_TEST_SUITE( SharedTest ); -//CPPUNIT_TEST(setUp); +CPPUNIT_TEST(setUp); //CPPUNIT_TEST( test1 ); // File operation testing - CPPUNIT_TEST( testFileNameOp ); - CPPUNIT_TEST( testFileHandleOp ); +// CPPUNIT_TEST( testFileNameOp ); +// CPPUNIT_TEST( testFileHandleOp ); CPPUNIT_TEST( testDirBasic ); + CPPUNIT_TEST( testCreateDeleteFile ); // Data block related testing - CPPUNIT_TEST( testCalculateRowIdBitmap ); - CPPUNIT_TEST( testBlockBuffer ); - CPPUNIT_TEST( testBitBasic ); - CPPUNIT_TEST( testBufferBit ); - CPPUNIT_TEST( testBitShift ); - CPPUNIT_TEST( testEmptyRowValue ); - CPPUNIT_TEST( testCorrectRowWidth ); +// CPPUNIT_TEST( testCalculateRowIdBitmap ); +// CPPUNIT_TEST( testBlockBuffer ); +// CPPUNIT_TEST( testBitBasic ); +// CPPUNIT_TEST( testBufferBit ); +// CPPUNIT_TEST( testBitShift ); +// CPPUNIT_TEST( testEmptyRowValue ); +// CPPUNIT_TEST( testCorrectRowWidth ); // DB File Block related testing - CPPUNIT_TEST( testDbBlock ); +// CPPUNIT_TEST( testDbBlock ); - CPPUNIT_TEST( testCopyDbFile ); +// CPPUNIT_TEST( testCopyDbFile ); +// Extent & dict related testing + CPPUNIT_TEST( testExtensionWOPrealloc ); + CPPUNIT_TEST( testDictExtensionWOPrealloc ); // Semaphore related testing - CPPUNIT_TEST( testSem ); +// CPPUNIT_TEST( testSem ); // Log related testing CPPUNIT_TEST( testLog ); // Version Buffer related testing - CPPUNIT_TEST( testHWM ); - CPPUNIT_TEST( testVB ); +// CPPUNIT_TEST( testHWM ); +// CPPUNIT_TEST( testVB ); // Disk manager related testing - CPPUNIT_TEST( testDM ); - CPPUNIT_TEST( tearDown ); +// CPPUNIT_TEST( testDM ); +// CPPUNIT_TEST( tearDown ); // Cache related testing - CPPUNIT_TEST( testCacheBasic ); - CPPUNIT_TEST( testCacheReadWrite ); +// CPPUNIT_TEST( testCacheBasic ); +// CPPUNIT_TEST( testCacheReadWrite ); CPPUNIT_TEST( testCleanup ); // NEVER COMMENT OUT THIS LINE CPPUNIT_TEST_SUITE_END(); @@ -113,8 +127,6 @@ public: void test1() { - //m_wrapper.test(); -// int numOfBlock = 10240; int numOfBlock = 1024; FILE* pFile; unsigned char writeBuf[BYTE_PER_BLOCK * 10]; @@ -139,7 +151,7 @@ public: } } } - +/* void testFileNameOp() { FileOp fileOp; @@ -342,23 +354,74 @@ public: } +*/ void testDirBasic() { FileOp fileOp; char dirName[30]; int rc; - strcpy( dirName, "testdir" ); - fileOp.removeDir( dirName ); + printf("\nRunning testDirBasic \n"); + idbdatafile::IDBPolicy::init(true, false, "", 0); + IDBFileSystem& fs = IDBPolicy::getFs( "/tmp" ); + strcpy( dirName, "/tmp/testdir42" ); + fs.remove( dirName ); CPPUNIT_ASSERT( fileOp.isDir( dirName ) == false ); rc = fileOp.createDir( dirName ); CPPUNIT_ASSERT( rc == NO_ERROR ); CPPUNIT_ASSERT( fileOp.isDir( dirName ) == true ); - fileOp.removeDir( dirName ); + fs.remove( dirName ); } + void testCreateDeleteFile() + { + IDBDataFile* pFile = NULL; + FileOp fileOp; + BlockOp blockOp; + char fileName[20]; + int rc; + char hdrs[ IDBCompressInterface::HDR_BUF_LEN * 2 ]; + + printf("\nRunning testCreateDeleteFile \n"); + idbdatafile::IDBPolicy::init(true, false, "", 0); + // Set to versionbuffer to satisfy IDBPolicy::getType + strcpy( fileName, "versionbuffer" ); + fileOp.compressionType(1); + + fileOp.deleteFile( fileName ); + CPPUNIT_ASSERT( fileOp.exists( fileName ) == false ); + + int width = blockOp.getCorrectRowWidth( execplan::CalpontSystemCatalog::BIGINT, 8 ); + int nBlocks = INITIAL_EXTENT_ROWS_TO_DISK / BYTE_PER_BLOCK * width; + uint64_t emptyVal = blockOp.getEmptyRowValue( execplan::CalpontSystemCatalog::BIGINT, 8 ); + // createFile runs IDBDataFile::open + initAbrevCompColumnExtent + // under the hood + // bigint column file + rc = fileOp.createFile( fileName, + nBlocks, // number of blocks + emptyVal, // NULL value + width, // width + 1 ); // dbroot + CPPUNIT_ASSERT( rc == NO_ERROR ); + + fileOp.closeFile(pFile); + + pFile = IDBDataFile::open(IDBPolicy::getType(fileName, + IDBPolicy::WRITEENG), fileName, "rb", 1); + + rc = pFile->seek(0, 0); + CPPUNIT_ASSERT(rc == NO_ERROR); + rc = fileOp.readHeaders(pFile, hdrs); + CPPUNIT_ASSERT( rc == NO_ERROR ); + // Couldn't use IDBDataFile->close() here w/o excplicit cast + fileOp.closeFile(pFile); + + fileOp.deleteFile( fileName ); + CPPUNIT_ASSERT( fileOp.exists( fileName ) == false ); + } +/* void testCalculateRowIdBitmap() { BlockOp blockOp; @@ -374,23 +437,23 @@ public: CPPUNIT_ASSERT( bio == 16 ); // Assuming 2048 per data block, 4 byte width - /* rowId = 2049; - CPPUNIT_ASSERT( blockOp.calculateRowId( rowId, 2048, 4, fbo, bio ) == true ); - CPPUNIT_ASSERT( fbo == 1 ); - CPPUNIT_ASSERT( bio == 16 ); - - // Assuming 4096 per data block, 2 byte width - rowId = 2049; - CPPUNIT_ASSERT( blockOp.calculateRowId( rowId, 4096, 2, fbo, bio ) == true ); - CPPUNIT_ASSERT( fbo == 1 ); - CPPUNIT_ASSERT( bio == 16 ); - - // Assuming 8192 per data block, 1 byte width - rowId = 2049; - CPPUNIT_ASSERT( blockOp.calculateRowId( rowId, 8192, 1, fbo, bio ) == true ); - CPPUNIT_ASSERT( fbo == 1 ); - CPPUNIT_ASSERT( bio == 16 ); - */ +// rowId = 2049; +// CPPUNIT_ASSERT( blockOp.calculateRowId( rowId, 2048, 4, fbo, bio ) == true ); +// CPPUNIT_ASSERT( fbo == 1 ); +// CPPUNIT_ASSERT( bio == 16 ); +// +// // Assuming 4096 per data block, 2 byte width +// rowId = 2049; +// CPPUNIT_ASSERT( blockOp.calculateRowId( rowId, 4096, 2, fbo, bio ) == true ); +// CPPUNIT_ASSERT( fbo == 1 ); +// CPPUNIT_ASSERT( bio == 16 ); +// +// // Assuming 8192 per data block, 1 byte width +// rowId = 2049; +// CPPUNIT_ASSERT( blockOp.calculateRowId( rowId, 8192, 1, fbo, bio ) == true ); +// CPPUNIT_ASSERT( fbo == 1 ); +// CPPUNIT_ASSERT( bio == 16 ); +// rowId = 65546; CPPUNIT_ASSERT( blockOp.calculateRowBitmap( rowId, BYTE_PER_BLOCK * 8, fbo, bio, bbo ) == true ); CPPUNIT_ASSERT( fbo == 1 ); @@ -550,19 +613,19 @@ public: curVal = blockOp.getEmptyRowValue( WriteEngine::DECIMAL, 8 ); CPPUNIT_ASSERT( curVal == 0x8000000000000001LL ); - /* - curVal = blockOp.getEmptyRowValue( WriteEngine::DECIMAL, 9 ); - CPPUNIT_ASSERT( curVal == 0x80000001 ); - - curVal = blockOp.getEmptyRowValue( WriteEngine::DECIMAL, 10 ); - CPPUNIT_ASSERT( curVal == 0x8000000000000001LL ); - - curVal = blockOp.getEmptyRowValue( WriteEngine::DECIMAL, 12 ); - CPPUNIT_ASSERT( curVal == 0x8000000000000001LL ); - - curVal = blockOp.getEmptyRowValue( WriteEngine::DECIMAL, 19 ); - CPPUNIT_ASSERT( curVal == 0xFFFFFFFFFFFFFFFFLL ); - */ + +// curVal = blockOp.getEmptyRowValue( WriteEngine::DECIMAL, 9 ); +// CPPUNIT_ASSERT( curVal == 0x80000001 ); +// +// curVal = blockOp.getEmptyRowValue( WriteEngine::DECIMAL, 10 ); +// CPPUNIT_ASSERT( curVal == 0x8000000000000001LL ); +// +// curVal = blockOp.getEmptyRowValue( WriteEngine::DECIMAL, 12 ); +// CPPUNIT_ASSERT( curVal == 0x8000000000000001LL ); +// +// curVal = blockOp.getEmptyRowValue( WriteEngine::DECIMAL, 19 ); +// CPPUNIT_ASSERT( curVal == 0xFFFFFFFFFFFFFFFFLL ); +// curVal = blockOp.getEmptyRowValue( WriteEngine::DATE, 4 ); CPPUNIT_ASSERT( curVal == 0xFFFFFFFF ); @@ -645,18 +708,18 @@ public: curVal = blockOp.getCorrectRowWidth( WriteEngine::DECIMAL, 8 ); CPPUNIT_ASSERT( curVal == 8 ); - /* curVal = blockOp.getCorrectRowWidth( WriteEngine::DECIMAL, 9 ); - CPPUNIT_ASSERT( curVal == 4 ); - - curVal = blockOp.getCorrectRowWidth( WriteEngine::DECIMAL, 10 ); - CPPUNIT_ASSERT( curVal == 8 ); - - curVal = blockOp.getCorrectRowWidth( WriteEngine::DECIMAL, 12 ); - CPPUNIT_ASSERT( curVal == 8 ); - - curVal = blockOp.getCorrectRowWidth( WriteEngine::DECIMAL, 19 ); - CPPUNIT_ASSERT( curVal == 8 ); - */ +// curVal = blockOp.getCorrectRowWidth( WriteEngine::DECIMAL, 9 ); +// CPPUNIT_ASSERT( curVal == 4 ); +// +// curVal = blockOp.getCorrectRowWidth( WriteEngine::DECIMAL, 10 ); +// CPPUNIT_ASSERT( curVal == 8 ); +// +// curVal = blockOp.getCorrectRowWidth( WriteEngine::DECIMAL, 12 ); +// CPPUNIT_ASSERT( curVal == 8 ); +// +// curVal = blockOp.getCorrectRowWidth( WriteEngine::DECIMAL, 19 ); +// CPPUNIT_ASSERT( curVal == 8 ); +// curVal = blockOp.getCorrectRowWidth( WriteEngine::DATE, 8 ); CPPUNIT_ASSERT( curVal == 4 ); @@ -892,126 +955,229 @@ public: dbFileOp.closeFile( pTargetFile ); } +*/ - void testSem() + void testExtensionWOPrealloc() { - SemOp semOp; + IDBDataFile* pFile = NULL; FileOp fileOp; + BlockOp blockOp; + char fileName[20]; int rc; - bool bSuccess; - key_t key; - int sid, totalNum = 5; - char fileName[100]; + char hdrs[ IDBCompressInterface::HDR_BUF_LEN * 2 ]; + int dbRoot = 1; - semOp.setMaxSemVal( 3 ); + printf("\nRunning testExtensionWOPrealloc \n"); + idbdatafile::IDBPolicy::init(true, false, "", 0); + // Set to versionbuffer to satisfy IDBPolicy::getType + strcpy( fileName, "versionbuffer" ); + fileOp.compressionType(1); - bSuccess = semOp.getKey( NULL, key ); - CPPUNIT_ASSERT( bSuccess == false ); + fileOp.deleteFile( fileName ); + CPPUNIT_ASSERT( fileOp.exists( fileName ) == false ); - rc = fileOp.getFileName( 9991, fileName ); + int width = blockOp.getCorrectRowWidth( execplan::CalpontSystemCatalog::BIGINT, 8 ); + int nBlocks = INITIAL_EXTENT_ROWS_TO_DISK / BYTE_PER_BLOCK * width; + uint64_t emptyVal = blockOp.getEmptyRowValue( execplan::CalpontSystemCatalog::BIGINT, 8 ); + // createFile runs IDBDataFile::open + initAbrevCompColumnExtent + // under the hood + // bigint column file + rc = fileOp.createFile( fileName, + nBlocks, // number of blocks + emptyVal, // NULL value + width, // width + dbRoot ); // dbroot CPPUNIT_ASSERT( rc == NO_ERROR ); - bSuccess = semOp.getKey( fileName, key ); - CPPUNIT_ASSERT( bSuccess == false ); + // open created compressed file and check its header + pFile = IDBDataFile::open(IDBPolicy::getType(fileName, + IDBPolicy::WRITEENG), fileName, "rb", dbRoot); - rc = fileOp.getFileName( 999, fileName ); + rc = pFile->seek(0, 0); + CPPUNIT_ASSERT(rc == NO_ERROR); + rc = fileOp.readHeaders(pFile, hdrs); CPPUNIT_ASSERT( rc == NO_ERROR ); - bSuccess = semOp.getKey( fileName, key ); - printf( "\nkey=%d", key ); - CPPUNIT_ASSERT( bSuccess == true ); + // Couldn't use IDBDataFile->close() here w/o excplicit cast + fileOp.closeFile(pFile); - if ( semOp.existSem( sid, key ) ) - semOp.deleteSem( sid ); + // Extend the extent up to 64MB + // first run w preallocation + idbdatafile::BufferedFile* bFile = new idbdatafile::BufferedFile(fileName, "r+b", 0); + pFile = dynamic_cast(bFile); + + rc = fileOp.initColumnExtent(pFile, + dbRoot, + BYTE_PER_BLOCK, // number of blocks + emptyVal, + width, + false, // use existing file + true, // expand the extent + false, // add full (not abbreviated) extent + false); // don't optimize extention - rc = semOp.createSem( sid, key, 1000 ); - CPPUNIT_ASSERT( rc == ERR_MAX_SEM ); - rc = semOp.createSem( sid, key, totalNum ); + CPPUNIT_ASSERT(rc == NO_ERROR); + CPPUNIT_ASSERT(bFile->size() == 67108864); + fileOp.closeFile(pFile); + // file has been extended delete the file before + // the second run + + fileOp.deleteFile( fileName ); + CPPUNIT_ASSERT(fileOp.exists( fileName ) == false); + + // second run with disabled preallocation + rc = fileOp.createFile( fileName, + nBlocks, // number of blocks + emptyVal, // NULL value + width, // width + dbRoot ); // dbroot CPPUNIT_ASSERT( rc == NO_ERROR ); - rc = semOp.createSem( sid, key, totalNum ); - CPPUNIT_ASSERT( rc == ERR_SEM_EXIST ); + // open created compressed file and check its header + pFile = IDBDataFile::open(IDBPolicy::getType(fileName, + IDBPolicy::WRITEENG), fileName, "rb", dbRoot); - rc = semOp.openSem( sid, key ); + rc = pFile->seek(0, 0); + CPPUNIT_ASSERT(rc == NO_ERROR); + rc = fileOp.readHeaders(pFile, hdrs); CPPUNIT_ASSERT( rc == NO_ERROR ); - semOp.printAllVal( sid ); + fileOp.closeFile(pFile); - // lock - printf( "\nlock one in 2" ); - rc = semOp.lockSem( sid, 2 ); - CPPUNIT_ASSERT( rc == NO_ERROR ); - CPPUNIT_ASSERT( semOp.getVal( sid, 2 ) == 2 ); - semOp.printAllVal( sid ); - - printf( "\nlock one in 2" ); - rc = semOp.lockSem( sid, 2 ); - CPPUNIT_ASSERT( rc == NO_ERROR ); - CPPUNIT_ASSERT( semOp.getVal( sid, 2 ) == 1 ); - semOp.printAllVal( sid ); - - printf( "\nlock one in 2" ); - rc = semOp.lockSem( sid, 2 ); - CPPUNIT_ASSERT( rc == NO_ERROR ); - CPPUNIT_ASSERT( semOp.getVal( sid, 2 ) == 0 ); - semOp.printAllVal( sid ); - - rc = semOp.lockSem( sid, 2 ); - CPPUNIT_ASSERT( rc == ERR_NO_SEM_RESOURCE ); - CPPUNIT_ASSERT( semOp.getVal( sid, 2 ) == 0 ); - - rc = semOp.lockSem( sid, -2 ); - CPPUNIT_ASSERT( rc == ERR_VALUE_OUTOFRANGE ); - - rc = semOp.lockSem( sid + 1, 1 ); - CPPUNIT_ASSERT( rc == ERR_LOCK_FAIL ); - - // unlock - rc = semOp.unlockSem( sid, -2 ); - CPPUNIT_ASSERT( rc == ERR_VALUE_OUTOFRANGE ); - - rc = semOp.unlockSem( sid, 1 ); - CPPUNIT_ASSERT( rc == ERR_NO_SEM_LOCK ); - - rc = semOp.unlockSem( sid + 1, 2 ); - CPPUNIT_ASSERT( rc == ERR_UNLOCK_FAIL ); - - printf( "\nunlock one in 2" ); - rc = semOp.unlockSem( sid, 2 ); - CPPUNIT_ASSERT( rc == NO_ERROR ); - CPPUNIT_ASSERT( semOp.getVal( sid, 2 ) == 1 ); - semOp.printAllVal( sid ); - - semOp.deleteSem( sid ); - CPPUNIT_ASSERT( semOp.existSem( sid, key ) == false ); - - CPPUNIT_ASSERT( semOp.getSemCount( sid + 1 ) == 0 ); + bFile = new idbdatafile::BufferedFile(fileName, "r+b", 0); + pFile = dynamic_cast(bFile); + + // disable disk space preallocation and extend + idbdatafile::IDBPolicy::setPreallocSpace(dbRoot); + rc = fileOp.initColumnExtent(pFile, + dbRoot, + BYTE_PER_BLOCK, // number of blocks + emptyVal, + width, + false, // use existing file + true, // expand the extent + false, // add full (not abbreviated) extent + true); // optimize extention + CPPUNIT_ASSERT(rc == NO_ERROR); + CPPUNIT_ASSERT(bFile->size() == 2105344); + fileOp.closeFile(pFile); + // file has been extended + + fileOp.deleteFile( fileName ); + CPPUNIT_ASSERT(fileOp.exists( fileName ) == false); } + // Create a dict file. Extend it w and w/o preallocation. + // Check the file sizes. + void testDictExtensionWOPrealloc() + { + FileOp fileOp; + BlockOp blockOp; + char fileName[20]; + int rc; + int dbRoot = 1; + int colWidth = 65535; + + DctnryCompress1 m_Dctnry; + // This is the magic for the stub in FileOp::oid2FileName + int oId = 42; + + printf("\nRunning testDictExtensionWOPrealloc "); + printf("There could be InetStreamSocket::connect errors \n"); + m_Dctnry.setDebugLevel( DEBUG_3 ); + + idbdatafile::IDBPolicy::init(true, false, "", 0); + // Set to versionbuffer to satisfy IDBPolicy::getType + strcpy( fileName, "versionbuffer" ); + + rc = m_Dctnry.dropDctnry(oId); + // FileOp::oid2FileName is called under the hood + // Dctnry::createDctnry could be used with running CS + // createDctnryFile also uses DBRM under the hood it works though. + IDBDataFile* m_dFile = m_Dctnry.createDctnryFileUnit(fileName, + colWidth, + "w+b", + DEFAULT_BUFSIZ); + + idbdatafile::BufferedFile* bFile = (idbdatafile::BufferedFile*)m_dFile; + CPPUNIT_ASSERT(m_dFile != NULL); + + const int m_totalHdrBytes = HDR_UNIT_SIZE + NEXT_PTR_BYTES + HDR_UNIT_SIZE + HDR_UNIT_SIZE; + + m_Dctnry.compressionType(1); + rc = m_Dctnry.initDctnryExtent( m_dFile, + dbRoot, + BYTE_PER_BLOCK, // 8192 + const_cast(m_Dctnry.getDctnryHeader2()), + m_totalHdrBytes, + false, + false ); //enable preallocation + // Check the file size and remove the file + CPPUNIT_ASSERT(bFile->size() == 67379200); + CPPUNIT_ASSERT(rc == NO_ERROR); + fileOp.deleteFile( fileName ); + CPPUNIT_ASSERT(fileOp.exists( fileName ) == false); + + // Create a Dictionary for the second time + m_dFile = m_Dctnry.createDctnryFileUnit(fileName, + colWidth, + "w+b", + DEFAULT_BUFSIZ); + + // Get the file size later + bFile = (idbdatafile::BufferedFile*)m_dFile; + CPPUNIT_ASSERT(m_dFile != NULL); + + // disable preallocation and create a Dictionary + idbdatafile::IDBPolicy::setPreallocSpace(dbRoot); + m_Dctnry.compressionType(1); + rc = m_Dctnry.initDctnryExtent( m_dFile, + dbRoot, + BYTE_PER_BLOCK, + const_cast(m_Dctnry.getDctnryHeader2()), + m_totalHdrBytes, + false, + true ); //skip preallocation + + // Check the size and remove the file. + CPPUNIT_ASSERT(bFile->size() == 483328); + CPPUNIT_ASSERT(rc == NO_ERROR); + fileOp.deleteFile(fileName); + CPPUNIT_ASSERT(fileOp.exists( fileName ) == false); + } + + + + + void testLog() { Log log; + FileOp fileOp; string msg; int iVal = 3; - float fVal = 2.0; + char logFile[] = "test1.log"; + char logErrFile[] = "test1err.log"; - log.setLogFileName( "test1.log", "test1err.log" ); + log.setLogFileName( logFile, logErrFile ); msg = Convertor::int2Str( iVal ); - log.logMsg( msg + " this is a info message", INFO ); + log.logMsg( msg + " this is a info message", MSGLVL_INFO1 ); msg = Convertor::getTimeStr(); - log.logMsg( Convertor::float2Str( fVal ) + " this is a warning message", WARNING ); - log.logMsg( "this is an error message ", 1011, ERROR ); - log.logMsg( "this is a critical message", 1211, CRITICAL ); - - //...Test formatting an unsigned 64 bit integer. - uint64_t i64Value(UINT64_MAX); - msg = Convertor::i64ToStr( i64Value ); - CPPUNIT_ASSERT( (msg == "18446744073709551615") ); - log.logMsg( msg + " this is an info message with the max uint64_t integer value", INFO ); + log.logMsg( " this is a warning message", MSGLVL_WARNING ); + log.logMsg( "this is an error message ", 1011, MSGLVL_ERROR ); + log.logMsg( "this is a critical message", 1211, MSGLVL_CRITICAL ); + CPPUNIT_ASSERT( fileOp.exists( logFile ) == true ); + CPPUNIT_ASSERT( fileOp.exists( logErrFile ) == true ); + fileOp.deleteFile( logFile ); + fileOp.deleteFile( logErrFile ); + CPPUNIT_ASSERT( fileOp.exists( logFile ) == false ); + CPPUNIT_ASSERT( fileOp.exists( logErrFile ) == false ); } +/* + void testHWM() { int rc ; @@ -1375,6 +1541,7 @@ public: } } +*/ void testCleanup() { diff --git a/writeengine/shared/we_define.h b/writeengine/shared/we_define.h index c26b926fd..8c3e85fe9 100644 --- a/writeengine/shared/we_define.h +++ b/writeengine/shared/we_define.h @@ -45,7 +45,8 @@ const short ROW_PER_BYTE = 8; // Rows/byte in bitmap file const int BYTE_PER_BLOCK = 8192; // Num bytes per data block const int BYTE_PER_SUBBLOCK = 256; // Num bytes per sub block const int ENTRY_PER_SUBBLOCK = 32; // Num entries per sub block -const int INITIAL_EXTENT_ROWS_TO_DISK = 256 * 1024; +const int INITIAL_EXTENT_ROWS_TO_DISK = 256 * 1024; // Used for initial number of blocks calculation +const int MAX_INITIAL_EXTENT_BLOCKS_TO_DISK = 256; // Number of blocks in abbrev extent for 8byte col. // Num rows reserved to disk for 'initial' extent const int FILE_NAME_SIZE = 200; // Max size of file name const long long MAX_ALLOW_ERROR_COUNT = 100000; //Max allowable error count diff --git a/writeengine/shared/we_fileop.cpp b/writeengine/shared/we_fileop.cpp index 7a2a2456b..784f8b4d9 100644 --- a/writeengine/shared/we_fileop.cpp +++ b/writeengine/shared/we_fileop.cpp @@ -18,7 +18,6 @@ // $Id: we_fileop.cpp 4737 2013-08-14 20:45:46Z bwilkinson $ #include "config.h" - #include #include #include @@ -63,7 +62,6 @@ namespace WriteEngine /*static*/ boost::mutex FileOp::m_createDbRootMutexes; /*static*/ boost::mutex FileOp::m_mkdirMutex; /*static*/ std::map FileOp::m_DbRootAddExtentMutexes; -const int MAX_NBLOCKS = 8192; // max number of blocks written to an extent // in 1 call to fwrite(), during initialization //StopWatch timer; @@ -541,7 +539,9 @@ bool FileOp::existsOIDDir( FID fid ) const * the applicable column segment file does not exist, it is created. * If this is the very first file for the specified DBRoot, then the * partition and segment number must be specified, else the selected - * partition and segment numbers are returned. + * partition and segment numbers are returned. This method tries to + * optimize full extents creation skiping disk space + * preallocation(if activated). * PARAMETERS: * oid - OID of the column to be extended * emptyVal - Empty value to be used for oid @@ -827,6 +827,7 @@ int FileOp::extendFile( return rc; // Initialize the contents of the extent. + // MCOL-498 optimize full extent creation. rc = initColumnExtent( pFile, dbRoot, allocSize, @@ -834,7 +835,8 @@ int FileOp::extendFile( width, newFile, // new or existing file false, // don't expand; new extent - false ); // add full (not abbreviated) extent + false, // add full (not abbreviated) extent + true); // try to optimize extent creation return rc; } @@ -982,6 +984,8 @@ int FileOp::addExtentExactFile( return rc; // Initialize the contents of the extent. + // CS doesn't optimize file operations to have a valid + // segment files with empty magics rc = initColumnExtent( pFile, dbRoot, allocSize, @@ -1006,6 +1010,9 @@ int FileOp::addExtentExactFile( * This function can be used to initialize an entirely new extent, or * to finish initializing an extent that has already been started. * nBlocks controls how many 8192-byte blocks are to be written out. + * If bOptExtension is set then method first checks config for + * DBRootX.Prealloc. If it is disabled then it skips disk space + * preallocation. * PARAMETERS: * pFile (in) - IDBDataFile* of column segment file to be written to * dbRoot (in) - DBRoot of pFile @@ -1016,6 +1023,7 @@ int FileOp::addExtentExactFile( * headers will be included "if" it is a compressed file. * bExpandExtent (in) - Expand existing extent, or initialize a new one * bAbbrevExtent(in) - if creating new extent, is it an abbreviated extent + * bOptExtension(in) - skip full extent preallocation. * RETURN: * returns ERR_FILE_WRITE if an error occurs, * else returns NO_ERROR. @@ -1028,7 +1036,8 @@ int FileOp::initColumnExtent( int width, bool bNewFile, bool bExpandExtent, - bool bAbbrevExtent ) + bool bAbbrevExtent, + bool bOptExtension) { if ((bNewFile) && (m_compressionType)) { @@ -1061,6 +1070,19 @@ int FileOp::initColumnExtent( // Create vector of mutexes used to serialize extent access per DBRoot initDbRootExtentMutexes( ); + // MCOL-498 Skip the huge preallocations if the option is set + // for the dbroot. This check is skiped for abbreviated extent. + // IMO it is better to check bool then to call a function. + if ( bOptExtension ) + { + bOptExtension = (idbdatafile::IDBPolicy::PreallocSpace(dbRoot)) + ? bOptExtension : false; + } + // Reduce number of blocks allocated for abbreviated extents thus + // CS writes less when creates a new table. This couldn't be zero + // b/c Snappy compressed file format doesn't tolerate empty files. + int realNBlocks = ( bOptExtension && nBlocks <= MAX_INITIAL_EXTENT_BLOCKS_TO_DISK ) ? 3 : nBlocks; + // Determine the number of blocks in each call to fwrite(), and the // number of fwrite() calls to make, based on this. In other words, // we put a cap on the "writeSize" so that we don't allocate and write @@ -1068,15 +1090,15 @@ int FileOp::initColumnExtent( // expanding an abbreviated 64M extent, we may not have an even // multiple of MAX_NBLOCKS to write; remWriteSize is the number of // blocks above and beyond loopCount*MAX_NBLOCKS. - int writeSize = nBlocks * BYTE_PER_BLOCK; // 1M and 8M row extent size + int writeSize = realNBlocks * BYTE_PER_BLOCK; // 1M and 8M row extent size int loopCount = 1; int remWriteSize = 0; - if (nBlocks > MAX_NBLOCKS) // 64M row extent size + if (realNBlocks > MAX_NBLOCKS) // 64M row extent size { writeSize = MAX_NBLOCKS * BYTE_PER_BLOCK; - loopCount = nBlocks / MAX_NBLOCKS; - remWriteSize = nBlocks - (loopCount * MAX_NBLOCKS); + loopCount = realNBlocks / MAX_NBLOCKS; + remWriteSize = realNBlocks - (loopCount * MAX_NBLOCKS); } // Allocate a buffer, initialize it, and use it to create the extent @@ -1096,69 +1118,77 @@ int FileOp::initColumnExtent( Stats::stopParseEvent(WE_STATS_WAIT_TO_EXPAND_COL_EXTENT); else Stats::stopParseEvent(WE_STATS_WAIT_TO_CREATE_COL_EXTENT); - - Stats::startParseEvent(WE_STATS_INIT_COL_EXTENT); #endif - - // Allocate buffer, and store in scoped_array to insure it's deletion. - // Create scope {...} to manage deletion of writeBuf. + // Skip space preallocation if configured so + // fallback to sequential write otherwise. + // Couldn't avoid preallocation for full extents, + // e.g. ADD COLUMN DDL b/c CS has to fill the file + // with empty magics. + if ( !bOptExtension ) { - unsigned char* writeBuf = new unsigned char[writeSize]; - boost::scoped_array writeBufPtr( writeBuf ); +#ifdef PROFILE + Stats::startParseEvent(WE_STATS_INIT_COL_EXTENT); +#endif + // Allocate buffer, store it in scoped_array to insure it's deletion. + // Create scope {...} to manage deletion of writeBuf. + { - setEmptyBuf( writeBuf, writeSize, emptyVal, width ); + unsigned char* writeBuf = new unsigned char[writeSize]; + boost::scoped_array writeBufPtr( writeBuf ); + + setEmptyBuf( writeBuf, writeSize, emptyVal, width ); + + #ifdef PROFILE + Stats::stopParseEvent(WE_STATS_INIT_COL_EXTENT); + + if (bExpandExtent) + Stats::startParseEvent(WE_STATS_EXPAND_COL_EXTENT); + else + Stats::startParseEvent(WE_STATS_CREATE_COL_EXTENT); + + #endif + + //std::ostringstream oss; + //oss << "initColExtent: width-" << width << + //"; loopCount-" << loopCount << + //"; writeSize-" << writeSize; + //std::cout << oss.str() << std::endl; + if (remWriteSize > 0) + { + if ( pFile->write( writeBuf, remWriteSize ) != remWriteSize ) + { + return ERR_FILE_WRITE; + } + } + + for (int j = 0; j < loopCount; j++) + { + if ( pFile->write( writeBuf, writeSize ) != writeSize ) + { + return ERR_FILE_WRITE; + } + } + } + + //@Bug 3219. update the compression header after the extent is expanded. + if ((!bNewFile) && (m_compressionType) && (bExpandExtent)) + { + updateColumnExtent(pFile, nBlocks); + } + + // @bug 2378. Synchronize here to avoid write buffer pile up too much, + // which could cause controllernode to timeout later when it needs to + // save a snapshot. + pFile->flush(); #ifdef PROFILE - Stats::stopParseEvent(WE_STATS_INIT_COL_EXTENT); - if (bExpandExtent) - Stats::startParseEvent(WE_STATS_EXPAND_COL_EXTENT); + Stats::stopParseEvent(WE_STATS_EXPAND_COL_EXTENT); else - Stats::startParseEvent(WE_STATS_CREATE_COL_EXTENT); - + Stats::stopParseEvent(WE_STATS_CREATE_COL_EXTENT); #endif - //std::ostringstream oss; - //oss << "initColExtent: width-" << width << - //"; loopCount-" << loopCount << - //"; writeSize-" << writeSize; - //std::cout << oss.str() << std::endl; - if (remWriteSize > 0) - { - if ( pFile->write( writeBuf, remWriteSize ) != remWriteSize ) - { - return ERR_FILE_WRITE; - } - } - - for (int j = 0; j < loopCount; j++) - { - if ( pFile->write( writeBuf, writeSize ) != writeSize ) - { - return ERR_FILE_WRITE; - } - } } - - //@Bug 3219. update the compression header after the extent is expanded. - if ((!bNewFile) && (m_compressionType) && (bExpandExtent)) - { - updateColumnExtent(pFile, nBlocks); - } - - // @bug 2378. Synchronize here to avoid write buffer pile up too much, - // which could cause controllernode to timeout later when it needs to - // save a snapshot. - pFile->flush(); - -#ifdef PROFILE - - if (bExpandExtent) - Stats::stopParseEvent(WE_STATS_EXPAND_COL_EXTENT); - else - Stats::stopParseEvent(WE_STATS_CREATE_COL_EXTENT); - -#endif } return NO_ERROR; @@ -1185,7 +1215,7 @@ int FileOp::initAbbrevCompColumnExtent( uint64_t emptyVal, int width) { - // Reserve disk space for full abbreviated extent + // Reserve disk space for optimized abbreviated extent int rc = initColumnExtent( pFile, dbRoot, nBlocks, @@ -1193,8 +1223,8 @@ int FileOp::initAbbrevCompColumnExtent( width, true, // new file false, // don't expand; add new extent - true ); // add abbreviated extent - + true, // add abbreviated extent + true); // optimize the initial extent if (rc != NO_ERROR) { return rc; @@ -1767,6 +1797,9 @@ int FileOp::writeHeaders(IDBDataFile* pFile, const char* controlHdr, * This function can be used to initialize an entirely new extent, or * to finish initializing an extent that has already been started. * nBlocks controls how many 8192-byte blocks are to be written out. + * If bOptExtension is set then method first checks config for + * DBRootX.Prealloc. If it is disabled then it skips disk space + * preallocation. * PARAMETERS: * pFile (in) - IDBDataFile* of column segment file to be written to * dbRoot (in) - DBRoot of pFile @@ -1774,6 +1807,7 @@ int FileOp::writeHeaders(IDBDataFile* pFile, const char* controlHdr, * blockHdrInit(in) - data used to initialize each block * blockHdrInitSize(in) - number of bytes in blockHdrInit * bExpandExtent (in) - Expand existing extent, or initialize a new one + * bOptExtension(in) - skip full extent preallocation. * RETURN: * returns ERR_FILE_WRITE if an error occurs, * else returns NO_ERROR. @@ -1784,7 +1818,8 @@ int FileOp::initDctnryExtent( int nBlocks, unsigned char* blockHdrInit, int blockHdrInitSize, - bool bExpandExtent ) + bool bExpandExtent, + bool bOptExtension ) { // @bug5769 Don't initialize extents or truncate db files on HDFS if (idbdatafile::IDBPolicy::useHdfs()) @@ -1801,6 +1836,21 @@ int FileOp::initDctnryExtent( // Create vector of mutexes used to serialize extent access per DBRoot initDbRootExtentMutexes( ); + // MCOL-498 Skip the huge preallocations if the option is set + // for the dbroot. This check is skiped for abbreviated extent. + // IMO it is better to check bool then to call a function. + // CS uses non-compressed dict files for its system catalog so + // CS doesn't optimize non-compressed dict creation. + if ( bOptExtension ) + { + bOptExtension = (idbdatafile::IDBPolicy::PreallocSpace(dbRoot) + && m_compressionType) ? bOptExtension : false; + } + // Reduce number of blocks allocated for abbreviated extents thus + // CS writes less when creates a new table. This couldn't be zero + // b/c Snappy compressed file format doesn't tolerate empty files. + int realNBlocks = ( bOptExtension && nBlocks <= MAX_INITIAL_EXTENT_BLOCKS_TO_DISK ) ? 1 : nBlocks; + // Determine the number of blocks in each call to fwrite(), and the // number of fwrite() calls to make, based on this. In other words, // we put a cap on the "writeSize" so that we don't allocate and write @@ -1808,99 +1858,101 @@ int FileOp::initDctnryExtent( // expanding an abbreviated 64M extent, we may not have an even // multiple of MAX_NBLOCKS to write; remWriteSize is the number of // blocks above and beyond loopCount*MAX_NBLOCKS. - int writeSize = nBlocks * BYTE_PER_BLOCK; // 1M and 8M row extent size + int writeSize = realNBlocks * BYTE_PER_BLOCK; // 1M and 8M row extent size int loopCount = 1; int remWriteSize = 0; - if (nBlocks > MAX_NBLOCKS) // 64M row extent size + if (realNBlocks > MAX_NBLOCKS) // 64M row extent size { writeSize = MAX_NBLOCKS * BYTE_PER_BLOCK; - loopCount = nBlocks / MAX_NBLOCKS; - remWriteSize = nBlocks - (loopCount * MAX_NBLOCKS); + loopCount = realNBlocks / MAX_NBLOCKS; + remWriteSize = realNBlocks - (loopCount * MAX_NBLOCKS); } // Allocate a buffer, initialize it, and use it to create the extent idbassert(dbRoot > 0); -#ifdef PROFILE +#ifdef PROFILE if (bExpandExtent) Stats::startParseEvent(WE_STATS_WAIT_TO_EXPAND_DCT_EXTENT); else Stats::startParseEvent(WE_STATS_WAIT_TO_CREATE_DCT_EXTENT); - #endif - boost::mutex::scoped_lock lk(*m_DbRootAddExtentMutexes[dbRoot]); -#ifdef PROFILE + boost::mutex::scoped_lock lk(*m_DbRootAddExtentMutexes[dbRoot]); + +#ifdef PROFILE if (bExpandExtent) Stats::stopParseEvent(WE_STATS_WAIT_TO_EXPAND_DCT_EXTENT); else Stats::stopParseEvent(WE_STATS_WAIT_TO_CREATE_DCT_EXTENT); - - Stats::startParseEvent(WE_STATS_INIT_DCT_EXTENT); #endif - - // Allocate buffer, and store in scoped_array to insure it's deletion. - // Create scope {...} to manage deletion of writeBuf. + // Skip space preallocation if configured so + // fallback to sequential write otherwise. + // Couldn't avoid preallocation for full extents, + // e.g. ADD COLUMN DDL b/c CS has to fill the file + // with empty magics. + if ( !bOptExtension ) { - unsigned char* writeBuf = new unsigned char[writeSize]; - boost::scoped_array writeBufPtr( writeBuf ); - - memset(writeBuf, 0, writeSize); - - for (int i = 0; i < nBlocks; i++) + // Allocate buffer, and store in scoped_array to insure it's deletion. + // Create scope {...} to manage deletion of writeBuf. { - memcpy( writeBuf + (i * BYTE_PER_BLOCK), - blockHdrInit, - blockHdrInitSize ); - } - #ifdef PROFILE - Stats::stopParseEvent(WE_STATS_INIT_DCT_EXTENT); - - if (bExpandExtent) - Stats::startParseEvent(WE_STATS_EXPAND_DCT_EXTENT); - else - Stats::startParseEvent(WE_STATS_CREATE_DCT_EXTENT); - + Stats::startParseEvent(WE_STATS_INIT_DCT_EXTENT); #endif - //std::ostringstream oss; - //oss << "initDctnryExtent: width-8(assumed)" << - //"; loopCount-" << loopCount << - //"; writeSize-" << writeSize; - //std::cout << oss.str() << std::endl; - if (remWriteSize > 0) - { - if (pFile->write( writeBuf, remWriteSize ) != remWriteSize) - { - return ERR_FILE_WRITE; - } - } + unsigned char* writeBuf = new unsigned char[writeSize]; + boost::scoped_array writeBufPtr( writeBuf ); - for (int j = 0; j < loopCount; j++) - { - if (pFile->write( writeBuf, writeSize ) != writeSize) - { - return ERR_FILE_WRITE; - } - } - } + memset(writeBuf, 0, writeSize); - if (m_compressionType) + for (int i = 0; i < realNBlocks; i++) + { + memcpy( writeBuf + (i * BYTE_PER_BLOCK), + blockHdrInit, + blockHdrInitSize ); + } + +#ifdef PROFILE + Stats::stopParseEvent(WE_STATS_INIT_DCT_EXTENT); + + if (bExpandExtent) + Stats::startParseEvent(WE_STATS_EXPAND_DCT_EXTENT); + else + Stats::startParseEvent(WE_STATS_CREATE_DCT_EXTENT); +#endif + + if (remWriteSize > 0) + { + if (pFile->write( writeBuf, remWriteSize ) != remWriteSize) + { + return ERR_FILE_WRITE; + } + } + + for (int j = 0; j < loopCount; j++) + { + if (pFile->write( writeBuf, writeSize ) != writeSize) + { + return ERR_FILE_WRITE; + } + } + // CS doesn't account flush timings. +#ifdef PROFILE + if (bExpandExtent) + Stats::stopParseEvent(WE_STATS_EXPAND_DCT_EXTENT); + else + Stats::stopParseEvent(WE_STATS_CREATE_DCT_EXTENT); +#endif + } + } // preallocation fallback end + + // MCOL-498 CS has to set a number of blocs in the chunk header + if ( m_compressionType ) + { updateDctnryExtent(pFile, nBlocks); - - // Synchronize to avoid write buffer pile up too much, which could cause - // controllernode to timeout later when it needs to save a snapshot. + } pFile->flush(); -#ifdef PROFILE - - if (bExpandExtent) - Stats::stopParseEvent(WE_STATS_EXPAND_DCT_EXTENT); - else - Stats::stopParseEvent(WE_STATS_CREATE_DCT_EXTENT); - -#endif } return NO_ERROR; @@ -2223,6 +2275,15 @@ int FileOp::oid2FileName( FID fid, #endif +// Need this stub to use ColumnOp::writeRow in the unit tests +#ifdef WITH_UNIT_TESTS + if (fid == 42) + { + sprintf(fullFileName, "./versionbuffer"); + return NO_ERROR; + } +#endif + /* If is a version buffer file, the format is different. */ if (fid < 1000) { @@ -2790,7 +2851,8 @@ int FileOp::expandAbbrevColumnExtent( int rc = FileOp::initColumnExtent(pFile, dbRoot, blksToAdd, emptyVal, width, false, // existing file true, // expand existing extent - false); // n/a since not adding new extent + false, // n/a since not adding new extent + true); // optimize segment file extension return rc; } diff --git a/writeengine/shared/we_fileop.h b/writeengine/shared/we_fileop.h index f4b7eac4a..e7f2fa576 100644 --- a/writeengine/shared/we_fileop.h +++ b/writeengine/shared/we_fileop.h @@ -50,6 +50,8 @@ #define EXPORT #endif +#define MAX_NBLOCKS 8192 + #include "brmtypes.h" /** Namespace WriteEngine */ @@ -90,6 +92,15 @@ public: execplan::CalpontSystemCatalog::ColDataType colDataType, uint64_t emptyVal = 0, int width = 1 ) ; + + /** + * @brief Create a file with a fixed file size by its name. + * Changed to public for UT. + */ + int createFile( const char* fileName, int fileSize, + uint64_t emptyVal, int width, + uint16_t dbRoot ); + /** * @brief Delete a file */ @@ -324,13 +335,15 @@ public: * @param blockHdrInit(in) - data used to initialize each block header * @param blockHdrInitSize(in) - number of bytes in blockHdrInit * @param bExpandExtent (in) - Expand existing extent, or initialize new one + * @param bOptExtension (in) - skip or optimize full extent preallocation */ EXPORT int initDctnryExtent( IDBDataFile* pFile, uint16_t dbRoot, int nBlocks, unsigned char* blockHdrInit, int blockHdrInitSize, - bool bExpandExtent ); + bool bExpandExtent, + bool bOptExtension = false ); /** * @brief Check whether it is an directory @@ -463,6 +476,25 @@ public: int compressionType() const; EXPORT virtual int flushFile(int rc, std::map& oids); + // Initialize an extent in a column segment file + // pFile (in) IDBDataFile* of column segment file to be written to + // dbRoot (in) - DBRoot of pFile + // nBlocks (in) - number of blocks to be written for an extent + // emptyVal(in) - empty value to be used for column data values + // width (in) - width of the applicable column + // bNewFile (in) - Adding extent to new file + // bExpandExtent (in) - Expand existing extent, or initialize new one + // bAbbrevExtent (in) - If adding new extent, is it abbreviated + // bOptExtension(in) - skip or optimize full extent preallocation + int initColumnExtent( IDBDataFile* pFile, + uint16_t dbRoot, + int nBlocks, + uint64_t emptyVal, + int width, + bool bNewFile, + bool bExpandExtent, + bool bAbbrevExtent, + bool bOptExtension=false ); protected: EXPORT virtual int updateColumnExtent(IDBDataFile* pFile, int nBlocks); @@ -475,10 +507,6 @@ private: FileOp(const FileOp& rhs); FileOp& operator=(const FileOp& rhs); - int createFile( const char* fileName, int fileSize, - uint64_t emptyVal, int width, - uint16_t dbRoot ); - int expandAbbrevColumnChunk( IDBDataFile* pFile, uint64_t emptyVal, int colWidth, @@ -491,24 +519,6 @@ private: uint64_t emptyVal, int width); - // Initialize an extent in a column segment file - // pFile (in) IDBDataFile* of column segment file to be written to - // dbRoot (in) - DBRoot of pFile - // nBlocks (in) - number of blocks to be written for an extent - // emptyVal(in) - empty value to be used for column data values - // width (in) - width of the applicable column - // bNewFile (in) - Adding extent to new file - // bExpandExtent (in) - Expand existing extent, or initialize new one - // bAbbrevExtent (in) - If adding new extent, is it abbreviated - int initColumnExtent( IDBDataFile* pFile, - uint16_t dbRoot, - int nBlocks, - uint64_t emptyVal, - int width, - bool bNewFile, - bool bExpandExtent, - bool bAbbrevExtent ); - static void initDbRootExtentMutexes(); static void removeDbRootExtentMutexes(); diff --git a/writeengine/wrapper/CMakeLists.txt b/writeengine/wrapper/CMakeLists.txt index 8bed5c065..205e3cd77 100644 --- a/writeengine/wrapper/CMakeLists.txt +++ b/writeengine/wrapper/CMakeLists.txt @@ -44,4 +44,3 @@ target_link_libraries(writeengine ${NETSNMP_LIBRARIES}) set_target_properties(writeengine PROPERTIES VERSION 1.0.0 SOVERSION 1) install(TARGETS writeengine DESTINATION ${ENGINE_LIBDIR} COMPONENT libs) - diff --git a/writeengine/wrapper/we_colop.cpp b/writeengine/wrapper/we_colop.cpp index 4f0e6a0d2..1359570e5 100644 --- a/writeengine/wrapper/we_colop.cpp +++ b/writeengine/wrapper/we_colop.cpp @@ -35,6 +35,7 @@ using namespace std; #include "idbcompress.h" #include "writeengine.h" #include "cacheutils.h" +#include "we_fileop.h" using namespace execplan; @@ -471,6 +472,15 @@ int ColumnOp::allocRowId(const TxnID& txnid, bool useStartingExtent, if ( rc != NO_ERROR) return rc; + // MCOL-498 This must be a first block in a new extent so + // fill the block up to its boundary with empties. Otherwise + // there could be fantom values. + { + uint64_t emptyVal = getEmptyRowValue(column.colDataType, column.colWidth); + setEmptyBuf(buf, BYTE_PER_BLOCK, emptyVal, column.colWidth); + } + + for (j = 0; j < totalRowPerBlock; j++) { if (isEmptyRow(buf, j, column)) @@ -1519,6 +1529,8 @@ void ColumnOp::setColParam(Column& column, * rowIdArray - the array of row id, for performance purpose, I am assuming the rowIdArray is sorted * valArray - the array of row values * oldValArray - the array of old value + * bDelete - yet. The flag must be useless b/c writeRows + * is used for deletion. * RETURN: * NO_ERROR if success, other number otherwise ***********************************************************/ @@ -1533,12 +1545,16 @@ int ColumnOp::writeRow(Column& curCol, uint64_t totalRow, const RID* rowIdArray, char charTmpBuf[8]; uint64_t emptyVal; int rc = NO_ERROR; + bool fillUpWEmptyVals = false; + bool firstRowInBlock = false; + bool lastRowInBlock = false; + uint16_t rowsInBlock = BYTE_PER_BLOCK / curCol.colWidth; while (!bExit) { curRowId = rowIdArray[i]; - calculateRowId(curRowId, BYTE_PER_BLOCK / curCol.colWidth, curCol.colWidth, dataFbo, dataBio); + calculateRowId(curRowId, rowsInBlock, curCol.colWidth, dataFbo, dataBio); // load another data block if necessary if (curDataFbo != dataFbo) @@ -1551,8 +1567,18 @@ int ColumnOp::writeRow(Column& curCol, uint64_t totalRow, const RID* rowIdArray, return rc; bDataDirty = false; + // MCOL-498 We got into the next block, so the row is first in that block + // - fill the block up with empty magics. + if ( curDataFbo != -1 && !bDelete ) + fillUpWEmptyVals = true; } + // MCOL-498 CS hasn't touched any block yet, + // but the row filled will be the first in the block. + firstRowInBlock = ( !(curRowId % (rowsInBlock)) ) ? true : false; + if( firstRowInBlock && !bDelete ) + fillUpWEmptyVals = true; + curDataFbo = dataFbo; rc = readBlock(curCol.dataFile.pFile, dataBuf, curDataFbo); @@ -1562,7 +1588,7 @@ int ColumnOp::writeRow(Column& curCol, uint64_t totalRow, const RID* rowIdArray, bDataDirty = true; } - // This is a awkward way to convert void* and get ith element, I just don't have a good solution for that + // This is a awkward way to convert void* and get its element, I just don't have a good solution for that // How about pVal = valArray + i*curCol.colWidth? switch (curCol.colType) { @@ -1676,8 +1702,54 @@ int ColumnOp::writeRow(Column& curCol, uint64_t totalRow, const RID* rowIdArray, // take care of the cleanup if (bDataDirty && curDataFbo >= 0) + { + if ( fillUpWEmptyVals ) + { + emptyVal = getEmptyRowValue(curCol.colDataType, curCol.colWidth); + int writeSize = BYTE_PER_BLOCK - ( dataBio + curCol.colWidth ); + // MCOL-498 Add the check though this is unlikely at the moment of writing. + if ( writeSize ) + setEmptyBuf( dataBuf + dataBio + curCol.colWidth, writeSize, + emptyVal, curCol.colWidth ); + } + rc = saveBlock(curCol.dataFile.pFile, dataBuf, curDataFbo); + if ( rc != NO_ERROR) + return rc; + + // MCOL-498 If it was the last row in a block fill the next block with + // empty vals, otherwise next ColumnOp::allocRowId() + // will fail on the next block. + lastRowInBlock = ( rowsInBlock - ( curRowId % rowsInBlock ) == 1 ) ? true : false; + if ( lastRowInBlock ) + { + if( !fillUpWEmptyVals ) + emptyVal = getEmptyRowValue(curCol.colDataType, curCol.colWidth); + // MCOL-498 Skip if this is the last block in an extent. + if ( curDataFbo % MAX_NBLOCKS != MAX_NBLOCKS - 1 ) + { + rc = saveBlock(curCol.dataFile.pFile, dataBuf, curDataFbo); + if ( rc != NO_ERROR) + return rc; + + curDataFbo += 1; + rc = readBlock(curCol.dataFile.pFile, dataBuf, curDataFbo); + if ( rc != NO_ERROR) + return rc; + + unsigned char zeroSubBlock[BYTE_PER_SUBBLOCK]; + std::memset(zeroSubBlock, 0, BYTE_PER_SUBBLOCK); + // The first subblock is made of 0 - fill the block with empty vals. + if ( !std::memcmp(dataBuf, zeroSubBlock, BYTE_PER_SUBBLOCK) ) + { + setEmptyBuf(dataBuf, BYTE_PER_BLOCK, emptyVal, curCol.colWidth); + rc = saveBlock(curCol.dataFile.pFile, dataBuf, curDataFbo); + } + } + } + + } return rc; } diff --git a/writeengine/wrapper/we_colop.h b/writeengine/wrapper/we_colop.h index 3637ec772..d1947d752 100644 --- a/writeengine/wrapper/we_colop.h +++ b/writeengine/wrapper/we_colop.h @@ -267,7 +267,7 @@ public: bool bDelete = false); /** - * @brief Write row(s) for update and delete @Bug 1886,2870 + * @brief Write row(s) for delete @Bug 1886,2870 */ EXPORT virtual int writeRows(Column& curCol, uint64_t totalRow, @@ -277,7 +277,7 @@ public: bool bDelete = false); /** - * @brief Write row(s) for update and delete @Bug 1886,2870 + * @brief Write row(s) for update @Bug 1886,2870 */ EXPORT virtual int writeRowsValues(Column& curCol, uint64_t totalRow,