Deep build refactoring phase 1 (#3562)

* configcpp refactored
* logging and datatypes refactored

* more dataconvert
* chore(build): massive removals, auto add files to debian install file
* chore(codemanagement): nodeps headers, potentioal library
* chore(build): configure before autobake
* chore(build): use custom cmake commands for components, mariadb-plugin-columnstore.install generated
* chore(build): install deps as separate step for build-packages
* more deps
* check  debian/mariadb-plugin-columnstore.install automatically
* chore(build): add option for multibracnh compilation
* Fix warning

.drone.jsonnet

@@ -631,9 +631,6 @@ local Pipeline(branch, platform, event, arch="amd64", server="10.6-enterprise",
              volumes: [pipeline._volumes.mdb],
              environment: {
                DEBIAN_FRONTEND: "noninteractive",
-               DEB_BUILD_OPTIONS: "parallel=4",
-               DH_BUILD_DDEBS: "1",
-               BUILDPACKAGE_FLAGS: "-b",  // Save time and produce only binary packages, not source
                AWS_ACCESS_KEY_ID: {
                  from_secret: "aws_access_key_id",
                },
@@ -656,7 +653,7 @@ local Pipeline(branch, platform, event, arch="amd64", server="10.6-enterprise",
                         'bash -c "set -o pipefail && bash /mdb/' + builddir + "/storage/columnstore/columnstore/build/bootstrap_mcs.sh " +
                          "--build-type RelWithDebInfo " +
                          "--distro " + platform + " " +
-                         "--build-packages --sccache " +
+                         "--build-packages --install-deps --sccache " +
                          " " + customBootstrapParams +
                          " " + customBootstrapParamsForExisitingPipelines(platform) + " | " +
                          "/mdb/" + builddir + "/storage/columnstore/columnstore/build/ansi2txt.sh " +

.gitignore

@@ -153,6 +153,7 @@ oam/install_scripts/mcs-workernode.service
 oam/install_scripts/mcs-writeengineserver.service
 oam/install_scripts/mcs-loadbrm.py
 oam/install_scripts/mcs-savebrm.py
+oam/install_scripts/mcssavebrm.py
 oam/install_scripts/mcs-storagemanager.service
 oam/install_scripts/mcs_module_installer.sh
 oam/install_scripts/slave-rep-columnstore.sh
@@ -186,4 +187,3 @@ versioning/BRM/shmkeys.cpp
 obj/
 build/build
 
-

build/bootstrap_mcs.sh

@@ -19,6 +19,9 @@ CONFIG_DIR=$RPM_CONFIG_DIR
 
 SCRIPT_LOCATION=$(dirname "$0")
 MDB_SOURCE_PATH=$(realpath "$SCRIPT_LOCATION"/../../../..)
+COLUMSNTORE_SOURCE_PATH=$(realpath "$SCRIPT_LOCATION"/../)
+
+DEFAULT_MARIA_BUILD_PATH=$(realpath "$MDB_SOURCE_PATH"/../BuildOf_$(basename "$MDB_SOURCE_PATH"))
 
 BUILD_TYPE_OPTIONS=("Debug" "RelWithDebInfo")
 DISTRO_OPTIONS=("ubuntu:20.04" "ubuntu:22.04" "ubuntu:24.04" "debian:11" "debian:12" "rockylinux:8" "rockylinux:9")
@@ -28,21 +31,10 @@ MDB_CMAKE_FLAGS=()
 
 source "$SCRIPT_LOCATION"/utils.sh
 
-if [ "$EUID" -ne 0 ]; then
-    error "Please run script as root to install MariaDb to system paths"
-    exit 1
-fi
-
-cd $SCRIPT_LOCATION
-CURRENT_BRANCH=$(git rev-parse --abbrev-ref HEAD)
-message "Columnstore will be built from $color_yellow$CURRENT_BRANCH$color_cyan branch"
-cd - >/dev/null
-
 echo "Arguments received: $@"
-message "Building Mariadb Server from $color_yellow$MDB_SOURCE_PATH$color_normal"
 
 optparse.define short=A long=asan desc="Build with ASAN" variable=ASAN default=false value=true
-optparse.define short=a long=build-path variable=MARIA_BUILD_PATH default=$(realpath "$MDB_SOURCE_PATH"/../BuildOf_$(basename "$MDB_SOURCE_PATH"))
+optparse.define short=a long=build-path desc="Path for build output" variable=MARIA_BUILD_PATH default=$DEFAULT_MARIA_BUILD_PATH
 optparse.define short=B long=run-microbench desc="Compile and run microbenchmarks " variable=RUN_BENCHMARKS default=false value=true
 optparse.define short=c long=cloud desc="Enable cloud storage" variable=CLOUD_STORAGE_ENABLED default=false value=true
 optparse.define short=C long=force-cmake-reconfig desc="Force cmake reconfigure" variable=FORCE_CMAKE_CONFIG default=false value=true
@@ -68,10 +60,26 @@ optparse.define short=T long=tsan desc="Build with TSAN" variable=TSAN default=f
 optparse.define short=u long=skip-unit-tests desc="Skip UnitTests" variable=SKIP_UNIT_TESTS default=false value=true
 optparse.define short=U long=ubsan desc="Build with UBSAN" variable=UBSAN default=false value=true
 optparse.define short=v long=verbose desc="Verbose makefile commands" variable=MAKEFILE_VERBOSE default=false value=true
+optparse.define short=V long=add-branch-name-to-outdir desc="Add branch name to build output directory" variable=BRANCH_NAME_TO_OUTDIR default=false value=true
 optparse.define short=W long=without-core-dumps desc="Do not produce core dumps" variable=WITHOUT_COREDUMPS default=false value=true
 
 source $(optparse.build)
 
+message "Building MariaDB Server from $color_yellow$MDB_SOURCE_PATH$color_normal"
+
+cd $COLUMSNTORE_SOURCE_PATH
+COLUMNSTORE_BRANCH=$(git rev-parse --abbrev-ref HEAD)
+message "Columnstore will be built from $color_yellow$COLUMNSTORE_BRANCH$color_cyan branch"
+
+cd $MDB_SOURCE_PATH
+MARIADB_BRANCH=$(git rev-parse --abbrev-ref HEAD)
+message "MariaDB will be built from $color_yellow$MARIADB_BRANCH$color_cyan branch"
+cd - >/dev/null
+
+if [[ ${BRANCH_NAME_TO_OUTDIR} = true ]]; then
+    MARIA_BUILD_PATH="${MARIA_BUILD_PATH}_${MARIADB_BRANCH}_${COLUMNSTORE_BRANCH}"
+fi
+
 if [[ ! " ${BUILD_TYPE_OPTIONS[*]} " =~ " ${MCS_BUILD_TYPE} " ]]; then
     getChoice -q "Select your Build Type" -o BUILD_TYPE_OPTIONS
     MCS_BUILD_TYPE=$selectedChoice
@@ -104,7 +112,7 @@ install_deps() {
       cppunit-devel cmake3 libxcrypt-devel xz-devel zlib-devel libzstd-devel glibc-devel"
 
     DEB_BUILD_DEPS="apt-get -y update && apt-get -y install build-essential automake libboost-all-dev \
-      bison cmake libncurses5-dev libaio-dev libsystemd-dev libpcre2-dev libperl-dev libssl-dev libxml2-dev \
+      bison cmake libncurses5-dev python3 libaio-dev libsystemd-dev libpcre2-dev libperl-dev libssl-dev libxml2-dev \
       libkrb5-dev flex libpam-dev git libsnappy-dev libcurl4-openssl-dev libgtest-dev libcppunit-dev googletest \
       libjemalloc-dev liblz-dev liblzo2-dev liblzma-dev liblz4-dev libbz2-dev libbenchmark-dev libdistro-info-perl \
       graphviz devscripts ccache equivs eatmydata curl"
@@ -259,8 +267,9 @@ construct_cmake_flags() {
         -DWITH_WSREP=NO
     )
 
-    if [[ MAINTAINER_MODE = true ]]; then
+    if [[ $MAINTAINER_MODE = true ]]; then
         MDB_CMAKE_FLAGS+=(-DCOLUMNSTORE_MAINTAINER=YES)
+        message "Columnstore mainteiner mode on"
     else
         warn "Maintainer mode is disabled, be careful, alien"
     fi
@@ -406,9 +415,15 @@ generate_svgs() {
 }
 
 build_package() {
+    cd $MDB_SOURCE_PATH
+
     if [[ $pkg_format == "rpm" ]]; then
         command="cmake ${MDB_CMAKE_FLAGS[@]} && make -j\$(nproc) package"
     else
+        export DEBIAN_FRONTEND="noninteractive"
+        export DEB_BUILD_OPTIONS="parallel=$(nproc)"
+        export DH_BUILD_DDEBS="1"
+        export BUILDPACKAGE_FLAGS="-b"
         command="mk-build-deps debian/control -t 'apt-get -y -o Debug::pkgProblemResolver=yes --no-install-recommends' -r -i && \
        CMAKEFLAGS=\"${MDB_CMAKE_FLAGS[@]}\" debian/autobake-deb.sh"
     fi
@@ -420,6 +435,14 @@ build_package() {
     check_errorcode
 }
 
+check_debian_install_file() {
+    message "checking debian/mariadb-plugin-columnstore.install"
+    message_split
+    python3 $COLUMSNTORE_SOURCE_PATH/build/debian_install_file_compare.py \
+        ${COLUMSNTORE_SOURCE_PATH}/debian/mariadb-plugin-columnstore.install \
+        $MARIA_BUILD_PATH/mariadb-plugin-columnstore.install.generated
+}
+
 build_binary() {
     MARIA_BUILD_PATH=$(realpath "$MARIA_BUILD_PATH")
     message_split
@@ -438,7 +461,7 @@ build_binary() {
     message "Configuring cmake silently"
     ${CMAKE_BIN_NAME} "${MDB_CMAKE_FLAGS[@]}" -S"$MDB_SOURCE_PATH" -B"$MARIA_BUILD_PATH" | spinner
     message_split
-
+    check_debian_install_file
     generate_svgs
 
     ${CMAKE_BIN_NAME} --build "$MARIA_BUILD_PATH" -j "$CPUS" | onelinearizator &&
@@ -553,6 +576,12 @@ make_dir() {
 
 install() {
     if [[ $RECOMPILE_ONLY = false ]]; then
+
+        if [ "$EUID" -ne 0 ]; then
+            error "Please run script as root to install MariaDb to system paths"
+            exit 1
+        fi
+
         message_split
         message "Installing MariaDB"
         disable_plugins_for_bootstrap
@@ -636,7 +665,7 @@ smoke() {
     fi
 }
 
-if [[ $INSTALL_DEPS = true || $BUILD_PACKAGES = true ]]; then
+if [[ $INSTALL_DEPS = true ]]; then
     install_deps
 fi
 

build/debian_install_file_compare.py

@@ -0,0 +1,81 @@
+#!/usr/bin/env python3
+
+import argparse
+import fnmatch
+import sys
+
+def read_patterns(filename):
+    with open(filename, 'r') as f:
+        return [line.strip() for line in f if line.strip()]
+
+def read_paths_with_comments(filename):
+    paths = []
+    with open(filename, 'r') as f:
+        for line in f:
+            raw = line.rstrip('\n')
+            if not raw.strip():
+                continue
+            # Split into path and comment
+            if '#' in raw:
+                path, comment = raw.split('#', 1)
+                path = path.strip()
+                comment = comment.strip()
+            else:
+                path = raw.strip()
+                comment = ''
+            if path:
+                paths.append((raw, path, comment))
+    return paths
+
+def print_red(msg):
+    if sys.stdout.isatty():
+        print("\033[31m" + msg + "\033[0m")
+    else:
+        print(msg)
+
+def main():
+    parser = argparse.ArgumentParser(description="Compare wildcard patterns to file paths.")
+    parser.add_argument('patterns_file', help='File with wildcard patterns (one per line)')
+    parser.add_argument('paths_file', help='File with file paths (one per line)')
+    args = parser.parse_args()
+
+    patterns = read_patterns(args.patterns_file)
+    paths_with_comments = read_paths_with_comments(args.paths_file)
+
+    # Track which paths are matched (by index)
+    path_matched = [False] * len(paths_with_comments)
+
+    # Collect patterns that don't match any path
+    unmatched_patterns = []
+    for pattern in patterns:
+        matched = False
+        for i, (raw, path, comment) in enumerate(paths_with_comments):
+            if fnmatch.fnmatch(path, pattern):
+                matched = True
+                path_matched[i] = True
+        if not matched:
+            unmatched_patterns.append(pattern)
+
+    if unmatched_patterns:
+        print_red("The files declared in debian/mariadb-plugin-columnstore.install "
+                  " are not added to CMakeLists via columnstore_* statements see cmake/ColumnstoreLibrary.cmake")
+        for pattern in unmatched_patterns:
+            print(f"- {pattern}")
+
+    # Collect paths that weren't matched by any pattern
+    unmatched_paths = []
+    for (raw, path, comment), matched in zip(paths_with_comments, path_matched):
+        if not matched:
+            unmatched_paths.append((path, comment))
+
+    if unmatched_paths:
+        print_red("The files added via columnstore_* statements from cmake/ColumnstoreLibrary.cmake "
+                  "are missing in debian/mariadb-plugin-columnstore.install file")
+        for path, comment in unmatched_paths:
+            if comment:
+                print(f"- {path} {comment}")
+            else:
+                print(f"- {path}")
+
+if __name__ == "__main__":
+    main()

cmake/ColumnstoreLibrary.cmake

@@ -1,8 +1,105 @@
-# Function to create either a static or shared library based on COLUMNSTORE_STATIC_LIBRARIES
+set(DEBIAN_INSTALL_FILE "${CMAKE_BINARY_DIR}/mariadb-plugin-columnstore.install.generated")
+file(WRITE ${DEBIAN_INSTALL_FILE} "#File is generated by ColumnstoreLibrary.cmake, do not edit\n")
+
+macro(add_to_debian_install_file file_path)
+    string(SUBSTRING "${file_path}" 1 -1 BINARY_ENTRY)
+    file(RELATIVE_PATH CMAKEFILE "${CMAKE_SOURCE_DIR}/storage/columnstore/columnstore" ${CMAKE_CURRENT_LIST_FILE})
+    string(STRIP "${BINARY_ENTRY}" BINARY_ENTRY)
+
+    file(APPEND ${DEBIAN_INSTALL_FILE} "${BINARY_ENTRY} # added in ${CMAKEFILE}\n")
+endmacro()
+
+function(get_target_output_filename TARGET_NAME OUTPUT_VAR)
+    # 1. Get the target's OUTPUT_NAME (falls back to TARGET_NAME)
+    get_target_property(OUTPUT_NAME ${TARGET_NAME} OUTPUT_NAME)
+    if(NOT OUTPUT_NAME)
+        set(OUTPUT_NAME "${TARGET_NAME}")
+    endif()
+
+    # 1. Get the correct suffix based on target type
+    get_target_property(TARGET_TYPE ${TARGET_NAME} TYPE)
+    if(TARGET_TYPE STREQUAL "EXECUTABLE")
+        set(SUFFIX "${CMAKE_EXECUTABLE_SUFFIX}")
+        set(PREFIX "") # No prefix for executables
+    elseif(TARGET_TYPE STREQUAL "SHARED_LIBRARY")
+        set(SUFFIX "${CMAKE_SHARED_LIBRARY_SUFFIX}")
+        set(PREFIX "lib")
+    elseif(TARGET_TYPE STREQUAL "STATIC_LIBRARY")
+        set(SUFFIX "${CMAKE_STATIC_LIBRARY_SUFFIX}")
+        set(PREFIX "lib")
+    else()
+        message(WARNING "Unknown target type for ${TARGET_NAME}")
+        set(SUFFIX "")
+    endif()
+
+    # 1. Combine into the final filename
+    set(${OUTPUT_VAR}
+        "${PREFIX}${OUTPUT_NAME}${SUFFIX}"
+        PARENT_SCOPE
+    )
+endfunction()
+
+macro(columnstore_install_target target destination)
+    install(
+        TARGETS ${target}
+        DESTINATION ${destination}
+        COMPONENT columnstore-engine
+    )
+    get_target_output_filename(${target} OUTPUT_FILENAME)
+    add_to_debian_install_file("${destination}/${OUTPUT_FILENAME}")
+endmacro()
+
+macro(columnstore_install_file file destination)
+    install(
+        FILES ${file}
+        DESTINATION ${destination}
+        COMPONENT columnstore-engine
+    )
+    get_filename_component(FILENAME ${file} NAME)
+    add_to_debian_install_file("${destination}/${FILENAME}")
+endmacro()
+
+macro(columnstore_install_program file destination)
+    install(
+        PROGRAMS ${file}
+        DESTINATION ${destination}
+        COMPONENT columnstore-engine
+    )
+    get_filename_component(FILENAME ${file} NAME)
+    add_to_debian_install_file(
+        "${destination}/${FILENAME}
+        "
+    )
+endmacro()
+
+macro(columnstore_static_library libname)
+    add_definitions(-fPIC -DPIC)
+    add_library(${libname} STATIC ${ARGN})
+endmacro()
+
+macro(columnstore_shared_library libname)
+    add_library(${libname} SHARED ${ARGN})
+    columnstore_install_target(${libname} ${ENGINE_LIBDIR})
+endmacro()
+
 macro(columnstore_library libname)
     if(COLUMNSTORE_STATIC_LIBRARIES)
-        add_library(${libname} STATIC ${ARGN})
+        columnstore_static_library(${libname} ${ARGN})
     else()
-        add_library(${libname} SHARED ${ARGN})
+        columnstore_shared_library(${libname} ${ARGN})
     endif()
 endmacro()
+
+macro(columnstore_mysql_plugin_library libname)
+    add_library(${libname} SHARED ${ARGN})
+    columnstore_install_target(${libname} ${MARIADB_PLUGINDIR})
+endmacro()
+
+macro(columnstore_link libname)
+    target_link_libraries(${libname} ${ARGN})
+endmacro()
+
+macro(columnstore_executable executable_name)
+    add_executable(${executable_name} ${ARGN})
+    columnstore_install_target(${executable_name} ${ENGINE_BINDIR})
+endmacro()

cmake/compiler_flags.cmake

@@ -51,6 +51,7 @@ set(COLUMNSTORE_MAINTAINER_FLAGS ${WERROR_FLAG})
 set(FLAGS_ALL
     -Wall
     -Wextra
+    -ggdb3
     -fno-omit-frame-pointer
     -fno-strict-aliasing
     -fsigned-char
@@ -59,9 +60,9 @@ set(FLAGS_ALL
     -DBOOST_BIND_GLOBAL_PLACEHOLDERS
 )
 
-set(FLAGS_RELEASE -g -O3 -DDBUG_OFF)
+set(FLAGS_RELEASE -O3 -DDBUG_OFF)
 
-set(FLAGS_DEBUG -ggdb3 -O0 -D_DEBUG)
+set(FLAGS_DEBUG -O0 -D_DEBUG)
 # } end Release, Debug and common flags
 
 # linker flags {

cmake/configureEngine.cmake

@@ -773,17 +773,10 @@ endif()
 
 # releasenum is used by external scripts for various tasks. Leave it alone.
 configure_file(${CMAKE_CURRENT_SOURCE_DIR}/build/releasenum.in ${CMAKE_CURRENT_BINARY_DIR}/build/releasenum IMMEDIATE)
-install(
+columnstore_install_file(${CMAKE_CURRENT_BINARY_DIR}/build/releasenum ${ENGINE_SUPPORTDIR})
-    FILES ${CMAKE_CURRENT_BINARY_DIR}/build/releasenum
+
-    DESTINATION ${ENGINE_SUPPORTDIR}
-    COMPONENT columnstore-engine
-)
 configure_file(${CMAKE_CURRENT_SOURCE_DIR}/columnstoreversion.h.in ${CMAKE_CURRENT_SOURCE_DIR}/columnstoreversion.h)
 configure_file(${CMAKE_CURRENT_SOURCE_DIR}/mcsconfig.h.in ${CMAKE_CURRENT_BINARY_DIR}/mcsconfig.h)
-
 configure_file(${CMAKE_CURRENT_SOURCE_DIR}/gitversionEngine.in ${CMAKE_CURRENT_BINARY_DIR}/gitversionEngine IMMEDIATE)
-install(
+
-    FILES ${CMAKE_CURRENT_BINARY_DIR}/gitversionEngine
+columnstore_install_file(${CMAKE_CURRENT_BINARY_DIR}/gitversionEngine ${ENGINE_SUPPORTDIR})
-    DESTINATION ${ENGINE_SUPPORTDIR}
-    COMPONENT columnstore-engine
-)

cmake/packages.cmake

@@ -14,7 +14,6 @@ if(NOT LEX_EXECUTABLE)
 endif()
 
 find_package(LibXml2)
-
 if(NOT LIBXML2_FOUND)
     message_once(CS_NO_LIBXML "Could not find a usable libxml2 development environment!")
     return()

datatypes/CMakeLists.txt

@@ -2,10 +2,5 @@ include_directories(${ENGINE_COMMON_INCLUDES})
 set(datatypes_LIB_SRCS mcs_int128.cpp mcs_decimal.cpp)
 
 columnstore_library(datatypes ${datatypes_LIB_SRCS})
-add_dependencies(datatypes loggingcpp external_boost)
+columnstore_link(datatypes PRIVATE loggingcpp)
-
+add_dependencies(datatypes external_boost)
-install(
-    TARGETS datatypes
-    DESTINATION ${ENGINE_LIBDIR}
-    COMPONENT columnstore-engine
-)

dbcon/ddlpackage/CMakeLists.txt

@@ -42,10 +42,4 @@ columnstore_library(
     ${FLEX_ddl_scan_OUTPUTS}
 )
 
-add_dependencies(ddlpackage loggingcpp)
+columnstore_link(ddlpackage loggingcpp)
-
-install(
-    TARGETS ddlpackage
-    DESTINATION ${ENGINE_LIBDIR}
-    COMPONENT columnstore-engine
-)

dbcon/ddlpackageproc/CMakeLists.txt

@@ -13,13 +13,4 @@ set(ddlpackageproc_LIB_SRCS
 )
 
 columnstore_library(ddlpackageproc ${ddlpackageproc_LIB_SRCS})
-
+columnstore_link(ddlpackageproc ${NETSNMP_LIBRARIES} loggingcpp)
-add_dependencies(ddlpackageproc loggingcpp)
-
-target_link_libraries(ddlpackageproc ${NETSNMP_LIBRARIES})
-
-install(
-    TARGETS ddlpackageproc
-    DESTINATION ${ENGINE_LIBDIR}
-    COMPONENT columnstore-engine
-)

dbcon/dmlpackage/CMakeLists.txt

@@ -41,10 +41,4 @@ columnstore_library(
     ${FLEX_dml_scan_OUTPUTS}
 )
 
-add_dependencies(dmlpackage loggingcpp)
+columnstore_link(dmlpackage loggingcpp)
-
-install(
-    TARGETS dmlpackage
-    DESTINATION ${ENGINE_LIBDIR}
-    COMPONENT columnstore-engine
-)

dbcon/dmlpackageproc/CMakeLists.txt

@@ -13,13 +13,4 @@ set(dmlpackageproc_LIB_SRCS
 )
 
 columnstore_library(dmlpackageproc ${dmlpackageproc_LIB_SRCS})
-
+columnstore_link(dmlpackageproc ${NETSNMP_LIBRARIES} loggingcpp)
-add_dependencies(dmlpackageproc loggingcpp)
-
-target_link_libraries(dmlpackageproc ${NETSNMP_LIBRARIES})
-
-install(
-    TARGETS dmlpackageproc
-    DESTINATION ${ENGINE_LIBDIR}
-    COMPONENT columnstore-engine
-)

dbcon/execplan/CMakeLists.txt

@@ -46,13 +46,4 @@ set(execplan_LIB_SRCS
 )
 
 columnstore_library(execplan ${execplan_LIB_SRCS})
-
+columnstore_link(execplan messageqcpp ${NETSNMP_LIBRARIES} ${ENGINE_DT_LIB} pron loggingcpp)
-add_dependencies(execplan loggingcpp)
-
-target_link_libraries(execplan messageqcpp ${NETSNMP_LIBRARIES} ${ENGINE_DT_LIB} pron)
-
-install(
-    TARGETS execplan
-    DESTINATION ${ENGINE_LIBDIR}
-    COMPONENT columnstore-engine
-)

dbcon/execplan/pseudocolumn.cpp

@@ -25,7 +25,7 @@
 #include <string>
 #include <sstream>
 using namespace std;
-
+#include "basic/string_utils.h"
 
 #include "bytestream.h"
 #include "collation.h"
@@ -138,8 +138,9 @@ string PseudoColumn::toCppCode(IncludeSet& includes) const
 {
   includes.insert("pseudocolumn.h");
   stringstream ss;
-  ss << "PseudoColumn(" << std::quoted(fSchemaName) << ", " << std::quoted(fTableName) << ", " << fisColumnStore << ", " << std::quoted(fColumnName)
+  ss << "PseudoColumn(" << std::quoted(fSchemaName) << ", " << std::quoted(fTableName) << ", "
-     << ", " << fPseudoType << ", " << sessionID() << ")";
+     << fisColumnStore << ", " << std::quoted(fColumnName) << ", " << fPseudoType << ", " << sessionID()
+     << ")";
 
   return ss.str();
 }

dbcon/execplan/windowfunctioncolumn.cpp

@@ -27,10 +27,13 @@
 #include <iostream>
 #include <sstream>
 using namespace std;
+#include "basic/string_utils.h"
 
 #include <boost/tokenizer.hpp>
 using namespace boost;
 
+#include "basic/string_utils.h"
+
 #include "bytestream.h"
 #include "windowfunctioncolumn.h"
 #include "constantcolumn.h"
@@ -398,23 +401,32 @@ void WindowFunctionColumn::adjustResultType()
 {
   if ((fResultType.colDataType == CalpontSystemCatalog::DECIMAL ||
        fResultType.colDataType == CalpontSystemCatalog::UDECIMAL) &&
-      !datatypes::ASCIIStringCaseInsensetiveEquals(fFunctionName, "COUNT") && !datatypes::ASCIIStringCaseInsensetiveEquals(fFunctionName, "COUNT(*)") &&
+      !datatypes::ASCIIStringCaseInsensetiveEquals(fFunctionName, "COUNT") &&
-      !datatypes::ASCIIStringCaseInsensetiveEquals(fFunctionName, "ROW_NUMBER") && !datatypes::ASCIIStringCaseInsensetiveEquals(fFunctionName, "RANK") &&
+      !datatypes::ASCIIStringCaseInsensetiveEquals(fFunctionName, "COUNT(*)") &&
-      !datatypes::ASCIIStringCaseInsensetiveEquals(fFunctionName, "PERCENT_RANK") && !datatypes::ASCIIStringCaseInsensetiveEquals(fFunctionName, "DENSE_RANK") &&
+      !datatypes::ASCIIStringCaseInsensetiveEquals(fFunctionName, "ROW_NUMBER") &&
-      !datatypes::ASCIIStringCaseInsensetiveEquals(fFunctionName, "CUME_DIST") && !datatypes::ASCIIStringCaseInsensetiveEquals(fFunctionName, "NTILE") &&
+      !datatypes::ASCIIStringCaseInsensetiveEquals(fFunctionName, "RANK") &&
+      !datatypes::ASCIIStringCaseInsensetiveEquals(fFunctionName, "PERCENT_RANK") &&
+      !datatypes::ASCIIStringCaseInsensetiveEquals(fFunctionName, "DENSE_RANK") &&
+      !datatypes::ASCIIStringCaseInsensetiveEquals(fFunctionName, "CUME_DIST") &&
+      !datatypes::ASCIIStringCaseInsensetiveEquals(fFunctionName, "NTILE") &&
       !datatypes::ASCIIStringCaseInsensetiveEquals(fFunctionName, "PERCENTILE") && !fFunctionParms.empty() &&
       fFunctionParms[0]->resultType().colDataType == CalpontSystemCatalog::DOUBLE)
     fResultType = fFunctionParms[0]->resultType();
 
-  if ((datatypes::ASCIIStringCaseInsensetiveEquals(fFunctionName, "LEAD") || datatypes::ASCIIStringCaseInsensetiveEquals(fFunctionName, "LAG") ||
+  if ((datatypes::ASCIIStringCaseInsensetiveEquals(fFunctionName, "LEAD") ||
-       datatypes::ASCIIStringCaseInsensetiveEquals(fFunctionName, "MIN") || datatypes::ASCIIStringCaseInsensetiveEquals(fFunctionName, "MAX") ||
+       datatypes::ASCIIStringCaseInsensetiveEquals(fFunctionName, "LAG") ||
-       datatypes::ASCIIStringCaseInsensetiveEquals(fFunctionName, "FIRST_VALUE") || datatypes::ASCIIStringCaseInsensetiveEquals(fFunctionName, "LAST_VALUE") ||
+       datatypes::ASCIIStringCaseInsensetiveEquals(fFunctionName, "MIN") ||
+       datatypes::ASCIIStringCaseInsensetiveEquals(fFunctionName, "MAX") ||
+       datatypes::ASCIIStringCaseInsensetiveEquals(fFunctionName, "FIRST_VALUE") ||
+       datatypes::ASCIIStringCaseInsensetiveEquals(fFunctionName, "LAST_VALUE") ||
        datatypes::ASCIIStringCaseInsensetiveEquals(fFunctionName, "NTH_VALUE")) &&
       !fFunctionParms.empty())
     fResultType = fFunctionParms[0]->resultType();
 
-  if (datatypes::ASCIIStringCaseInsensetiveEquals(fFunctionName, "SUM") || datatypes::ASCIIStringCaseInsensetiveEquals(fFunctionName, "AVG") ||
+  if (datatypes::ASCIIStringCaseInsensetiveEquals(fFunctionName, "SUM") ||
-      datatypes::ASCIIStringCaseInsensetiveEquals(fFunctionName, "AVG_DISTINCT") || datatypes::ASCIIStringCaseInsensetiveEquals(fFunctionName, "PERCENTILE"))
+      datatypes::ASCIIStringCaseInsensetiveEquals(fFunctionName, "AVG") ||
+      datatypes::ASCIIStringCaseInsensetiveEquals(fFunctionName, "AVG_DISTINCT") ||
+      datatypes::ASCIIStringCaseInsensetiveEquals(fFunctionName, "PERCENTILE"))
   {
     if (fFunctionParms[0]->resultType().colDataType == CalpontSystemCatalog::DECIMAL ||
         fFunctionParms[0]->resultType().colDataType == CalpontSystemCatalog::UDECIMAL)
@@ -531,7 +543,7 @@ void WindowFunctionColumn::evaluate(Row& row, bool& isNull)
             fResult.strVal.assign(str.unsafeStringRef());
 
           // stringColVal is padded with '\0' to colWidth so can't use str.length()
-          //if (strlen(fResult.strVal.str()) == 0)
+          // if (strlen(fResult.strVal.str()) == 0)
           //  isNull = true;
 
           break;

dbcon/joblist/CMakeLists.txt

@@ -57,8 +57,8 @@ set(joblist_LIB_SRCS
 
 # ########## next target ###############
 
-add_library(joblist_server STATIC ${joblist_LIB_SRCS})
+columnstore_static_library(joblist_server ${joblist_LIB_SRCS})
-add_dependencies(joblist_server loggingcpp)
+columnstore_link(joblist_server loggingcpp)
 
 # ########## next target ###############
 
@@ -68,20 +68,9 @@ columnstore_library(joblist ${joblist_LIB_SRCS})
 target_include_directories(
     joblist BEFORE PUBLIC ${OPENSSL_INCLUDE_DIR} ${LIBMARIADB_BININC_DIR} ${LIBMARIADB_SRCINC_DIR}
 )
-add_dependencies(joblist loggingcpp)
+columnstore_link(joblist loggingcpp)
-
-install(
-    TARGETS joblist
-    DESTINATION ${ENGINE_LIBDIR}
-    COMPONENT columnstore-engine
-)
 
 if(WITH_ORDERBY_UT)
-    add_executable(job_orderby_tests orderby-tests.cpp)
+    columnstore_executable(job_orderby_tests orderby-tests.cpp)
-    target_link_libraries(job_orderby_tests ${ENGINE_LDFLAGS} ${ENGINE_WRITE_LIBS} ${CPPUNIT_LIBRARIES} cppunit)
+    columnstore_link(job_orderby_tests ${ENGINE_LDFLAGS} ${ENGINE_WRITE_LIBS} ${CPPUNIT_LIBRARIES} cppunit)
-    install(
-        TARGETS job_orderby_tests
-        DESTINATION ${ENGINE_BINDIR}
-        COMPONENT columnstore-engine
-    )
 endif()

dbcon/mysql/CMakeLists.txt

@@ -71,16 +71,17 @@ if(COMMAND mysql_add_plugin)
         CONFIG
         columnstore.cnf
     )
+    add_to_debian_install_file(${MARIADB_MYCNFDIR}/columnstore.cnf)
+    add_to_debian_install_file("${MARIADB_PLUGINDIR}/ha_columnstore.so")
+
 else()
     message("Adding plugin via CMake")
-    add_library(ha_columnstore SHARED ${libcalmysql_SRCS})
+    columnstore_shared_library(ha_columnstore ${libcalmysql_SRCS})
     set_target_properties(ha_columnstore PROPERTIES PREFIX "")
 
-    add_dependencies(ha_columnstore loggingcpp)
-    add_dependencies(ha_columnstore marias3)
     add_definitions(-DMYSQL_DYNAMIC_PLUGIN -DPLUGIN_COLUMNSTORE_VERSION="${PACKAGE_VERSION}")
 
-    target_link_libraries(
+    columnstore_link(
         ha_columnstore
         ${S3API_DEPS}
         ${ENGINE_LDFLAGS}
@@ -88,33 +89,20 @@ else()
         ${NETSNMP_LIBRARIES}
         ${SERVER_BUILD_DIR}/libservices/libmysqlservices.a
         threadpool
+        loggingcpp
+        marias3
     )
-
-    install(
-        TARGETS ha_columnstore
-        DESTINATION ${MARIADB_PLUGINDIR}
-        COMPONENT columnstore-engine
-    )
-
     # define this dummy target for standalone builds (ie, when mysql_add_plugin doesn't exist)
     add_custom_target(columnstore DEPENDS ha_columnstore)
-    install(
+    columnstore_install_file(columnstore.cnf ${MARIADB_MYCNFDIR})
-        FILES columnstore.cnf
-        DESTINATION ${MARIADB_MYCNFDIR}
-        COMPONENT columnstore-engine
-    )
 endif()
 
 if(TARGET columnstore)
-    install(
+    columnstore_install_file(syscatalog_mysql.sql ${ENGINE_SUPPORTDIR})
-        FILES syscatalog_mysql.sql dumpcat_mysql.sql calsetuserpriority.sql calremoveuserpriority.sql
+    columnstore_install_file(dumpcat_mysql.sql ${ENGINE_SUPPORTDIR})
-              calshowprocesslist.sql columnstore_info.sql
+    columnstore_install_file(calsetuserpriority.sql ${ENGINE_SUPPORTDIR})
-        DESTINATION ${ENGINE_SUPPORTDIR}
+    columnstore_install_file(calremoveuserpriority.sql ${ENGINE_SUPPORTDIR})
-        COMPONENT columnstore-engine
+    columnstore_install_file(calshowprocesslist.sql ${ENGINE_SUPPORTDIR})
-    )
+    columnstore_install_file(columnstore_info.sql ${ENGINE_SUPPORTDIR})
-    install(
+    columnstore_install_program(install_mcs_mysql.sh ${ENGINE_SBINDIR})
-        PROGRAMS install_mcs_mysql.sh
-        DESTINATION ${ENGINE_SBINDIR}
-        COMPONENT columnstore-engine
-    )
 endif()

dbcon/mysql/ha_mcs_client_udfs.cpp

@@ -45,6 +45,7 @@ using namespace logging;
 
 #include "columnstoreversion.h"
 #include "ha_mcs_sysvars.h"
+#include "basic/string_utils.h"
 
 extern "C"
 {

dbcon/mysql/ha_mcs_impl_if.h

@@ -26,7 +26,7 @@
 #include <boost/shared_ptr.hpp>
 #include <stack>
 #include <vector>
-
+#include "basic/string_utils.h"
 #include "idb_mysql.h"
 #include "ha_mcs_sysvars.h"
 
@@ -36,6 +36,8 @@ struct st_ha_create_information;
 class ha_columnstore_select_handler;
 class ha_columnstore_derived_handler;
 
+#include "basic/string_utils.h"
+
 #include "configcpp.h"
 #include "idberrorinfo.h"
 #include "calpontselectexecutionplan.h"
@@ -117,7 +119,8 @@ struct gp_walk_info
   std::vector<execplan::ReturnedColumn*> localCols;
   std::stack<execplan::ReturnedColumn*> rcWorkStack;
   std::stack<execplan::ParseTree*> ptWorkStack;
-  boost::shared_ptr<execplan::SimpleColumn> scsp; // while defined as SSCP, it is used as SRCP, nothing specific to SimpleColumn is used in use sites.
+  boost::shared_ptr<execplan::SimpleColumn> scsp;  // while defined as SSCP, it is used as SRCP, nothing
+                                                   // specific to SimpleColumn is used in use sites.
   uint32_t sessionid;
   bool fatalParseError;
   std::string parseErrorText;
@@ -227,7 +230,6 @@ struct gp_walk_info
   {
   }
   ~gp_walk_info();
-
 };
 
 struct SubQueryChainHolder;
@@ -237,7 +239,7 @@ struct ext_cond_info
   // circular dependency on header inclusion with ha_subquery.h.
   boost::shared_ptr<SubQueryChainHolder> chainHolder;
   gp_walk_info gwi;
-  ext_cond_info(long timeZone); // needs knowledge on SubQueryChainHolder, will be defined elsewhere
+  ext_cond_info(long timeZone);  // needs knowledge on SubQueryChainHolder, will be defined elsewhere
 };
 
 struct cal_table_info

ddlproc/CMakeLists.txt

@@ -4,14 +4,6 @@ include_directories(${ENGINE_COMMON_INCLUDES})
 
 set(DDLProc_SRCS ddlproc.cpp ddlprocessor.cpp ../utils/common/crashtrace.cpp)
 
-add_executable(DDLProc ${DDLProc_SRCS})
+columnstore_executable(DDLProc ${DDLProc_SRCS})
 
-add_dependencies(DDLProc loggingcpp)
+columnstore_link(DDLProc ${ENGINE_LDFLAGS} ${ENGINE_WRITE_LIBS} ${NETSNMP_LIBRARIES} threadpool loggingcpp)
-
-target_link_libraries(DDLProc ${ENGINE_LDFLAGS} ${ENGINE_WRITE_LIBS} ${NETSNMP_LIBRARIES} threadpool)
-
-install(
-    TARGETS DDLProc
-    DESTINATION ${ENGINE_BINDIR}
-    COMPONENT columnstore-engine
-)

dmlproc/CMakeLists.txt

@@ -6,11 +6,9 @@ set(DMLProc_SRCS dmlproc.cpp dmlprocessor.cpp dmlresultbuffer.cpp batchinsertpro
                  ../utils/common/crashtrace.cpp
 )
 
-add_executable(DMLProc ${DMLProc_SRCS})
+columnstore_executable(DMLProc ${DMLProc_SRCS})
 
-add_dependencies(DMLProc loggingcpp)
+columnstore_link(
-
-target_link_libraries(
     DMLProc
     ${ENGINE_LDFLAGS}
     ${ENGINE_WRITE_LIBS}
@@ -18,10 +16,5 @@ target_link_libraries(
     threadpool
     ddlcleanuputil
     batchloader
-)
+    loggingcpp
-
-install(
-    TARGETS DMLProc
-    DESTINATION ${ENGINE_BINDIR}
-    COMPONENT columnstore-engine
 )

oam/etc/CMakeLists.txt

@@ -1,5 +1 @@
-install(
+columnstore_install_file(Columnstore.xml ${ENGINE_SYSCONFDIR}/columnstore)
-    FILES Columnstore.xml
-    DESTINATION ${ENGINE_SYSCONFDIR}/columnstore
-    COMPONENT columnstore-engine
-)

oam/install_scripts/CMakeLists.txt

@@ -131,49 +131,37 @@ configure_file("${CMAKE_CURRENT_SOURCE_DIR}/mcs-savebrm.py.in" "${CMAKE_CURRENT_
 configure_file("${CMAKE_CURRENT_SOURCE_DIR}/mcs-savebrm.py.in" "${CMAKE_CURRENT_SOURCE_DIR}/mcssavebrm.py" @ONLY)
 configure_file("${CMAKE_CURRENT_SOURCE_DIR}/columnstoreSyslog.in" "${CMAKE_CURRENT_SOURCE_DIR}/columnstoreSyslog" @ONLY)
 
-install(
+columnstore_install_program(columnstore-post-install ${ENGINE_BINDIR})
-    PROGRAMS columnstore-post-install
+columnstore_install_program(columnstore-pre-uninstall ${ENGINE_BINDIR})
-             columnstore-pre-uninstall
+columnstore_install_program(columnstore_run.sh ${ENGINE_BINDIR})
-             columnstore_run.sh
+columnstore_install_program(post-mysql-install ${ENGINE_BINDIR})
-             post-mysql-install
+columnstore_install_program(post-mysqld-install ${ENGINE_BINDIR})
-             post-mysqld-install
+columnstore_install_program(columnstoreSyslogSetup.sh ${ENGINE_BINDIR})
-             columnstoreSyslogSetup.sh
+columnstore_install_program(mcs-stop-controllernode.sh ${ENGINE_BINDIR})
-             mcs-stop-controllernode.sh
+columnstore_install_program(mcs-loadbrm.py ${ENGINE_BINDIR})
-             mcs-loadbrm.py
+columnstore_install_program(mcs-savebrm.py ${ENGINE_BINDIR})
-             mcs-savebrm.py
+columnstore_install_program(mariadb-columnstore-start.sh ${ENGINE_BINDIR})
-             mariadb-columnstore-start.sh
+columnstore_install_program(mariadb-columnstore-stop.sh ${ENGINE_BINDIR})
-             mariadb-columnstore-stop.sh
+columnstore_install_program(loop_process_starter.sh ${ENGINE_BINDIR})
-             loop_process_starter.sh
-    DESTINATION ${ENGINE_BINDIR}
-    COMPONENT columnstore-engine
-)
 
-install(
+columnstore_install_file(mariadb-columnstore.service ${ENGINE_SUPPORTDIR})
-    FILES mariadb-columnstore.service
+columnstore_install_file(columnstoreAlias ${ENGINE_SUPPORTDIR})
-          columnstoreAlias
+columnstore_install_file(columnstoreSyslog ${ENGINE_SUPPORTDIR})
-          columnstoreSyslog
+columnstore_install_file(columnstoreSyslog7 ${ENGINE_SUPPORTDIR})
-          columnstoreSyslog7
+columnstore_install_file(columnstoreSyslog-ng ${ENGINE_SUPPORTDIR})
-          columnstoreSyslog-ng
+columnstore_install_file(columnstoreLogRotate ${ENGINE_SUPPORTDIR})
-          columnstoreLogRotate
+columnstore_install_file(myCnf-include-args.text ${ENGINE_SUPPORTDIR})
-          myCnf-include-args.text
+columnstore_install_file(myCnf-exclude-args.text ${ENGINE_SUPPORTDIR})
-          myCnf-exclude-args.text
+columnstore_install_file(mcs-workernode.service ${ENGINE_SUPPORTDIR})
-          mcs-workernode.service
+columnstore_install_file(mcs-controllernode.service ${ENGINE_SUPPORTDIR})
-          mcs-controllernode.service
+columnstore_install_file(mcs-primproc.service ${ENGINE_SUPPORTDIR})
-          mcs-primproc.service
+columnstore_install_file(mcs-writeengineserver.service ${ENGINE_SUPPORTDIR})
-          mcs-writeengineserver.service
+columnstore_install_file(mcs-dmlproc.service ${ENGINE_SUPPORTDIR})
-          mcs-dmlproc.service
+columnstore_install_file(mcs-ddlproc.service ${ENGINE_SUPPORTDIR})
-          mcs-ddlproc.service
+columnstore_install_file(mcs-loadbrm.service ${ENGINE_SUPPORTDIR})
-          mcs-loadbrm.service
+columnstore_install_file(mcs-storagemanager.service ${ENGINE_SUPPORTDIR})
-          mcs-storagemanager.service
-    DESTINATION ${ENGINE_SUPPORTDIR}
-    COMPONENT columnstore-engine
-)
 
-install(
+columnstore_install_file(module ${ENGINE_DATADIR}/local)
-    FILES module
-    DESTINATION ${ENGINE_DATADIR}/local
-    COMPONENT columnstore-engine
-)
 
 find_package(Python3 COMPONENTS Interpreter)
 if(Python3_FOUND)

oam/oamcpp/CMakeLists.txt

@@ -5,15 +5,6 @@ include_directories(${ENGINE_COMMON_INCLUDES})
 set(oamcpp_LIB_SRCS liboamcpp.cpp oamcache.cpp)
 
 columnstore_library(oamcpp ${oamcpp_LIB_SRCS})
-
+columnstore_link(oamcpp loggingcpp)
-add_dependencies(oamcpp loggingcpp)
-
-target_link_libraries(oamcpp)
 
 target_compile_options(oamcpp PRIVATE -Wno-unused-result)
-
-install(
-    TARGETS oamcpp
-    DESTINATION ${ENGINE_LIBDIR}
-    COMPONENT columnstore-engine
-)

oamapps/columnstoreDB/CMakeLists.txt

@@ -4,12 +4,6 @@ include_directories(${ENGINE_COMMON_INCLUDES})
 
 set(columnstoreDBWrite_SRCS columnstoreDB.cpp)
 
-add_executable(columnstoreDBWrite ${columnstoreDBWrite_SRCS})
+columnstore_executable(columnstoreDBWrite ${columnstoreDBWrite_SRCS})
 
-target_link_libraries(columnstoreDBWrite ${ENGINE_LDFLAGS} ncurses ${ENGINE_EXEC_LIBS})
+columnstore_link(columnstoreDBWrite ${ENGINE_LDFLAGS} ncurses ${ENGINE_EXEC_LIBS})
-
-install(
-    TARGETS columnstoreDBWrite
-    DESTINATION ${ENGINE_BINDIR}
-    COMPONENT columnstore-engine
-)

oamapps/columnstoreSupport/CMakeLists.txt

@@ -4,20 +4,13 @@ include_directories(${ENGINE_COMMON_INCLUDES})
 
 set(columnstoreSupport_SRCS columnstoreSupport.cpp mcsSupportUtil.cpp)
 
-add_executable(columnstoreSupport ${columnstoreSupport_SRCS})
+columnstore_executable(columnstoreSupport ${columnstoreSupport_SRCS})
-
 target_compile_options(columnstoreSupport PRIVATE -Wno-unused-result)
+columnstore_link(columnstoreSupport ${ENGINE_LDFLAGS} ncurses ${ENGINE_EXEC_LIBS})
 
-target_link_libraries(columnstoreSupport ${ENGINE_LDFLAGS} ncurses ${ENGINE_EXEC_LIBS})
+columnstore_install_program(dbmsReport.sh, ${ENGINE_BINDIR})
-
+columnstore_install_program(bulklogReport.sh, ${ENGINE_BINDIR})
-install(
+columnstore_install_program(configReport.sh, ${ENGINE_BINDIR})
-    TARGETS columnstoreSupport
+columnstore_install_program(hardwareReport.sh, ${ENGINE_BINDIR})
-    DESTINATION ${ENGINE_BINDIR}
+columnstore_install_program(logReport.sh, ${ENGINE_BINDIR})
-    COMPONENT columnstore-engine
+columnstore_install_program(resourceReport.sh, ${ENGINE_BINDIR})
-)
-
-install(
-    PROGRAMS dbmsReport.sh bulklogReport.sh configReport.sh hardwareReport.sh logReport.sh resourceReport.sh
-    DESTINATION ${ENGINE_BINDIR}
-    COMPONENT columnstore-engine
-)

oamapps/postConfigure/CMakeLists.txt

@@ -4,14 +4,6 @@ include_directories(${ENGINE_COMMON_INCLUDES})
 
 set(mycnfUpgrade_SRCS mycnfUpgrade.cpp)
 
-add_executable(mycnfUpgrade ${mycnfUpgrade_SRCS})
+columnstore_executable(mycnfUpgrade ${mycnfUpgrade_SRCS})
 
-target_compile_options(mycnfUpgrade PRIVATE -Wno-unused-result)
+columnstore_link(mycnfUpgrade ${ENGINE_LDFLAGS} ${ENGINE_READLINE_LIBRARY} ncurses ${ENGINE_EXEC_LIBS})
-
-target_link_libraries(mycnfUpgrade ${ENGINE_LDFLAGS} ${ENGINE_READLINE_LIBRARY} ncurses ${ENGINE_EXEC_LIBS})
-
-install(
-    TARGETS mycnfUpgrade
-    DESTINATION ${ENGINE_BINDIR}
-    COMPONENT columnstore-engine
-)

oamapps/postConfigure/mycnfUpgrade.cpp

@@ -46,6 +46,7 @@
 #include <stdio.h>
 #include <string>
 #include <unistd.h>
+#include <tuple>
 #include <vector>
 
 #include <sys/sysinfo.h>
@@ -282,7 +283,7 @@ int main(int argc, char* argv[])
     USER = p;
 
   string cmd = "chown " + USER + ":" + USER + " " + mycnfFile;
-  system(cmd.c_str());
+  std::ignore = std::system(cmd.c_str());
 
   exit(0);
 }

primitives/blockcache/CMakeLists.txt

@@ -13,13 +13,5 @@ set(dbbc_STAT_SRCS
     stats.cpp
     fsutils.cpp
 )
-
+columnstore_static_library(dbbc ${dbbc_STAT_SRCS})
-# libdbbc_a_CXXFLAGS = $(march_flags) $(AM_CXXFLAGS)
+columnstore_link(dbbc ${NETSNMP_LIBRARIES} loggingcpp)
-
-add_library(dbbc STATIC ${dbbc_STAT_SRCS})
-
-add_dependencies(dbbc loggingcpp)
-
-target_link_libraries(dbbc ${NETSNMP_LIBRARIES})
-
-install(TARGETS dbbc DESTINATION ${ENGINE_LIBDIR})

primitives/linux-port/CMakeLists.txt

@@ -4,10 +4,5 @@ include_directories(${ENGINE_COMMON_INCLUDES} ../blockcache ../primproc)
 
 set(processor_STAT_SRCS primitiveprocessor.cpp dictionary.cpp column.cpp)
 
-add_library(processor STATIC ${processor_STAT_SRCS})
+columnstore_static_library(processor ${processor_STAT_SRCS})
-
+columnstore_link(processor ${NETSNMP_LIBRARIES} loggingcpp)
-add_dependencies(processor loggingcpp)
-
-target_link_libraries(processor ${NETSNMP_LIBRARIES})
-
-install(TARGETS processor DESTINATION ${ENGINE_LIBDIR})

primitives/primproc/CMakeLists.txt

@@ -25,11 +25,10 @@ set(PrimProc_SRCS
     ../../utils/common/crashtrace.cpp
 )
 
-add_executable(PrimProc ${PrimProc_SRCS})
+columnstore_executable(PrimProc ${PrimProc_SRCS})
 
-add_dependencies(PrimProc loggingcpp)
 target_include_directories(PrimProc PRIVATE ${Boost_INCLUDE_DIRS})
-target_link_libraries(
+columnstore_link(
     PrimProc
     ${ENGINE_LDFLAGS}
     ${NETSNMP_LIBRARIES}
@@ -38,10 +37,5 @@ target_link_libraries(
     cacheutils
     dbbc
     processor
-)
+    loggingcpp
-
-install(
-    TARGETS PrimProc
-    DESTINATION ${ENGINE_BINDIR}
-    COMPONENT columnstore-engine
 )

storage-manager/CMakeLists.txt

@@ -64,11 +64,11 @@ configure_file(
 link_directories(${CMAKE_BINARY_DIR}/lib)
 set(CMAKE_INSTALL_RPATH $ORIGIN $ORIGIN/../lib)
 
-add_library(storagemanager SHARED ${storagemanager_SRCS})
+columnstore_library(storagemanager SHARED ${storagemanager_SRCS})
 add_dependencies(storagemanager marias3 external_boost)
 
 target_compile_definitions(storagemanager PUBLIC BOOST_NO_CXX11_SCOPED_ENUMS)
-target_link_libraries(
+columnstore_link(
     storagemanager
     boost_chrono
     boost_system
@@ -80,8 +80,8 @@ target_link_libraries(
 )
 target_include_directories(storagemanager PRIVATE ${Boost_INCLUDE_DIRS})
 
-add_executable(StorageManager src/main.cpp)
+columnstore_executable(StorageManager src/main.cpp)
-target_link_libraries(StorageManager storagemanager)
+columnstore_link(StorageManager storagemanager)
 set_property(TARGET StorageManager PROPERTY CXX_STANDARD 20)
 
 set(TMPDIR ${CMAKE_RUNTIME_OUTPUT_DIRECTORY})
@@ -89,11 +89,11 @@ set(CMAKE_RUNTIME_OUTPUT_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR})
 
 add_executable(unit_tests src/unit_tests.cpp)
 target_compile_definitions(unit_tests PUBLIC BOOST_NO_CXX11_SCOPED_ENUMS)
-target_link_libraries(unit_tests storagemanager)
+columnstore_link(unit_tests storagemanager)
 
-add_executable(testS3Connection src/testS3Connection.cpp)
+columnstore_executable(testS3Connection src/testS3Connection.cpp)
 target_compile_definitions(testS3Connection PUBLIC BOOST_NO_CXX11_SCOPED_ENUMS)
-target_link_libraries(testS3Connection storagemanager)
+columnstore_link(testS3Connection storagemanager)
 
 set(CMAKE_RUNTIME_OUTPUT_DIRECTORY ${TMPDIR})
 
@@ -112,42 +112,25 @@ add_custom_command(
 # The includes and lib linkages required to link against cloudio ... pretty crazy.  When lib dependencies are eventually
 # config'd right, change this to only include and link against cloudio.
 include_directories(${ENGINE_SRC_DIR}/utils/cloudio ${ENGINE_COMMON_INCLUDES})
-add_executable(smcat src/smcat.cpp)
+columnstore_executable(smcat src/smcat.cpp)
-target_link_libraries(smcat storagemanager cloudio ${ENGINE_LDFLAGS} ${ENGINE_EXEC_LIBS})
+columnstore_link(smcat storagemanager cloudio ${ENGINE_LDFLAGS} ${ENGINE_EXEC_LIBS})
 
-add_executable(smput src/smput.cpp)
+columnstore_executable(smput src/smput.cpp)
-target_link_libraries(smput storagemanager cloudio ${ENGINE_LDFLAGS} ${ENGINE_EXEC_LIBS})
+columnstore_link(smput storagemanager cloudio ${ENGINE_LDFLAGS} ${ENGINE_EXEC_LIBS})
 
-add_executable(smls src/smls.cpp)
+columnstore_executable(smls src/smls.cpp)
-target_link_libraries(smls storagemanager cloudio ${ENGINE_LDFLAGS} ${ENGINE_EXEC_LIBS})
+columnstore_link(smls storagemanager cloudio ${ENGINE_LDFLAGS} ${ENGINE_EXEC_LIBS})
 
-add_executable(smrm src/smrm.cpp)
+columnstore_executable(smrm src/smrm.cpp)
-target_link_libraries(smrm storagemanager cloudio ${ENGINE_LDFLAGS} ${ENGINE_EXEC_LIBS})
+columnstore_link(smrm storagemanager cloudio ${ENGINE_LDFLAGS} ${ENGINE_EXEC_LIBS})
 
-add_executable(smps src/smps.cpp)
+columnstore_executable(smps src/smps.cpp)
-target_link_libraries(smps storagemanager cloudio ${ENGINE_LDFLAGS} ${ENGINE_EXEC_LIBS})
+columnstore_link(smps storagemanager cloudio ${ENGINE_LDFLAGS} ${ENGINE_EXEC_LIBS})
 
-add_executable(smkill src/smkill.cpp)
+columnstore_executable(smkill src/smkill.cpp)
-target_link_libraries(smkill storagemanager cloudio ${ENGINE_LDFLAGS} ${ENGINE_EXEC_LIBS})
+columnstore_link(smkill storagemanager cloudio ${ENGINE_LDFLAGS} ${ENGINE_EXEC_LIBS})
-install(TARGETS storagemanager LIBRARY DESTINATION ${ENGINE_LIBDIR} COMPONENT columnstore-engine)
 
-install(
+columnstore_install_file(storagemanager.cnf ${ENGINE_SYSCONFDIR}/columnstore)
-    TARGETS StorageManager
-            smcat
-            smkill
-            smps
-            smput
-            smls
-            smrm
-            testS3Connection
-    RUNTIME DESTINATION ${ENGINE_BINDIR} COMPONENT columnstore-engine
-)
-
-install(
-    FILES storagemanager.cnf
-    DESTINATION ${ENGINE_SYSCONFDIR}/columnstore
-    COMPONENT columnstore-engine
-)
 
 install(
     FILES storagemanager.cnf

tests/CMakeLists.txt

@@ -30,82 +30,80 @@ if(WITH_UNITTESTS)
     add_executable(rowgroup_tests rowgroup-tests.cpp)
     target_compile_options(rowgroup_tests PRIVATE -Wno-sign-compare)
     add_dependencies(rowgroup_tests googletest)
-    target_link_libraries(rowgroup_tests ${ENGINE_LDFLAGS} ${GTEST_LIBRARIES} ${ENGINE_EXEC_LIBS})
+    columnstore_link(rowgroup_tests ${ENGINE_LDFLAGS} ${GTEST_LIBRARIES} ${ENGINE_EXEC_LIBS})
     gtest_add_tests(TARGET rowgroup_tests TEST_PREFIX columnstore:)
 
     add_executable(rewritetest rewritetest.cpp)
     add_dependencies(rewritetest googletest)
-    target_link_libraries(rewritetest ${ENGINE_LDFLAGS} ${GTEST_LIBRARIES} ${ENGINE_EXEC_LIBS})
+    columnstore_link(rewritetest ${ENGINE_LDFLAGS} ${GTEST_LIBRARIES} ${ENGINE_EXEC_LIBS})
     gtest_add_tests(TARGET rewritetest TEST_PREFIX columnstore:)
 
     add_executable(mcs_decimal_tests mcs_decimal-tests.cpp)
     add_dependencies(mcs_decimal_tests googletest)
-    target_link_libraries(mcs_decimal_tests ${ENGINE_LDFLAGS} ${GTEST_LIBRARIES} ${ENGINE_EXEC_LIBS})
+    columnstore_link(mcs_decimal_tests ${ENGINE_LDFLAGS} ${GTEST_LIBRARIES} ${ENGINE_EXEC_LIBS})
     gtest_add_tests(TARGET mcs_decimal_tests TEST_PREFIX columnstore:)
 
     add_executable(dataconvert_tests dataconvert-tests.cpp)
     add_dependencies(dataconvert_tests googletest)
-    target_link_libraries(dataconvert_tests ${ENGINE_LDFLAGS} ${GTEST_LIBRARIES} ${ENGINE_EXEC_LIBS})
+    columnstore_link(dataconvert_tests ${ENGINE_LDFLAGS} ${GTEST_LIBRARIES} ${ENGINE_EXEC_LIBS})
     gtest_add_tests(TARGET dataconvert_tests TEST_PREFIX columnstore:)
 
     add_executable(rebuild_em_tests rebuild-em-tests.cpp)
     add_dependencies(rebuild_em_tests googletest)
-    target_link_libraries(rebuild_em_tests ${ENGINE_LDFLAGS} ${GTEST_LIBRARIES} ${ENGINE_WRITE_LIBS})
+    columnstore_link(rebuild_em_tests ${ENGINE_LDFLAGS} ${GTEST_LIBRARIES} ${ENGINE_WRITE_LIBS})
     gtest_add_tests(TARGET rebuild_em_tests TEST_PREFIX columnstore:)
 
     add_executable(compression_tests compression-tests.cpp)
     add_dependencies(compression_tests googletest)
-    target_link_libraries(compression_tests ${ENGINE_LDFLAGS} ${GTEST_LIBRARIES} ${ENGINE_WRITE_LIBS})
+    columnstore_link(compression_tests ${ENGINE_LDFLAGS} ${GTEST_LIBRARIES} ${ENGINE_WRITE_LIBS})
     gtest_add_tests(TARGET compression_tests TEST_PREFIX columnstore:)
 
     add_executable(column_scan_filter_tests primitives_column_scan_and_filter.cpp)
     target_compile_options(column_scan_filter_tests PRIVATE -Wno-error -Wno-sign-compare)
     add_dependencies(column_scan_filter_tests googletest)
-    target_link_libraries(
+    columnstore_link(column_scan_filter_tests ${ENGINE_LDFLAGS} ${ENGINE_WRITE_LIBS} ${GTEST_LIBRARIES} processor dbbc)
-        column_scan_filter_tests ${ENGINE_LDFLAGS} ${ENGINE_WRITE_LIBS} ${GTEST_LIBRARIES} processor dbbc
-    )
     gtest_add_tests(TARGET column_scan_filter_tests TEST_PREFIX columnstore:)
 
     add_executable(simd_processors simd_processors.cpp)
     target_compile_options(simd_processors PRIVATE -Wno-error)
     add_dependencies(simd_processors googletest)
-    target_link_libraries(simd_processors ${ENGINE_LDFLAGS} ${ENGINE_WRITE_LIBS} ${GTEST_LIBRARIES} processor dbbc)
+    columnstore_link(simd_processors ${ENGINE_LDFLAGS} ${ENGINE_WRITE_LIBS} ${GTEST_LIBRARIES} processor dbbc)
     gtest_add_tests(TARGET simd_processors TEST_PREFIX columnstore:)
 
     add_executable(fair_threadpool_test fair_threadpool.cpp)
     add_dependencies(fair_threadpool_test googletest)
-    target_link_libraries(fair_threadpool_test ${ENGINE_LDFLAGS} ${ENGINE_WRITE_LIBS} ${GTEST_LIBRARIES} processor dbbc)
+    columnstore_link(fair_threadpool_test ${ENGINE_LDFLAGS} ${ENGINE_WRITE_LIBS} ${GTEST_LIBRARIES} processor dbbc)
     gtest_add_tests(TARGET fair_threadpool_test TEST_PREFIX columnstore:)
 
     add_executable(counting_allocator counting_allocator.cpp)
     target_compile_options(counting_allocator PRIVATE -Wno-sign-compare)
     add_dependencies(counting_allocator googletest)
-    target_link_libraries(counting_allocator ${ENGINE_LDFLAGS} ${ENGINE_WRITE_LIBS} ${GTEST_LIBRARIES})
+    columnstore_link(counting_allocator ${ENGINE_LDFLAGS} ${ENGINE_WRITE_LIBS} ${GTEST_LIBRARIES})
     gtest_add_tests(TARGET counting_allocator TEST_PREFIX columnstore:)
 
     add_executable(poolallocator poolallocator.cpp)
     target_compile_options(poolallocator PRIVATE -Wno-sign-compare)
     add_dependencies(poolallocator googletest)
-    target_link_libraries(poolallocator ${ENGINE_LDFLAGS} ${ENGINE_WRITE_LIBS} ${GTEST_LIBRARIES})
+    columnstore_link(poolallocator ${ENGINE_LDFLAGS} ${ENGINE_WRITE_LIBS} ${GTEST_LIBRARIES})
     gtest_add_tests(TARGET poolallocator TEST_PREFIX columnstore:)
 
     add_executable(stlpoolallocator stlpoolallocator.cpp)
     target_compile_options(stlpoolallocator PRIVATE -Wno-sign-compare)
     add_dependencies(stlpoolallocator googletest)
-    target_link_libraries(stlpoolallocator ${ENGINE_LDFLAGS} ${ENGINE_WRITE_LIBS} ${GTEST_LIBRARIES})
+    columnstore_link(stlpoolallocator ${ENGINE_LDFLAGS} ${ENGINE_WRITE_LIBS} ${GTEST_LIBRARIES})
     gtest_add_tests(TARGET stlpoolallocator TEST_PREFIX columnstore:)
 
     add_executable(comparators_tests comparators-tests.cpp)
-    target_link_libraries(comparators_tests ${ENGINE_LDFLAGS} ${ENGINE_WRITE_LIBS} ${CPPUNIT_LIBRARIES} cppunit)
+    columnstore_link(comparators_tests ${ENGINE_LDFLAGS} ${ENGINE_WRITE_LIBS} ${CPPUNIT_LIBRARIES} cppunit)
     add_test(NAME columnstore:comparators_tests COMMAND comparators_tests)
 
     add_executable(bytestream bytestream.cpp)
-    target_link_libraries(bytestream ${ENGINE_LDFLAGS} ${ENGINE_WRITE_LIBS} ${CPPUNIT_LIBRARIES} cppunit)
+    columnstore_link(bytestream ${ENGINE_LDFLAGS} ${ENGINE_WRITE_LIBS} ${CPPUNIT_LIBRARIES} cppunit)
     add_test(NAME columnstore:bytestream COMMAND bytestream)
 
     # standalone EM routines test
     add_executable(brm_em_standalone brm-em-standalone.cpp)
-    target_link_libraries(brm_em_standalone ${ENGINE_LDFLAGS} ${ENGINE_WRITE_LIBS} ${CPPUNIT_LIBRARIES} cppunit)
+    columnstore_link(brm_em_standalone ${ENGINE_LDFLAGS} ${ENGINE_WRITE_LIBS} ${CPPUNIT_LIBRARIES} cppunit)
     add_test(NAME columnstore:brm_em_standalone COMMAND brm_em_standalone)
     set_tests_properties(columnstore:brm_em_standalone PROPERTIES DISABLED True)
 endif()
@@ -116,7 +114,7 @@ if(WITH_MICROBENCHMARKS AND (NOT CMAKE_BUILD_TYPE STREQUAL "Debug"))
     target_include_directories(
         primitives_scan_bench PUBLIC ${ENGINE_COMMON_INCLUDES} ${ENGINE_BLOCKCACHE_INCLUDE} ${ENGINE_PRIMPROC_INCLUDE}
     )
-    target_link_libraries(
+    columnstore_link(
         primitives_scan_bench
         ${ENGINE_LDFLAGS}
         ${ENGINE_WRITE_LIBS}

tools/bincvt/li2bin.cpp

@@ -1,258 +0,0 @@
-#include <iostream>
-#include <inttypes.h>
-#include <string>
-#include <cstring>
-#include <cstdlib>
-//#define NDEBUG
-#include <cassert>
-using namespace std;
-
-#include <boost/tokenizer.hpp>
-#include <boost/static_assert.hpp>
-using namespace boost;
-
-
-// 1|155190|7706|1|17|21168.23|0.04|0.02|N|O|1996-03-13|1996-02-12|1996-03-22|DELIVER IN PERSON|TRUCK|egular
-// courts above the|
-
-#pragma pack(1)
-struct lineitem_image
-{
-  int32_t l_orderkey;
-  int32_t l_partkey;
-  int32_t l_suppkey;  // 12
-  int64_t l_linenumber;
-  int64_t l_quantity;
-  int64_t l_extendedprice;
-  int64_t l_discount;
-  int64_t l_tax;  // 40
-  char l_returnflag;
-  char l_linestatus;  // 2
-  int32_t l_shipdate;
-  int32_t l_commitdate;
-  int32_t l_receiptdate;  // 12
-  char l_shipinstruct[25];
-  char l_shipmode[10];
-  char l_comment[44];  // 79
-};
-BOOST_STATIC_ASSERT(sizeof(struct lineitem_image) == 145);
-
-// version of lineitem_image that treats some integer columns as unsigned ints.
-// Decimal, date, and date/time columns are still treated as integer.
-struct unsigned_lineitem_image
-{
-  uint32_t l_orderkey;
-  uint32_t l_partkey;
-  uint32_t l_suppkey;  // 12
-  uint64_t l_linenumber;
-  int64_t l_quantity;
-  int64_t l_extendedprice;
-  int64_t l_discount;
-  int64_t l_tax;  // 40
-  char l_returnflag;
-  char l_linestatus;  // 2
-  int32_t l_shipdate;
-  int32_t l_commitdate;
-  int32_t l_receiptdate;  // 12
-  char l_shipinstruct[25];
-  char l_shipmode[10];
-  char l_comment[44];  // 79
-};
-BOOST_STATIC_ASSERT(sizeof(struct unsigned_lineitem_image) == 145);
-
-struct Date
-{
-  unsigned spare : 6;
-  unsigned day : 6;
-  unsigned month : 4;
-  unsigned year : 16;
-};
-BOOST_STATIC_ASSERT(sizeof(struct Date) == 4);
-
-union date_image
-{
-  struct Date d;
-  int32_t i;
-};
-
-namespace
-{
-typedef boost::tokenizer<boost::char_separator<char> > tokenizer;
-
-int64_t idbbigdec2(const string& str)
-{
-  int64_t val = 0;
-  string::size_type idx = string::npos;
-  string tmp(str);
-  idx = tmp.find('.');
-
-  if (idx != string::npos)
-    tmp.erase(idx, 1);
-  else
-    tmp.append("00");
-
-  val = atoll(tmp.c_str());
-  return val;
-}
-
-int32_t idbdate(const string& str)
-{
-  date_image di;
-  di.i = 0;
-  boost::char_separator<char> sep("-");
-  tokenizer tokens(str, sep);
-  tokenizer::iterator tok_iter = tokens.begin();
-
-  // Note that the spare bits must be set, instead of leaving them
-  // initialized to 0; to be compatible with internal date format.
-  if (distance(tok_iter, tokens.end()) == 3)
-  {
-    di.d.spare = 0x3E;
-    di.d.year = atoi(tok_iter->c_str());
-    ++tok_iter;
-    di.d.month = atoi(tok_iter->c_str());
-    ++tok_iter;
-    di.d.day = atoi(tok_iter->c_str());
-    ++tok_iter;
-  }
-
-  return di.i;
-}
-
-int parseinto(lineitem_image& img, const string& buf)
-{
-  memset(&img, 0, sizeof(img));
-
-  boost::char_separator<char> sep("|");
-  tokenizer tokens(buf, sep);
-  tokenizer::iterator tok_iter = tokens.begin();
-
-  if (distance(tok_iter, tokens.end()) < 16)
-    return -1;
-
-  img.l_orderkey = atoi(tok_iter->c_str());
-  ++tok_iter;
-  img.l_partkey = atoi(tok_iter->c_str());
-  ++tok_iter;
-  img.l_suppkey = atoi(tok_iter->c_str());
-  ++tok_iter;
-  img.l_linenumber = atoll(tok_iter->c_str());
-  ++tok_iter;
-  img.l_quantity = idbbigdec2(tok_iter->c_str());
-  ++tok_iter;
-  img.l_extendedprice = idbbigdec2(tok_iter->c_str());
-  ++tok_iter;
-  img.l_discount = idbbigdec2(tok_iter->c_str());
-  ++tok_iter;
-  img.l_tax = idbbigdec2(tok_iter->c_str());
-  ++tok_iter;
-  img.l_returnflag = tok_iter->at(0);
-  ++tok_iter;
-  img.l_linestatus = tok_iter->at(0);
-  ++tok_iter;
-  img.l_shipdate = idbdate(tok_iter->c_str());
-  ++tok_iter;
-  img.l_commitdate = idbdate(tok_iter->c_str());
-  ++tok_iter;
-  img.l_receiptdate = idbdate(tok_iter->c_str());
-  ++tok_iter;
-  memcpy(&img.l_shipinstruct[0], tok_iter->c_str(), tok_iter->size());
-  ++tok_iter;
-  memcpy(&img.l_shipmode[0], tok_iter->c_str(), tok_iter->size());
-  ++tok_iter;
-  memcpy(&img.l_comment[0], tok_iter->c_str(), tok_iter->size());
-  ++tok_iter;
-
-  return 0;
-}
-
-int unsigned_parseinto(unsigned_lineitem_image& img, const string& buf)
-{
-  memset(&img, 0, sizeof(img));
-
-  boost::char_separator<char> sep("|");
-  tokenizer tokens(buf, sep);
-  tokenizer::iterator tok_iter = tokens.begin();
-
-  if (distance(tok_iter, tokens.end()) < 16)
-    return -1;
-
-  img.l_orderkey = strtoul(tok_iter->c_str(), 0, 10);
-  ++tok_iter;
-  img.l_partkey = strtoul(tok_iter->c_str(), 0, 10);
-  ++tok_iter;
-  img.l_suppkey = strtoul(tok_iter->c_str(), 0, 10);
-  ++tok_iter;
-  img.l_linenumber = strtoull(tok_iter->c_str(), 0, 10);
-  ++tok_iter;
-  img.l_quantity = idbbigdec2(tok_iter->c_str());
-  ++tok_iter;
-  img.l_extendedprice = idbbigdec2(tok_iter->c_str());
-  ++tok_iter;
-  img.l_discount = idbbigdec2(tok_iter->c_str());
-  ++tok_iter;
-  img.l_tax = idbbigdec2(tok_iter->c_str());
-  ++tok_iter;
-  img.l_returnflag = tok_iter->at(0);
-  ++tok_iter;
-  img.l_linestatus = tok_iter->at(0);
-  ++tok_iter;
-  img.l_shipdate = idbdate(tok_iter->c_str());
-  ++tok_iter;
-  img.l_commitdate = idbdate(tok_iter->c_str());
-  ++tok_iter;
-  img.l_receiptdate = idbdate(tok_iter->c_str());
-  ++tok_iter;
-  memcpy(&img.l_shipinstruct[0], tok_iter->c_str(), tok_iter->size());
-  ++tok_iter;
-  memcpy(&img.l_shipmode[0], tok_iter->c_str(), tok_iter->size());
-  ++tok_iter;
-  memcpy(&img.l_comment[0], tok_iter->c_str(), tok_iter->size());
-  ++tok_iter;
-
-  return 0;
-}
-
-}  // namespace
-
-int main(int argc, char** argv)
-{
-  if ((argc > 1) && (strcmp(argv[1], "-h") == 0))
-  {
-    std::cerr << "li2bin [-u]" << std::endl;
-    std::cerr << "  -u Create first 4 fields as unsigned integers" << std::endl;
-    return 0;
-  }
-
-  string input;
-
-
-  getline(cin, input);
-
-  if ((argc > 1) && (strcmp(argv[1], "-u") == 0))
-  {
-    unsigned_lineitem_image i;
-
-    while (!cin.eof())
-    {
-      if (unsigned_parseinto(i, input) == 0)
-        cout.write(reinterpret_cast<const char*>(&i), sizeof(i));
-
-      getline(cin, input);
-    }
-  }
-  else
-  {
-    lineitem_image i;
-
-    while (!cin.eof())
-    {
-      if (parseinto(i, input) == 0)
-        cout.write(reinterpret_cast<const char*>(&i), sizeof(i));
-
-      getline(cin, input);
-    }
-  }
-
-  return 0;
-}

tools/cfread/CMakeLists.txt

@@ -1,77 +0,0 @@
-include_directories(${KDE4_INCLUDES} ${KDE4_INCLUDE_DIR} ${QT_INCLUDES})
-
-# ########## next target ###############
-
-set(cfread_SRCS cfread.cpp)
-
-kde4_add_executable(cfread ${cfread_SRCS})
-
-target_link_libraries(
-    cfread
-    ${KDE4_KDECORE_LIBS}
-    pthread
-    writeengine
-    brm
-    rwlock
-    messageqcpp
-    dl
-    configcpp
-    xml2
-    loggingcpp
-    cacheutils
-    boost_idb
-    pthread
-    rt
-)
-
-install(TARGETS cfread ${INSTALL_TARGETS_DEFAULT_ARGS})
-
-# ########## next target ###############
-
-set(mtread_SRCS mtread.cpp)
-
-kde4_add_executable(mtread ${mtread_SRCS})
-
-target_link_libraries(
-    mtread
-    ${KDE4_KDECORE_LIBS}
-    pthread
-    writeengine
-    brm
-    rwlock
-    messageqcpp
-    dl
-    configcpp
-    xml2
-    loggingcpp
-    cacheutils
-    boost_idb
-    pthread
-    rt
-)
-
-install(TARGETS mtread ${INSTALL_TARGETS_DEFAULT_ARGS})
-
-# ########## install files ###############
-
-# original Makefile.am contents follow:
-
-# $Id: Makefile.am 333 2009-04-03 20:35:04Z rdempsey $ Process this file with automake to produce Makefile.in
-#
-# AM_CPPFLAGS = $(idb_cppflags) AM_CFLAGS = $(idb_cflags) AM_CXXFLAGS = $(idb_cxxflags) AM_LDFLAGS = $(idb_ldflags)
-# bin_PROGRAMS = cfread mtread cfread_SOURCES = cfread.cpp cfread_CPPFLAGS = @idb_common_includes@ $(AM_CPPFLAGS)
-# cfread_LDFLAGS = @idb_common_ldflags@ -lpthread -lwriteengine -lbrm -lrwlock -lmessageqcpp -ldl -lconfigcpp -lxml2
-# -lloggingcpp -lcacheutils -lboost_idb -lpthread -lrt $(AM_LDFLAGS) mtread_SOURCES = mtread.cpp mtread_CPPFLAGS =
-# @idb_common_includes@ $(AM_CPPFLAGS) mtread_LDFLAGS = @idb_common_ldflags@ -lpthread -lwriteengine -lbrm -lrwlock
-# -lmessageqcpp -ldl -lconfigcpp -lxml2 -lloggingcpp -lcacheutils -lboost_idb -lpthread -lrt $(AM_LDFLAGS)
-#
-# test:
-#
-# coverage:
-#
-# leakcheck:
-#
-# docs:
-#
-# bootstrap: install-data-am
-#

tools/cfread/cfread.cpp

@@ -1,183 +0,0 @@
-//
-
-#include <iostream>
-#include <iomanip>
-#include <boost/scoped_array.hpp>
-#include <errno.h>
-#define NDEBUG
-#include <cassert>
-#include "writeengine.h"
-
-using namespace std;
-
-void timespec_sub(const struct timespec& tv1, const struct timespec& tv2, struct timespec& diff)
-{
-  if (tv2.tv_nsec < tv1.tv_nsec)
-  {
-    diff.tv_sec = tv2.tv_sec - tv1.tv_sec - 1;
-    diff.tv_nsec = tv1.tv_nsec - tv2.tv_nsec;
-  }
-  else
-  {
-    diff.tv_sec = tv2.tv_sec - tv1.tv_sec;
-    diff.tv_nsec = tv2.tv_nsec - tv1.tv_nsec;
-  }
-}
-
-// TODO:
-// add getopt for CLAs
-// add threads for reading a file
-// add threads for reading multiple files
-
-// main()
-//
-int main(int argc, char** argv)
-{
-  uint64_t acc = 0;
-  uint64_t readBlocks = 0;  // read size ib blocks
-  uint64_t readSize = 0;    // read size ib bytes
-  uint64_t readBufferSz = 0;
-  const uint64_t blockSize = 8192;
-  char* alignedbuff = 0;
-  boost::scoped_array<char> realbuff;
-  const unsigned pageSize = 4096;  // getpagesize();
-  WriteEngine::FileOp fFileOp;
-  BRM::OID_t oid;
-  char fname[256];
-  struct timespec tm;
-  struct timespec tm2;
-  struct timespec tm3;
-  struct timespec starttm;
-  struct timespec endtm;
-  struct timespec tottm;
-  bool odirect = true;
-  int fd = 0;
-  char response = 'Y';
-
-  if (argc <= 1)
-  {
-    cerr << "usage: testread <oid> <buffer size in blocks>" << endl;
-    return -1;
-  }
-
-  oid = atoi(argv[1]);
-
-  if (oid <= 0)
-    exit(-1);
-
-  if (argc >= 2)
-  {
-    readBlocks = atoi(argv[2]);
-
-    if (readBlocks <= 0)
-      readBlocks = 8;
-  }
-
-  if (argc >= 4)
-  {
-    odirect = false;
-  }
-
-  readSize = readBlocks * blockSize;
-  readBufferSz = readSize + pageSize;
-
-  realbuff.reset(new char[readBufferSz]);
-
-  if (realbuff.get() == 0)
-  {
-    cerr << "thr_popper: Can't allocate space for a whole extent in memory" << endl;
-    return 0;
-  }
-
-  if (fFileOp.getFileName(oid, fname) != WriteEngine::NO_ERROR)
-  {
-    fname[0] = 0;
-    throw std::runtime_error("fileOp.getFileName failed");
-  }
-  else
-  {
-    cout << "Reading oid: " << oid << " od: " << odirect << " file: " << fname << endl;
-  }
-
-#if __LP64__
-  alignedbuff = (char*)((((ptrdiff_t)realbuff.get() >> 12) << 12) + pageSize);
-#else
-  alignedbuff = (char*)(((((ptrdiff_t)realbuff.get() >> 12) << 12) & 0xffffffff) + pageSize);
-#endif
-  idbassert(((ptrdiff_t)alignedbuff - (ptrdiff_t)realbuff.get()) < (ptrdiff_t)pageSize);
-  idbassert(((ptrdiff_t)alignedbuff % pageSize) == 0);
-
-  if (odirect)
-    fd = open(fname, O_RDONLY | O_DIRECT | O_LARGEFILE | O_NOATIME);
-  else
-    fd = open(fname, O_RDONLY | O_LARGEFILE | O_NOATIME);
-
-  if (fd < 0)
-  {
-    cerr << "Open failed" << endl;
-    perror("open");
-    throw runtime_error("Error opening file");
-  }
-
-  while (toupper(response) != 'N')
-  {
-    uint64_t i = 1;
-    uint64_t rCnt = 0;
-    clock_gettime(CLOCK_REALTIME, &starttm);
-
-    while (i != 0)
-    {
-      // clock_gettime(CLOCK_REALTIME, &tm);
-      i = pread(fd, alignedbuff, readSize, acc);
-      // clock_gettime(CLOCK_REALTIME, &tm2);
-      idbassert(i == 0 || i == readSize);
-      idbassert(i % pageSize == 0);
-      idbassert(acc % pageSize == 0);
-
-      if (i < 0 && errno == EINTR)
-      {
-        timespec_sub(tm, tm2, tm3);
-        cout << "* " << i << " " << right << setw(2) << setfill(' ') << tm3.tv_sec << "." << right << setw(9)
-             << setfill('0') << tm3.tv_nsec << endl;
-        continue;
-      }
-      else if (i < 0)
-      {
-        timespec_sub(tm, tm2, tm3);
-        cout << "* i: " << i << " sz: " << readSize << " acc: " << acc << right << setw(2) << setfill(' ')
-             << tm3.tv_sec << " " << right << tm3.tv_nsec << endl;
-        perror("pread");
-        // make loop exit
-        i = 0;
-      }
-
-      acc += i;
-
-      if (i > 0)
-        rCnt++;
-
-      // timespec_sub(tm, tm2, tm3);
-      // cout
-      //	<< i << " "
-      //	<< right << setw(2) << setfill(' ') << tm3.tv_sec << " "
-      //	<< right << tm3.tv_nsec
-      //	<< endl;
-
-    }  // while(acc...
-
-    clock_gettime(CLOCK_REALTIME, &endtm);
-    timespec_sub(starttm, endtm, tottm);
-
-    cout << "Total reads: " << rCnt << " sz: " << acc / (1024 * 1024) << "MB"
-         << " tm: " << tottm.tv_sec << "secs " << tottm.tv_nsec << "ns" << endl;
-
-    cout << "Repeat the last scan[Y,N]?" << endl;
-    cin >> response;
-    acc = 0;
-
-  }  // while response...
-
-  close(fd);
-  return 0;
-
-}  // main

tools/cfread/mtread.cpp

@@ -1,269 +0,0 @@
-//
-
-#include <iostream>
-#include <iomanip>
-#include <getopt.h>
-#include <errno.h>
-#include <boost/thread.hpp>
-#define NDEBUG
-#include <cassert>
-#include "writeengine.h"
-
-using namespace std;
-
-void timespec_sub(const struct timespec& tv1, const struct timespec& tv2, struct timespec& diff)
-{
-  if (tv2.tv_nsec < tv1.tv_nsec)
-  {
-    diff.tv_sec = tv2.tv_sec - tv1.tv_sec - 1;
-    diff.tv_nsec = tv1.tv_nsec - tv2.tv_nsec;
-  }
-  else
-  {
-    diff.tv_sec = tv2.tv_sec - tv1.tv_sec;
-    diff.tv_nsec = tv2.tv_nsec - tv1.tv_nsec;
-  }
-}
-
-// TODO:
-// add threads for reading a file
-// add threads for reading multiple files
-
-struct readThr
-{
- public:
-  readThr(const int oid, const int rSize)
-  {
-    fblockSize = 8192;
-    fpageSize = getpagesize();
-    freadBlocks = rSize;
-    freadSize = rSize * fblockSize;  // read size ib bytes
-    freadBufferSz = freadSize + fpageSize;
-    falignedbuff = 0;
-    foid = oid;
-    facc = 0;
-    memset(fname, 0, sizeof(fname));
-    memset((char*)&ftm, 0, sizeof(ftm));
-    memset((char*)&ftm2, 0, sizeof(ftm2));
-    memset((char*)&ftm3, 0, sizeof(ftm3));
-    memset((char*)&fstarttm, 0, sizeof(fstarttm));
-    memset((char*)&fendtm, 0, sizeof(fendtm));
-    memset((char*)&ftottm, 0, sizeof(ftottm));
-    fodirect = true;
-    fd = 0;
-    cout << "o: " << foid << " r: " << freadBlocks << " b: " << freadBufferSz << endl;
-  }
-
-  void operator()()
-  {
-    WriteEngine::FileOp fFileOp;
-    char frealbuff[freadBufferSz];
-    memset(frealbuff, 0, freadBufferSz);
-
-    if (frealbuff == 0)
-    {
-      cerr << "thr_popper: Can't allocate space for a whole extent in memory" << endl;
-      return;
-    }
-
-    if (fFileOp.getFileName(foid, fname) != WriteEngine::NO_ERROR)
-    {
-      fname[0] = 0;
-      throw std::runtime_error("fileOp.getFileName failed");
-    }
-    else
-    {
-      cout << "Reading oid: " << foid << " od: " << fodirect << " file: " << fname << endl;
-    }
-
-#if __LP64__
-    falignedbuff = (char*)((((ptrdiff_t)frealbuff >> 12) << 12) + fpageSize);
-#else
-    falignedbuff = (char*)(((((ptrdiff_t)frealbuff >> 12) << 12) & 0xffffffff) + fpageSize);
-#endif
-    idbassert(((ptrdiff_t)falignedbuff - (ptrdiff_t)frealbuff) < (ptrdiff_t)fpageSize);
-    idbassert(((ptrdiff_t)falignedbuff % fpageSize) == 0);
-
-    if (fodirect)
-      fd = open(fname, O_RDONLY | O_DIRECT | O_LARGEFILE | O_NOATIME);
-    else
-      fd = open(fname, O_RDONLY | O_LARGEFILE | O_NOATIME);
-
-    if (fd < 0)
-    {
-      cerr << "Open failed" << endl;
-      perror("open");
-      throw runtime_error("Error opening file");
-    }
-
-    uint64_t i = 1;
-    uint64_t rCnt = 0;
-
-    clock_gettime(CLOCK_REALTIME, &fstarttm);
-
-    while (i > 0)
-    {
-      clock_gettime(CLOCK_REALTIME, &ftm);
-      i = pread(fd, falignedbuff, freadSize, facc);
-      clock_gettime(CLOCK_REALTIME, &ftm2);
-
-      idbassert(i == 0 || i == freadSize);
-      idbassert(i % fpageSize == 0);
-      idbassert(facc % fpageSize == 0);
-
-      if (i < 0 && errno == EINTR)
-      {
-        timespec_sub(ftm, ftm2, ftm3);
-        cout << "* " << i << " " << right << setw(2) << setfill(' ') << ftm3.tv_sec << "." << right << setw(9)
-             << setfill('0') << ftm3.tv_nsec << endl;
-        continue;
-      }
-      else if (i < 0)
-      {
-        timespec_sub(ftm, ftm2, ftm3);
-        cout << "* i: " << i << " sz: " << freadSize << " acc: " << facc << right << setw(2) << setfill(' ')
-             << ftm3.tv_sec << " " << right << ftm3.tv_nsec << endl;
-        perror("pread");
-      }
-
-      facc += i;
-
-      if (i > 0)
-        rCnt++;
-
-      /**
-      timespec_sub(ftm, ftm2, ftm3);
-      cout
-          << rCnt << " " << facc/(1024*1024)
-          << right << setw(2) << setfill(' ') << ftm3.tv_sec << "."
-          << right << ftm3.tv_nsec << " i: " << i/(1024*1024)
-          << endl;
-      **/
-
-    }  // while(acc...
-
-    clock_gettime(CLOCK_REALTIME, &fendtm);
-    timespec_sub(fstarttm, fendtm, ftottm);
-
-    cout << "Total reads: " << rCnt << " sz: " << facc / (1024 * 1024) << "MB"
-         << " tm: " << ftottm.tv_sec << "secs " << ftottm.tv_nsec << "ns" << endl;
-
-    facc = 0;
-    close(fd);
-  }  // operator()
-
- public:
-  uint64_t facc;
-  uint64_t freadBlocks;  // read size ib blocks
-  uint64_t freadSize;    // read size ib bytes
-  uint64_t freadBufferSz;
-  uint64_t fblockSize;
-  char* falignedbuff;
-  unsigned fpageSize;
-  BRM::OID_t foid;
-  char fname[256];
-  struct timespec ftm;
-  struct timespec ftm2;
-  struct timespec ftm3;
-  struct timespec fstarttm;
-  struct timespec fendtm;
-  struct timespec ftottm;
-  bool fodirect;
-  int fd;
-
-};  // struct readThr
-
-//
-void usage()
-{
-  cerr << "usage: mtread -o <oid> -s <read size in blocks>" << endl;
-}
-
-// main()
-//
-int main(int argc, char** argv)
-{
-  int ch = 0;
-  int readBlocks = 0;
-  BRM::OID_t oid = 0;
-  std::vector<BRM::OID_t> oidList;
-
-  enum CLA_ENUM
-  {
-    OID = (int)0,
-    READSIZE
-  };
-
-  // longopt struct
-  // struct option {
-  // const char *name;
-  // int has_arg;
-  // int *flag;
-  // int val;
-  //};
-
-  static struct option long_options[] = {//{const char *name, 	int has_arg, 	int *flag,	int val},
-                                         {"oid", required_argument, NULL, OID},
-                                         {"rsize", required_argument, NULL, READSIZE},
-                                         {0, 0, 0, 0}};
-
-  if (argc <= 1)
-  {
-    return -1;
-  }
-
-  // process command line arguments
-  while ((ch = getopt_long_only(argc, argv, "o:s:", long_options, NULL)) != -1)
-  {
-    // pid_t pidId = getpid();
-    switch (ch)
-    {
-      case OID:
-      case 'o':
-        oid = atoi(optarg);
-        oidList.push_back(oid);
-        cout << "oid: " << optarg << endl;
-        break;
-
-      case READSIZE:
-      case 's':
-        readBlocks = atoi(optarg);
-        cout << "read size: " << optarg << endl;
-
-        if (readBlocks <= 0)
-          readBlocks = 1;
-
-        break;
-
-      case '?':
-      default:
-        cout << "optarg " << optarg << endl;
-        usage();
-        break;
-
-    }  // switch
-
-  }  // while...
-
-  uint32_t idx = 0;
-  std::vector<boost::thread*> thrList;
-
-  while (idx < oidList.size())
-  {
-    struct readThr rdr(oidList[idx++], readBlocks);
-    boost::thread* thr = new boost::thread(rdr);
-    thrList.push_back(thr);
-  }
-
-  idx = 0;
-
-  while (idx < thrList.size())
-  {
-    boost::thread* thr = thrList[idx++];
-    thr->join();
-    delete thr;
-  }
-
-  thrList.clear();
-
-}  // main

tools/clearShm/CMakeLists.txt

@@ -4,12 +4,6 @@ include_directories(${ENGINE_COMMON_INCLUDES})
 
 set(clearShm_SRCS main.cpp)
 
-add_executable(clearShm ${clearShm_SRCS})
+columnstore_executable(clearShm ${clearShm_SRCS})
 
-target_link_libraries(clearShm ${ENGINE_LDFLAGS} ${ENGINE_EXEC_LIBS})
+columnstore_link(clearShm ${ENGINE_LDFLAGS} ${ENGINE_EXEC_LIBS})
-
-install(
-    TARGETS clearShm
-    DESTINATION ${ENGINE_BINDIR}
-    COMPONENT columnstore-engine
-)

tools/cleartablelock/CMakeLists.txt

@@ -4,12 +4,6 @@ include_directories(${ENGINE_COMMON_INCLUDES})
 
 set(cleartablelock_SRCS cleartablelock.cpp cleartablelockthread.cpp)
 
-add_executable(cleartablelock ${cleartablelock_SRCS})
+columnstore_executable(cleartablelock ${cleartablelock_SRCS})
 
-target_link_libraries(cleartablelock ${ENGINE_LDFLAGS} ${NETSNMP_LIBRARIES} ${ENGINE_WRITE_LIBS})
+columnstore_link(cleartablelock ${ENGINE_LDFLAGS} ${NETSNMP_LIBRARIES} ${ENGINE_WRITE_LIBS})
-
-install(
-    TARGETS cleartablelock
-    DESTINATION ${ENGINE_BINDIR}
-    COMPONENT columnstore-engine
-)

tools/configMgt/CMakeLists.txt

@@ -5,7 +5,4 @@ include_directories(${ENGINE_COMMON_INCLUDES})
 set(autoConfigure_SRCS autoConfigure.cpp)
 
 add_executable(autoConfigure ${autoConfigure_SRCS})
-
+columnstore_link(autoConfigure ${ENGINE_LDFLAGS} ${NETSNMP_LIBRARIES} ${ENGINE_EXEC_LIBS})
-target_link_libraries(autoConfigure ${ENGINE_LDFLAGS} ${NETSNMP_LIBRARIES} ${ENGINE_EXEC_LIBS})
-
-# install(TARGETS autoConfigure DESTINATION ${ENGINE_BINDIR} COMPONENT columnstore-engine)

tools/cplogger/CMakeLists.txt

@@ -4,12 +4,6 @@ include_directories(${ENGINE_COMMON_INCLUDES})
 
 set(cplogger_SRCS main.cpp)
 
-add_executable(cplogger ${cplogger_SRCS})
+columnstore_executable(cplogger ${cplogger_SRCS})
 
-target_link_libraries(cplogger ${ENGINE_LDFLAGS} ${ENGINE_EXEC_LIBS})
+columnstore_link(cplogger ${ENGINE_LDFLAGS} ${ENGINE_EXEC_LIBS})
-
-install(
-    TARGETS cplogger
-    DESTINATION ${ENGINE_BINDIR}
-    COMPONENT columnstore-engine
-)

tools/dbbuilder/CMakeLists.txt

@@ -4,12 +4,6 @@ include_directories(${ENGINE_COMMON_INCLUDES})
 
 set(dbbuilder_SRCS dbbuilder.cpp systemcatalog.cpp)
 
-add_executable(dbbuilder ${dbbuilder_SRCS})
+columnstore_executable(dbbuilder ${dbbuilder_SRCS})
 
-target_link_libraries(dbbuilder ${ENGINE_LDFLAGS} ${NETSNMP_LIBRARIES} ${ENGINE_WRITE_LIBS})
+columnstore_link(dbbuilder ${ENGINE_LDFLAGS} ${NETSNMP_LIBRARIES} ${ENGINE_WRITE_LIBS})
-
-install(
-    TARGETS dbbuilder
-    DESTINATION ${ENGINE_BINDIR}
-    COMPONENT columnstore-engine
-)

tools/dbbuilder/dbbuilder.cpp

@@ -26,6 +26,7 @@
 #include <stdexcept>
 using namespace std;
 
+#include "basic/string_utils.h"
 #include <boost/algorithm/string.hpp>
 
 #include "mcsconfig.h"
@@ -202,7 +203,11 @@ int main(int argc, char* argv[])
             (iter->second).second = true;
             isUpgrade = true;
           }
-          messageHandler("", std::string("Upgrade flag is ") + std::to_string(isUpgrade) + std::string(" after checking upgrade candidate OID ") + oam.itoa(iter->first) + std::string(" "), false);
+          messageHandler("",
+                         std::string("Upgrade flag is ") + std::to_string(isUpgrade) +
+                             std::string(" after checking upgrade candidate OID ") + oam.itoa(iter->first) +
+                             std::string(" "),
+                         false);
         }
 
         if (!isUpgrade)

tools/dbloadxml/CMakeLists.txt

@@ -4,20 +4,14 @@ include_directories(${ENGINE_COMMON_INCLUDES})
 
 set(dbload_STAT_SRCS inputmgr.cpp)
 
-add_library(dbload STATIC ${dbload_STAT_SRCS})
+columnstore_static_library(dbload ${dbload_STAT_SRCS})
 
-add_dependencies(dbload loggingcpp)
+columnstore_link(dbload loggingcpp)
 
 # ########## next target ###############
 
 set(colxml_SRCS colxml.cpp)
 
-add_executable(colxml ${colxml_SRCS})
+columnstore_executable(colxml ${colxml_SRCS})
 
-target_link_libraries(colxml ${ENGINE_LDFLAGS} dbload ${ENGINE_WRITE_LIBS})
+columnstore_link(colxml ${ENGINE_LDFLAGS} dbload ${ENGINE_WRITE_LIBS})
-
-install(
-    TARGETS colxml
-    DESTINATION ${ENGINE_BINDIR}
-    COMPONENT columnstore-engine
-)

tools/ddlcleanup/CMakeLists.txt

@@ -4,12 +4,5 @@ include_directories(${ENGINE_COMMON_INCLUDES})
 
 set(ddlcleanup_SRCS ddlcleanup.cpp)
 
-add_executable(ddlcleanup ${ddlcleanup_SRCS})
+columnstore_executable(ddlcleanup ${ddlcleanup_SRCS})
-
+columnstore_link(ddlcleanup ${ENGINE_LDFLAGS} ${NETSNMP_LIBRARIES} ${ENGINE_WRITE_LIBS} ddlcleanuputil)
-target_link_libraries(ddlcleanup ${ENGINE_LDFLAGS} ${NETSNMP_LIBRARIES} ${ENGINE_WRITE_LIBS} ddlcleanuputil)
-
-install(
-    TARGETS ddlcleanup
-    DESTINATION ${ENGINE_BINDIR}
-    COMPONENT columnstore-engine
-)

tools/ddldriver/CMakeLists.txt

@@ -1,43 +0,0 @@
-include_directories(${KDE4_INCLUDES} ${KDE4_INCLUDE_DIR} ${QT_INCLUDES})
-
-# ########## next target ###############
-
-set(ddldriver_SRCS ddldriver.cpp)
-
-kde4_add_executable(ddldriver ${ddldriver_SRCS})
-
-target_link_libraries(ddldriver ${KDE4_KDECORE_LIBS})
-
-install(TARGETS ddldriver ${INSTALL_TARGETS_DEFAULT_ARGS})
-
-# ########## install files ###############
-
-# original Makefile.am contents follow:
-
-# Copyright (C) 2014 InfiniDB, Inc.
-#
-# This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public
-# License as published by the Free Software Foundation; version 2 of the License.
-#
-# This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied
-# warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License along with this program; if not, write to the Free
-# Software Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
-#
-# $Id: Makefile.am 333 2009-04-03 20:35:04Z rdempsey $ Process this file with automake to produce Makefile.in
-#
-# AM_CPPFLAGS = $(idb_cppflags) AM_CFLAGS = $(idb_cflags) AM_CXXFLAGS = $(idb_cxxflags) AM_LDFLAGS = $(idb_ldflags)
-# bin_PROGRAMS = ddldriver ddldriver_SOURCES = ddldriver.cpp ddldriver_CPPFLAGS = @idb_common_includes@ $(AM_CPPFLAGS)
-# ddldriver_LDFLAGS = @idb_common_ldflags@ @idb_write_libs@ @idb_common_libs@ @netsnmp_libs@ $(AM_LDFLAGS)
-#
-# test:
-#
-# coverage:
-#
-# leakcheck:
-#
-# docs:
-#
-# bootstrap: install-data-am
-#

tools/ddldriver/ddldriver.cpp

@@ -1,133 +0,0 @@
-/* Copyright (C) 2014 InfiniDB, Inc.
-
-   This program is free software; you can redistribute it and/or
-   modify it under the terms of the GNU General Public License
-   as published by the Free Software Foundation; version 2 of
-   the License.
-
-   This program is distributed in the hope that it will be useful,
-   but WITHOUT ANY WARRANTY; without even the implied warranty of
-   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-   GNU General Public License for more details.
-
-   You should have received a copy of the GNU General Public License
-   along with this program; if not, write to the Free Software
-   Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
-   MA 02110-1301, USA. */
-
-// $Id: ddldriver.cpp 2101 2013-01-21 14:12:52Z rdempsey $
-
-#include <unistd.h>
-#include <cstdio>
-#include <iostream>
-#include <cctype>
-#include <sstream>
-using namespace std;
-
-#include "ddlpkg.h"
-#include "sqlparser.h"
-using namespace ddlpackage;
-
-#include "bytestream.h"
-#include "messagequeue.h"
-using namespace messageqcpp;
-
-namespace
-{
-void usage()
-{
-  cout << "usage: ddlriver [-h] schema sql_text" << endl;
-}
-const string toupper_(const string& in)
-{
-  string::const_iterator iter = in.begin();
-  string::const_iterator end = in.end();
-  ostringstream oss;
-
-  while (iter != end)
-  {
-    oss << static_cast<char>(toupper(*iter));
-    ++iter;
-  }
-
-  return oss.str();
-}
-}  // namespace
-
-int main(int argc, char** argv)
-{
-  int c;
-
-  opterr = 0;
-
-  while ((c = getopt(argc, argv, "h")) != EOF)
-    switch (c)
-    {
-      case 'h':
-      case '?':
-      default:
-        usage();
-        return (c == 'h' ? 0 : 1);
-        break;
-    }
-
-  if (argc - optind < 2)
-  {
-    usage();
-    return 1;
-  }
-
-  string owner(toupper_(argv[optind++]));
-
-  SqlParser parser;
-  parser.setDefaultSchema(owner);
-
-  string stmtStr(toupper_(argv[optind++]));
-  parser.Parse(stmtStr.c_str());
-
-  if (!parser.Good())
-  {
-    cerr << "Failed to parse statement: " << stmtStr << endl;
-    return 1;
-  }
-
-  const ParseTree& ptree = parser.GetParseTree();
-  SqlStatement& stmt = *ptree.fList[0];
-
-  stmt.fSessionID = 1;
-  stmt.fSql = stmtStr;
-  stmt.fOwner = owner;
-
-  ByteStream bytestream;
-  bytestream << stmt.fSessionID;
-  stmt.serialize(bytestream);
-  MessageQueueClient mq("DDLProc");
-  ByteStream::byte b;
-  string errorMsg;
-
-  try
-  {
-    mq.write(bytestream);
-    bytestream = mq.read();
-    bytestream >> b;
-    bytestream >> errorMsg;
-  }
-  catch (runtime_error& rex)
-  {
-    cerr << "runtime_error in engine: " << rex.what() << endl;
-    return 1;
-  }
-  catch (...)
-  {
-    cerr << "uknown error in engine" << endl;
-    return 1;
-  }
-
-  if (b != 0)
-  {
-    cerr << "DDLProc error: " << errorMsg << endl;
-    return 1;
-  }
-
-  return 0;
-}

tools/dmldriver/CMakeLists.txt

@@ -1,44 +0,0 @@
-include_directories(${KDE4_INCLUDES} ${KDE4_INCLUDE_DIR} ${QT_INCLUDES})
-
-# ########## next target ###############
-
-set(dmldriver_SRCS dmldriver.cpp tpchrf2.cpp dmlif.cpp)
-
-kde4_add_executable(dmldriver ${dmldriver_SRCS})
-
-target_link_libraries(dmldriver ${KDE4_KDECORE_LIBS})
-
-install(TARGETS dmldriver ${INSTALL_TARGETS_DEFAULT_ARGS})
-
-# ########## install files ###############
-
-# original Makefile.am contents follow:
-
-# Copyright (C) 2014 InfiniDB, Inc.
-#
-# This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public
-# License as published by the Free Software Foundation; version 2 of the License.
-#
-# This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied
-# warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License along with this program; if not, write to the Free
-# Software Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
-#
-# $Id: Makefile.am 333 2009-04-03 20:35:04Z rdempsey $ Process this file with automake to produce Makefile.in
-#
-# AM_CPPFLAGS = $(idb_cppflags) AM_CFLAGS = $(idb_cflags) AM_CXXFLAGS = $(idb_cxxflags) AM_LDFLAGS = $(idb_ldflags)
-# bin_PROGRAMS = dmldriver dmldriver_SOURCES = dmldriver.cpp tpchrf2.cpp dmlif.cpp dmldriver_CPPFLAGS =
-# @idb_common_includes@ $(AM_CPPFLAGS) dmldriver_LDFLAGS = @idb_common_ldflags@ @idb_write_libs@ @idb_common_libs@
-# @netsnmp_libs@ $(AM_LDFLAGS)
-#
-# test:
-#
-# coverage:
-#
-# leakcheck:
-#
-# docs:
-#
-# bootstrap: install-data-am
-#

tools/dmldriver/dmldriver.cpp

@@ -1,205 +0,0 @@
-/* Copyright (C) 2014 InfiniDB, Inc.
-
-   This program is free software; you can redistribute it and/or
-   modify it under the terms of the GNU General Public License
-   as published by the Free Software Foundation; version 2 of
-   the License.
-
-   This program is distributed in the hope that it will be useful,
-   but WITHOUT ANY WARRANTY; without even the implied warranty of
-   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-   GNU General Public License for more details.
-
-   You should have received a copy of the GNU General Public License
-   along with this program; if not, write to the Free Software
-   Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
-   MA 02110-1301, USA. */
-
-// $Id: dmldriver.cpp 2101 2013-01-21 14:12:52Z rdempsey $
-
-//#define NDEBUG
-#include <cassert>
-#include <iostream>
-#include <string>
-#include <unistd.h>
-#include <cctype>
-#include <sstream>
-#include <fstream>
-#include <limits>
-using namespace std;
-
-#include <boost/scoped_array.hpp>
-#include <boost/tokenizer.hpp>
-using namespace boost;
-
-#include "tpchrf2.h"
-#include "dmlif.h"
-
-namespace
-{
-bool vflg;
-bool dflg;
-
-void usage()
-{
-  cout << "usage: dmldriver [-vhd] [-c intvl] [-f file] [-s sid] [-t flgs] [-r file] [-p cnt] [sql_text]"
-       << endl;
-  cout << "   -c intvl \tcommit every intvl statements" << endl;
-  cout << "   -f file  \tread statements from file (max 15KB/stmt)" << endl;
-  cout << "   -s sid   \tset sid as session id" << endl;
-  cout << "   -t flgs  \tset trace flags" << endl;
-  cout << "   -v       \tdisplay affected row count(s)" << endl;
-  cout << "   -d       \tdisplay debug info" << endl;
-  cout << "   -r file  \tread orderkeys from file for TPC-H RF2" << endl;
-  cout << "   -p cnt   \tpack cnt orderkeys into each delete stmt (only w/ -r)" << endl;
-  cout << "   -e schema\tset the schema name (only w/ -r)" << endl;
-  cout << "   -h       \tdisplay this help text" << endl;
-}
-}  // namespace
-
-int main(int argc, char** argv)
-{
-  int c;
-
-  opterr = 0;
-
-  vflg = false;
-  dflg = false;
-
-  bool fflg = false;
-  string infilename;
-
-  int cIntvl = numeric_limits<int>::max();
-
-  uint32_t sessionID = time(0) & 0x7fffffff;
-
-  bool rflg = false;
-
-  int packCnt = 1;
-
-  uint32_t tflg = 0;
-
-  string schema;
-
-  while ((c = getopt(argc, argv, "e:t:s:c:f:r:p:vhd")) != EOF)
-    switch (c)
-    {
-      case 'v': vflg = true; break;
-
-      case 'd': dflg = true; break;
-
-      case 'f':
-        fflg = true;
-        infilename = optarg;
-        break;
-
-      case 'c': cIntvl = static_cast<int>(strtol(optarg, 0, 0)); break;
-
-      case 's': sessionID = static_cast<uint32_t>(strtoul(optarg, 0, 0)); break;
-
-      case 't': tflg = static_cast<uint32_t>(strtoul(optarg, 0, 0)); break;
-
-      case 'r':
-        rflg = true;
-        infilename = optarg;
-        break;
-
-      case 'p': packCnt = static_cast<int>(strtol(optarg, 0, 0)); break;
-
-      case 'e': schema = optarg; break;
-
-      case 'h':
-      case '?':
-      default:
-        usage();
-        return (c == 'h' ? 0 : 1);
-        break;
-    }
-
-  if (!fflg && !rflg && ((argc - optind) < 1))
-  {
-    usage();
-    return 1;
-  }
-
-  if (fflg && rflg)
-  {
-    cout << "-f and -r are mutually exclusive!" << endl << endl;
-    usage();
-    return 1;
-  }
-
-  if (!schema.empty() && !rflg)
-  {
-    cout << "-e requires -r!" << endl << endl;
-    usage();
-    return 1;
-  }
-
-  string stmtStr;
-
-  if (!fflg && !rflg)
-    stmtStr = argv[optind++];
-
-  int rc = 0;
-
-  dmlif::DMLIF dmlif(sessionID, tflg, dflg, vflg);
-
-  if (fflg)
-  {
-    ifstream ifs(infilename.c_str());
-
-    if (!ifs.good())
-    {
-      cerr << "Error accessing file " << infilename << endl;
-      return 1;
-    }
-
-    const streamsize ilinelen = 15 * 1024;
-    scoped_array<char> iline(new char[ilinelen]);
-    int cnt = 0;
-
-    for (;;)
-    {
-      ifs.getline(iline.get(), ilinelen);
-
-      if (ifs.eof())
-        break;
-
-      rc = dmlif.sendOne(iline.get());
-
-      if (rc != 0)
-        break;
-
-      cnt++;
-
-      if ((cnt % cIntvl) == 0)
-        dmlif.sendOne("COMMIT;");
-    }
-  }
-  else if (rflg)
-  {
-    ifstream ifs(infilename.c_str());
-
-    if (!ifs.good())
-    {
-      cerr << "Error accessing file " << infilename << endl;
-      return 1;
-    }
-
-    if (schema.empty())
-      schema = "tpch";
-
-    tpch::RF2 rf2(schema, sessionID, tflg, cIntvl, packCnt, dflg, vflg);
-    rc = rf2.run(ifs);
-  }
-  else
-  {
-    rc = dmlif.sendOne(stmtStr);
-  }
-
-  if (rc == 0)
-    dmlif.sendOne("COMMIT;");
-
-  return 0;
-}

tools/dmldriver/dmlif.cpp

@@ -1,384 +0,0 @@
-/* Copyright (C) 2014 InfiniDB, Inc.
-
-   This program is free software; you can redistribute it and/or
-   modify it under the terms of the GNU General Public License
-   as published by the Free Software Foundation; version 2 of
-   the License.
-
-   This program is distributed in the hope that it will be useful,
-   but WITHOUT ANY WARRANTY; without even the implied warranty of
-   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-   GNU General Public License for more details.
-
-   You should have received a copy of the GNU General Public License
-   along with this program; if not, write to the Free Software
-   Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
-   MA 02110-1301, USA. */
-
-// $Id: dmlif.cpp 2101 2013-01-21 14:12:52Z rdempsey $
-
-//#define NDEBUG
-#include <cassert>
-#include <string>
-#include <sstream>
-using namespace std;
-
-#include <boost/tokenizer.hpp>
-using namespace boost;
-
-#include "vendordmlstatement.h"
-#include "calpontdmlpackage.h"
-#include "calpontdmlfactory.h"
-using namespace dmlpackage;
-
-#include "bytestream.h"
-#include "messagequeue.h"
-using namespace messageqcpp;
-
-#include "simplecolumn.h"
-#include "calpontselectexecutionplan.h"
-#include "sessionmanager.h"
-#include "simplefilter.h"
-#include "constantcolumn.h"
-#include "constantfilter.h"
-using namespace execplan;
-
-#include "brmtypes.h"
-
-#include "dmlif.h"
-using namespace dmlif;
-
-namespace dmlif
-{
-DMLIF::DMLIF(uint32_t sessionid, uint32_t tflg, bool dflg, bool vflg)
- : fSessionID(sessionid), fTflg(tflg), fDflg(dflg), fVflg(vflg), fOPt(0), fLPt(0)
-{
-  fMqp.reset(new MessageQueueClient("DMLProc"));
-}
-
-DMLIF::~DMLIF()
-{
-}
-
-int DMLIF::sendOne(const string& stmt)
-{
-  int rc;
-
-  string tStmt(stmt);
-
-  if (*tStmt.rbegin() != ';')
-    tStmt += ";";
-
-  VendorDMLStatement dmlStmt(tStmt, fSessionID);
-  CalpontDMLPackage* pDMLPackage = CalpontDMLFactory::makeCalpontDMLPackage(dmlStmt);
-
-  if (pDMLPackage == 0)
-  {
-    cerr << "Failed to parse statement: " << tStmt << endl;
-    return -1;
-  }
-
-  string queryString = pDMLPackage->get_QueryString();
-
-  if (fDflg)
-    cout << "qs: >" << queryString << '<' << endl;
-
-  string dmlStatement = pDMLPackage->get_DMLStatement();
-
-  if (fDflg)
-    cout << "DML: " << dmlStatement << endl;
-
-  bool isDML = true;
-
-  if (dmlStatement == "COMMIT" || dmlStatement == "ROLLBACK")
-  {
-    isDML = false;
-  }
-
-  if (isDML)
-  {
-    char_separator<char> sep(" ");
-    tokenizer<char_separator<char> > tok(queryString, sep);
-    tokenizer<char_separator<char> >::iterator iter = tok.begin();
-    idbassert(iter != tok.end());
-    string where = *iter;
-    ++iter;
-    idbassert(iter != tok.end());
-    string col1 = *iter;
-    ++iter;
-    idbassert(iter != tok.end());
-    string op = *iter;
-    ++iter;
-    idbassert(iter != tok.end());
-    string col2 = *iter;
-    ++iter;
-    idbassert(iter == tok.end());
-
-    if (fDflg)
-      cout << "SQL: " << pDMLPackage->get_SQLStatement() << endl;
-
-    if (fDflg)
-      cout << "hf: " << pDMLPackage->HasFilter() << endl;
-
-    DMLTable* tp = pDMLPackage->get_Table();
-
-    if (fDflg)
-      cout << "sn: " << tp->get_SchemaName() << " tn: " << tp->get_TableName() << endl;
-
-    if (fDflg)
-      cout << "row count: " << tp->get_RowList().size() << endl;
-
-    SRCP srcp(new SimpleColumn(tp->get_SchemaName(), tp->get_TableName(), col1, fSessionID));
-    CalpontSelectExecutionPlan::ColumnMap cm;
-    cm.insert(make_pair(col1, srcp));
-    pDMLPackage->get_ExecutionPlan()->columnMap(cm);
-    CalpontSelectExecutionPlan::ReturnedColumnList rcl;
-    rcl.push_back(srcp);
-    pDMLPackage->get_ExecutionPlan()->returnedCols(rcl);
-    pDMLPackage->get_ExecutionPlan()->sessionID(fSessionID);
-    pDMLPackage->get_ExecutionPlan()->traceFlags(fTflg);
-    SessionManager sm;
-    BRM::TxnID txnid = sm.getTxnID(fSessionID);
-
-    if (!txnid.valid)
-      txnid = sm.newTxnID(fSessionID);
-
-    pDMLPackage->get_ExecutionPlan()->txnID(txnid.id);
-    pDMLPackage->get_ExecutionPlan()->verID(sm.verID());
-    ParseTree* pt = new ParseTree();
-    ReturnedColumn* rc1 = srcp->clone();
-    ReturnedColumn* rc2 = new ConstantColumn(col2, ConstantColumn::NUM);
-    SOP sop(new Operator(op));
-    SimpleFilter* sf = new SimpleFilter(sop, rc1, rc2);
-    pt->data(sf);
-    pDMLPackage->get_ExecutionPlan()->filters(pt);
-
-    if (fDflg)
-      cout << "ep: " << *pDMLPackage->get_ExecutionPlan() << endl;
-  }
-
-  ByteStream bytestream;
-  pDMLPackage->write(bytestream);
-  delete pDMLPackage;
-  ByteStream::octbyte rows;
-
-  rc = DMLSend(bytestream, rows);
-
-  if (isDML && fVflg)
-    cout << rows << " rows affected" << endl;
-
-  return rc;
-}
-
-int DMLIF::DMLSend(ByteStream& bytestream, ByteStream::octbyte& rows)
-{
-  ByteStream::byte b;
-  string errorMsg;
-
-  try
-  {
-    fMqp->connect();
-    fMqp->write(bytestream);
-    bytestream = fMqp->read();
-    fMqp->shutdown();
-
-    if (fDflg)
-      cout << "read " << bytestream.length() << " bytes from DMLProc" << endl;
-
-    bytestream >> b;
-
-    if (fDflg)
-      cout << "b = " << (int)b << endl;
-
-    bytestream >> rows;
-
-    if (fDflg)
-      cout << "rows = " << rows << endl;
-
-    bytestream >> errorMsg;
-
-    if (fDflg)
-      cout << "errorMsg = " << errorMsg << endl;
-  }
-  catch (runtime_error& rex)
-  {
-    cerr << "runtime_error in engine: " << rex.what() << endl;
-    return -1;
-  }
-  catch (...)
-  {
-    cerr << "uknown error in engine" << endl;
-    return -1;
-  }
-
-  if (b != 0)
-  {
-    cerr << "DMLProc error: " << errorMsg << endl;
-    return -1;
-  }
-
-  return 0;
-}
-
-void DMLIF::rf2Start(const string& sn)
-{
-  fSchema = sn;
-  fOFilterStr = "";
-  fLFilterStr = "";
-  fOPt = 0;
-  fLPt = 0;
-}
-
-void DMLIF::rf2Add(int64_t okey)
-{
-  ostringstream oss;
-  oss << okey;
-  string okeyStr(oss.str());
-
-  if (fOFilterStr.empty())
-  {
-    fOFilterStr = "o_orderkey=" + okeyStr;
-    ReturnedColumn* rc1 = new SimpleColumn(fSchema, "orders", "o_orderkey", fSessionID);
-    ReturnedColumn* rc2 = new ConstantColumn(okeyStr, ConstantColumn::NUM);
-    SOP sop(new Operator("="));
-    ConstantFilter* cf = new ConstantFilter(sop, rc1, rc2);
-    sop.reset(new Operator("or"));
-    cf->op(sop);
-    fOPt = new ParseTree(cf);
-  }
-  else
-  {
-    fOFilterStr += " or o_orderkey=" + okeyStr;
-    ReturnedColumn* rc1 = new SimpleColumn(fSchema, "orders", "o_orderkey", fSessionID);
-    ReturnedColumn* rc2 = new ConstantColumn(okeyStr, ConstantColumn::NUM);
-    SOP sop(new Operator("="));
-    ConstantFilter* cf = dynamic_cast<ConstantFilter*>(fOPt->data());
-    cf->pushFilter(new SimpleFilter(sop, rc1, rc2));
-  }
-
-  if (fLFilterStr.empty())
-  {
-    fLFilterStr = "l_orderkey=" + okeyStr;
-    ReturnedColumn* rc1 = new SimpleColumn(fSchema, "lineitem", "l_orderkey", fSessionID);
-    ReturnedColumn* rc2 = new ConstantColumn(okeyStr, ConstantColumn::NUM);
-    SOP sop(new Operator("="));
-    ConstantFilter* cf = new ConstantFilter(sop, rc1, rc2);
-    sop.reset(new Operator("or"));
-    cf->op(sop);
-    fLPt = new ParseTree(cf);
-  }
-  else
-  {
-    fLFilterStr += " or l_orderkey=" + okeyStr;
-    ReturnedColumn* rc1 = new SimpleColumn(fSchema, "lineitem", "l_orderkey", fSessionID);
-    ReturnedColumn* rc2 = new ConstantColumn(okeyStr, ConstantColumn::NUM);
-    SOP sop(new Operator("="));
-    ConstantFilter* cf = dynamic_cast<ConstantFilter*>(fLPt->data());
-    cf->pushFilter(new SimpleFilter(sop, rc1, rc2));
-  }
-}
-
-int DMLIF::rf2Send()
-{
-  if (fOFilterStr.empty())
-    return -1;
-
-  int rc = 0;
-  string dmlstr;
-  dmlstr = "delete from " + fSchema + ".orders where " + fOFilterStr + ';';
-
-  if (fDflg)
-    cout << dmlstr << endl;
-
-  VendorDMLStatement dmlStmt(dmlstr, fSessionID);
-  CalpontDMLPackage* pDMLPackage = CalpontDMLFactory::makeCalpontDMLPackage(dmlStmt);
-
-  if (pDMLPackage == 0)
-  {
-    cerr << "Failed to parse statement: " << dmlstr << endl;
-    return -1;
-  }
-
-  SRCP srcp(new SimpleColumn(fSchema, "orders", "o_orderkey", fSessionID));
-  CalpontSelectExecutionPlan::ColumnMap cm;
-  cm.insert(make_pair("o_orderkey", srcp));
-  pDMLPackage->get_ExecutionPlan()->columnMap(cm);
-  CalpontSelectExecutionPlan::ReturnedColumnList rcl;
-  rcl.push_back(srcp);
-  pDMLPackage->get_ExecutionPlan()->returnedCols(rcl);
-  pDMLPackage->get_ExecutionPlan()->sessionID(fSessionID);
-  pDMLPackage->get_ExecutionPlan()->traceFlags(fTflg);
-  SessionManager sm;
-  BRM::TxnID txnid = sm.getTxnID(fSessionID);
-
-  if (!txnid.valid)
-    txnid = sm.newTxnID(fSessionID);
-
-  pDMLPackage->get_ExecutionPlan()->txnID(txnid.id);
-  pDMLPackage->get_ExecutionPlan()->verID(sm.verID());
-  pDMLPackage->get_ExecutionPlan()->filters(fOPt);
-
-  if (fDflg)
-    cout << "ep: " << *pDMLPackage->get_ExecutionPlan() << endl;
-
-  ByteStream bytestream;
-  pDMLPackage->write(bytestream);
-  delete pDMLPackage;
-  pDMLPackage = 0;
-  ByteStream::octbyte rows = 0;
-
-  rc = DMLSend(bytestream, rows);
-
-  if (fVflg)
-    cout << rows << " rows affected" << endl;
-
-  dmlstr = "delete from " + fSchema + ".lineitem where " + fLFilterStr + ';';
-
-  if (fDflg)
-    cout << dmlstr << endl;
-
-  VendorDMLStatement dmlStmt1(dmlstr, fSessionID);
-  pDMLPackage = CalpontDMLFactory::makeCalpontDMLPackage(dmlStmt1);
-
-  if (pDMLPackage == 0)
-  {
-    cerr << "Failed to parse statement: " << dmlstr << endl;
-    return -1;
-  }
-
-  srcp.reset(new SimpleColumn(fSchema, "lineitem", "l_orderkey", fSessionID));
-  cm.clear();
-  cm.insert(make_pair("l_orderkey", srcp));
-  pDMLPackage->get_ExecutionPlan()->columnMap(cm);
-  rcl.clear();
-  rcl.push_back(srcp);
-  pDMLPackage->get_ExecutionPlan()->returnedCols(rcl);
-  pDMLPackage->get_ExecutionPlan()->sessionID(fSessionID);
-  pDMLPackage->get_ExecutionPlan()->traceFlags(fTflg);
-  txnid = sm.getTxnID(fSessionID);
-
-  if (!txnid.valid)
-    txnid = sm.newTxnID(fSessionID);
-
-  pDMLPackage->get_ExecutionPlan()->txnID(txnid.id);
-  pDMLPackage->get_ExecutionPlan()->verID(sm.verID());
-  pDMLPackage->get_ExecutionPlan()->filters(fLPt);
-
-  if (fDflg)
-    cout << "ep: " << *pDMLPackage->get_ExecutionPlan() << endl;
-
-  bytestream.reset();
-  pDMLPackage->write(bytestream);
-  delete pDMLPackage;
-  pDMLPackage = 0;
-  rows = 0;
-
-  rc = DMLSend(bytestream, rows);
-
-  if (fVflg)
-    cout << rows << " rows affected" << endl;
-
-  return 0;
-}
-
-}  // namespace dmlif

tools/dmldriver/dmlif.h

@@ -1,65 +0,0 @@
-/* Copyright (C) 2014 InfiniDB, Inc.
-
-   This program is free software; you can redistribute it and/or
-   modify it under the terms of the GNU General Public License
-   as published by the Free Software Foundation; version 2 of
-   the License.
-
-   This program is distributed in the hope that it will be useful,
-   but WITHOUT ANY WARRANTY; without even the implied warranty of
-   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-   GNU General Public License for more details.
-
-   You should have received a copy of the GNU General Public License
-   along with this program; if not, write to the Free Software
-   Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
-   MA 02110-1301, USA. */
-
-// $Id: dmlif.h 2101 2013-01-21 14:12:52Z rdempsey $
-
-#pragma once
-
-#include <string>
-#include <cstddef>
-#include <boost/scoped_ptr.hpp>
-
-#include "messagequeue.h"
-#include "expressionparser.h"
-
-namespace dmlif
-{
-class DMLIF
-{
- public:
-  DMLIF(uint32_t sessionid, uint32_t tflg = 0, bool dflg = false, bool vflg = false);
-  ~DMLIF();
-
-  int sendOne(const std::string& stmt);
-
-  void rf2Start(const std::string& sn);
-  void rf2Add(int64_t okey);
-  int rf2Send();
-
- protected:
-  int DMLSend(messageqcpp::ByteStream& bytestream, messageqcpp::ByteStream::octbyte& rows);
-
- private:
-  // DMLIF(const DMLIF& rhs);
-  // DMLIF& operator=(const DMLIF& rhs);
-
-  uint32_t fSessionID;
-  uint32_t fTflg;
-  bool fDflg;
-  bool fVflg;
-
-  boost::scoped_ptr<messageqcpp::MessageQueueClient> fMqp;
-
-  std::string fSchema;
-  std::string fOFilterStr;
-  std::string fLFilterStr;
-
-  execplan::ParseTree* fOPt;
-  execplan::ParseTree* fLPt;
-};
-
-}  // namespace dmlif

tools/dmldriver/tpchrf2.cpp

@@ -1,86 +0,0 @@
-/* Copyright (C) 2014 InfiniDB, Inc.
-
-   This program is free software; you can redistribute it and/or
-   modify it under the terms of the GNU General Public License
-   as published by the Free Software Foundation; version 2 of
-   the License.
-
-   This program is distributed in the hope that it will be useful,
-   but WITHOUT ANY WARRANTY; without even the implied warranty of
-   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-   GNU General Public License for more details.
-
-   You should have received a copy of the GNU General Public License
-   along with this program; if not, write to the Free Software
-   Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
-   MA 02110-1301, USA. */
-
-// $Id: tpchrf2.cpp 2101 2013-01-21 14:12:52Z rdempsey $
-
-#include <istream>
-using namespace std;
-
-#include <boost/scoped_array.hpp>
-#include <boost/tokenizer.hpp>
-using namespace boost;
-
-#include "tpchrf2.h"
-#include "dmlif.h"
-
-namespace tpch
-{
-RF2::RF2(const string& sn, uint32_t sid, uint32_t tflg, int c, int p, bool d, bool v)
- : fSchema(sn), fSessionID(sid), fTflg(tflg), fIntvl(c), fPack(p), fDflg(d), fVflg(v)
-{
-}
-
-RF2::~RF2()
-{
-}
-
-int RF2::run(istream& in)
-{
-  const streamsize ilinelen = 1024;
-  scoped_array<char> iline(new char[ilinelen]);
-  int cnt = 0;
-  dmlif::DMLIF dmlif(fSessionID, fTflg, fDflg, fVflg);
-
-  for (;;)
-  {
-    dmlif.rf2Start(fSchema);
-
-    for (int i = 0; i < fPack; i++)
-    {
-      in.getline(iline.get(), ilinelen);
-
-      if (in.eof())
-        break;
-
-      typedef char_separator<char> cs;
-      typedef tokenizer<cs> tk;
-      cs sep("|");
-      tk tok(string(iline.get()), sep);
-      tk::iterator iter = tok.begin();
-      idbassert(iter != tok.end());
-      string keystr = *iter;
-      ++iter;
-      // idbassert(iter == tok.end());
-      int64_t okey = strtol(keystr.c_str(), 0, 0);
-      dmlif.rf2Add(okey);
-    }
-
-    dmlif.rf2Send();
-    cnt++;
-
-    if ((cnt % fIntvl) == 0)
-      dmlif.sendOne("COMMIT;");
-
-    if (in.eof())
-      break;
-  }
-
-  dmlif.sendOne("COMMIT;");
-  return 0;
-}
-
-}  // namespace tpch

tools/dmldriver/tpchrf2.h

@@ -1,52 +0,0 @@
-/* Copyright (C) 2014 InfiniDB, Inc.
-
-   This program is free software; you can redistribute it and/or
-   modify it under the terms of the GNU General Public License
-   as published by the Free Software Foundation; version 2 of
-   the License.
-
-   This program is distributed in the hope that it will be useful,
-   but WITHOUT ANY WARRANTY; without even the implied warranty of
-   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-   GNU General Public License for more details.
-
-   You should have received a copy of the GNU General Public License
-   along with this program; if not, write to the Free Software
-   Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
-   MA 02110-1301, USA. */
-
-// $Id: tpchrf2.h 2101 2013-01-21 14:12:52Z rdempsey $
-
-#pragma once
-
-#include <string>
-#include <istream>
-#include <cstddef>
-#include <limits>
-#include <stdint.h>
-
-namespace tpch
-{
-class RF2
-{
- public:
-  RF2(const std::string& sn, uint32_t sid, uint32_t tflg = 0, int c = std::numeric_limits<int>::max(),
-      int p = 1, bool d = false, bool v = false);
-  ~RF2();
-
-  int run(std::istream& in);
-
- private:
-  // RF2(const RF2& rhs);
-  // RF2& operator=(const RF2& rhs);
-
-  std::string fSchema;
-  uint32_t fSessionID;
-  uint32_t fTflg;
-  int fIntvl;
-  int fPack;
-  bool fDflg;
-  bool fVflg;
-};
-
-}  // namespace tpch

tools/editem/CMakeLists.txt

@@ -4,12 +4,5 @@ include_directories(${ENGINE_COMMON_INCLUDES})
 
 set(editem_SRCS editem.cpp)
 
-add_executable(editem ${editem_SRCS})
+columnstore_executable(editem ${editem_SRCS})
-
+columnstore_link(editem ${ENGINE_LDFLAGS} ${NETSNMP_LIBRARIES} ${ENGINE_EXEC_LIBS})
-target_link_libraries(editem ${ENGINE_LDFLAGS} ${NETSNMP_LIBRARIES} ${ENGINE_EXEC_LIBS})
-
-install(
-    TARGETS editem
-    DESTINATION ${ENGINE_BINDIR}
-    COMPONENT columnstore-engine
-)

tools/fixdate/fixdate.cpp

@@ -1,173 +0,0 @@
-/* Copyright (C) 2014 InfiniDB, Inc.
-
-   This program is free software; you can redistribute it and/or
-   modify it under the terms of the GNU General Public License
-   as published by the Free Software Foundation; version 2 of
-   the License.
-
-   This program is distributed in the hope that it will be useful,
-   but WITHOUT ANY WARRANTY; without even the implied warranty of
-   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-   GNU General Public License for more details.
-
-   You should have received a copy of the GNU General Public License
-   along with this program; if not, write to the Free Software
-   Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
-   MA 02110-1301, USA. */
-
-/*
- * $Id: fixdate.cpp 2101 2013-01-21 14:12:52Z rdempsey $
- *
- */
-
-#include <iostream>
-#include <fstream>
-#include <cassert>
-#include <vector>
-#include <algorithm>
-#include <iterator>
-using namespace std;
-
-#include <unistd.h>
-
-#include <boost/format.hpp>
-#include <boost/scoped_array.hpp>
-using namespace boost;
-
-#include "bytestream.h"
-using namespace messageqcpp;
-
-#include "dmlpackageprocessor.h"
-using namespace dmlpackageprocessor;
-
-namespace
-{
-const streamsize blkSz = 8192;
-
-void usage()
-{
-}
-
-u_int64_t fixDate(ByteStream& bs, ostream& out)
-{
-  ByteStream fixed;
-  ByteStream::quadbyte o;
-  u_int64_t cnt = 0;
-#if 0
-    DMLPackageProcessor::Date minDate;
-    DMLPackageProcessor::Date maxDate;
-    minDate.year = 1992;
-    minDate.month = 1;
-    minDate.day = 2;
-    maxDate.year = 1998;
-    maxDate.month = 12;
-    maxDate.day = 25;
-    ByteStream::quadbyte mxd;
-    ByteStream::quadbyte mnd;
-    mxd = *(reinterpret_cast<ByteStream::quadbyte*>(&maxDate));
-    mnd = *(reinterpret_cast<ByteStream::quadbyte*>(&minDate));
-#endif
-  DMLPackageProcessor::Date fixDate;
-  fixDate.spare = 0;
-  ByteStream::quadbyte f;
-
-  while (bs.length() > 0)
-  {
-    bs >> o;
-
-    if (o >= 0xfffffffe)
-    {
-      fixed << o;
-      continue;
-    }
-
-    f = o & 0xffff;
-    fixDate.year = f;
-    o >>= 16;
-    f = o & 0xf;
-    fixDate.month = f;
-    o >>= 4;
-    f = o & 0x3f;
-    fixDate.day = f;
-    // o >>= 6;
-    o = *(reinterpret_cast<ByteStream::quadbyte*>(&fixDate));
-    fixed << o;
-#if 0
-        cout << DMLPackageProcessor::dateToString(o) << endl;
-        idbassert(o >= mnd && o <= mxd);
-        cnt++;
-#endif
-  }
-
-  out << fixed;
-  return cnt;
-}
-
-}  // namespace
-
-int main(int argc, char* argv[])
-{
-  int c;
-
-  opterr = 0;
-
-  while ((c = getopt(argc, argv, "h")) != EOF)
-    switch (c)
-    {
-      case 'h':
-        usage();
-        return 0;
-        break;
-
-      default:
-        usage();
-        return 1;
-        break;
-    }
-
-  if ((argc - optind) < 1)
-  {
-    usage();
-    return 1;
-  }
-
-  ByteStream bs;
-
-  ifstream ifs(argv[optind + 0]);
-  ByteStream::byte inbuf[blkSz];
-  streampos fLen;
-  u_int64_t blkNo = 0;
-
-  ifs.seekg(0, ios_base::end);
-  fLen = ifs.tellg();
-  ifs.seekg(0, ios_base::beg);
-
-  idbassert((fLen % blkSz) == 0);
-  u_int64_t numBlks = fLen / blkSz;
-  cout << numBlks << " blocks to fix..." << endl;
-
-  ofstream ofs("fixdate.cdf");
-
-  cout << "pct done:    " << setw(3);
-
-  for (;;)
-  {
-    ifs.read(reinterpret_cast<char*>(inbuf), blkSz);
-
-    if (ifs.eof())
-      break;
-
-    bs.load(inbuf, blkSz);
-
-    fixDate(bs, ofs);
-    cout << "\b\b\b" << setw(3) << (u_int64_t)(blkNo * 100 / numBlks);
-    // cout << setw(3) << (u_int64_t)(blkNo * 100 / numBlks) << endl;
-
-    blkNo++;
-  }
-
-  cout << "\b\b\b" << setw(3) << 100 << endl;
-  // cout << setw(3) << 100 << endl;
-
-  return 0;
-}

tools/getConfig/CMakeLists.txt

@@ -4,12 +4,6 @@ include_directories(${ENGINE_COMMON_INCLUDES})
 
 set(getConfig_SRCS main.cpp)
 
-add_executable(mcsGetConfig ${getConfig_SRCS})
+columnstore_executable(mcsGetConfig ${getConfig_SRCS})
 
-target_link_libraries(mcsGetConfig ${ENGINE_LDFLAGS} ${ENGINE_EXEC_LIBS})
+columnstore_link(mcsGetConfig ${ENGINE_LDFLAGS} ${ENGINE_EXEC_LIBS})
-
-install(
-    TARGETS mcsGetConfig
-    DESTINATION ${ENGINE_BINDIR}
-    COMPONENT columnstore-engine
-)

tools/idb_comp/main.cpp

@@ -1,265 +0,0 @@
-#include <unistd.h>
-#include <sys/types.h>
-#include <sys/stat.h>
-#include <fcntl.h>
-#include <cerrno>
-//#define NDEBUG
-#include <cassert>
-#include <cstring>
-
-#include <lzo/lzoconf.h>
-#include <lzo/lzo1x.h>
-
-#include <iostream>
-#include <iomanip>
-#include <sstream>
-#include <inttypes.h>
-#include <cstdlib>
-using namespace std;
-
-#include <boost/scoped_array.hpp>
-
-namespace
-{
-size_t exp_buf_size;
-unsigned vflg;
-
-const string lzo_rctos(int r)
-{
-  switch (r)
-  {
-    case LZO_E_INPUT_NOT_CONSUMED: return "LZO_E_INPUT_NOT_CONSUMED";
-
-    default: break;
-  }
-
-  return "Unknown Error!";
-}
-
-void usage()
-{
-  cout << "usage: idb_comp [-e size][-h] file(s)" << endl
-       << "\t-e size size (in KB) of expansion buffer" << endl
-       << "\t-h      display this help" << endl;
-}
-
-int doit(const string& infile)
-{
-  int rc = 0;
-  int ifd = open(infile.c_str(), O_RDONLY | O_LARGEFILE | O_DIRECT);
-
-  if (ifd < 0)
-  {
-    cerr << infile << ": open: " << strerror(errno) << endl;
-    return 1;
-  }
-
-  struct stat istatbuf;
-
-  fstat(ifd, &istatbuf);
-
-  string outname(infile);
-
-  string::size_type ptr;
-
-  ptr = outname.find_last_of('.');
-
-  if (ptr != string::npos)
-    outname.erase(ptr);
-
-  ptr = outname.find_last_of('/');
-
-  if (ptr != string::npos)
-    outname.erase(0, ptr + 1);
-
-  outname = "./" + outname + ".cmp";
-
-  int ofd = open(outname.c_str(), O_WRONLY | O_CREAT | O_TRUNC | O_LARGEFILE | O_DIRECT, 0644);
-
-  if (ofd < 0)
-  {
-    cerr << outname << ": open: " << strerror(errno) << endl;
-    close(ifd);
-    return 1;
-  }
-
-  lzo_init();
-
-  ssize_t nread = -1;
-  ssize_t nwritten = -1;
-  lzo_bytep ibuf;
-  lzo_bytep tibuf;
-  lzo_bytep cbuf;
-  lzo_bytep tcbuf;
-  lzo_bytep wkmem;
-  lzo_uint32 ibuf_len = 0;
-  lzo_uint cbuf_len = 0;
-
-  ibuf_len = 512 * 1024 * 8;
-  tibuf = new lzo_byte[ibuf_len + 4095];
-
-  if ((ptrdiff_t)tibuf & 0xfffULL)
-    ibuf = (lzo_bytep)((ptrdiff_t)tibuf & 0xfffffffffffff000ULL) + 4096;
-  else
-    ibuf = tibuf;
-
-  cbuf_len = 512 * 1024 * 8;
-  tcbuf = new lzo_byte[cbuf_len + 4095 + exp_buf_size * 1024];
-
-  if ((ptrdiff_t)tcbuf & 0xfff)
-    cbuf = (lzo_bytep)((ptrdiff_t)tcbuf & 0xfffffffffffff000ULL) + 4096;
-  else
-    cbuf = tcbuf;
-
-  wkmem = new lzo_byte[LZO1X_1_15_MEM_COMPRESS];
-
-  int r = LZO_E_OK;
-
-  const int TOTAL_HDR_LEN = 4096 * 2;
-  char* hdrbuf = new char[TOTAL_HDR_LEN + 4095];
-  memset(hdrbuf, 0, TOTAL_HDR_LEN + 4095);
-  char* hdrbufp = 0;
-
-  if ((ptrdiff_t)hdrbuf & 0xfff)
-    hdrbufp = (char*)((ptrdiff_t)hdrbuf & 0xfffffffffffff000ULL) + 4096;
-  else
-    hdrbufp = hdrbuf;
-
-  struct compHdr
-  {
-    uint64_t ptrs[512];
-  };
-
-  idbassert(sizeof(compHdr) <= 4096);
-
-  compHdr* hdrptr1 = (compHdr*)hdrbufp;
-  compHdr* hdrptr = hdrptr1 + 1;  // advance to 2nd hdr to store compression ptrs
-  lseek(ofd, TOTAL_HDR_LEN, SEEK_SET);
-
-  nread = read(ifd, ibuf, ibuf_len);
-
-  int idx = 0;
-  off_t cmpoff = TOTAL_HDR_LEN;
-
-  while (nread > 0)
-  {
-    cbuf_len = 512 * 1024 * 8;
-    memset(cbuf, 0, cbuf_len);
-    r = lzo1x_1_15_compress(ibuf, nread, cbuf, &cbuf_len, wkmem);
-
-    if (r != LZO_E_OK)
-    {
-      cerr << "compression failed!: " << r << endl;
-      rc = 1;
-      goto out;
-    }
-
-    if (cbuf_len > (unsigned)nread)
-    {
-      cerr << "WARNING: expansion detected! (output grew by " << (cbuf_len - nread) << " bytes)" << endl;
-      idbassert((cbuf_len - nread) <= exp_buf_size * 1024);
-    }
-
-    if (cbuf_len & 0xfff)
-      cbuf_len = (cbuf_len & 0xfffffffffffff000ULL) + 4096;
-
-    // cbuf_len = 512 * 1024 * 8;
-    nwritten = write(ofd, cbuf, cbuf_len);
-
-    if (nwritten < 0 || (unsigned)nwritten != cbuf_len)
-    {
-      cerr << outname << ": write: " << strerror(errno) << " (" << nwritten << ')' << endl;
-      rc = 1;
-      goto out;
-    }
-
-    if (vflg > 0)
-    {
-      lzo_bytep tbuf;
-      lzo_uint tbuflen = 4 * 1024 * 1024 + 4;
-      boost::scoped_array<lzo_byte> tbuf_sa(new lzo_byte[tbuflen]);
-      tbuf = tbuf_sa.get();
-      cout << "idx: " << idx << " off: " << cmpoff << " size: " << cbuf_len;
-      r = lzo1x_decompress(cbuf, cbuf_len, tbuf, &tbuflen, 0);
-      cout << " r: " << lzo_rctos(r) << " size: " << tbuflen << endl;
-    }
-
-    hdrptr->ptrs[idx] = cmpoff;
-    idx++;
-    cmpoff += cbuf_len;
-
-    nread = read(ifd, ibuf, ibuf_len);
-  }
-
-  if (nread < 0)
-  {
-    cerr << infile << ": read: " << strerror(errno) << endl;
-    rc = 1;
-    goto out;
-  }
-
-  hdrptr->ptrs[idx] = cmpoff;
-  idbassert(idx <= 64);
-
-  // Fill in meta-data information in first header
-  hdrptr1->ptrs[0] = 0xfdc119a384d0778eULL;
-  hdrptr1->ptrs[1] = 1;
-  hdrptr1->ptrs[2] = 1;
-
-  nwritten = pwrite(ofd, hdrbufp, TOTAL_HDR_LEN, 0);
-  idbassert(nwritten == TOTAL_HDR_LEN);
-
-out:
-  delete[] wkmem;
-  delete[] tcbuf;
-  delete[] tibuf;
-  fsync(ofd);
-  struct stat ostatbuf;
-  fstat(ofd, &ostatbuf);
-  idbassert(ostatbuf.st_size == (signed)hdrptr->ptrs[idx]);
-  delete[] hdrbuf;
-  cout << infile << ": Input Size: " << istatbuf.st_size << " Output size: " << ostatbuf.st_size
-       << " Compression: " << (100LL - (ostatbuf.st_size * 100LL / istatbuf.st_size)) << '%' << endl;
-  close(ofd);
-  close(ifd);
-
-  return rc;
-}
-
-}  // namespace
-
-int main(int argc, char** argv)
-{
-  opterr = 0;
-  int c;
-  exp_buf_size = 128;
-  vflg = 0;
-
-  while ((c = getopt(argc, argv, "e:vh")) != -1)
-    switch (c)
-    {
-      case 'e': exp_buf_size = atoi(optarg); break;
-
-      case 'v': vflg++; break;
-
-      case 'h':
-      default:
-        usage();
-        return (c == 'h' ? 0 : 1);
-        break;
-    }
-
-  if ((argc - optind) < 1)
-  {
-    usage();
-    return 1;
-  }
-
-  int rc = 0;
-
-  for (int i = optind; i < argc; i++)
-    if (doit(argv[i]))
-      rc = 1;
-
-  return rc;
-}

tools/idbmeminfo/CMakeLists.txt

@@ -4,12 +4,5 @@ include_directories(${ENGINE_COMMON_INCLUDES})
 
 set(idbmeminfo_SRCS idbmeminfo.cpp)
 
-add_executable(idbmeminfo ${idbmeminfo_SRCS})
+columnstore_executable(idbmeminfo ${idbmeminfo_SRCS})
-
+columnstore_link(idbmeminfo ${ENGINE_LDFLAGS})
-target_link_libraries(idbmeminfo ${ENGINE_LDFLAGS})
-
-install(
-    TARGETS idbmeminfo
-    DESTINATION ${ENGINE_BINDIR}
-    COMPONENT columnstore-engine
-)

tools/passwd/CMakeLists.txt

@@ -4,21 +4,10 @@ include_directories(${ENGINE_COMMON_INCLUDES} ${ENGINE_UTILS_COMMON_INCLUDE})
 set(cspasswd_SRCS cspasswd.cpp secrets.cpp)
 set(cskeys_SRCS cskeys.cpp secrets.cpp)
 
-add_executable(cspasswd ${cspasswd_SRCS})
+columnstore_executable(cspasswd ${cspasswd_SRCS})
-add_executable(cskeys ${cskeys_SRCS})
+columnstore_executable(cskeys ${cskeys_SRCS})
 
 target_include_directories(cspasswd BEFORE PUBLIC ${OPENSSL_INCLUDE_DIR})
 target_include_directories(cskeys BEFORE PUBLIC ${OPENSSL_INCLUDE_DIR})
-target_link_libraries(cspasswd ${ENGINE_LDFLAGS} ${ENGINE_EXEC_LIBS} ${SSL_LIBRARIES})
+columnstore_link(cspasswd ${ENGINE_LDFLAGS} ${ENGINE_EXEC_LIBS} ${SSL_LIBRARIES})
-target_link_libraries(cskeys ${ENGINE_LDFLAGS} ${ENGINE_EXEC_LIBS} ${SSL_LIBRARIES})
+columnstore_link(cskeys ${ENGINE_LDFLAGS} ${ENGINE_EXEC_LIBS} ${SSL_LIBRARIES})
-
-install(
-    TARGETS cspasswd
-    DESTINATION ${ENGINE_BINDIR}
-    COMPONENT columnstore-engine
-)
-install(
-    TARGETS cskeys
-    DESTINATION ${ENGINE_BINDIR}
-    COMPONENT columnstore-engine
-)

tools/qfe/CMakeLists.txt

@@ -1,55 +0,0 @@
-include_directories(${KDE4_INCLUDES} ${KDE4_INCLUDE_DIR} ${QT_INCLUDES})
-
-# ########## next target ###############
-
-set(QFE_SRCS
-    cseputils.cpp
-    ddlstmts.cpp
-    parsequery.cpp
-    returnedrows.cpp
-    sendcsep.cpp
-    server.cpp
-    socketio.cpp
-    qfelexer.lpp
-    qfeparser.ypp
-)
-
-kde4_add_executable(QFE ${QFE_SRCS})
-
-target_link_libraries(QFE ${KDE4_KDECORE_LIBS})
-
-install(TARGETS QFE ${INSTALL_TARGETS_DEFAULT_ARGS})
-
-# ########## install files ###############
-
-# original Makefile.am contents follow:
-
-# Copyright (C) 2014 InfiniDB, Inc.
-#
-# This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public
-# License as published by the Free Software Foundation; version 2 of the License.
-#
-# This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied
-# warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License along with this program; if not, write to the Free
-# Software Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
-#
-# $Id$ Process this file with automake to produce Makefile.in
-#
-# AM_CPPFLAGS = $(idb_cppflags) AM_CFLAGS = $(idb_cflags) AM_CXXFLAGS = $(idb_cxxflags) AM_LDFLAGS = $(idb_ldflags)
-# AM_YFLAGS = -d -p qfe AM_LFLAGS = -i -Pqfe -olex.yy.c bin_PROGRAMS = QFE QFE_SOURCES = \ cseputils.cpp \ ddlstmts.cpp
-# \ parsequery.cpp \ returnedrows.cpp \ sendcsep.cpp \ server.cpp \ socketio.cpp \ qfelexer.lpp \ qfeparser.ypp
-# QFE_CPPFLAGS = @idb_common_includes@ $(AM_CPPFLAGS) QFE_LDFLAGS = @idb_common_ldflags@ @idb_write_libs@
-# @idb_exec_libs@ $(AM_LDFLAGS) BUILT_SOURCES = qfeparser.cpp qfelexer.cpp qfeparser.h
-#
-# test:
-#
-# coverage:
-#
-# leakcheck:
-#
-# docs:
-#
-# bootstrap: install-data-am
-#

tools/qfe/cli.cpp

@@ -1,148 +0,0 @@
-#include <poll.h>
-#include <sys/socket.h>
-#include <netinet/in.h>
-#include <arpa/inet.h>
-#include <netdb.h>
-#include <iostream>
-#include <string>
-#include <sstream>
-#include <iomanip>
-#include <sys/types.h>
-#include <sys/stat.h>
-#include <fcntl.h>
-#include <unistd.h>
-#include <cerrno>
-#include <cstring>
-#include <cstdlib>
-using namespace std;
-
-#include "socktype.h"
-#include "socketio.h"
-using namespace qfe;
-
-namespace
-{
-void usage()
-{
-  cout << "usage: cli [-h] [-s serverip] [-p port] [-c schema] <query>" << endl;
-  cout << '\t' << "-s serverip connect to serverip instead of 127.0.0.1" << endl;
-  cout << '\t' << "-p port     connect on port instead of 9198" << endl;
-  cout << '\t' << "-c schema   use schema as default instead of tpch1" << endl;
-  cout << '\t' << "-h          display this help" << endl;
-}
-}  // namespace
-
-int main(int argc, char** argv)
-{
-  opterr = 0;
-  int c;
-  string serverip("127.0.0.1");
-  string schema("tpch1");
-  short PortNo = 9198;
-
-  while ((c = getopt(argc, argv, "s:c:p:h")) != -1)
-    switch (c)
-    {
-      case 's': serverip = optarg; break;
-
-      case 'c': schema = optarg; break;
-
-      case 'p':
-        PortNo = atoi(optarg);
-
-        if (PortNo == 0)
-        {
-          usage();
-          return 1;
-        }
-
-        break;
-
-      case 'h':
-      case '?':
-      default:
-        usage();
-        return 0;
-        break;
-    }
-
-  if (argc - optind < 1)
-  {
-    usage();
-    return 1;
-  }
-
-  string query(argv[optind + 0]);
-
-  SockType fd = -1;
-
-  fd = ::socket(PF_INET, SOCK_STREAM, IPPROTO_TCP);
-
-  if (fd < 0)
-  {
-    cerr << "socket create error: " << strerror(errno) << endl;
-    return 1;
-  }
-
-  int rc = 0;
-  struct sockaddr_in serv_addr;
-  struct addrinfo hints;
-  struct addrinfo* res = 0;
-  memset(&hints, 0, sizeof(struct addrinfo));
-  hints.ai_family = PF_INET;
-  hints.ai_socktype = SOCK_STREAM;
-  hints.ai_protocol = IPPROTO_TCP;
-  rc = getaddrinfo(serverip.c_str(), 0, &hints, &res);
-
-  if (rc != 0)
-  {
-    cerr << "Error resolving '" << serverip << "': " << gai_strerror(rc) << endl;
-    return 1;
-  }
-
-  sockaddr_in* sain = 0;
-  sain = reinterpret_cast<sockaddr_in*>(res->ai_addr);
-  memset(&serv_addr, 0, sizeof(serv_addr));
-  serv_addr.sin_family = PF_INET;
-  serv_addr.sin_addr = sain->sin_addr;
-  serv_addr.sin_port = htons(PortNo);
-  freeaddrinfo(res);
-
-  rc = ::connect(fd, (sockaddr*)&serv_addr, sizeof(serv_addr));
-
-  if (rc < 0)
-  {
-    cerr << "socket connect error: " << strerror(errno) << endl;
-    return 1;
-  }
-
-  socketio::writeString(fd, schema);
-  socketio::writeString(fd, query);
-
-  uint32_t flag = 0;
-  string row;
-  row = socketio::readString(fd);
-
-  if (row != "OK")
-  {
-    cerr << "query failed: " << row << endl;
-    goto bailout;
-  }
-
-  row = socketio::readString(fd);
-
-  while (!row.empty())
-  {
-    cout << row << endl;
-    row = socketio::readString(fd);
-  }
-
-  flag = 0;
-  SockWriteFcn(fd, &flag, 4);
-bailout:
-  ::shutdown(fd, SHUT_RDWR);
-  ::close(fd);
-  fd = -1;
-
-  return 0;
-}

tools/qfe/cseputils.cpp

@@ -1,141 +0,0 @@
-#include <unistd.h>
-#include <string>
-using namespace std;
-
-#include "constantcolumn.h"
-#include "simplefilter.h"
-#include "calpontsystemcatalog.h"
-#include "parsetree.h"
-#include "simplecolumn.h"
-#include "calpontselectexecutionplan.h"
-using namespace execplan;
-
-#include "cseputils.h"
-
-namespace qfe
-{
-extern string DefaultSchema;
-
-namespace utils
-{
-ConstantColumn* createConstCol(const string& valstr)
-{
-  ConstantColumn* cc = new ConstantColumn(valstr);
-  cc->alias(valstr);
-  return cc;
-}
-
-template <typename T>
-ConstantColumn* createConstCol(const string& valstr, T val)
-{
-  ConstantColumn* cc = new ConstantColumn(valstr, val);
-  cc->alias(valstr);
-  return cc;
-}
-
-SimpleFilter* createSimpleFilter(boost::shared_ptr<CalpontSystemCatalog>& csc,
-                                 const CalpontSystemCatalog::TableColName& tcn, const string& opstr,
-                                 ConstantColumn* cc)
-{
-  SimpleFilter* lsf = new SimpleFilter();
-
-  Operator* op = new Operator();
-  op->data(opstr);
-  CalpontSystemCatalog::ColType ccct;
-  ccct = op->resultType();
-  ccct.colDataType = cc->resultType().colDataType;
-  op->operationType(ccct);
-
-  SOP sop(op);
-  lsf->op(sop);
-
-  CalpontSystemCatalog::OID oid = csc->lookupOID(tcn);
-  CalpontSystemCatalog::ColType ct = csc->colType(oid);
-
-  SimpleColumn* sc = new SimpleColumn();
-  sc->schemaName(tcn.schema);
-  sc->tableName(tcn.table);
-  sc->tableAlias(tcn.table);
-  sc->columnName(tcn.column);
-  sc->oid(oid);
-  sc->resultType(ct);
-  sc->alias(tcn.toString());
-
-  lsf->lhs(sc);
-  lsf->rhs(cc);
-
-  return lsf;
-}
-
-void appendSimpleFilter(ParseTree*& ptree, SimpleFilter* filter)
-{
-  if (ptree->data() == 0)
-  {
-    // degenerate case, this filter goes at this node
-    ptree->data(filter);
-  }
-  else if (ptree->right() == 0 && ptree->left() == 0)
-  {
-    // this will be the case when there is a single node in the tree
-    // that contains a filter.  Here we want to make the root node an
-    // 'and' operator, push the existing down to the lhs and make a
-    // new node for the new filter
-    ParseTree* newLhs = new ParseTree(ptree->data());
-    ParseTree* newRhs = new ParseTree(filter);
-
-    Operator* op = new Operator();
-    op->data("and");
-
-    ptree->data(op);
-    ptree->left(newLhs);
-    ptree->right(newRhs);
-  }
-  else
-  {
-    // this will be the case once we have a tree with an 'and' at the
-    // root node, a filter in the lhs, and an arbitrary height tree
-    // with the same properties on the rhs.  Because all operators
-    // are guaranteed to be and for now we simply insert a new rhs
-    // node and "push down" the existing tree
-    Operator* op = new Operator();
-    op->data("and");
-
-    ParseTree* newRhs = new ParseTree(op);
-    newRhs->left(new ParseTree(filter));
-    newRhs->right(ptree->right());
-    ptree->right(newRhs);
-  }
-}
-
-void updateParseTree(boost::shared_ptr<execplan::CalpontSystemCatalog>& csc,
-                     execplan::CalpontSelectExecutionPlan*& csep, execplan::SimpleColumn* sc,
-                     const std::string& relop, pair<int, string> cval)
-{
-  execplan::ConstantColumn* cc = 0;
-
-  if (cval.first == 0)
-    cc = createConstCol(cval.second, static_cast<int64_t>(atoll(cval.second.c_str())));
-  else
-    cc = createConstCol(cval.second);
-
-  if (sc->schemaName() == "infinidb_unknown" && !DefaultSchema.empty())
-    sc->schemaName(DefaultSchema);
-
-  execplan::SimpleFilter* sf = 0;
-  sf = createSimpleFilter(csc, execplan::make_tcn(sc->schemaName(), sc->tableName(), sc->columnName()), relop,
-                          cc);
-  execplan::ParseTree* ptp = 0;
-  ptp = csep->filters();
-
-  if (ptp == 0)
-    ptp = new execplan::ParseTree();
-
-  appendSimpleFilter(ptp, sf);
-  csep->filters(ptp);
-}
-
-// template instantiations
-template ConstantColumn* createConstCol<int64_t>(const string& valstr, int64_t val);
-
-}  // namespace utils
-}  // namespace qfe

tools/qfe/cseputils.h

@@ -1,34 +0,0 @@
-#pragma once
-
-#include <string>
-
-namespace execplan
-{
-class ConstantColumn;
-class SimpleFilter;
-class ParseTree;
-class SimpleColumn;
-}  // namespace execplan
-#include "calpontsystemcatalog.h"
-
-namespace qfe
-{
-namespace utils
-{
-execplan::ConstantColumn* createConstCol(const std::string& valstr);
-
-template <typename T>
-execplan::ConstantColumn* createConstCol(const std::string& valstr, T val);
-
-execplan::SimpleFilter* createSimpleFilter(boost::shared_ptr<execplan::CalpontSystemCatalog>& csc,
-                                           const execplan::CalpontSystemCatalog::TableColName& tcn,
-                                           const std::string& opstr, execplan::ConstantColumn* cc);
-
-void appendSimpleFilter(execplan::ParseTree*& ptree, execplan::SimpleFilter* filter);
-
-void updateParseTree(boost::shared_ptr<execplan::CalpontSystemCatalog>&,
-                     execplan::CalpontSelectExecutionPlan*&, execplan::SimpleColumn*, const std::string&,
-                     pair<int, string>);
-
-}  // namespace utils
-}  // namespace qfe

tools/qfe/ddlstmts.cpp

@@ -1,80 +0,0 @@
-#include <unistd.h>
-#include <string>
-#include <stdexcept>
-using namespace std;
-
-#include <boost/scoped_ptr.hpp>
-using namespace boost;
-
-#include "ddlpkg.h"
-#include "sqlparser.h"
-using namespace ddlpackage;
-
-#include "bytestream.h"
-#include "messagequeue.h"
-using namespace messageqcpp;
-
-#include "ddlstmts.h"
-
-namespace qfe
-{
-extern string DefaultSchema;
-}
-
-namespace
-{
-using namespace qfe;
-
-int processDDLStmt(const string& stmt, uint32_t sid)
-{
-  MessageQueueClient* mq = 0;
-  SqlParser parser;
-
-  parser.setDefaultSchema(DefaultSchema);
-  parser.Parse(stmt.c_str());
-
-  if (parser.Good())
-  {
-    const ddlpackage::ParseTree& ptree = parser.GetParseTree();
-    SqlStatement& ddlstmt = *ptree.fList[0];
-    ddlstmt.fSessionID = sid;
-    ddlstmt.fSql = stmt;
-    ddlstmt.fOwner = DefaultSchema;
-    ByteStream bytestream;
-    bytestream << ddlstmt.fSessionID;
-    ddlstmt.serialize(bytestream);
-    mq = new MessageQueueClient("DDLProc");
-    scoped_ptr<MessageQueueClient> smq(mq);
-    ByteStream::byte b = 0;
-    mq->write(bytestream);
-    bytestream = mq->read();
-    bytestream >> b;
-    string emsg;
-    bytestream >> emsg;
-
-    if (b != 0)
-      throw runtime_error(emsg);
-  }
-  else
-  {
-    throw runtime_error("syntax error");
-  }
-
-  return 0;
-}
-
-}  // namespace
-
-namespace qfe
-{
-void processCreateStmt(const string& stmt, uint32_t sid)
-{
-  processDDLStmt(stmt, sid);
-}
-
-void processDropStmt(const string& stmt, uint32_t sid)
-{
-  processDDLStmt(stmt, sid);
-}
-
-}  // namespace qfe

tools/qfe/ddlstmts.h

@@ -1,10 +0,0 @@
-#pragma once
-
-#include <unistd.h>
-#include <string>
-
-namespace qfe
-{
-void processCreateStmt(const std::string&, uint32_t);
-void processDropStmt(const std::string&, uint32_t);
-}  // namespace qfe

tools/qfe/parsequery.cpp

@@ -1,97 +0,0 @@
-#include <unistd.h>
-#include <string>
-#include <memory>
-using namespace std;
-
-#include <boost/thread/mutex.hpp>
-#include <boost/scoped_ptr.hpp>
-using namespace boost;
-
-#include "cseputils.h"
-#include "parsequery.h"
-
-#include "calpontselectexecutionplan.h"
-#include "simplecolumn.h"
-#include "simplefilter.h"
-#include "constantcolumn.h"
-#include "calpontsystemcatalog.h"
-#include "sessionmanager.h"
-using namespace execplan;
-
-#include "brmtypes.h"
-using namespace BRM;
-
-extern int qfeparse();
-extern int qfedebug;
-
-struct yy_buffer_state;
-extern yy_buffer_state* qfe_scan_string(const char*);
-extern void qfe_delete_buffer(yy_buffer_state*);
-extern CalpontSelectExecutionPlan* ParserCSEP;
-extern boost::shared_ptr<CalpontSystemCatalog> ParserCSC;
-
-namespace
-{
-mutex ParserMutex;
-}  // namespace
-
-namespace qfe
-{
-extern string DefaultSchema;
-
-CalpontSelectExecutionPlan* parseQuery(const string& query, const uint32_t sid)
-{
-  // We're going to make parsing the query single-threaded for now. This makes it a lot
-  // easier to interface with the parser and doesn;t materially affect overall query
-  // performance (I think)
-  mutex::scoped_lock lk(ParserMutex);
-
-  boost::shared_ptr<CalpontSystemCatalog> csc = CalpontSystemCatalog::makeCalpontSystemCatalog(sid);
-  CalpontSelectExecutionPlan* csep = 0;
-  csep = new CalpontSelectExecutionPlan();
-  // we use an auto_ptr here with some trepidation. We only want auto delete on an execption.
-  // If the parseing and plan build succeed, we want the ptr to stay around. boost::scoped_ptr<>
-  // doesn't have an API to release ownership, so we use auto_ptr...
-  auto_ptr<CalpontSelectExecutionPlan> scsep(csep);
-
-  yy_buffer_state* ybs = 0;
-  ybs = qfe_scan_string(query.c_str());
-
-  if (ybs != 0)
-  {
-    ParserCSEP = csep;
-    ParserCSC = csc;
-
-    if (qfeparse() != 0)
-      throw runtime_error("syntax error");
-
-    qfe_delete_buffer(ybs);
-  }
-  else
-    throw runtime_error("Internal parser memory error");
-
-  csep->data(query);
-
-  SessionManager sm;
-  TxnID txnID;
-  txnID = sm.getTxnID(sid);
-
-  if (!txnID.valid)
-  {
-    txnID.id = 0;
-    txnID.valid = true;
-  }
-
-  QueryContext verID;
-  verID = sm.verID();
-
-  csep->txnID(txnID.id);
-  csep->verID(verID);
-  csep->sessionID(sid);
-
-  // cout << *csep << endl;
-  scsep.release();
-  return csep;
-}
-
-}  // namespace qfe

tools/qfe/parsequery.h

@@ -1,12 +0,0 @@
-#pragma once
-
-#include <unistd.h>
-#include <string>
-
-#include "calpontselectexecutionplan.h"
-
-namespace qfe
-{
-execplan::CalpontSelectExecutionPlan* parseQuery(const std::string&, uint32_t);
-
-}

tools/qfe/qfe-cli.py

@@ -1,51 +0,0 @@
-#!/usr/bin/python
-'''
-Created on May 1, 2013
-
-@author: rtw
-'''
-
-import struct
-import binascii
-import socket
-import sys
-
-QFE_HOST = 'localhost'
-#QFE_HOST = 'winsvr2008r2.calpont.com'
-QFE_SOCK = 9198
-
-def do_query(query):
-    client_socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
-    client_socket.connect((QFE_HOST, QFE_SOCK))
-
-    defschem = "tpch1";
-    lenstruct = struct.pack("I", len(defschem))
-    client_socket.send(lenstruct)
-    client_socket.send(defschem)
-    
-    lenstruct = struct.pack("I", len(query))
-    client_socket.send(lenstruct)
-    client_socket.send(query)
-    
-    rowlen = 1
-    while rowlen:
-        rowlen = client_socket.recv(4)
-        if len(rowlen) < 4:
-            break
-        rowval = struct.unpack("I",rowlen)[0]
-	if rowval == 0:
-		lenstruct = struct.pack("I", 0);
-		client_socket.send(lenstruct)
-		break
-        row = client_socket.recv(rowval)
-        print row
-
-    client_socket.close()
-        
-if __name__ == '__main__':
-    if len(sys.argv) < 2:
-        query = raw_input ( "QFE-cli > " )
-    else:
-        query = sys.argv[1]
-        
-    sys.exit(do_query(query))

tools/qfe/qfelexer.lpp

@@ -1,65 +0,0 @@
-%{
-#include <utility>
-#include <string>
-#include "returnedcolumn.h"
-#ifndef _MSC_VER
-#include "qfeparser.h"
-#else
-#include "bison-win.h"
-#endif
-
-extern int qfeerror(const char *s);
-
-extern std::string* newstr(const char* cp);
-
-%}
-
-%option noyywrap
-%option yylineno
-%option nounput
-
-%x CCONST
-
-digit		[0-9]
-int_const	[+-]?{digit}+
-letter		[a-zA-Z_]
-objname		{letter}({letter}|{digit})*
-
-%%
-
-"SELECT"	{ return QFEP_SELECT; }
-"FROM"		{ return QFEP_FROM; }
-"WHERE"		{ return QFEP_WHERE; }
-"GROUP BY"	{ return GROUPBY; }
-"AND"		{ qfelval.cp = newstr(qfetext); return LOGICOP; }
-"OR"		{ qfelval.cp = newstr(qfetext); return LOGICOP; }
-"LIMIT"		{ return LIMIT; }
-"ORDER BY"	{ return ORDERBY; }
-"ASC"		{ return ASC; }
-"DESC"		{ return DESC; }
-"AS"		{ return AS; }
-"SUM"		{ qfelval.cp = newstr(qfetext); return FUNC; }
-"MIN"		{ qfelval.cp = newstr(qfetext); return FUNC; }
-"MAX"		{ qfelval.cp = newstr(qfetext); return FUNC; }
-"AVG"		{ qfelval.cp = newstr(qfetext); return FUNC; }
-
-{objname}	{ qfelval.cp = newstr(qfetext); return OBJNAME; }
-
-"'"		{ BEGIN(CCONST); }
-<CCONST>[^']*	{ qfelval.cp = newstr(qfetext); return CHAR_CONST; }
-<CCONST>"'"	{ BEGIN(INITIAL); }
-
-{int_const}	{ qfelval.cp = newstr(qfetext); return INT_CONST; }
-
-">"		{ qfelval.cp = newstr(qfetext); return RELOP; }
-">="		{ qfelval.cp = newstr(qfetext); return RELOP; }
-"<"		{ qfelval.cp = newstr(qfetext); return RELOP; }
-"<="		{ qfelval.cp = newstr(qfetext); return RELOP; }
-"<>"		{ qfelval.cp = newstr(qfetext); return RELOP; }
-"="		{ qfelval.cp = newstr(qfetext); return RELOP; }
-
-[ \t]*		{}
-[\n]		{ qfelineno++; }
-
-.		{ return qfetext[0]; }
-

tools/qfe/qfeparser.ypp

@@ -1,306 +0,0 @@
-%{
-#include <iostream>
-#include <string>
-#include <utility>
-using namespace std;
-
-namespace qfe
-{
-extern string DefaultSchema;
-}
-
-int qfeerror(const char *s);
-int qfelex(void);
-
-#include "calpontselectexecutionplan.h"
-#include "simplecolumn.h"
-#include "calpontsystemcatalog.h"
-#include "aggregatecolumn.h"
-execplan::CalpontSelectExecutionPlan* ParserCSEP;
-execplan::CalpontSystemCatalog* ParserCSC;
-namespace execplan
-{
-class ReturnedColumn;
-}
-
-#include "cseputils.h"
-
-%}
-
-%union
-{
-	execplan::ReturnedColumn* rcp;
-	std::string* cp;
-	std::pair<int, std::string>* cvp;
-}
-
-//Some of these defines cause conflicts in calpontselectexecutionplan.h, so we prepend QFEP_
-%token CHAR_CONST RELOP LOGICOP QFEP_SELECT QFEP_FROM QFEP_WHERE GROUPBY OBJNAME INT_CONST LIMIT ORDERBY ASC
-%token DESC AS FUNC
-
-%type <cp> select_column_list table_name filter_predicate_list obj_name_aux opt_alias opt_as
-%type <cp> opt_limit_clause
-%type <cp> OBJNAME CHAR_CONST INT_CONST RELOP LOGICOP FUNC AS
-
-%type <rcp> aliased_column_name select_column_name obj_name func_col groupby_item
-
-%type <cvp> constant
-
-%destructor { delete $$; } OBJNAME CHAR_CONST INT_CONST RELOP LOGICOP FUNC AS
-%destructor { delete $$; } aliased_column_name select_column_name obj_name func_col groupby_item
-%destructor { delete $$; } constant
-
-%%
-
-statement:	QFEP_SELECT select_column_list
-			QFEP_FROM table_spec
-			opt_where_clause
-			opt_groupby_clause
-			opt_orderby_clause
-			opt_limit_clause ';'
-		;
-
-select_column_list:	aliased_column_name
-			{
-				execplan::SRCP rcp($1);
-				ParserCSEP->returnedCols().push_back(rcp);
-				$$ = 0;
-			}
-			| select_column_list ',' aliased_column_name
-			{
-				execplan::SRCP rcp($3);
-				ParserCSEP->returnedCols().push_back(rcp);
-				$$ = 0;
-			}
-			;
-
-select_column_name:	obj_name
-			{
-				execplan::SimpleColumn* sc = dynamic_cast<execplan::SimpleColumn*>($1);
-				if (sc->schemaName() == "infinidb_unknown" && !qfe::DefaultSchema.empty())
-					sc->schemaName(qfe::DefaultSchema);
-				sc->setOID();
-				sc->tableAlias(sc->tableName());
-				sc->alias(execplan::make_tcn(sc->schemaName(), sc->tableName(), sc->columnName()).toString());
-				$$ = $1;
-			}
-			| func_col
-			;
-
-func_col:	FUNC '(' select_column_name ')'
-		{
-			execplan::AggregateColumn* ag = new execplan::AggregateColumn(execplan::AggregateColumn::agname2num(*($1)), $3);
-			delete $1;
-			$$ = ag;
-		}
-		;
-
-aliased_column_name:	select_column_name opt_alias
-			;
-
-opt_alias:	/* empty */
-		{ $$ = 0; }
-		| opt_as OBJNAME
-		{
-			delete $2;
-			$$ = $1;
-		}
-		| opt_as CHAR_CONST
-		{
-			delete $2;
-			$$ = $1;
-		}
-		;
-
-opt_as:	/* empty */
-	{ $$ = 0; }
-	| AS
-	;
-
-table_spec:	aliased_table_name
-		;
-
-/* this is too loose for a table name, but it's easier to parse */
-table_name:	obj_name
-		{
-			execplan::SimpleColumn* sc=0;
-			sc = dynamic_cast<execplan::SimpleColumn*>($1);
-			execplan::CalpontSystemCatalog::TableAliasName tan;
-			tan.schema = sc->tableName();
-			if (tan.schema == "infinidb_unknown" && !qfe::DefaultSchema.empty())
-				tan.schema = qfe::DefaultSchema;
-			if (qfe::DefaultSchema.empty())
-				qfe::DefaultSchema = tan.schema;
-			tan.table = sc->columnName();
-			tan.alias = tan.table;
-			execplan::CalpontSelectExecutionPlan::TableList tl;
-			tl.push_back(tan);
-			ParserCSEP->tableList(tl);
-			delete sc;
-			$$ = 0;
-		}
-		;
-
-aliased_table_name:	table_name opt_alias
-			;
-
-opt_where_clause:	/* empty */
-			| QFEP_WHERE filter_predicate_list
-			;
-
-filter_predicate_list:	obj_name RELOP constant
-			{
-				pair<int, string> cval = *($3);
-				delete $3;
-				execplan::SimpleColumn* sc=0;
-				sc = dynamic_cast<execplan::SimpleColumn*>($1);
-				qfe::utils::updateParseTree(ParserCSC, ParserCSEP,
-					sc, *($2), cval);
-				delete sc;
-				delete $2;
-				$$ = 0;
-			}
-			| filter_predicate_list LOGICOP obj_name RELOP constant
-			{
-				//string logicop = *($2);
-				delete $2;
-				pair<int, string> cval = *($5);
-				delete $5;
-				execplan::SimpleColumn* sc=0;
-				sc = dynamic_cast<execplan::SimpleColumn*>($3);
-				qfe::utils::updateParseTree(ParserCSC, ParserCSEP,
-					sc, *($4), cval);
-				delete sc;
-				delete $4;
-				$$ = 0;
-			}
-			;
-
-obj_name:	obj_name_aux
-		{
-			//This is possibly a table name, but we shove it into a SimpleColumn. We'll
-			// fix this in the table_spec production
-			execplan::SimpleColumn* sc = new execplan::SimpleColumn("infinidb_unknown", "infinidb_unknown", *($1));
-			sc->tableAlias("infinidb_unknown");
-			sc->alias(execplan::make_tcn("infinidb_unknown", "infinidb_unknown", *($1)).toString());
-			//cerr << "inside parser: " << *sc << endl;
-			delete $1;
-			$$ = sc;
-		}
-		| obj_name_aux '.' obj_name_aux
-		{
-			//This is possibly a table name, but we shove it into a SimpleColumn. We'll
-			// fix this in the table_spec production
-			execplan::SimpleColumn* sc = new execplan::SimpleColumn("infinidb_unknown", *($1), *($3));
-			sc->tableAlias(*($1));
-			sc->alias(execplan::make_tcn("infinidb_unknown", *($1), *($3)).toString());
-			//cerr << "inside parser: " << *sc << endl;
-			delete $1;
-			delete $3;
-			$$ = sc;
-		}
-		| obj_name_aux '.' obj_name_aux '.' obj_name_aux
-		{
-			if (qfe::DefaultSchema.empty())
-				qfe::DefaultSchema = *($1);
-			execplan::SimpleColumn* sc = new execplan::SimpleColumn(*($1), *($3), *($5));
-			sc->tableAlias(*($3));
-			sc->alias(execplan::make_tcn(*($1), *($3), *($5)).toString());
-			//cerr << "inside parser: " << *sc << endl;
-			delete $1;
-			delete $3;
-			delete $5;
-			$$ = sc;
-		}
-		;
-
-obj_name_aux:	OBJNAME
-		| '`' OBJNAME '`'
-		{
-			$$ = $2;
-		}
-		;
-
-constant:	CHAR_CONST
-		{
-			pair<int, string>* p = new pair<int, string>(1, *($1));
-			delete $1;
-			$$ = p;
-		}
-		| INT_CONST
-		{
-			pair<int, string>* p = new pair<int, string>(0, *($1));
-			delete $1;
-			$$ = p;
-		}
-		;
-
-opt_groupby_clause:	/* empty */
-			| GROUPBY groupby_items_list
-			{ YYERROR; }
-			;
-
-groupby_items_list:	groupby_item
-			| groupby_items_list ',' groupby_item
-			{ delete $3; }
-			;
-
-groupby_item:	obj_name
-		| INT_CONST
-		{
-			delete $1;
-			$$ = 0;
-		}
-		;
-
-opt_orderby_clause:	/* empty */
-			| ORDERBY groupby_items_list opt_direction
-			{ YYERROR; }
-			;
-
-opt_direction:	/* empty */
-		| ASC
-		| DESC
-		;
-
-opt_limit_clause:	/* empty */
-			{ $$ = 0; }
-			| LIMIT INT_CONST
-			{
-				delete $2;
-				YYERROR;
-				$$ = 0;
-			}
-			| LIMIT INT_CONST ',' INT_CONST
-			{
-				delete $2;
-				delete $4;
-				YYERROR;
-				$$ = 0;
-			}
-			;
-
-%%
-
-int qfeerror(const string& s)
-{
-  extern int qfelineno;	// defined and maintained in lex.c
-  extern char *qfetext;	// defined and maintained in lex.c
-  
-  cerr << "ERROR: " << s << " at symbol \"" << qfetext;
-  cerr << "\" on line " << qfelineno << endl;
-  return -1;
-}
-
-int qfeerror(const char *s)
-{
-  return qfeerror(string(s));
-}
-
-string* newstr(const char* cp)
-{
-	string* strp;
-	strp = new string(cp);
-	return strp;
-}
-

tools/qfe/returnedrows.cpp

@@ -1,95 +0,0 @@
-#include <unistd.h>
-#include <iostream>
-#include <string>
-#include <cerrno>
-using namespace std;
-
-#include <boost/scoped_ptr.hpp>
-#include <boost/scoped_array.hpp>
-using namespace boost;
-
-#include "mcsconfig.h"
-
-#include "socktype.h"
-
-#include "exceptclasses.h"
-
-#include "socketio.h"
-
-#include "bytestream.h"
-#include "messagequeue.h"
-using namespace messageqcpp;
-
-#include "rowgroup.h"
-using namespace rowgroup;
-
-namespace qfe
-{
-void processReturnedRows(MessageQueueClient* mqc, SockType fd)
-{
-  scoped_ptr<MessageQueueClient> cleaner(mqc);
-  SBS sbs;
-  sbs = mqc->read();
-  // cerr << "got a bs of " << sbs->length() << " bytes" << endl;
-
-  RowGroup rg;
-  rg.deserialize(*sbs);
-
-  // cerr << "got a base rowgroup with rows of " << rg.getRowSize() << " bytes" << endl;
-  // cerr << rg.toString() << endl;
-
-  ByteStream bs;
-  ByteStream::quadbyte tableOID = 100;
-  bs.reset();
-  bs << tableOID;
-  mqc->write(bs);
-
-  sbs = mqc->read();
-  // cerr << "got a bs of " << sbs->length() << " bytes" << endl;
-  RGData rgd;
-  rgd.deserialize(*sbs, true);
-  rg.setData(&rgd);
-  // cerr << "got a rowgroup with: " << rg.getRowCount() << " rows" << endl;
-
-  socketio::writeString(fd, "OK");
-  Row r;
-
-  while (rg.getRowCount() > 0)
-  {
-    rg.initRow(&r);
-    rg.getRow(0, &r);
-    string csv;
-    bs.reset();
-
-    for (unsigned i = 0; i < rg.getRowCount(); i++)
-    {
-      csv = r.toCSV();
-      bs << csv;
-      r.nextRow();
-    }
-
-    // cerr << "writing " << bs.length() << " bytes back to client" << endl;
-    SockWriteFcn(fd, bs.buf(), bs.length());
-
-    bs.reset();
-    bs << tableOID;
-    mqc->write(bs);
-
-    sbs = mqc->read();
-    // cerr << "got a bs of " << sbs->length() << " bytes" << endl;
-    rgd.deserialize(*sbs, true);
-    rg.setData(&rgd);
-    // cerr << "got a rowgroup with: " << rg.getRowCount() << " rows" << endl;
-  }
-
-  tableOID = 0;
-  bs.reset();
-  bs << tableOID;
-  mqc->write(bs);
-
-  // sync with the client on end-of-results
-  SockWriteFcn(fd, &tableOID, 4);
-  SockReadFcn(fd, &tableOID, 4);
-}
-
-}  // namespace qfe

tools/qfe/returnedrows.h

@@ -1,10 +0,0 @@
-#pragma once
-
-#include "socktype.h"
-#include "messagequeue.h"
-
-namespace qfe
-{
-void processReturnedRows(messageqcpp::MessageQueueClient*, SockType);
-
-}

tools/qfe/sendcsep.cpp

@@ -1,53 +0,0 @@
-#include <unistd.h>
-#include <stdexcept>
-#include <memory>
-using namespace std;
-
-#include <boost/scoped_ptr.hpp>
-using namespace boost;
-
-#include "calpontselectexecutionplan.h"
-using namespace execplan;
-#include "bytestream.h"
-#include "messagequeue.h"
-using namespace messageqcpp;
-
-namespace qfe
-{
-MessageQueueClient* sendCSEP(CalpontSelectExecutionPlan* csep)
-{
-  scoped_ptr<CalpontSelectExecutionPlan> cleaner(csep);
-
-  ByteStream bs;
-
-  MessageQueueClient* mqc = 0;
-
-  mqc = new MessageQueueClient("ExeMgr1");
-  auto_ptr<MessageQueueClient> smqc(mqc);
-
-  bs.reset();
-  ByteStream::quadbyte wantTuples = 4;
-  bs << wantTuples;
-  mqc->write(bs);
-
-  bs.reset();
-  csep->serialize(bs);
-  mqc->write(bs);
-
-  SBS sbs;
-  sbs = mqc->read();
-  *sbs >> wantTuples;
-  // cerr << "got flag: " << wantTuples << endl;
-  string msg;
-  sbs = mqc->read();
-  *sbs >> msg;
-  // cerr << "got msg: " << msg << endl;
-
-  if (wantTuples != 0)
-    throw runtime_error(msg);
-
-  smqc.release();
-  return mqc;
-}
-
-}  // namespace qfe

tools/qfe/sendcsep.h

@@ -1,12 +0,0 @@
-#pragma once
-
-#include "messagequeue.h"
-#include "calpontselectexecutionplan.h"
-
-namespace qfe
-{
-// Takes ownership of the alloc'd ptr
-// Returns an alloc'd mqc
-messageqcpp::MessageQueueClient* sendCSEP(execplan::CalpontSelectExecutionPlan*);
-
-}  // namespace qfe

tools/qfe/server.cpp

@@ -1,429 +0,0 @@
-/* Copyright (C) 2014 InfiniDB, Inc.
-
-   This program is free software; you can redistribute it and/or
-   modify it under the terms of the GNU General Public License
-   as published by the Free Software Foundation; version 2 of
-   the License.
-
-   This program is distributed in the hope that it will be useful,
-   but WITHOUT ANY WARRANTY; without even the implied warranty of
-   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-   GNU General Public License for more details.
-
-   You should have received a copy of the GNU General Public License
-   along with this program; if not, write to the Free Software
-   Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
-   MA 02110-1301, USA. */
-
-/*
-        Protocol definition:
-*/
-
-#include <poll.h>
-#include <sys/socket.h>
-#include <netinet/in.h>
-#include <arpa/inet.h>
-#include <iostream>
-#include <string>
-#include <stdexcept>
-#include <sstream>
-#include <iomanip>
-#include <sys/types.h>
-#include <sys/stat.h>
-#include <fcntl.h>
-#include <unistd.h>
-#include <signal.h>
-#include <cerrno>
-using namespace std;
-
-#include <boost/thread.hpp>
-#include <boost/scoped_ptr.hpp>
-#include <boost/scoped_array.hpp>
-#include <boost/tokenizer.hpp>
-#include <boost/algorithm/string.hpp>
-using namespace boost;
-
-#include "mcsconfig.h"
-
-#include "socktype.h"
-#include "parsequery.h"
-#include "sendcsep.h"
-#include "returnedrows.h"
-#include "socketio.h"
-#include "ddlstmts.h"
-
-#include "exceptclasses.h"  //brings in idbassert_s macro
-
-#include "messagequeue.h"
-using namespace messageqcpp;
-
-#include "atomicops.h"
-
-namespace execplan
-{
-class CalpontSelectExecutionPlan;
-}
-
-#define SINGLE_THREADED
-
-namespace qfe
-{
-string DefaultSchema;
-}
-
-namespace
-{
-using namespace qfe;
-
-enum StmtType
-{
-  UNKNOWN,
-  QUERY,
-  CREATE,
-  DROP,
-  SHOW,
-};
-
-volatile uint32_t SystemSID;
-
-void log(const string& s)
-{
-  cerr << s << endl;
-}
-
-struct QueryMessage
-{
-  QueryMessage() : isValid(false)
-  {
-  }
-  ~QueryMessage()
-  {
-  }
-
-  string toString() const;
-
-  bool isValid;
-  string queryText;
-  string defaultSchema;
-};
-
-string QueryMessage::toString() const
-{
-  ostringstream oss;
-  oss << "valid: " << boolalpha << isValid << ", "
-      << "queryText: " << queryText << ", "
-      << "defaultSchema: " << defaultSchema;
-  return oss.str();
-}
-
-ostream& operator<<(ostream& os, const QueryMessage& rhs)
-{
-  os << rhs.toString();
-  return os;
-}
-
-class ThreadFunc
-{
- public:
-  ThreadFunc(SockType fd) : fFd(fd)
-  {
-  }
-  ~ThreadFunc()
-  {
-  }
-
-  void run();
-  void operator()()
-  {
-    run();
-  }
-
- private:
-  ThreadFunc(const ThreadFunc& rhs);
-  ThreadFunc& operator=(const ThreadFunc& rhs);
-
-  SockType fFd;
-};
-
-bool serverInit()
-{
-
-  setsid();
-
-  // Handle certain signals (we want these to return EINTR so we can throw)
-  // SIGPIPE
-  // I don't think we'll get any of these from init (except possibly HUP, but that's an indication
-  // of bad things anyway)
-  // SIGHUP?
-  // SIGUSR1?
-  // SIGUSR2?
-  // SIGPOLL?
-  struct sigaction sa;
-  memset(&sa, 0, sizeof(struct sigaction));
-  sa.sa_handler = SIG_IGN;
-  sigaction(SIGPIPE, &sa, 0);
-  sigaction(SIGHUP, &sa, 0);
-  sigaction(SIGUSR1, &sa, 0);
-  sigaction(SIGUSR2, &sa, 0);
-#ifndef __FreeBSD__
-  sigaction(SIGPOLL, &sa, 0);
-#endif
-#if 0
-    int fd;
-    close(2);
-    fd = open("/tmp/qfe.err", O_CREAT | O_TRUNC | O_WRONLY, 0644);
-
-    if (fd >= 0 && fd != 2)
-    {
-        dup2(fd, 2);
-        close(fd);
-    }
-
-#endif
-  return true;
-}
-
-SockType initListenSock(short portNo)
-{
-  SockType listenSock = -1;
-  listenSock = socket(PF_INET, SOCK_STREAM, IPPROTO_TCP);
-  idbassert_s(listenSock >= 0, string("socket create error: ") + strerror(errno));
-  // if (listenSock < 0) throw runtime_error(string("socket create error: ") + strerror(errno));
-  int optval = 1;
-  setsockopt(listenSock, SOL_SOCKET, SO_REUSEADDR, reinterpret_cast<const char*>(&optval), sizeof(optval));
-  int rc = 0;
-  struct sockaddr_in serv_addr;
-  memset(&serv_addr, 0, sizeof(serv_addr));
-  serv_addr.sin_family = AF_INET;
-  serv_addr.sin_addr.s_addr = INADDR_ANY;
-  serv_addr.sin_port = htons(portNo);
-  const int MaxTries = 5 * 60 / 10;
-  int tries = 0;
-again:
-  rc = ::bind(listenSock, (sockaddr*)&serv_addr, sizeof(serv_addr));
-
-  if (rc < 0)
-  {
-    if (errno == EADDRINUSE)
-    {
-      // cerr << "Addr in use..." << endl;
-      if (++tries >= MaxTries)
-      {
-        log("Waited too long for socket to bind...giving up");
-        // cerr << "Waited too long for socket to bind...giving up" << endl;
-        exit(1);
-      }
-
-      sleep(10);
-      goto again;
-    }
-
-    idbassert_s(0, string("socket bind error: ") + strerror(errno));
-    // throw runtime_error(string("socket bind error: ") + strerror(errno));
-  }
-
-  rc = listen(listenSock, 16);
-  idbassert_s(rc >= 0, string("socket listen error") + strerror(errno));
-  // if (rc < 0) throw runtime_error(string("socket listen error") + strerror(errno));
-
-  return listenSock;
-}
-
-QueryMessage getNextMsg(SockType fd)
-{
-  QueryMessage msg;
-
-  try
-  {
-    msg.defaultSchema = socketio::readString(fd);
-    msg.queryText = socketio::readString(fd);
-    msg.isValid = true;
-  }
-  catch (runtime_error& rex)
-  {
-    cerr << "re reading ctl msg: " << rex.what() << endl;
-    msg.queryText = "";
-  }
-  catch (...)
-  {
-    cerr << "ex reading ctl msg" << endl;
-    msg.queryText = "";
-  }
-
-  return msg;
-}
-
-StmtType guessStatementType(const string& stmt)
-{
-  typedef boost::tokenizer<boost::char_separator<char> > tokenizer;
-  char_separator<char> sep;
-  tokenizer tokens(stmt, sep);
-  tokenizer::iterator tok_iter = tokens.begin();
-  string first_word;
-  first_word = *tok_iter;
-  algorithm::to_lower(first_word);
-
-  if (first_word == "select")
-    return QUERY;
-
-  if (first_word == "create")
-    return CREATE;
-
-  if (first_word == "drop")
-    return DROP;
-
-  if (first_word == "show")
-    return SHOW;
-
-  return UNKNOWN;
-}
-
-struct ScopedCleaner
-{
-  ScopedCleaner(SockType fd = -1) : fFd(fd)
-  {
-  }
-  ~ScopedCleaner()
-  {
-    if (fFd >= 0)
-      shutdown(fFd, SHUT_RDWR);
-
-    close(fFd);
-  }
-
-  SockType fFd;
-};
-
-void ThreadFunc::run()
-{
-  QueryMessage m;
-  execplan::CalpontSelectExecutionPlan* csep = 0;
-  MessageQueueClient* msgqcl;
-
-  ScopedCleaner cleaner(fFd);
-
-  uint32_t sid = 1;
-  sid = atomicops::atomicInc(&SystemSID);
-
-  try
-  {
-    m = getNextMsg(fFd);
-
-    if (m.isValid)
-    {
-      DefaultSchema = m.defaultSchema;
-      StmtType st = guessStatementType(m.queryText);
-
-      switch (st)
-      {
-        case QUERY:
-          csep = parseQuery(m.queryText, sid);
-          // sendCSEP takes ownership of the ptr from parseQuery
-          msgqcl = sendCSEP(csep);
-          // processReturnedRows takes ownership of the ptr from sendCSEP
-          processReturnedRows(msgqcl, fFd);
-          break;
-
-        case CREATE: processCreateStmt(m.queryText, sid); break;
-
-        case DROP: processDropStmt(m.queryText, sid); break;
-
-        case SHOW:
-        {
-          ostringstream oss;
-          oss << "select calpontsys.systable.tablename from calpontsys.systable where "
-                 "calpontsys.systable.schema='"
-              << m.defaultSchema << "';";
-          csep = parseQuery(oss.str(), sid);
-          msgqcl = sendCSEP(csep);
-          processReturnedRows(msgqcl, fFd);
-          break;
-        }
-
-        default: throw runtime_error("couldn't guess the statement type"); break;
-      }
-    }
-  }
-  catch (std::exception& ex)
-  {
-    socketio::writeString(fFd, ex.what());
-    throw;  // in a multi-threaded server this will simply cause this thread to exit
-  }
-  catch (...)
-  {
-    socketio::writeString(fFd, "internal query processing error");
-    throw;
-  }
-}
-
-}  // namespace
-
-int main(int argc, char** argv)
-{
-  int c;
-  SockType listenSock;
-  short portNo;
-
-  portNo = 0;
-  char* p = getenv("IDB_QFE_PORT");
-
-  if (p && *p)
-    portNo = atoi(p);
-
-  if (portNo <= 0)
-    portNo = 9198;
-
-  listenSock = -1;
-  opterr = 0;
-
-  while ((c = getopt(argc, argv, "p:")) != -1)
-    switch (c)
-    {
-      case 'p': portNo = atoi(optarg); break;
-
-      case '?':
-      default: break;
-    }
-
-  if (!serverInit())
-  {
-    log("Could not initialize the QFE Server!");
-    cerr << "Could not initialize the QFE Server!" << endl;
-    return 1;
-  }
-
-  listenSock = initListenSock(portNo);
-
-  SystemSID = 0;
-
-  for (;;)
-  {
-    int querySock = -1;
-    querySock = accept(listenSock, 0, 0);
-    idbassert_s(querySock >= 0, string("socket accept error: ") + strerror(errno));
-
-    // ThreadFunc now owns querySock and is responsible for cleaning it up
-    ThreadFunc tf(querySock);
-
-#ifdef SINGLE_THREADED
-
-    try
-    {
-      tf.run();
-    }
-    catch (std::exception& ex)
-    {
-      cerr << "ThreadFunc run threw an exception: " << ex.what() << endl;
-    }
-    catch (...)
-    {
-      cerr << "ThreadFunc run threw an exception" << endl;
-    }
-
-#else
-    thread t(tf);
-#endif
-  }
-
-  return 0;
-}

tools/qfe/socketio.cpp

@@ -1,133 +0,0 @@
-#include <unistd.h>
-#include <poll.h>
-#include <sys/socket.h>
-#include <netinet/in.h>
-#include <arpa/inet.h>
-#include <cerrno>
-#include <sstream>
-using namespace std;
-
-#include <boost/scoped_array.hpp>
-using namespace boost;
-
-#include "mcsconfig.h"
-
-#include "exceptclasses.h"
-
-#include "socketio.h"
-
-namespace qfe
-{
-namespace socketio
-{
-void readn(int fd, void* buf, const size_t wanted)
-{
-  size_t needed = wanted;
-  size_t sofar = 0;
-  char* p = static_cast<char*>(buf);
-  ssize_t rrc = -1;
-  pollfd fds[1];
-  int en = 0;
-
-  fds[0].fd = fd;
-  fds[0].events = POLLIN;
-
-  while (wanted > sofar)
-  {
-    fds[0].revents = 0;
-    poll(fds, 1, -1);
-    errno = 0;
-    rrc = read(fd, (p + sofar), needed);
-    en = errno;
-
-    if (rrc < 0)
-    {
-      if (en == EAGAIN || en == EINTR || en == 512)
-        continue;
-
-      ostringstream oss;
-      oss << "qfe: readn: read() returned " << rrc << " (" << strerror(en) << ")";
-      idbassert_s(0, oss.str());
-    }
-
-    needed -= rrc;
-    sofar += rrc;
-  }
-}
-
-size_t writen(int fd, const void* data, const size_t nbytes)
-{
-  size_t nleft;
-  ssize_t nwritten;
-  const char* bufp = static_cast<const char*>(data);
-  nleft = nbytes;
-
-  while (nleft > 0)
-  {
-    // the O_NONBLOCK flag is not set, this is a blocking I/O.
-    if ((nwritten = ::write(fd, bufp, nleft)) < 0)
-    {
-      if (errno == EINTR)
-        nwritten = 0;
-      else
-      {
-        // save the error no first
-        int e = errno;
-        string errorMsg = "qfe: writen: write() error: ";
-        scoped_array<char> buf(new char[80]);
-#if STRERROR_R_CHAR_P
-        const char* p;
-
-        if ((p = strerror_r(e, buf.get(), 80)) != 0)
-          errorMsg += p;
-
-#else
-        int p;
-
-        if ((p = strerror_r(e, buf.get(), 80)) == 0)
-          errorMsg += buf.get();
-
-#endif
-        idbassert_s(0, errorMsg);
-      }
-    }
-
-    nleft -= nwritten;
-    bufp += nwritten;
-  }
-
-  return nbytes;
-}
-
-uint32_t readNumber32(SockType fd)
-{
-  uint32_t np;
-  SockReadFcn(fd, &np, 4);
-  return np;
-}
-
-string readString(SockType fd)
-{
-  string s;
-  uint32_t len = readNumber32(fd);
-
-  if (len > 0)
-  {
-    scoped_array<char> buf(new char[len + 1]);
-    SockReadFcn(fd, buf.get(), len);
-    buf[len] = 0;
-    s = buf.get();
-  }
-
-  return s;
-}
-
-void writeString(SockType fd, const string& data)
-{
-  uint32_t len = data.length();
-  SockWriteFcn(fd, &len, 4);
-  SockWriteFcn(fd, data.c_str(), len);
-}
-
-}  // namespace socketio
-}  // namespace qfe

tools/qfe/socketio.h

@@ -1,20 +0,0 @@
-#pragma once
-
-#include <stdint.h>
-#include <unistd.h>
-#include <string>
-
-#include "socktype.h"
-
-namespace qfe
-{
-namespace socketio
-{
-void readn(int fd, void* buf, const size_t wanted);
-size_t writen(int fd, const void* data, const size_t nbytes);
-uint32_t readNumber32(SockType);
-std::string readString(SockType);
-void writeString(SockType, const std::string&);
-
-}  // namespace socketio
-}  // namespace qfe

tools/qfe/socktype.h

@@ -1,6 +0,0 @@
-#pragma once
-
-
-typedef int SockType;
-#define SockReadFcn qfe::socketio::readn
-#define SockWriteFcn qfe::socketio::writen

tools/rebuildEM/CMakeLists.txt

@@ -1,10 +1,5 @@
 include_directories(${ENGINE_COMMON_INCLUDES})
 
 set(rebuildEM_SRCS main.cpp rebuildEM.cpp)
-add_executable(mcsRebuildEM ${rebuildEM_SRCS})
+columnstore_executable(mcsRebuildEM ${rebuildEM_SRCS})
-target_link_libraries(mcsRebuildEM ${ENGINE_LDFLAGS} ${ENGINE_WRITE_LIBS} boost_system boost_filesystem)
+columnstore_link(mcsRebuildEM ${ENGINE_LDFLAGS} ${ENGINE_WRITE_LIBS} boost_system boost_filesystem)
-install(
-    TARGETS mcsRebuildEM
-    DESTINATION ${ENGINE_BINDIR}
-    COMPONENT columnstore-engine
-)

tools/rgprint/CMakeLists.txt

@@ -4,12 +4,6 @@ include_directories(${ENGINE_COMMON_INCLUDES})
 
 set(rgprint_SRCS rgprint.cpp)
 
-add_executable(rgprint ${rgprint_SRCS})
+columnstore_executable(rgprint ${rgprint_SRCS})
 
-target_link_libraries(rgprint ${ENGINE_LDFLAGS} ${NETSNMP_LIBRARIES} ${ENGINE_WRITE_LIBS})
+columnstore_link(rgprint ${ENGINE_LDFLAGS} ${NETSNMP_LIBRARIES} ${ENGINE_WRITE_LIBS})
-
-install(
-    TARGETS rgprint
-    DESTINATION ${ENGINE_BINDIR}
-    COMPONENT columnstore-engine
-)

tools/sendPlan/CMakeLists.txt

@@ -1,32 +0,0 @@
-include_directories(${KDE4_INCLUDES} ${KDE4_INCLUDE_DIR} ${QT_INCLUDES})
-
-# ########## next target ###############
-
-set(sendPlan_SRCS sendplan.cpp)
-
-kde4_add_executable(sendPlan ${sendPlan_SRCS})
-
-target_link_libraries(sendPlan ${KDE4_KDECORE_LIBS})
-
-install(TARGETS sendPlan ${INSTALL_TARGETS_DEFAULT_ARGS})
-
-# ########## install files ###############
-
-# original Makefile.am contents follow:
-
-# $Id: Makefile.am 333 2009-04-03 20:35:04Z rdempsey $ Process this file with automake to produce Makefile.in
-#
-# AM_CPPFLAGS = $(idb_cppflags) AM_CFLAGS = $(idb_cflags) AM_CXXFLAGS = $(idb_cxxflags) AM_LDFLAGS = $(idb_ldflags)
-# bin_PROGRAMS = sendPlan sendPlan_SOURCES = sendplan.cpp sendPlan_CPPFLAGS = @idb_common_includes@ $(AM_CPPFLAGS)
-# sendPlan_LDFLAGS = @idb_common_ldflags@ @idb_exec_libs@ $(AM_LDFLAGS)
-#
-# test:
-#
-# coverage:
-#
-# leakcheck:
-#
-# docs:
-#
-# bootstrap: install-data-am
-#

tools/sendPlan/sendplan.cpp

@@ -1,392 +0,0 @@
-// $Id: sendplan.cpp 1739 2012-03-22 12:57:59Z pleblanc $
-#include <iostream>
-#include <fstream>
-#include <sstream>
-#include <unistd.h>
-#include <sys/time.h>
-#include <iomanip>
-using namespace std;
-
-#include "bytestream.h"
-using namespace messageqcpp;
-
-#include "calpontselectexecutionplan.h"
-#include "sessionmanager.h"
-using namespace execplan;
-
-#include "calpontsystemcatalog.h"
-using namespace execplan;
-
-#include "tableband.h"
-#include "joblist.h"
-#include "joblistfactory.h"
-using namespace joblist;
-
-#include "configcpp.h"
-#include "errorcodes.h"
-
-#include "rowgroup.h"
-using namespace rowgroup;
-
-typedef CalpontSystemCatalog::OID OID;
-
-namespace
-{
-bool vflg;
-
-void usage()
-{
-  cout << "usage: sendPlan [-v|h|d|B] [-t lvl] [-s sid] plan_file ..." << endl;
-  cout << "-v verbose output" << endl;
-  cout << "-t lvl set trace level to lvl" << endl;
-  cout << "-s sid set session id to sid" << endl;
-  cout << "-d display the query, but don't run it" << endl;
-  cout << "-B Bob's personal preferred output format" << endl;
-  cout << "-h display this help" << endl;
-}
-
-double tm_diff(const struct timeval* st, const struct timeval* et)
-{
-  double sd = (double)st->tv_sec + (double)st->tv_usec / 1000000.0;
-  double ed = (double)et->tv_sec + (double)et->tv_usec / 1000000.0;
-
-  return (ed - sd);
-}
-
-//...Extract the stats string to be printed and the runtime start time from
-//...the full stats string we receive from ExeMgr.
-void parseStatsString(const string& fullStatsString, string& printStatsString,
-                      struct timeval& queryRunStartTime)
-{
-  string::size_type delimPos = fullStatsString.find('|');
-  printStatsString = fullStatsString.substr(0, delimPos);
-
-  istringstream startTimeString(fullStatsString.substr(delimPos + 1));
-  startTimeString >> queryRunStartTime.tv_sec >> queryRunStartTime.tv_usec;
-}
-
-}  // namespace
-
-int main(int argc, char** argv)
-{
-  vflg = false;
-  uint32_t tlvl = 0;
-  bool dflg = false;
-  int c;
-  int32_t sid = -1;
-  bool Bflg = false;
-
-  opterr = 0;
-
-  while ((c = getopt(argc, argv, "vt:ds:Bh")) != EOF)
-    switch (c)
-    {
-      case 't': tlvl = static_cast<uint32_t>(strtoul(optarg, 0, 0)); break;
-
-      case 'v': vflg = true; break;
-
-      case 'd': dflg = true; break;
-
-      case 's': sid = static_cast<int32_t>(strtol(optarg, 0, 0)); break;
-
-      case 'B': Bflg = true; break;
-
-      case 'h':
-      case '?':
-      default:
-        usage();
-        return (c == 'h' ? 0 : 1);
-        break;
-    }
-
-  if (dflg)
-    vflg = true;
-
-  if ((argc - optind) < 1)
-  {
-    usage();
-    return 1;
-  }
-
-  ifstream inputf;
-  ByteStream bs;
-  ByteStream dbs;
-  ByteStream eoq;
-  ByteStream tbs;
-  ByteStream statsStream;
-  ByteStream::quadbyte q = 0;
-  eoq << q;
-  uint32_t sessionid;
-  time_t t;
-  SJLP jl;
-  DeliveredTableMap tm;
-  DeliveredTableMap::iterator iter;
-  DeliveredTableMap::iterator end;
-  CalpontSelectExecutionPlan csep;
-  struct timeval start_time;
-  struct timeval end_time;
-
-  MessageQueueClient* mqc = 0;
-
-  if (!dflg)
-    mqc = new MessageQueueClient("ExeMgr1");
-
-  if (sid == -1)
-  {
-    time(&t);
-    sessionid = static_cast<uint32_t>(t);
-  }
-  else
-  {
-    sessionid = static_cast<uint32_t>(sid);
-  }
-
-  sessionid &= 0x7fffffff;
-  logging::ErrorCodes errorCodes;
-
-  for (; optind < argc; optind++)
-  {
-    inputf.open(argv[optind]);
-
-    if (!inputf.good())
-    {
-      cerr << "error opening plan stream " << argv[optind] << endl;
-      return 1;
-    }
-
-    bs.reset();
-    inputf >> bs;
-
-    inputf.close();
-
-    csep.unserialize(bs);
-
-    csep.sessionID(sessionid);
-    SessionManager sm;
-    csep.verID(sm.verID());
-
-    csep.traceFlags(0);
-    ResourceManager* rm = ResourceManager::instance();
-    jl = JobListFactory::makeJobList(&csep, rm);
-    csep.traceFlags(tlvl);
-
-    if (vflg)
-    {
-      if (dflg)
-        cout << endl << "Query:" << endl;
-      else
-      {
-        cout << endl << "Session: " << sessionid << ", Sending Query";
-
-        if (Bflg)
-          cout << " (" << argv[optind] << ')';
-
-        cout << ':' << endl;
-      }
-
-      if (!Bflg)
-        cout << csep.data() << endl << endl;
-    }
-
-    if (dflg)
-      continue;
-
-    try
-    {
-      dbs.reset();
-      csep.serialize(dbs);
-
-      gettimeofday(&start_time, 0);
-
-      // try tuples first, but expect the worst...
-      bool expectTuples = false;
-      ByteStream tbs;
-      ByteStream::quadbyte tqb = 4;
-      tbs << tqb;
-      mqc->write(tbs);
-
-      // send the CSEP
-      mqc->write(dbs);
-
-      // read the response to the tuple request
-      tbs = mqc->read();
-      idbassert(tbs.length() == 4);
-      tbs >> tqb;
-
-      if (tqb == 4)
-        expectTuples = true;
-
-      if (!expectTuples)
-        cout << "Using TableBand I/F" << endl;
-      else
-        cout << "Using tuple I/F" << endl;
-
-      tm = jl->deliveredTables();
-
-      iter = tm.begin();
-      end = tm.end();
-
-      OID toid;
-      uint64_t rowTot;
-      bool reported = false;
-      bool needRGCtor = true;
-
-      while (iter != end)
-      {
-        toid = iter->first;
-        q = static_cast<ByteStream::quadbyte>(toid);
-        tbs.reset();
-        tbs << q;
-        mqc->write(tbs);
-
-        ByteStream tbbs;
-        TableBand tb;
-        RowGroup rg;
-        rowTot = 0;
-        uint16_t status = 0;
-        TableBand::VBA::size_type rc;
-        ofstream out;
-
-        for (;;)
-        {
-          tbbs = mqc->read();
-#if 0
-                    cout << tbbs.length() << endl;
-                    out.open("bs1.dat");
-                    idbassert(out.good());
-                    out << tbbs;
-                    out.close();
-                    tbbs = mqc->read();
-                    cout << tbbs.length() << endl;
-                    out.open("bs2.dat");
-                    idbassert(out.good());
-                    out << tbbs;
-                    out.close();
-                    tbbs = mqc->read();
-                    cout << tbbs.length() << endl;
-                    out.open("bs3.dat");
-                    idbassert(out.good());
-                    out << tbbs;
-                    out.close();
-#endif
-
-          if (tbbs.length())
-          {
-            if (!expectTuples)
-              tb.unserialize(tbbs);
-            else
-            {
-              if (needRGCtor)
-              {
-                rg.deserialize(tbbs);
-                needRGCtor = false;
-                tbbs = mqc->read();
-              }
-
-              rg.setData((uint8_t*)tbbs.buf());
-            }
-          }
-          else
-          {
-            //@bug 1346
-            if (!status)
-              status = logging::makeJobListErr;
-
-            break;
-          }
-
-          if (!expectTuples)
-          {
-            rc = tb.getRowCount();
-            status = tb.getStatus();
-          }
-          else
-          {
-            rc = rg.getRowCount();
-            status = rg.getStatus();
-
-            if (rc == 0)
-              status = 0;
-          }
-
-          if (rc == 0)
-            break;
-
-          rowTot += rc;
-        }
-
-        BatchPrimitive* step = dynamic_cast<BatchPrimitive*>(iter->second.get());
-
-        if (vflg && step)
-        {
-          cout << "For table " << step->tableName();
-
-          if (!Bflg)
-            cout << " " << toid;
-
-          cout << ": read " << rowTot << " rows" << endl;
-        }
-
-        if (status && !reported)
-        {
-          cout << "### Query failed: " << errorCodes.errorString(status) << "  Check crit.log\n";
-          reported = true;
-        }
-
-        if (!step && !reported)
-        {
-          cout << "### Query failed: Did not return project BatchPrimitive. Check crit.log\n";
-          reported = true;
-        }
-
-        ++iter;
-      }
-
-      if (vflg)
-      {
-        gettimeofday(&end_time, 0);
-        cout << "Query time: " << fixed << setprecision(1) << tm_diff(&start_time, &end_time) << " secs"
-             << endl;
-
-        //...Ask for query stats through special table id of 3
-        const OID TABLE_ID_TO_GET_QUERY_STATS = 3;
-
-        if (!Bflg)
-          cout << "Retrieving stats..." << endl;
-
-        toid = TABLE_ID_TO_GET_QUERY_STATS;
-        q = static_cast<ByteStream::quadbyte>(toid);
-        statsStream.reset();
-        statsStream << q;
-        mqc->write(statsStream);
-
-        ByteStream bs_statsString;
-        bs_statsString = mqc->read();
-        string statsString;
-        bs_statsString >> statsString;
-
-        string printStatsString;
-        struct timeval startRunTime;
-        parseStatsString(statsString, printStatsString, startRunTime);
-        cout << printStatsString << "; QuerySetupTime-" << tm_diff(&start_time, &startRunTime) << "secs"
-             << endl;
-      }
-
-      //...Close this query/session
-      mqc->write(eoq);
-      jl.reset();
-    }
-    catch (const exception& ex)
-    {
-      cout << "### SendPlan caught an exception: " << ex.what() << endl;
-    }
-  }
-
-  // 	jl.reset();
-  CalpontSystemCatalog::removeCalpontSystemCatalog(sessionid);
-  config::Config::deleteInstanceMap();
-
-  delete mqc;
-
-  return 0;
-}

tools/setConfig/CMakeLists.txt

@@ -4,12 +4,5 @@ include_directories(${ENGINE_COMMON_INCLUDES})
 
 set(setConfig_SRCS main.cpp)
 
-add_executable(mcsSetConfig ${setConfig_SRCS})
+columnstore_executable(mcsSetConfig ${setConfig_SRCS})
-
+columnstore_link(mcsSetConfig ${ENGINE_LDFLAGS} ${NETSNMP_LIBRARIES} ${ENGINE_EXEC_LIBS})
-target_link_libraries(mcsSetConfig ${ENGINE_LDFLAGS} ${NETSNMP_LIBRARIES} ${ENGINE_EXEC_LIBS})
-
-install(
-    TARGETS mcsSetConfig
-    DESTINATION ${ENGINE_BINDIR}
-    COMPONENT columnstore-engine
-)

tools/snconvert/snconvert.pl

@@ -1,132 +0,0 @@
-#!/usr/bin/perl -w -s
-
-
-###################################
-# Control parameters
-###################################
-
-# location of the data files to process
-$srcDir="";
-$srcFileName="";
-
-if ( $#ARGV>-1 )
-{
-	$srcDir = $ARGV[0];
-	$srcFileName= $ARGV[1];
-	print "\n\nCLA $srcDir $srcFileName\n\n";
-}
-else
-{
-	$srcDir = ".";
-	$srcFileName = "FILE000.cdf";
-}
-
-# full path of the data file
-$srcFile = $srcDir . $srcFileName;
-
-# name of the new file/extent file
-$newFileName = $srcFileName . '_';
-
-# get current directory
-$currDir = `pwd`;
-chomp $currDir;
-
-# number of file system files
-$fsFileCnt=8;
-
-# once file data has been reorganized they will be moved
-# to these locations.
-@pmDest = (
-	"root\@srvswdev2\:\/tmp",
-	"root\@srvswdev2\:\/tmp",
-	"root\@srvswdev2\:\/tmp",
-	"root\@srvswdev2\:\/tmp"
-);
-
-# split column file into seperate extent files
-@sysCmd = ("split", "-d", "-b", "8m", "$srcFile", "$newFileName");
-print "@sysCmd\n";
-print "Starting split . . . .\n";
-system(@sysCmd);
-print "Split complete.\n";
-
-# get names of extent files
-@files = <$newFileName*>;
-
-$extentCnt = @files;
-$i=0;
-
-# list of extent files to place into a file system file
-@extList = ("");
-
-#list of file system files
-@fsFiles = ("");
-
-#init file lists
-while ( $i<$fsFileCnt )
-{
-	$fsFiles[$i]="fsFile" . $i;
-	$extList[$i]="";
-	$i++;
-}
-
-# organize extents into files per file systems
-#in this case there are 8 file systems
-$i=0;
-$j=0;
-while ( $i<$extentCnt )
-{
-	$j=0;
-	while ( $j<$fsFileCnt && $i<$extentCnt )
-	{
-		$extList[$j] = $extList[$j] . " " . $files[$i];
-		$i++;
-		$j++;
-	}
-}
-
-$i=0;
-while ( $i < $fsFileCnt )
-{
-	print "List $i: $extList[$i]\n";
-	$Cmd="cat $extList[$i] >> $fsFiles[$i]";
-	print "concat $i: $Cmd\n";
-	`$Cmd`;
-	$i++;
-}
-
-# copy the file system files to the appropriate PM
-# in this case there are 4 PMs
-$pmCnt=4;
-$i=0;
-$j=0;
-@pmFiles =("");
-
-while ( $i<$fsFileCnt )
-{
-	$j=0;
-	while ( $j < $pmCnt  && $i < $fsFileCnt)
-	{
-#		$pmFiles[$j] = $pmFiles[$j] . " " . $fsFiles[$i++];
-		$Cmd = "./remote_scp_put.sh $fsFiles[$i++] $pmDest[$j++] Calpont1";
-		print "scp command: $Cmd\n";
-		`$Cmd`;
-	}
-}
-
-#$j=0;
-#while ( $j < $pmCnt )
-#{
-#	print "pmFileList $j : $pmFiles[$j]\n";
-#	$j++;
-#}
-#
-#$j=0;
-#while ( $j < $pmCnt )
-#{
-#	$Cmd = "scp $pmFiles[$j] $pmDest[$j]";
-#	print "scp command: $Cmd\n";
-#	system($Cmd);
-#	$j++;
-#}
-

tools/updateGenii/listRepos.pl

@@ -1,58 +0,0 @@
-#!/usr/bin/perl -w
-#
-# $Id: listRepos.pl 192 2007-05-14 14:42:49Z rdempsey $
-#
-
-use File::Basename;
-
-$reporoot = '/Calpont/repos/genii/*';
-
-$opt_a = 0;
-if (defined($ENV{'QUERY_STRING'}))
-{
-	$opt_a = 1 if ($ENV{'QUERY_STRING'} eq "All");
-}
-
-sub findRepos
-{
-	my ($root) = @_;
-
-	my @repolist = ();
-
-	my @repodirs = glob($root);
-
-	my $dir;
-
-	for $dir (@repodirs)
-	{
-		$dir = basename($dir);
-
-		if ($opt_a == 0)
-		{
-			next if ($dir eq "net-snmp");
-			# deprecated projects
-			next if ($dir eq "altera");
-			next if ($dir eq "diskmgr");
-			next if ($dir eq "emulator");
-			next if ($dir eq "message");
-			next if ($dir eq "sqlengine");
-		}
-		next if ($dir eq "demo");
-		next if ($dir eq "doc");
-
-		push(@repolist, $dir);
-	}
-
-	return @repolist;
-}
-
-@repos = findRepos($reporoot);
-
-print 'content-type: text/plain', "\r\n";
-print "\r\n";
-
-foreach $repo (@repos)
-{
-	print $repo, "\r\n";
-}
-

tools/updateGenii/test.dat

@@ -1,14 +0,0 @@
-A      OID3046.dat
-?      brmtest/brmtest
-?      dumpcol/dumpcol
-?      fixdate/fixdate.cdf
-?      fixdate/f1.cdf
-?      fixdate/fixdate
-?      evalcol/evalcol
-?      editem/editem
-?      dbbuilder/dbbuilder
-?      oid2file/oid2file
-M      Makefile
-G      updateGenii
-C      updateGenii/updateGenii.pl
-G      updateGenii/listRepos.pl

tools/updateGenii/updateGenii.pl

@@ -1,186 +0,0 @@
-#!/usr/bin/perl -w
-#
-# $Id: updateGenii.pl 1265 2010-09-17 12:25:42Z rdempsey $
-#
-
-use Getopt::Std;
-
-sub usage
-{
-	my $msg = <<'EOD'
-usage: updateGenii.pl [-h] [-b|-t branch/tag] [-r root] [-a]
-   -h        Display this help
-   -b branch Update a branch wc
-   -t tag    Update a tag wc
-   -r root   Use root as the starting point rather than $HOME
-   -a        Update wc for all svn projects, even deprecated ones
-EOD
-;
-	print "$msg\n";
-}
-
-sub findRepos
-{
-	my @repolist = ();
-	my $all = '';
-
-	$all = '?All' if ($opt_a);
-
-	open(FH, 'curl -s http://srvengcm1.calpont.com/cgi-bin/listRepos.pl' . $all . ' |');
-	while (<FH>)
-	{
-		$_ =~ s/[\r\n]*$//;
-		push(@repolist, $_);
-	}
-	close(FH);
-
-	return @repolist;
-}
-
-if (!getopts('hab:r:t:'))
-{
-	usage();
-	exit 1;
-}
-
-if (defined($opt_h))
-{
-	$opt_h = 1;
-	usage();
-	exit 0;
-}
-
-$opt_a = 1 if (defined($opt_a));
-
-$prjroot = 'genii';
-$prjroot = $opt_r if (defined($opt_r));
-$prjroot = $opt_b if (defined($opt_b));
-$prjroot = $opt_t if (defined($opt_t));
-
-if (!defined($opt_b) && !defined($opt_t) && $#ARGV >= 0)
-{
-	usage();
-	exit 1;
-}
-
-if (defined($opt_b) || defined($opt_t))
-{
-	$cmd = 'svn list http://srvengcm1.calpont.com/svn/genii/dbcon/';
-	if (defined($opt_b))
-	{
-		$cmd = $cmd . 'branches';
-	}
-	else
-	{
-		$cmd = $cmd . 'tags';
-	}
-	$listcmd = $cmd;
-	$tmp = $prjroot;
-	$tmp =~ s/\./\\./g;
-	$cmd = $cmd . " 2>/dev/null | egrep '^" . $tmp . "/' | wc -l";
-	$brcnt = `$cmd`;
-	if ($brcnt != 1)
-	{
-		print "branch/tag $prjroot not found. Choices are:\n";
-		$listcmd = $listcmd . ' | sed s,/\$,,';
-		system($listcmd);
-		exit 1;
-	}
-}
-
-chdir $ENV{'HOME'} or die;
-
-$destDir = $prjroot;
-if ((defined($opt_b) || defined($opt_t)) && defined($opt_r))
-{
-	$destDir = $opt_r;
-}
-mkdir $destDir if (! -d $destDir);
-
-chdir $destDir or die;
-
-@repos = findRepos();
-@mergelines = ();
-@conflictlines = ();
-
-foreach $repo (@repos)
-{
-	#print $repo, "\n"; next;
-	if (-d $repo . '/.svn')
-	{
-		print $repo, ": \n";
-		chdir $repo or die;
-		open FH, 'svn update |';
-		@svnlines = <FH>;
-		close FH;
-		foreach $line (@svnlines)
-		{
-			print "$line";
-			if ($line =~ /^C/)
-			{
-				push(@conflictlines, $repo . ": " . $line);
-			}
-			elsif ($line =~ /^G/)
-			{
-				push(@mergelines, $repo . ": " . $line);
-			}
-		}
-
-		chdir '..' or die;
-	}
-	else
-	{
-		print $repo, ": \n";
-		if (defined($opt_b))
-		{
-			$svncmd = 'svn checkout http://srvengcm1.calpont.com/svn/genii/' . $repo . '/branches/' .
-				$prjroot . ' ' . $repo;
-		}
-		elsif (defined($opt_t))
-		{
-			$svncmd = 'svn checkout http://srvengcm1.calpont.com/svn/genii/' . $repo . '/tags/' .
-				$prjroot . ' ' . $repo;
-		}
-		else
-		{
-			$svncmd = 'svn checkout http://srvengcm1.calpont.com/svn/genii/' . $repo . '/trunk/' . ' ' .
-				$repo;
-		}
-		system $svncmd;
-	}
-}
-
-unlink 'merges.txt';
-unlink 'conflicts.txt';
-
-if ($#mergelines >= 0)
-{
-	print "\n** Warning! merges:\n";
-	foreach $merge (@mergelines)
-	{
-		print "$merge";
-	}
-	open FH, ">merges.txt";
-	print FH @mergelines;
-	close FH;
-	print "This information has been saved in merges.txt\n";
-}
-
-if ($#conflictlines >= 0)
-{
-	print "\n** Warning! conflicts:\n";
-	foreach $conflict (@conflictlines)
-	{
-		print "$conflict";
-	}
-	open FH, ">conflicts.txt";
-	print FH @conflictlines;
-	close FH;
-	print "This information has been saved in conflicts.txt\n";
-}
-
-if (! -e 'Makefile')
-{
-	system './build/bootstrap';
-}
-

tools/valprint/valprint.cpp

@@ -1,48 +0,0 @@
-/* Copyright (C) 2014 InfiniDB, Inc.
-
-   This program is free software; you can redistribute it and/or
-   modify it under the terms of the GNU General Public License
-   as published by the Free Software Foundation; version 2 of
-   the License.
-
-   This program is distributed in the hope that it will be useful,
-   but WITHOUT ANY WARRANTY; without even the implied warranty of
-   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-   GNU General Public License for more details.
-
-   You should have received a copy of the GNU General Public License
-   along with this program; if not, write to the Free Software
-   Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
-   MA 02110-1301, USA. */
-
-// $Id: valprint.cpp 2101 2013-01-21 14:12:52Z rdempsey $
-
-#include <iostream>
-#include <cstdlib>
-using namespace std;
-
-#include "configcpp.h"
-using namespace config;
-
-int main(int argc, char** argv)
-{
-  int c;
-  opterr = 0;
-
-  while ((c = getopt(argc, argv, "")) != EOF)
-    switch (c)
-    {
-      case '?':
-      default: break;
-    }
-
-  string s;
-
-  for (int i = optind; i < argc; i++)
-  {
-    s = argv[i];
-    cout << s << " = " << Config::fromText(s) << endl;
-  }
-
-  return 0;
-}

tools/viewtablelock/CMakeLists.txt

@@ -4,12 +4,5 @@ include_directories(${ENGINE_COMMON_INCLUDES})
 
 set(viewtablelock_SRCS viewtablelock.cpp)
 
-add_executable(viewtablelock ${viewtablelock_SRCS})
+columnstore_executable(viewtablelock ${viewtablelock_SRCS})
-
+columnstore_link(viewtablelock ${ENGINE_LDFLAGS} ${ENGINE_EXEC_LIBS})
-target_link_libraries(viewtablelock ${ENGINE_LDFLAGS} ${ENGINE_EXEC_LIBS})
-
-install(
-    TARGETS viewtablelock
-    DESTINATION ${ENGINE_BINDIR}
-    COMPONENT columnstore-engine
-)

utils/basic/string_utils.h

@@ -0,0 +1,15 @@
+#pragma once
+
+#include <algorithm>
+#include <string>
+
+namespace datatypes
+{
+inline bool ASCIIStringCaseInsensetiveEquals(const std::string& left, const std::string& right)
+{
+  auto asciiHigher = [](char c) { return (c >= 'a' && c <= 'z') ? c - 'a' + 'A' : c; };
+  return left.size() == right.size() &&
+         std::equal(left.begin(), left.end(), right.begin(),
+                    [&asciiHigher](char l, char r) { return asciiHigher(l) == asciiHigher(r); });
+}
+}  // namespace datatypes

utils/batchloader/CMakeLists.txt

@@ -6,12 +6,4 @@ set(batchloader_LIB_SRCS batchloader.cpp)
 
 columnstore_library(batchloader ${batchloader_LIB_SRCS})
 
-add_dependencies(batchloader loggingcpp)
+columnstore_link(batchloader ${NETSNMP_LIBRARIES} loggingcpp)
-
-target_link_libraries(batchloader ${NETSNMP_LIBRARIES})
-
-install(
-    TARGETS batchloader
-    DESTINATION ${ENGINE_LIBDIR}
-    COMPONENT columnstore-engine
-)

utils/cacheutils/CMakeLists.txt

@@ -5,11 +5,4 @@ include_directories(${ENGINE_COMMON_INCLUDES})
 set(cacheutils_LIB_SRCS cacheutils.cpp)
 
 columnstore_library(cacheutils ${cacheutils_LIB_SRCS})
-
+columnstore_link(cacheutils messageqcpp)
-add_dependencies(cacheutils loggingcpp)
-
-install(
-    TARGETS cacheutils
-    DESTINATION ${ENGINE_LIBDIR}
-    COMPONENT columnstore-engine
-)

utils/cloudio/CMakeLists.txt

@@ -14,22 +14,15 @@ columnstore_library(cloudio ${cloudio_LIB_SRCS})
 
 # IDBDataFile currently depends on cloudio, which is backward. Once cloudio has been turned into a proper plugin for
 # idbdatafile, we should be able to reverse the dependency like so:
-target_link_libraries(cloudio idbdatafile messageqcpp loggingcpp)
+columnstore_link(cloudio idbdatafile messageqcpp)
-
-install(
-    TARGETS cloudio
-    DESTINATION ${ENGINE_LIBDIR}
-    COMPONENT columnstore-engine
-)
 
 add_executable(cloudio_component_test component_test.cpp)
-
+add_executable(cloudio_end_to_end_test end_to_end_test.cpp)
-add_executable(end_to_end_test end_to_end_test.cpp)
 
 # see the comment above and change this dependency to cloudio.  Hm our lib dependencies seem not to be declared. Punting
 # on that, maybe in the future we can have some poor unfortunate intern untangle all of that and declare lib
 # dependencies properly. For now I'm going to do like the other executables, which means nearly everything AFAICT.
-target_link_libraries(cloudio_component_test ${ENGINE_LDFLAGS} ${ENGINE_EXEC_LIBS} cloudio)
+columnstore_link(cloudio_component_test ${ENGINE_LDFLAGS} ${ENGINE_EXEC_LIBS} cloudio)
-target_link_libraries(end_to_end_test ${ENGINE_LDFLAGS} ${ENGINE_EXEC_LIBS} cloudio)
+columnstore_link(cloudio_end_to_end_test ${ENGINE_LDFLAGS} ${ENGINE_EXEC_LIBS} cloudio)
-# target_link_libraries(cloudio_component_test cloudio) Copy end_to_end_test dataFile to binary directory
+# columnstore_link(cloudio_component_test cloudio) Copy end_to_end_test dataFile to binary directory
 file(COPY testData DESTINATION ${CMAKE_CURRENT_BINARY_DIR}/)

utils/common/CMakeLists.txt

@@ -15,13 +15,4 @@ set(common_LIB_SRCS
 )
 
 columnstore_library(common ${common_LIB_SRCS})
-
+columnstore_link(common boost_filesystem configcpp loggingcpp messageqcpp ${MARIADB_STRING_LIBS})
-target_link_libraries(common boost_filesystem)
-
-add_dependencies(common loggingcpp)
-
-install(
-    TARGETS common
-    DESTINATION ${ENGINE_LIBDIR}
-    COMPONENT columnstore-engine
-)

utils/common/collation.h

@@ -275,12 +275,4 @@ class CollationAwareComparator : public Charset
   }
 };
 
-inline bool ASCIIStringCaseInsensetiveEquals(const std::string& left, const std::string& right)
-{
-  auto asciiHigher = [](char c) { return (c >= 'a' && c <= 'z') ? c - 'a' + 'A' : c; };
-  return left.size() == right.size() &&
-         std::equal(left.begin(), left.end(), right.begin(),
-                    [&asciiHigher](char l, char r) { return asciiHigher(l) == asciiHigher(r); });
-}
-
 }  // end of namespace datatypes

utils/compress/CMakeLists.txt

@@ -11,19 +11,10 @@ set_property(DIRECTORY PROPERTY INCLUDE_DIRECTORIES "${dirs}")
 
 set(compress_LIB_SRCS idbcompress.cpp)
 
-add_definitions(-DNDEBUG)
-
 columnstore_library(compress ${compress_LIB_SRCS})
-add_dependencies(compress loggingcpp external_boost)
+columnstore_link(compress PRIVATE loggingcpp boost_filesystem ${SNAPPY_LIBRARIES})
 
-target_link_libraries(compress ${SNAPPY_LIBRARIES})
 if(HAVE_LZ4)
     message_once(STATUS "LINK WITH LZ4")
-    target_link_libraries(compress ${LZ4_LIBRARIES})
+    columnstore_link(compress PRIVATE ${LZ4_LIBRARIES})
 endif()
-
-install(
-    TARGETS compress
-    DESTINATION ${ENGINE_LIBDIR}
-    COMPONENT columnstore-engine
-)

utils/configcpp/CMakeLists.txt

@@ -1,16 +1,4 @@
+set(configcpp_LIB_SRCS configcpp.cpp xmlparser.cpp)
 include_directories(${ENGINE_COMMON_INCLUDES})
-
-# ########## next target ###############
-
-set(configcpp_LIB_SRCS configcpp.cpp xmlparser.cpp configstream.cpp)
-
 columnstore_library(configcpp ${configcpp_LIB_SRCS})
-add_dependencies(configcpp loggingcpp)
+columnstore_link(configcpp PRIVATE LibXml2::LibXml2 boost_filesystem)
-
-target_compile_definitions(configcpp PUBLIC BOOST_NO_CXX11_SCOPED_ENUMS)
-
-install(
-    TARGETS configcpp
-    DESTINATION ${ENGINE_LIBDIR}
-    COMPONENT columnstore-engine
-)

utils/configcpp/Calpont.xml

@@ -1,12 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<Calpont Version="V1.0.0">
-	<Message>
-		<Name>Message</Name>
-		<Dir>.</Dir>
-		<Length>875</Length>
-		<Version>V1.0</Version>
-	</Message>
-	<SystemConfig>
-		<SystemVersion>V1.0</SystemVersion>
-	</SystemConfig>
-</Calpont>