database/mariadb-columnstore-engine
1
0
Fork 0
mirror of https://github.com/mariadb-corporation/mariadb-columnstore-engine.git synced 2025-07-30 19:23:07 +03:00
Code Activity

Deep build refactoring phase 1 (#3562)

Browse Source 
* configcpp refactored
* logging and datatypes refactored

* more dataconvert
* chore(build): massive removals, auto add files to debian install file
* chore(codemanagement): nodeps headers, potentioal library
* chore(build): configure before autobake
* chore(build): use custom cmake commands for components, mariadb-plugin-columnstore.install generated
* chore(build): install deps as separate step for build-packages
* more deps
* check  debian/mariadb-plugin-columnstore.install automatically
* chore(build): add option for multibracnh compilation
* Fix warning
This commit is contained in:
Leonid Fedorov
2025-05-30 14:05:21 +04:00
committed by GitHub
parent 06ae3c8d6a
commit aa7e0fb9b4
176 changed files with 557 additions and 19155 deletions
Download Patch File Download Diff File Expand all files Collapse all files

5
.drone.jsonnet
View File

@ -631,9 +631,6 @@ local Pipeline(branch, platform, event, arch="amd64", server="10.6-enterprise",
volumes: [pipeline._volumes.mdb],
environment: {
DEBIAN_FRONTEND: "noninteractive",
DEB_BUILD_OPTIONS: "parallel=4",
DH_BUILD_DDEBS: "1",
BUILDPACKAGE_FLAGS: "-b", // Save time and produce only binary packages, not source
AWS_ACCESS_KEY_ID: {
from_secret: "aws_access_key_id",
},
@ -656,7 +653,7 @@ local Pipeline(branch, platform, event, arch="amd64", server="10.6-enterprise",
'bash -c "set -o pipefail && bash /mdb/' + builddir + "/storage/columnstore/columnstore/build/bootstrap_mcs.sh " +
"--build-type RelWithDebInfo " +
"--distro " + platform + " " +
"--build-packages --sccache " +
"--build-packages --install-deps --sccache " +
" " + customBootstrapParams +
" " + customBootstrapParamsForExisitingPipelines(platform) + " | " +
"/mdb/" + builddir + "/storage/columnstore/columnstore/build/ansi2txt.sh " +

2
.gitignore vendored
View File

@ -153,6 +153,7 @@ oam/install_scripts/mcs-workernode.service
oam/install_scripts/mcs-writeengineserver.service
oam/install_scripts/mcs-loadbrm.py
oam/install_scripts/mcs-savebrm.py
oam/install_scripts/mcssavebrm.py
oam/install_scripts/mcs-storagemanager.service
oam/install_scripts/mcs_module_installer.sh
oam/install_scripts/slave-rep-columnstore.sh
@ -186,4 +187,3 @@ versioning/BRM/shmkeys.cpp
obj/
build/build

61
build/bootstrap_mcs.sh
View File

@ -19,6 +19,9 @@ CONFIG_DIR=$RPM_CONFIG_DIR
SCRIPT_LOCATION=$(dirname "$0")
MDB_SOURCE_PATH=$(realpath "$SCRIPT_LOCATION"/../../../..)
COLUMSNTORE_SOURCE_PATH=$(realpath "$SCRIPT_LOCATION"/../)
DEFAULT_MARIA_BUILD_PATH=$(realpath "$MDB_SOURCE_PATH"/../BuildOf_$(basename "$MDB_SOURCE_PATH"))
BUILD_TYPE_OPTIONS=("Debug" "RelWithDebInfo")
DISTRO_OPTIONS=("ubuntu:20.04" "ubuntu:22.04" "ubuntu:24.04" "debian:11" "debian:12" "rockylinux:8" "rockylinux:9")
@ -28,21 +31,10 @@ MDB_CMAKE_FLAGS=()
source "$SCRIPT_LOCATION"/utils.sh
if [ "$EUID" -ne 0 ]; then
error "Please run script as root to install MariaDb to system paths"
exit 1
fi
cd $SCRIPT_LOCATION
CURRENT_BRANCH=$(git rev-parse --abbrev-ref HEAD)
message "Columnstore will be built from $color_yellow$CURRENT_BRANCH$color_cyan branch"
cd - >/dev/null
echo "Arguments received: $@"
message "Building Mariadb Server from $color_yellow$MDB_SOURCE_PATH$color_normal"
optparse.define short=A long=asan desc="Build with ASAN" variable=ASAN default=false value=true
optparse.define short=a long=build-path variable=MARIA_BUILD_PATH default=$(realpath "$MDB_SOURCE_PATH"/../BuildOf_$(basename "$MDB_SOURCE_PATH"))
optparse.define short=a long=build-path desc="Path for build output" variable=MARIA_BUILD_PATH default=$DEFAULT_MARIA_BUILD_PATH
optparse.define short=B long=run-microbench desc="Compile and run microbenchmarks " variable=RUN_BENCHMARKS default=false value=true
optparse.define short=c long=cloud desc="Enable cloud storage" variable=CLOUD_STORAGE_ENABLED default=false value=true
optparse.define short=C long=force-cmake-reconfig desc="Force cmake reconfigure" variable=FORCE_CMAKE_CONFIG default=false value=true
@ -68,10 +60,26 @@ optparse.define short=T long=tsan desc="Build with TSAN" variable=TSAN default=f
optparse.define short=u long=skip-unit-tests desc="Skip UnitTests" variable=SKIP_UNIT_TESTS default=false value=true
optparse.define short=U long=ubsan desc="Build with UBSAN" variable=UBSAN default=false value=true
optparse.define short=v long=verbose desc="Verbose makefile commands" variable=MAKEFILE_VERBOSE default=false value=true
optparse.define short=V long=add-branch-name-to-outdir desc="Add branch name to build output directory" variable=BRANCH_NAME_TO_OUTDIR default=false value=true
optparse.define short=W long=without-core-dumps desc="Do not produce core dumps" variable=WITHOUT_COREDUMPS default=false value=true
source $(optparse.build)
message "Building MariaDB Server from $color_yellow$MDB_SOURCE_PATH$color_normal"
cd $COLUMSNTORE_SOURCE_PATH
COLUMNSTORE_BRANCH=$(git rev-parse --abbrev-ref HEAD)
message "Columnstore will be built from $color_yellow$COLUMNSTORE_BRANCH$color_cyan branch"
cd $MDB_SOURCE_PATH
MARIADB_BRANCH=$(git rev-parse --abbrev-ref HEAD)
message "MariaDB will be built from $color_yellow$MARIADB_BRANCH$color_cyan branch"
cd - >/dev/null
if [[ ${BRANCH_NAME_TO_OUTDIR} = true ]]; then
MARIA_BUILD_PATH="${MARIA_BUILD_PATH}_${MARIADB_BRANCH}_${COLUMNSTORE_BRANCH}"
fi
if [[ ! " ${BUILD_TYPE_OPTIONS[*]} " =~ " ${MCS_BUILD_TYPE} " ]]; then
getChoice -q "Select your Build Type" -o BUILD_TYPE_OPTIONS
MCS_BUILD_TYPE=$selectedChoice
@ -104,7 +112,7 @@ install_deps() {
cppunit-devel cmake3 libxcrypt-devel xz-devel zlib-devel libzstd-devel glibc-devel"
DEB_BUILD_DEPS="apt-get -y update && apt-get -y install build-essential automake libboost-all-dev \
bison cmake libncurses5-dev libaio-dev libsystemd-dev libpcre2-dev libperl-dev libssl-dev libxml2-dev \
bison cmake libncurses5-dev python3 libaio-dev libsystemd-dev libpcre2-dev libperl-dev libssl-dev libxml2-dev \
libkrb5-dev flex libpam-dev git libsnappy-dev libcurl4-openssl-dev libgtest-dev libcppunit-dev googletest \
libjemalloc-dev liblz-dev liblzo2-dev liblzma-dev liblz4-dev libbz2-dev libbenchmark-dev libdistro-info-perl \
graphviz devscripts ccache equivs eatmydata curl"
@ -259,8 +267,9 @@ construct_cmake_flags() {
-DWITH_WSREP=NO
)
if [[ MAINTAINER_MODE = true ]]; then
if [[ $MAINTAINER_MODE = true ]]; then
MDB_CMAKE_FLAGS+=(-DCOLUMNSTORE_MAINTAINER=YES)
message "Columnstore mainteiner mode on"
else
warn "Maintainer mode is disabled, be careful, alien"
fi
@ -406,9 +415,15 @@ generate_svgs() {
}
build_package() {
cd $MDB_SOURCE_PATH
if [[ $pkg_format == "rpm" ]]; then
command="cmake ${MDB_CMAKE_FLAGS[@]} && make -j\$(nproc) package"
else
export DEBIAN_FRONTEND="noninteractive"
export DEB_BUILD_OPTIONS="parallel=$(nproc)"
export DH_BUILD_DDEBS="1"
export BUILDPACKAGE_FLAGS="-b"
command="mk-build-deps debian/control -t 'apt-get -y -o Debug::pkgProblemResolver=yes --no-install-recommends' -r -i && \
CMAKEFLAGS=\"${MDB_CMAKE_FLAGS[@]}\" debian/autobake-deb.sh"
fi
@ -420,6 +435,14 @@ build_package() {
check_errorcode
}
check_debian_install_file() {
message "checking debian/mariadb-plugin-columnstore.install"
message_split
python3 $COLUMSNTORE_SOURCE_PATH/build/debian_install_file_compare.py \
${COLUMSNTORE_SOURCE_PATH}/debian/mariadb-plugin-columnstore.install \
$MARIA_BUILD_PATH/mariadb-plugin-columnstore.install.generated
}
build_binary() {
MARIA_BUILD_PATH=$(realpath "$MARIA_BUILD_PATH")
message_split
@ -438,7 +461,7 @@ build_binary() {
message "Configuring cmake silently"
${CMAKE_BIN_NAME} "${MDB_CMAKE_FLAGS[@]}" -S"$MDB_SOURCE_PATH" -B"$MARIA_BUILD_PATH" | spinner
message_split
check_debian_install_file
generate_svgs
${CMAKE_BIN_NAME} --build "$MARIA_BUILD_PATH" -j "$CPUS" | onelinearizator &&
@ -553,6 +576,12 @@ make_dir() {
install() {
if [[ $RECOMPILE_ONLY = false ]]; then
if [ "$EUID" -ne 0 ]; then
error "Please run script as root to install MariaDb to system paths"
exit 1
fi
message_split
message "Installing MariaDB"
disable_plugins_for_bootstrap
@ -636,7 +665,7 @@ smoke() {
fi
}
if [[ $INSTALL_DEPS = true || $BUILD_PACKAGES = true ]]; then
if [[ $INSTALL_DEPS = true ]]; then
install_deps
fi

81
build/debian_install_file_compare.py Normal file
View File

@ -0,0 +1,81 @@
#!/usr/bin/env python3
import argparse
import fnmatch
import sys
def read_patterns(filename):
with open(filename, 'r') as f:
return [line.strip() for line in f if line.strip()]
def read_paths_with_comments(filename):
paths = []
with open(filename, 'r') as f:
for line in f:
raw = line.rstrip('\n')
if not raw.strip():
continue
# Split into path and comment
if '#' in raw:
path, comment = raw.split('#', 1)
path = path.strip()
comment = comment.strip()
else:
path = raw.strip()
comment = ''
if path:
paths.append((raw, path, comment))
return paths
def print_red(msg):
if sys.stdout.isatty():
print("\033[31m" + msg + "\033[0m")
else:
print(msg)
def main():
parser = argparse.ArgumentParser(description="Compare wildcard patterns to file paths.")
parser.add_argument('patterns_file', help='File with wildcard patterns (one per line)')
parser.add_argument('paths_file', help='File with file paths (one per line)')
args = parser.parse_args()
patterns = read_patterns(args.patterns_file)
paths_with_comments = read_paths_with_comments(args.paths_file)
# Track which paths are matched (by index)
path_matched = [False] * len(paths_with_comments)
# Collect patterns that don't match any path
unmatched_patterns = []
for pattern in patterns:
matched = False
for i, (raw, path, comment) in enumerate(paths_with_comments):
if fnmatch.fnmatch(path, pattern):
matched = True
path_matched[i] = True
if not matched:
unmatched_patterns.append(pattern)
if unmatched_patterns:
print_red("The files declared in debian/mariadb-plugin-columnstore.install "
" are not added to CMakeLists via columnstore_* statements see cmake/ColumnstoreLibrary.cmake")
for pattern in unmatched_patterns:
print(f"- {pattern}")
# Collect paths that weren't matched by any pattern
unmatched_paths = []
for (raw, path, comment), matched in zip(paths_with_comments, path_matched):
if not matched:
unmatched_paths.append((path, comment))
if unmatched_paths:
print_red("The files added via columnstore_* statements from cmake/ColumnstoreLibrary.cmake "
"are missing in debian/mariadb-plugin-columnstore.install file")
for path, comment in unmatched_paths:
if comment:
print(f"- {path} {comment}")
else:
print(f"- {path}")
if __name__ == "__main__":
main()

103
cmake/ColumnstoreLibrary.cmake
View File

@ -1,8 +1,105 @@
# Function to create either a static or shared library based on COLUMNSTORE_STATIC_LIBRARIES
set(DEBIAN_INSTALL_FILE "${CMAKE_BINARY_DIR}/mariadb-plugin-columnstore.install.generated")
file(WRITE ${DEBIAN_INSTALL_FILE} "#File is generated by ColumnstoreLibrary.cmake, do not edit\n")
macro(add_to_debian_install_file file_path)
string(SUBSTRING "${file_path}" 1 -1 BINARY_ENTRY)
file(RELATIVE_PATH CMAKEFILE "${CMAKE_SOURCE_DIR}/storage/columnstore/columnstore" ${CMAKE_CURRENT_LIST_FILE})
string(STRIP "${BINARY_ENTRY}" BINARY_ENTRY)
file(APPEND ${DEBIAN_INSTALL_FILE} "${BINARY_ENTRY} # added in ${CMAKEFILE}\n")
endmacro()
function(get_target_output_filename TARGET_NAME OUTPUT_VAR)
# 1. Get the target's OUTPUT_NAME (falls back to TARGET_NAME)
get_target_property(OUTPUT_NAME ${TARGET_NAME} OUTPUT_NAME)
if(NOT OUTPUT_NAME)
set(OUTPUT_NAME "${TARGET_NAME}")
endif()
# 1. Get the correct suffix based on target type
get_target_property(TARGET_TYPE ${TARGET_NAME} TYPE)
if(TARGET_TYPE STREQUAL "EXECUTABLE")
set(SUFFIX "${CMAKE_EXECUTABLE_SUFFIX}")
set(PREFIX "") # No prefix for executables
elseif(TARGET_TYPE STREQUAL "SHARED_LIBRARY")
set(SUFFIX "${CMAKE_SHARED_LIBRARY_SUFFIX}")
set(PREFIX "lib")
elseif(TARGET_TYPE STREQUAL "STATIC_LIBRARY")
set(SUFFIX "${CMAKE_STATIC_LIBRARY_SUFFIX}")
set(PREFIX "lib")
else()
message(WARNING "Unknown target type for ${TARGET_NAME}")
set(SUFFIX "")
endif()
# 1. Combine into the final filename
set(${OUTPUT_VAR}
"${PREFIX}${OUTPUT_NAME}${SUFFIX}"
PARENT_SCOPE
)
endfunction()
macro(columnstore_install_target target destination)
install(
TARGETS ${target}
DESTINATION ${destination}
COMPONENT columnstore-engine
)
get_target_output_filename(${target} OUTPUT_FILENAME)
add_to_debian_install_file("${destination}/${OUTPUT_FILENAME}")
endmacro()
macro(columnstore_install_file file destination)
install(
FILES ${file}
DESTINATION ${destination}
COMPONENT columnstore-engine
)
get_filename_component(FILENAME ${file} NAME)
add_to_debian_install_file("${destination}/${FILENAME}")
endmacro()
macro(columnstore_install_program file destination)
install(
PROGRAMS ${file}
DESTINATION ${destination}
COMPONENT columnstore-engine
)
get_filename_component(FILENAME ${file} NAME)
add_to_debian_install_file(
"${destination}/${FILENAME}
"
)
endmacro()
macro(columnstore_static_library libname)
add_definitions(-fPIC -DPIC)
add_library(${libname} STATIC ${ARGN})
endmacro()
macro(columnstore_shared_library libname)
add_library(${libname} SHARED ${ARGN})
columnstore_install_target(${libname} ${ENGINE_LIBDIR})
endmacro()
macro(columnstore_library libname)
if(COLUMNSTORE_STATIC_LIBRARIES)
add_library(${libname} STATIC ${ARGN})
columnstore_static_library(${libname} ${ARGN})
else()
add_library(${libname} SHARED ${ARGN})
columnstore_shared_library(${libname} ${ARGN})
endif()
endmacro()
macro(columnstore_mysql_plugin_library libname)
add_library(${libname} SHARED ${ARGN})
columnstore_install_target(${libname} ${MARIADB_PLUGINDIR})
endmacro()
macro(columnstore_link libname)
target_link_libraries(${libname} ${ARGN})
endmacro()
macro(columnstore_executable executable_name)
add_executable(${executable_name} ${ARGN})
columnstore_install_target(${executable_name} ${ENGINE_BINDIR})
endmacro()

5
cmake/compiler_flags.cmake
View File

@ -51,6 +51,7 @@ set(COLUMNSTORE_MAINTAINER_FLAGS ${WERROR_FLAG})
set(FLAGS_ALL
-Wall
-Wextra
-ggdb3
-fno-omit-frame-pointer
-fno-strict-aliasing
-fsigned-char
@ -59,9 +60,9 @@ set(FLAGS_ALL
-DBOOST_BIND_GLOBAL_PLACEHOLDERS
)
set(FLAGS_RELEASE -g -O3 -DDBUG_OFF)
set(FLAGS_RELEASE -O3 -DDBUG_OFF)
set(FLAGS_DEBUG -ggdb3 -O0 -D_DEBUG)
set(FLAGS_DEBUG -O0 -D_DEBUG)
# } end Release, Debug and common flags
# linker flags {

15
cmake/configureEngine.cmake
View File

@ -773,17 +773,10 @@ endif()
# releasenum is used by external scripts for various tasks. Leave it alone.
configure_file(${CMAKE_CURRENT_SOURCE_DIR}/build/releasenum.in ${CMAKE_CURRENT_BINARY_DIR}/build/releasenum IMMEDIATE)
install(
FILES ${CMAKE_CURRENT_BINARY_DIR}/build/releasenum
DESTINATION ${ENGINE_SUPPORTDIR}
COMPONENT columnstore-engine
)
columnstore_install_file(${CMAKE_CURRENT_BINARY_DIR}/build/releasenum ${ENGINE_SUPPORTDIR})
configure_file(${CMAKE_CURRENT_SOURCE_DIR}/columnstoreversion.h.in ${CMAKE_CURRENT_SOURCE_DIR}/columnstoreversion.h)
configure_file(${CMAKE_CURRENT_SOURCE_DIR}/mcsconfig.h.in ${CMAKE_CURRENT_BINARY_DIR}/mcsconfig.h)
configure_file(${CMAKE_CURRENT_SOURCE_DIR}/gitversionEngine.in ${CMAKE_CURRENT_BINARY_DIR}/gitversionEngine IMMEDIATE)
install(
FILES ${CMAKE_CURRENT_BINARY_DIR}/gitversionEngine
DESTINATION ${ENGINE_SUPPORTDIR}
COMPONENT columnstore-engine
)
columnstore_install_file(${CMAKE_CURRENT_BINARY_DIR}/gitversionEngine ${ENGINE_SUPPORTDIR})

1
cmake/packages.cmake
View File

@ -14,7 +14,6 @@ if(NOT LEX_EXECUTABLE)
endif()
find_package(LibXml2)
if(NOT LIBXML2_FOUND)
message_once(CS_NO_LIBXML "Could not find a usable libxml2 development environment!")
return()

9
datatypes/CMakeLists.txt
View File

@ -2,10 +2,5 @@ include_directories(${ENGINE_COMMON_INCLUDES})
set(datatypes_LIB_SRCS mcs_int128.cpp mcs_decimal.cpp)
columnstore_library(datatypes ${datatypes_LIB_SRCS})
add_dependencies(datatypes loggingcpp external_boost)
install(
TARGETS datatypes
DESTINATION ${ENGINE_LIBDIR}
COMPONENT columnstore-engine
)
columnstore_link(datatypes PRIVATE loggingcpp)
add_dependencies(datatypes external_boost)

8
dbcon/ddlpackage/CMakeLists.txt
View File

@ -42,10 +42,4 @@ columnstore_library(
${FLEX_ddl_scan_OUTPUTS}
)
add_dependencies(ddlpackage loggingcpp)
install(
TARGETS ddlpackage
DESTINATION ${ENGINE_LIBDIR}
COMPONENT columnstore-engine
)
columnstore_link(ddlpackage loggingcpp)

11
dbcon/ddlpackageproc/CMakeLists.txt
View File

@ -13,13 +13,4 @@ set(ddlpackageproc_LIB_SRCS
)
columnstore_library(ddlpackageproc ${ddlpackageproc_LIB_SRCS})
add_dependencies(ddlpackageproc loggingcpp)
target_link_libraries(ddlpackageproc ${NETSNMP_LIBRARIES})
install(
TARGETS ddlpackageproc
DESTINATION ${ENGINE_LIBDIR}
COMPONENT columnstore-engine
)
columnstore_link(ddlpackageproc ${NETSNMP_LIBRARIES} loggingcpp)

8
dbcon/dmlpackage/CMakeLists.txt
View File

@ -41,10 +41,4 @@ columnstore_library(
${FLEX_dml_scan_OUTPUTS}
)
add_dependencies(dmlpackage loggingcpp)
install(
TARGETS dmlpackage
DESTINATION ${ENGINE_LIBDIR}
COMPONENT columnstore-engine
)
columnstore_link(dmlpackage loggingcpp)

11
dbcon/dmlpackageproc/CMakeLists.txt
View File

@ -13,13 +13,4 @@ set(dmlpackageproc_LIB_SRCS
)
columnstore_library(dmlpackageproc ${dmlpackageproc_LIB_SRCS})
add_dependencies(dmlpackageproc loggingcpp)
target_link_libraries(dmlpackageproc ${NETSNMP_LIBRARIES})
install(
TARGETS dmlpackageproc
DESTINATION ${ENGINE_LIBDIR}
COMPONENT columnstore-engine
)
columnstore_link(dmlpackageproc ${NETSNMP_LIBRARIES} loggingcpp)

11
dbcon/execplan/CMakeLists.txt
View File

@ -46,13 +46,4 @@ set(execplan_LIB_SRCS
)
columnstore_library(execplan ${execplan_LIB_SRCS})
add_dependencies(execplan loggingcpp)
target_link_libraries(execplan messageqcpp ${NETSNMP_LIBRARIES} ${ENGINE_DT_LIB} pron)
install(
TARGETS execplan
DESTINATION ${ENGINE_LIBDIR}
COMPONENT columnstore-engine
)
columnstore_link(execplan messageqcpp ${NETSNMP_LIBRARIES} ${ENGINE_DT_LIB} pron loggingcpp)

7
dbcon/execplan/pseudocolumn.cpp
View File

@ -25,7 +25,7 @@
#include <string>
#include <sstream>
using namespace std;
#include "basic/string_utils.h"
#include "bytestream.h"
#include "collation.h"
@ -138,8 +138,9 @@ string PseudoColumn::toCppCode(IncludeSet& includes) const
{
includes.insert("pseudocolumn.h");
stringstream ss;
ss << "PseudoColumn(" << std::quoted(fSchemaName) << ", " << std::quoted(fTableName) << ", " << fisColumnStore << ", " << std::quoted(fColumnName)
<< ", " << fPseudoType << ", " << sessionID() << ")";
ss << "PseudoColumn(" << std::quoted(fSchemaName) << ", " << std::quoted(fTableName) << ", "
<< fisColumnStore << ", " << std::quoted(fColumnName) << ", " << fPseudoType << ", " << sessionID()
<< ")";
return ss.str();
}

30
dbcon/execplan/windowfunctioncolumn.cpp
View File

@ -27,10 +27,13 @@
#include <iostream>
#include <sstream>
using namespace std;
#include "basic/string_utils.h"
#include <boost/tokenizer.hpp>
using namespace boost;
#include "basic/string_utils.h"
#include "bytestream.h"
#include "windowfunctioncolumn.h"
#include "constantcolumn.h"
@ -398,23 +401,32 @@ void WindowFunctionColumn::adjustResultType()
{
if ((fResultType.colDataType == CalpontSystemCatalog::DECIMAL ||
fResultType.colDataType == CalpontSystemCatalog::UDECIMAL) &&
!datatypes::ASCIIStringCaseInsensetiveEquals(fFunctionName, "COUNT") && !datatypes::ASCIIStringCaseInsensetiveEquals(fFunctionName, "COUNT(*)") &&
!datatypes::ASCIIStringCaseInsensetiveEquals(fFunctionName, "ROW_NUMBER") && !datatypes::ASCIIStringCaseInsensetiveEquals(fFunctionName, "RANK") &&
!datatypes::ASCIIStringCaseInsensetiveEquals(fFunctionName, "PERCENT_RANK") && !datatypes::ASCIIStringCaseInsensetiveEquals(fFunctionName, "DENSE_RANK") &&
!datatypes::ASCIIStringCaseInsensetiveEquals(fFunctionName, "CUME_DIST") && !datatypes::ASCIIStringCaseInsensetiveEquals(fFunctionName, "NTILE") &&
!datatypes::ASCIIStringCaseInsensetiveEquals(fFunctionName, "COUNT") &&
!datatypes::ASCIIStringCaseInsensetiveEquals(fFunctionName, "COUNT(*)") &&
!datatypes::ASCIIStringCaseInsensetiveEquals(fFunctionName, "ROW_NUMBER") &&
!datatypes::ASCIIStringCaseInsensetiveEquals(fFunctionName, "RANK") &&
!datatypes::ASCIIStringCaseInsensetiveEquals(fFunctionName, "PERCENT_RANK") &&
!datatypes::ASCIIStringCaseInsensetiveEquals(fFunctionName, "DENSE_RANK") &&
!datatypes::ASCIIStringCaseInsensetiveEquals(fFunctionName, "CUME_DIST") &&
!datatypes::ASCIIStringCaseInsensetiveEquals(fFunctionName, "NTILE") &&
!datatypes::ASCIIStringCaseInsensetiveEquals(fFunctionName, "PERCENTILE") && !fFunctionParms.empty() &&
fFunctionParms[0]->resultType().colDataType == CalpontSystemCatalog::DOUBLE)
fResultType = fFunctionParms[0]->resultType();
if ((datatypes::ASCIIStringCaseInsensetiveEquals(fFunctionName, "LEAD") || datatypes::ASCIIStringCaseInsensetiveEquals(fFunctionName, "LAG") ||
datatypes::ASCIIStringCaseInsensetiveEquals(fFunctionName, "MIN") || datatypes::ASCIIStringCaseInsensetiveEquals(fFunctionName, "MAX") ||
datatypes::ASCIIStringCaseInsensetiveEquals(fFunctionName, "FIRST_VALUE") || datatypes::ASCIIStringCaseInsensetiveEquals(fFunctionName, "LAST_VALUE") ||
if ((datatypes::ASCIIStringCaseInsensetiveEquals(fFunctionName, "LEAD") ||
datatypes::ASCIIStringCaseInsensetiveEquals(fFunctionName, "LAG") ||
datatypes::ASCIIStringCaseInsensetiveEquals(fFunctionName, "MIN") ||
datatypes::ASCIIStringCaseInsensetiveEquals(fFunctionName, "MAX") ||
datatypes::ASCIIStringCaseInsensetiveEquals(fFunctionName, "FIRST_VALUE") ||
datatypes::ASCIIStringCaseInsensetiveEquals(fFunctionName, "LAST_VALUE") ||
datatypes::ASCIIStringCaseInsensetiveEquals(fFunctionName, "NTH_VALUE")) &&
!fFunctionParms.empty())
fResultType = fFunctionParms[0]->resultType();
if (datatypes::ASCIIStringCaseInsensetiveEquals(fFunctionName, "SUM") || datatypes::ASCIIStringCaseInsensetiveEquals(fFunctionName, "AVG") ||
datatypes::ASCIIStringCaseInsensetiveEquals(fFunctionName, "AVG_DISTINCT") || datatypes::ASCIIStringCaseInsensetiveEquals(fFunctionName, "PERCENTILE"))
if (datatypes::ASCIIStringCaseInsensetiveEquals(fFunctionName, "SUM") ||
datatypes::ASCIIStringCaseInsensetiveEquals(fFunctionName, "AVG") ||
datatypes::ASCIIStringCaseInsensetiveEquals(fFunctionName, "AVG_DISTINCT") ||
datatypes::ASCIIStringCaseInsensetiveEquals(fFunctionName, "PERCENTILE"))
{
if (fFunctionParms[0]->resultType().colDataType == CalpontSystemCatalog::DECIMAL ||
fFunctionParms[0]->resultType().colDataType == CalpontSystemCatalog::UDECIMAL)

21
dbcon/joblist/CMakeLists.txt
View File

@ -57,8 +57,8 @@ set(joblist_LIB_SRCS
# ########## next target ###############
add_library(joblist_server STATIC ${joblist_LIB_SRCS})
add_dependencies(joblist_server loggingcpp)
columnstore_static_library(joblist_server ${joblist_LIB_SRCS})
columnstore_link(joblist_server loggingcpp)
# ########## next target ###############
@ -68,20 +68,9 @@ columnstore_library(joblist ${joblist_LIB_SRCS})
target_include_directories(
joblist BEFORE PUBLIC ${OPENSSL_INCLUDE_DIR} ${LIBMARIADB_BININC_DIR} ${LIBMARIADB_SRCINC_DIR}
)
add_dependencies(joblist loggingcpp)
install(
TARGETS joblist
DESTINATION ${ENGINE_LIBDIR}
COMPONENT columnstore-engine
)
columnstore_link(joblist loggingcpp)
if(WITH_ORDERBY_UT)
add_executable(job_orderby_tests orderby-tests.cpp)
target_link_libraries(job_orderby_tests ${ENGINE_LDFLAGS} ${ENGINE_WRITE_LIBS} ${CPPUNIT_LIBRARIES} cppunit)
install(
TARGETS job_orderby_tests
DESTINATION ${ENGINE_BINDIR}
COMPONENT columnstore-engine
)
columnstore_executable(job_orderby_tests orderby-tests.cpp)
columnstore_link(job_orderby_tests ${ENGINE_LDFLAGS} ${ENGINE_WRITE_LIBS} ${CPPUNIT_LIBRARIES} cppunit)
endif()

42
dbcon/mysql/CMakeLists.txt
View File

@ -71,16 +71,17 @@ if(COMMAND mysql_add_plugin)
CONFIG
columnstore.cnf
)
add_to_debian_install_file(${MARIADB_MYCNFDIR}/columnstore.cnf)
add_to_debian_install_file("${MARIADB_PLUGINDIR}/ha_columnstore.so")
else()
message("Adding plugin via CMake")
add_library(ha_columnstore SHARED ${libcalmysql_SRCS})
columnstore_shared_library(ha_columnstore ${libcalmysql_SRCS})
set_target_properties(ha_columnstore PROPERTIES PREFIX "")
add_dependencies(ha_columnstore loggingcpp)
add_dependencies(ha_columnstore marias3)
add_definitions(-DMYSQL_DYNAMIC_PLUGIN -DPLUGIN_COLUMNSTORE_VERSION="${PACKAGE_VERSION}")
target_link_libraries(
columnstore_link(
ha_columnstore
${S3API_DEPS}
${ENGINE_LDFLAGS}
@ -88,33 +89,20 @@ else()
${NETSNMP_LIBRARIES}
${SERVER_BUILD_DIR}/libservices/libmysqlservices.a
threadpool
loggingcpp
marias3
)
install(
TARGETS ha_columnstore
DESTINATION ${MARIADB_PLUGINDIR}
COMPONENT columnstore-engine
)
# define this dummy target for standalone builds (ie, when mysql_add_plugin doesn't exist)
add_custom_target(columnstore DEPENDS ha_columnstore)
install(
FILES columnstore.cnf
DESTINATION ${MARIADB_MYCNFDIR}
COMPONENT columnstore-engine
)
columnstore_install_file(columnstore.cnf ${MARIADB_MYCNFDIR})
endif()
if(TARGET columnstore)
install(
FILES syscatalog_mysql.sql dumpcat_mysql.sql calsetuserpriority.sql calremoveuserpriority.sql
calshowprocesslist.sql columnstore_info.sql
DESTINATION ${ENGINE_SUPPORTDIR}
COMPONENT columnstore-engine
)
install(
PROGRAMS install_mcs_mysql.sh
DESTINATION ${ENGINE_SBINDIR}
COMPONENT columnstore-engine
)
columnstore_install_file(syscatalog_mysql.sql ${ENGINE_SUPPORTDIR})
columnstore_install_file(dumpcat_mysql.sql ${ENGINE_SUPPORTDIR})
columnstore_install_file(calsetuserpriority.sql ${ENGINE_SUPPORTDIR})
columnstore_install_file(calremoveuserpriority.sql ${ENGINE_SUPPORTDIR})
columnstore_install_file(calshowprocesslist.sql ${ENGINE_SUPPORTDIR})
columnstore_install_file(columnstore_info.sql ${ENGINE_SUPPORTDIR})
columnstore_install_program(install_mcs_mysql.sh ${ENGINE_SBINDIR})
endif()

1
dbcon/mysql/ha_mcs_client_udfs.cpp
View File

@ -45,6 +45,7 @@ using namespace logging;
#include "columnstoreversion.h"
#include "ha_mcs_sysvars.h"
#include "basic/string_utils.h"
extern "C"
{

8
dbcon/mysql/ha_mcs_impl_if.h
View File

@ -26,7 +26,7 @@
#include <boost/shared_ptr.hpp>
#include <stack>
#include <vector>
#include "basic/string_utils.h"
#include "idb_mysql.h"
#include "ha_mcs_sysvars.h"
@ -36,6 +36,8 @@ struct st_ha_create_information;
class ha_columnstore_select_handler;
class ha_columnstore_derived_handler;
#include "basic/string_utils.h"
#include "configcpp.h"
#include "idberrorinfo.h"
#include "calpontselectexecutionplan.h"
@ -117,7 +119,8 @@ struct gp_walk_info
std::vector<execplan::ReturnedColumn*> localCols;
std::stack<execplan::ReturnedColumn*> rcWorkStack;
std::stack<execplan::ParseTree*> ptWorkStack;
boost::shared_ptr<execplan::SimpleColumn> scsp; // while defined as SSCP, it is used as SRCP, nothing specific to SimpleColumn is used in use sites.
boost::shared_ptr<execplan::SimpleColumn> scsp; // while defined as SSCP, it is used as SRCP, nothing
// specific to SimpleColumn is used in use sites.
uint32_t sessionid;
bool fatalParseError;
std::string parseErrorText;
@ -227,7 +230,6 @@ struct gp_walk_info
{
}
~gp_walk_info();
};
struct SubQueryChainHolder;

12
ddlproc/CMakeLists.txt
View File

@ -4,14 +4,6 @@ include_directories(${ENGINE_COMMON_INCLUDES})
set(DDLProc_SRCS ddlproc.cpp ddlprocessor.cpp ../utils/common/crashtrace.cpp)
add_executable(DDLProc ${DDLProc_SRCS})
columnstore_executable(DDLProc ${DDLProc_SRCS})
add_dependencies(DDLProc loggingcpp)
target_link_libraries(DDLProc ${ENGINE_LDFLAGS} ${ENGINE_WRITE_LIBS} ${NETSNMP_LIBRARIES} threadpool)
install(
TARGETS DDLProc
DESTINATION ${ENGINE_BINDIR}
COMPONENT columnstore-engine
)
columnstore_link(DDLProc ${ENGINE_LDFLAGS} ${ENGINE_WRITE_LIBS} ${NETSNMP_LIBRARIES} threadpool loggingcpp)

13
dmlproc/CMakeLists.txt
View File

@ -6,11 +6,9 @@ set(DMLProc_SRCS dmlproc.cpp dmlprocessor.cpp dmlresultbuffer.cpp batchinsertpro
../utils/common/crashtrace.cpp
)
add_executable(DMLProc ${DMLProc_SRCS})
columnstore_executable(DMLProc ${DMLProc_SRCS})
add_dependencies(DMLProc loggingcpp)
target_link_libraries(
columnstore_link(
DMLProc
${ENGINE_LDFLAGS}
${ENGINE_WRITE_LIBS}
@ -18,10 +16,5 @@ target_link_libraries(
threadpool
ddlcleanuputil
batchloader
)
install(
TARGETS DMLProc
DESTINATION ${ENGINE_BINDIR}
COMPONENT columnstore-engine
loggingcpp
)

6
oam/etc/CMakeLists.txt
View File

@ -1,5 +1 @@
install(
FILES Columnstore.xml
DESTINATION ${ENGINE_SYSCONFDIR}/columnstore
COMPONENT columnstore-engine
)
columnstore_install_file(Columnstore.xml ${ENGINE_SYSCONFDIR}/columnstore)

70
oam/install_scripts/CMakeLists.txt
View File

@ -131,49 +131,37 @@ configure_file("${CMAKE_CURRENT_SOURCE_DIR}/mcs-savebrm.py.in" "${CMAKE_CURRENT_
configure_file("${CMAKE_CURRENT_SOURCE_DIR}/mcs-savebrm.py.in" "${CMAKE_CURRENT_SOURCE_DIR}/mcssavebrm.py" @ONLY)
configure_file("${CMAKE_CURRENT_SOURCE_DIR}/columnstoreSyslog.in" "${CMAKE_CURRENT_SOURCE_DIR}/columnstoreSyslog" @ONLY)
install(
PROGRAMS columnstore-post-install
columnstore-pre-uninstall
columnstore_run.sh
post-mysql-install
post-mysqld-install
columnstoreSyslogSetup.sh
mcs-stop-controllernode.sh
mcs-loadbrm.py
mcs-savebrm.py
mariadb-columnstore-start.sh
mariadb-columnstore-stop.sh
loop_process_starter.sh
DESTINATION ${ENGINE_BINDIR}
COMPONENT columnstore-engine
)
columnstore_install_program(columnstore-post-install ${ENGINE_BINDIR})
columnstore_install_program(columnstore-pre-uninstall ${ENGINE_BINDIR})
columnstore_install_program(columnstore_run.sh ${ENGINE_BINDIR})
columnstore_install_program(post-mysql-install ${ENGINE_BINDIR})
columnstore_install_program(post-mysqld-install ${ENGINE_BINDIR})
columnstore_install_program(columnstoreSyslogSetup.sh ${ENGINE_BINDIR})
columnstore_install_program(mcs-stop-controllernode.sh ${ENGINE_BINDIR})
columnstore_install_program(mcs-loadbrm.py ${ENGINE_BINDIR})
columnstore_install_program(mcs-savebrm.py ${ENGINE_BINDIR})
columnstore_install_program(mariadb-columnstore-start.sh ${ENGINE_BINDIR})
columnstore_install_program(mariadb-columnstore-stop.sh ${ENGINE_BINDIR})
columnstore_install_program(loop_process_starter.sh ${ENGINE_BINDIR})
install(
FILES mariadb-columnstore.service
columnstoreAlias
columnstoreSyslog
columnstoreSyslog7
columnstoreSyslog-ng
columnstoreLogRotate
myCnf-include-args.text
myCnf-exclude-args.text
mcs-workernode.service
mcs-controllernode.service
mcs-primproc.service
mcs-writeengineserver.service
mcs-dmlproc.service
mcs-ddlproc.service
mcs-loadbrm.service
mcs-storagemanager.service
DESTINATION ${ENGINE_SUPPORTDIR}
COMPONENT columnstore-engine
)
columnstore_install_file(mariadb-columnstore.service ${ENGINE_SUPPORTDIR})
columnstore_install_file(columnstoreAlias ${ENGINE_SUPPORTDIR})
columnstore_install_file(columnstoreSyslog ${ENGINE_SUPPORTDIR})
columnstore_install_file(columnstoreSyslog7 ${ENGINE_SUPPORTDIR})
columnstore_install_file(columnstoreSyslog-ng ${ENGINE_SUPPORTDIR})
columnstore_install_file(columnstoreLogRotate ${ENGINE_SUPPORTDIR})
columnstore_install_file(myCnf-include-args.text ${ENGINE_SUPPORTDIR})
columnstore_install_file(myCnf-exclude-args.text ${ENGINE_SUPPORTDIR})
columnstore_install_file(mcs-workernode.service ${ENGINE_SUPPORTDIR})
columnstore_install_file(mcs-controllernode.service ${ENGINE_SUPPORTDIR})
columnstore_install_file(mcs-primproc.service ${ENGINE_SUPPORTDIR})
columnstore_install_file(mcs-writeengineserver.service ${ENGINE_SUPPORTDIR})
columnstore_install_file(mcs-dmlproc.service ${ENGINE_SUPPORTDIR})
columnstore_install_file(mcs-ddlproc.service ${ENGINE_SUPPORTDIR})
columnstore_install_file(mcs-loadbrm.service ${ENGINE_SUPPORTDIR})
columnstore_install_file(mcs-storagemanager.service ${ENGINE_SUPPORTDIR})
install(
FILES module
DESTINATION ${ENGINE_DATADIR}/local
COMPONENT columnstore-engine
)
columnstore_install_file(module ${ENGINE_DATADIR}/local)
find_package(Python3 COMPONENTS Interpreter)
if(Python3_FOUND)

11
oam/oamcpp/CMakeLists.txt
View File

@ -5,15 +5,6 @@ include_directories(${ENGINE_COMMON_INCLUDES})
set(oamcpp_LIB_SRCS liboamcpp.cpp oamcache.cpp)
columnstore_library(oamcpp ${oamcpp_LIB_SRCS})
add_dependencies(oamcpp loggingcpp)
target_link_libraries(oamcpp)
columnstore_link(oamcpp loggingcpp)
target_compile_options(oamcpp PRIVATE -Wno-unused-result)
install(
TARGETS oamcpp
DESTINATION ${ENGINE_LIBDIR}
COMPONENT columnstore-engine
)

10
oamapps/columnstoreDB/CMakeLists.txt
View File

@ -4,12 +4,6 @@ include_directories(${ENGINE_COMMON_INCLUDES})
set(columnstoreDBWrite_SRCS columnstoreDB.cpp)
add_executable(columnstoreDBWrite ${columnstoreDBWrite_SRCS})
columnstore_executable(columnstoreDBWrite ${columnstoreDBWrite_SRCS})
target_link_libraries(columnstoreDBWrite ${ENGINE_LDFLAGS} ncurses ${ENGINE_EXEC_LIBS})
install(
TARGETS columnstoreDBWrite
DESTINATION ${ENGINE_BINDIR}
COMPONENT columnstore-engine
)
columnstore_link(columnstoreDBWrite ${ENGINE_LDFLAGS} ncurses ${ENGINE_EXEC_LIBS})

23
oamapps/columnstoreSupport/CMakeLists.txt
View File

@ -4,20 +4,13 @@ include_directories(${ENGINE_COMMON_INCLUDES})
set(columnstoreSupport_SRCS columnstoreSupport.cpp mcsSupportUtil.cpp)
add_executable(columnstoreSupport ${columnstoreSupport_SRCS})
columnstore_executable(columnstoreSupport ${columnstoreSupport_SRCS})
target_compile_options(columnstoreSupport PRIVATE -Wno-unused-result)
columnstore_link(columnstoreSupport ${ENGINE_LDFLAGS} ncurses ${ENGINE_EXEC_LIBS})
target_link_libraries(columnstoreSupport ${ENGINE_LDFLAGS} ncurses ${ENGINE_EXEC_LIBS})
install(
TARGETS columnstoreSupport
DESTINATION ${ENGINE_BINDIR}
COMPONENT columnstore-engine
)
install(
PROGRAMS dbmsReport.sh bulklogReport.sh configReport.sh hardwareReport.sh logReport.sh resourceReport.sh
DESTINATION ${ENGINE_BINDIR}
COMPONENT columnstore-engine
)
columnstore_install_program(dbmsReport.sh, ${ENGINE_BINDIR})
columnstore_install_program(bulklogReport.sh, ${ENGINE_BINDIR})
columnstore_install_program(configReport.sh, ${ENGINE_BINDIR})
columnstore_install_program(hardwareReport.sh, ${ENGINE_BINDIR})
columnstore_install_program(logReport.sh, ${ENGINE_BINDIR})
columnstore_install_program(resourceReport.sh, ${ENGINE_BINDIR})

12
oamapps/postConfigure/CMakeLists.txt
View File

@ -4,14 +4,6 @@ include_directories(${ENGINE_COMMON_INCLUDES})
set(mycnfUpgrade_SRCS mycnfUpgrade.cpp)
add_executable(mycnfUpgrade ${mycnfUpgrade_SRCS})
columnstore_executable(mycnfUpgrade ${mycnfUpgrade_SRCS})
target_compile_options(mycnfUpgrade PRIVATE -Wno-unused-result)
target_link_libraries(mycnfUpgrade ${ENGINE_LDFLAGS} ${ENGINE_READLINE_LIBRARY} ncurses ${ENGINE_EXEC_LIBS})
install(
TARGETS mycnfUpgrade
DESTINATION ${ENGINE_BINDIR}
COMPONENT columnstore-engine
)
columnstore_link(mycnfUpgrade ${ENGINE_LDFLAGS} ${ENGINE_READLINE_LIBRARY} ncurses ${ENGINE_EXEC_LIBS})

3
oamapps/postConfigure/mycnfUpgrade.cpp
View File

@ -46,6 +46,7 @@
#include <stdio.h>
#include <string>
#include <unistd.h>
#include <tuple>
#include <vector>
#include <sys/sysinfo.h>
@ -282,7 +283,7 @@ int main(int argc, char* argv[])
USER = p;
string cmd = "chown " + USER + ":" + USER + " " + mycnfFile;
system(cmd.c_str());
std::ignore = std::system(cmd.c_str());
exit(0);
}

12
primitives/blockcache/CMakeLists.txt
View File

@ -13,13 +13,5 @@ set(dbbc_STAT_SRCS
stats.cpp
fsutils.cpp
)
# libdbbc_a_CXXFLAGS = $(march_flags) $(AM_CXXFLAGS)
add_library(dbbc STATIC ${dbbc_STAT_SRCS})
add_dependencies(dbbc loggingcpp)
target_link_libraries(dbbc ${NETSNMP_LIBRARIES})
install(TARGETS dbbc DESTINATION ${ENGINE_LIBDIR})
columnstore_static_library(dbbc ${dbbc_STAT_SRCS})
columnstore_link(dbbc ${NETSNMP_LIBRARIES} loggingcpp)

9
primitives/linux-port/CMakeLists.txt
View File

@ -4,10 +4,5 @@ include_directories(${ENGINE_COMMON_INCLUDES} ../blockcache ../primproc)
set(processor_STAT_SRCS primitiveprocessor.cpp dictionary.cpp column.cpp)
add_library(processor STATIC ${processor_STAT_SRCS})
add_dependencies(processor loggingcpp)
target_link_libraries(processor ${NETSNMP_LIBRARIES})
install(TARGETS processor DESTINATION ${ENGINE_LIBDIR})
columnstore_static_library(processor ${processor_STAT_SRCS})
columnstore_link(processor ${NETSNMP_LIBRARIES} loggingcpp)

12
primitives/primproc/CMakeLists.txt
View File

@ -25,11 +25,10 @@ set(PrimProc_SRCS
../../utils/common/crashtrace.cpp
)
add_executable(PrimProc ${PrimProc_SRCS})
columnstore_executable(PrimProc ${PrimProc_SRCS})
add_dependencies(PrimProc loggingcpp)
target_include_directories(PrimProc PRIVATE ${Boost_INCLUDE_DIRS})
target_link_libraries(
columnstore_link(
PrimProc
${ENGINE_LDFLAGS}
${NETSNMP_LIBRARIES}
@ -38,10 +37,5 @@ target_link_libraries(
cacheutils
dbbc
processor
)
install(
TARGETS PrimProc
DESTINATION ${ENGINE_BINDIR}
COMPONENT columnstore-engine
loggingcpp
)

57
storage-manager/CMakeLists.txt
View File

@ -64,11 +64,11 @@ configure_file(
link_directories(${CMAKE_BINARY_DIR}/lib)
set(CMAKE_INSTALL_RPATH $ORIGIN $ORIGIN/../lib)
add_library(storagemanager SHARED ${storagemanager_SRCS})
columnstore_library(storagemanager SHARED ${storagemanager_SRCS})
add_dependencies(storagemanager marias3 external_boost)
target_compile_definitions(storagemanager PUBLIC BOOST_NO_CXX11_SCOPED_ENUMS)
target_link_libraries(
columnstore_link(
storagemanager
boost_chrono
boost_system
@ -80,8 +80,8 @@ target_link_libraries(
)
target_include_directories(storagemanager PRIVATE ${Boost_INCLUDE_DIRS})
add_executable(StorageManager src/main.cpp)
target_link_libraries(StorageManager storagemanager)
columnstore_executable(StorageManager src/main.cpp)
columnstore_link(StorageManager storagemanager)
set_property(TARGET StorageManager PROPERTY CXX_STANDARD 20)
set(TMPDIR ${CMAKE_RUNTIME_OUTPUT_DIRECTORY})
@ -89,11 +89,11 @@ set(CMAKE_RUNTIME_OUTPUT_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR})
add_executable(unit_tests src/unit_tests.cpp)
target_compile_definitions(unit_tests PUBLIC BOOST_NO_CXX11_SCOPED_ENUMS)
target_link_libraries(unit_tests storagemanager)
columnstore_link(unit_tests storagemanager)
add_executable(testS3Connection src/testS3Connection.cpp)
columnstore_executable(testS3Connection src/testS3Connection.cpp)
target_compile_definitions(testS3Connection PUBLIC BOOST_NO_CXX11_SCOPED_ENUMS)
target_link_libraries(testS3Connection storagemanager)
columnstore_link(testS3Connection storagemanager)
set(CMAKE_RUNTIME_OUTPUT_DIRECTORY ${TMPDIR})
@ -112,42 +112,25 @@ add_custom_command(
# The includes and lib linkages required to link against cloudio ... pretty crazy. When lib dependencies are eventually
# config'd right, change this to only include and link against cloudio.
include_directories(${ENGINE_SRC_DIR}/utils/cloudio ${ENGINE_COMMON_INCLUDES})
add_executable(smcat src/smcat.cpp)
target_link_libraries(smcat storagemanager cloudio ${ENGINE_LDFLAGS} ${ENGINE_EXEC_LIBS})
columnstore_executable(smcat src/smcat.cpp)
columnstore_link(smcat storagemanager cloudio ${ENGINE_LDFLAGS} ${ENGINE_EXEC_LIBS})
add_executable(smput src/smput.cpp)
target_link_libraries(smput storagemanager cloudio ${ENGINE_LDFLAGS} ${ENGINE_EXEC_LIBS})
columnstore_executable(smput src/smput.cpp)
columnstore_link(smput storagemanager cloudio ${ENGINE_LDFLAGS} ${ENGINE_EXEC_LIBS})
add_executable(smls src/smls.cpp)
target_link_libraries(smls storagemanager cloudio ${ENGINE_LDFLAGS} ${ENGINE_EXEC_LIBS})
columnstore_executable(smls src/smls.cpp)
columnstore_link(smls storagemanager cloudio ${ENGINE_LDFLAGS} ${ENGINE_EXEC_LIBS})
add_executable(smrm src/smrm.cpp)
target_link_libraries(smrm storagemanager cloudio ${ENGINE_LDFLAGS} ${ENGINE_EXEC_LIBS})
columnstore_executable(smrm src/smrm.cpp)
columnstore_link(smrm storagemanager cloudio ${ENGINE_LDFLAGS} ${ENGINE_EXEC_LIBS})
add_executable(smps src/smps.cpp)
target_link_libraries(smps storagemanager cloudio ${ENGINE_LDFLAGS} ${ENGINE_EXEC_LIBS})
columnstore_executable(smps src/smps.cpp)
columnstore_link(smps storagemanager cloudio ${ENGINE_LDFLAGS} ${ENGINE_EXEC_LIBS})
add_executable(smkill src/smkill.cpp)
target_link_libraries(smkill storagemanager cloudio ${ENGINE_LDFLAGS} ${ENGINE_EXEC_LIBS})
install(TARGETS storagemanager LIBRARY DESTINATION ${ENGINE_LIBDIR} COMPONENT columnstore-engine)
columnstore_executable(smkill src/smkill.cpp)
columnstore_link(smkill storagemanager cloudio ${ENGINE_LDFLAGS} ${ENGINE_EXEC_LIBS})
install(
TARGETS StorageManager
smcat
smkill
smps
smput
smls
smrm
testS3Connection
RUNTIME DESTINATION ${ENGINE_BINDIR} COMPONENT columnstore-engine
)
install(
FILES storagemanager.cnf
DESTINATION ${ENGINE_SYSCONFDIR}/columnstore
COMPONENT columnstore-engine
)
columnstore_install_file(storagemanager.cnf ${ENGINE_SYSCONFDIR}/columnstore)
install(
FILES storagemanager.cnf

34
tests/CMakeLists.txt
View File

@ -30,82 +30,80 @@ if(WITH_UNITTESTS)
add_executable(rowgroup_tests rowgroup-tests.cpp)
target_compile_options(rowgroup_tests PRIVATE -Wno-sign-compare)
add_dependencies(rowgroup_tests googletest)
target_link_libraries(rowgroup_tests ${ENGINE_LDFLAGS} ${GTEST_LIBRARIES} ${ENGINE_EXEC_LIBS})
columnstore_link(rowgroup_tests ${ENGINE_LDFLAGS} ${GTEST_LIBRARIES} ${ENGINE_EXEC_LIBS})
gtest_add_tests(TARGET rowgroup_tests TEST_PREFIX columnstore:)
add_executable(rewritetest rewritetest.cpp)
add_dependencies(rewritetest googletest)
target_link_libraries(rewritetest ${ENGINE_LDFLAGS} ${GTEST_LIBRARIES} ${ENGINE_EXEC_LIBS})
columnstore_link(rewritetest ${ENGINE_LDFLAGS} ${GTEST_LIBRARIES} ${ENGINE_EXEC_LIBS})
gtest_add_tests(TARGET rewritetest TEST_PREFIX columnstore:)
add_executable(mcs_decimal_tests mcs_decimal-tests.cpp)
add_dependencies(mcs_decimal_tests googletest)
target_link_libraries(mcs_decimal_tests ${ENGINE_LDFLAGS} ${GTEST_LIBRARIES} ${ENGINE_EXEC_LIBS})
columnstore_link(mcs_decimal_tests ${ENGINE_LDFLAGS} ${GTEST_LIBRARIES} ${ENGINE_EXEC_LIBS})
gtest_add_tests(TARGET mcs_decimal_tests TEST_PREFIX columnstore:)
add_executable(dataconvert_tests dataconvert-tests.cpp)
add_dependencies(dataconvert_tests googletest)
target_link_libraries(dataconvert_tests ${ENGINE_LDFLAGS} ${GTEST_LIBRARIES} ${ENGINE_EXEC_LIBS})
columnstore_link(dataconvert_tests ${ENGINE_LDFLAGS} ${GTEST_LIBRARIES} ${ENGINE_EXEC_LIBS})
gtest_add_tests(TARGET dataconvert_tests TEST_PREFIX columnstore:)
add_executable(rebuild_em_tests rebuild-em-tests.cpp)
add_dependencies(rebuild_em_tests googletest)
target_link_libraries(rebuild_em_tests ${ENGINE_LDFLAGS} ${GTEST_LIBRARIES} ${ENGINE_WRITE_LIBS})
columnstore_link(rebuild_em_tests ${ENGINE_LDFLAGS} ${GTEST_LIBRARIES} ${ENGINE_WRITE_LIBS})
gtest_add_tests(TARGET rebuild_em_tests TEST_PREFIX columnstore:)
add_executable(compression_tests compression-tests.cpp)
add_dependencies(compression_tests googletest)
target_link_libraries(compression_tests ${ENGINE_LDFLAGS} ${GTEST_LIBRARIES} ${ENGINE_WRITE_LIBS})
columnstore_link(compression_tests ${ENGINE_LDFLAGS} ${GTEST_LIBRARIES} ${ENGINE_WRITE_LIBS})
gtest_add_tests(TARGET compression_tests TEST_PREFIX columnstore:)
add_executable(column_scan_filter_tests primitives_column_scan_and_filter.cpp)
target_compile_options(column_scan_filter_tests PRIVATE -Wno-error -Wno-sign-compare)
add_dependencies(column_scan_filter_tests googletest)
target_link_libraries(
column_scan_filter_tests ${ENGINE_LDFLAGS} ${ENGINE_WRITE_LIBS} ${GTEST_LIBRARIES} processor dbbc
)
columnstore_link(column_scan_filter_tests ${ENGINE_LDFLAGS} ${ENGINE_WRITE_LIBS} ${GTEST_LIBRARIES} processor dbbc)
gtest_add_tests(TARGET column_scan_filter_tests TEST_PREFIX columnstore:)
add_executable(simd_processors simd_processors.cpp)
target_compile_options(simd_processors PRIVATE -Wno-error)
add_dependencies(simd_processors googletest)
target_link_libraries(simd_processors ${ENGINE_LDFLAGS} ${ENGINE_WRITE_LIBS} ${GTEST_LIBRARIES} processor dbbc)
columnstore_link(simd_processors ${ENGINE_LDFLAGS} ${ENGINE_WRITE_LIBS} ${GTEST_LIBRARIES} processor dbbc)
gtest_add_tests(TARGET simd_processors TEST_PREFIX columnstore:)
add_executable(fair_threadpool_test fair_threadpool.cpp)
add_dependencies(fair_threadpool_test googletest)
target_link_libraries(fair_threadpool_test ${ENGINE_LDFLAGS} ${ENGINE_WRITE_LIBS} ${GTEST_LIBRARIES} processor dbbc)
columnstore_link(fair_threadpool_test ${ENGINE_LDFLAGS} ${ENGINE_WRITE_LIBS} ${GTEST_LIBRARIES} processor dbbc)
gtest_add_tests(TARGET fair_threadpool_test TEST_PREFIX columnstore:)
add_executable(counting_allocator counting_allocator.cpp)
target_compile_options(counting_allocator PRIVATE -Wno-sign-compare)
add_dependencies(counting_allocator googletest)
target_link_libraries(counting_allocator ${ENGINE_LDFLAGS} ${ENGINE_WRITE_LIBS} ${GTEST_LIBRARIES})
columnstore_link(counting_allocator ${ENGINE_LDFLAGS} ${ENGINE_WRITE_LIBS} ${GTEST_LIBRARIES})
gtest_add_tests(TARGET counting_allocator TEST_PREFIX columnstore:)
add_executable(poolallocator poolallocator.cpp)
target_compile_options(poolallocator PRIVATE -Wno-sign-compare)
add_dependencies(poolallocator googletest)
target_link_libraries(poolallocator ${ENGINE_LDFLAGS} ${ENGINE_WRITE_LIBS} ${GTEST_LIBRARIES})
columnstore_link(poolallocator ${ENGINE_LDFLAGS} ${ENGINE_WRITE_LIBS} ${GTEST_LIBRARIES})
gtest_add_tests(TARGET poolallocator TEST_PREFIX columnstore:)
add_executable(stlpoolallocator stlpoolallocator.cpp)
target_compile_options(stlpoolallocator PRIVATE -Wno-sign-compare)
add_dependencies(stlpoolallocator googletest)
target_link_libraries(stlpoolallocator ${ENGINE_LDFLAGS} ${ENGINE_WRITE_LIBS} ${GTEST_LIBRARIES})
columnstore_link(stlpoolallocator ${ENGINE_LDFLAGS} ${ENGINE_WRITE_LIBS} ${GTEST_LIBRARIES})
gtest_add_tests(TARGET stlpoolallocator TEST_PREFIX columnstore:)
add_executable(comparators_tests comparators-tests.cpp)
target_link_libraries(comparators_tests ${ENGINE_LDFLAGS} ${ENGINE_WRITE_LIBS} ${CPPUNIT_LIBRARIES} cppunit)
columnstore_link(comparators_tests ${ENGINE_LDFLAGS} ${ENGINE_WRITE_LIBS} ${CPPUNIT_LIBRARIES} cppunit)
add_test(NAME columnstore:comparators_tests COMMAND comparators_tests)
add_executable(bytestream bytestream.cpp)
target_link_libraries(bytestream ${ENGINE_LDFLAGS} ${ENGINE_WRITE_LIBS} ${CPPUNIT_LIBRARIES} cppunit)
columnstore_link(bytestream ${ENGINE_LDFLAGS} ${ENGINE_WRITE_LIBS} ${CPPUNIT_LIBRARIES} cppunit)
add_test(NAME columnstore:bytestream COMMAND bytestream)
# standalone EM routines test
add_executable(brm_em_standalone brm-em-standalone.cpp)
target_link_libraries(brm_em_standalone ${ENGINE_LDFLAGS} ${ENGINE_WRITE_LIBS} ${CPPUNIT_LIBRARIES} cppunit)
columnstore_link(brm_em_standalone ${ENGINE_LDFLAGS} ${ENGINE_WRITE_LIBS} ${CPPUNIT_LIBRARIES} cppunit)
add_test(NAME columnstore:brm_em_standalone COMMAND brm_em_standalone)
set_tests_properties(columnstore:brm_em_standalone PROPERTIES DISABLED True)
endif()
@ -116,7 +114,7 @@ if(WITH_MICROBENCHMARKS AND (NOT CMAKE_BUILD_TYPE STREQUAL "Debug"))
target_include_directories(
primitives_scan_bench PUBLIC ${ENGINE_COMMON_INCLUDES} ${ENGINE_BLOCKCACHE_INCLUDE} ${ENGINE_PRIMPROC_INCLUDE}
)
target_link_libraries(
columnstore_link(
primitives_scan_bench
${ENGINE_LDFLAGS}
${ENGINE_WRITE_LIBS}

258
tools/bincvt/li2bin.cpp
View File

@ -1,258 +0,0 @@
#include <iostream>
#include <inttypes.h>
#include <string>
#include <cstring>
#include <cstdlib>
//#define NDEBUG
#include <cassert>
using namespace std;
#include <boost/tokenizer.hpp>
#include <boost/static_assert.hpp>
using namespace boost;
// 1|155190|7706|1|17|21168.23|0.04|0.02|N|O|1996-03-13|1996-02-12|1996-03-22|DELIVER IN PERSON|TRUCK|egular
// courts above the|
#pragma pack(1)
struct lineitem_image
{
int32_t l_orderkey;
int32_t l_partkey;
int32_t l_suppkey; // 12
int64_t l_linenumber;
int64_t l_quantity;
int64_t l_extendedprice;
int64_t l_discount;
int64_t l_tax; // 40
char l_returnflag;
char l_linestatus; // 2
int32_t l_shipdate;
int32_t l_commitdate;
int32_t l_receiptdate; // 12
char l_shipinstruct[25];
char l_shipmode[10];
char l_comment[44]; // 79
};
BOOST_STATIC_ASSERT(sizeof(struct lineitem_image) == 145);
// version of lineitem_image that treats some integer columns as unsigned ints.
// Decimal, date, and date/time columns are still treated as integer.
struct unsigned_lineitem_image
{
uint32_t l_orderkey;
uint32_t l_partkey;
uint32_t l_suppkey; // 12
uint64_t l_linenumber;
int64_t l_quantity;
int64_t l_extendedprice;
int64_t l_discount;
int64_t l_tax; // 40
char l_returnflag;
char l_linestatus; // 2
int32_t l_shipdate;
int32_t l_commitdate;
int32_t l_receiptdate; // 12
char l_shipinstruct[25];
char l_shipmode[10];
char l_comment[44]; // 79
};
BOOST_STATIC_ASSERT(sizeof(struct unsigned_lineitem_image) == 145);
struct Date
{
unsigned spare : 6;
unsigned day : 6;
unsigned month : 4;
unsigned year : 16;
};
BOOST_STATIC_ASSERT(sizeof(struct Date) == 4);
union date_image
{
struct Date d;
int32_t i;
};
namespace
{
typedef boost::tokenizer<boost::char_separator<char> > tokenizer;
int64_t idbbigdec2(const string& str)
{
int64_t val = 0;
string::size_type idx = string::npos;
string tmp(str);
idx = tmp.find('.');
if (idx != string::npos)
tmp.erase(idx, 1);
else
tmp.append("00");
val = atoll(tmp.c_str());
return val;
}
int32_t idbdate(const string& str)
{
date_image di;
di.i = 0;
boost::char_separator<char> sep("-");
tokenizer tokens(str, sep);
tokenizer::iterator tok_iter = tokens.begin();
// Note that the spare bits must be set, instead of leaving them
// initialized to 0; to be compatible with internal date format.
if (distance(tok_iter, tokens.end()) == 3)
{
di.d.spare = 0x3E;
di.d.year = atoi(tok_iter->c_str());
++tok_iter;
di.d.month = atoi(tok_iter->c_str());
++tok_iter;
di.d.day = atoi(tok_iter->c_str());
++tok_iter;
}
return di.i;
}
int parseinto(lineitem_image& img, const string& buf)
{
memset(&img, 0, sizeof(img));
boost::char_separator<char> sep("|");
tokenizer tokens(buf, sep);
tokenizer::iterator tok_iter = tokens.begin();
if (distance(tok_iter, tokens.end()) < 16)
return -1;
img.l_orderkey = atoi(tok_iter->c_str());
++tok_iter;
img.l_partkey = atoi(tok_iter->c_str());
++tok_iter;
img.l_suppkey = atoi(tok_iter->c_str());
++tok_iter;
img.l_linenumber = atoll(tok_iter->c_str());
++tok_iter;
img.l_quantity = idbbigdec2(tok_iter->c_str());
++tok_iter;
img.l_extendedprice = idbbigdec2(tok_iter->c_str());
++tok_iter;
img.l_discount = idbbigdec2(tok_iter->c_str());
++tok_iter;
img.l_tax = idbbigdec2(tok_iter->c_str());
++tok_iter;
img.l_returnflag = tok_iter->at(0);
++tok_iter;
img.l_linestatus = tok_iter->at(0);
++tok_iter;
img.l_shipdate = idbdate(tok_iter->c_str());
++tok_iter;
img.l_commitdate = idbdate(tok_iter->c_str());
++tok_iter;
img.l_receiptdate = idbdate(tok_iter->c_str());
++tok_iter;
memcpy(&img.l_shipinstruct[0], tok_iter->c_str(), tok_iter->size());
++tok_iter;
memcpy(&img.l_shipmode[0], tok_iter->c_str(), tok_iter->size());
++tok_iter;
memcpy(&img.l_comment[0], tok_iter->c_str(), tok_iter->size());
++tok_iter;
return 0;
}
int unsigned_parseinto(unsigned_lineitem_image& img, const string& buf)
{
memset(&img, 0, sizeof(img));
boost::char_separator<char> sep("|");
tokenizer tokens(buf, sep);
tokenizer::iterator tok_iter = tokens.begin();
if (distance(tok_iter, tokens.end()) < 16)
return -1;
img.l_orderkey = strtoul(tok_iter->c_str(), 0, 10);
++tok_iter;
img.l_partkey = strtoul(tok_iter->c_str(), 0, 10);
++tok_iter;
img.l_suppkey = strtoul(tok_iter->c_str(), 0, 10);
++tok_iter;
img.l_linenumber = strtoull(tok_iter->c_str(), 0, 10);
++tok_iter;
img.l_quantity = idbbigdec2(tok_iter->c_str());
++tok_iter;
img.l_extendedprice = idbbigdec2(tok_iter->c_str());
++tok_iter;
img.l_discount = idbbigdec2(tok_iter->c_str());
++tok_iter;
img.l_tax = idbbigdec2(tok_iter->c_str());
++tok_iter;
img.l_returnflag = tok_iter->at(0);
++tok_iter;
img.l_linestatus = tok_iter->at(0);
++tok_iter;
img.l_shipdate = idbdate(tok_iter->c_str());
++tok_iter;
img.l_commitdate = idbdate(tok_iter->c_str());
++tok_iter;
img.l_receiptdate = idbdate(tok_iter->c_str());
++tok_iter;
memcpy(&img.l_shipinstruct[0], tok_iter->c_str(), tok_iter->size());
++tok_iter;
memcpy(&img.l_shipmode[0], tok_iter->c_str(), tok_iter->size());
++tok_iter;
memcpy(&img.l_comment[0], tok_iter->c_str(), tok_iter->size());
++tok_iter;
return 0;
}
} // namespace
int main(int argc, char** argv)
{
if ((argc > 1) && (strcmp(argv[1], "-h") == 0))
{
std::cerr << "li2bin [-u]" << std::endl;
std::cerr << " -u Create first 4 fields as unsigned integers" << std::endl;
return 0;
}
string input;
getline(cin, input);
if ((argc > 1) && (strcmp(argv[1], "-u") == 0))
{
unsigned_lineitem_image i;
while (!cin.eof())
{
if (unsigned_parseinto(i, input) == 0)
cout.write(reinterpret_cast<const char*>(&i), sizeof(i));
getline(cin, input);
}
}
else
{
lineitem_image i;
while (!cin.eof())
{
if (parseinto(i, input) == 0)
cout.write(reinterpret_cast<const char*>(&i), sizeof(i));
getline(cin, input);
}
}
return 0;
}

77
tools/cfread/CMakeLists.txt
View File

@ -1,77 +0,0 @@
include_directories(${KDE4_INCLUDES} ${KDE4_INCLUDE_DIR} ${QT_INCLUDES})
# ########## next target ###############
set(cfread_SRCS cfread.cpp)
kde4_add_executable(cfread ${cfread_SRCS})
target_link_libraries(
cfread
${KDE4_KDECORE_LIBS}
pthread
writeengine
brm
rwlock
messageqcpp
dl
configcpp
xml2
loggingcpp
cacheutils
boost_idb
pthread
rt
)
install(TARGETS cfread ${INSTALL_TARGETS_DEFAULT_ARGS})
# ########## next target ###############
set(mtread_SRCS mtread.cpp)
kde4_add_executable(mtread ${mtread_SRCS})
target_link_libraries(
mtread
${KDE4_KDECORE_LIBS}
pthread
writeengine
brm
rwlock
messageqcpp
dl
configcpp
xml2
loggingcpp
cacheutils
boost_idb
pthread
rt
)
install(TARGETS mtread ${INSTALL_TARGETS_DEFAULT_ARGS})
# ########## install files ###############
# original Makefile.am contents follow:
# $Id: Makefile.am 333 2009-04-03 20:35:04Z rdempsey $ Process this file with automake to produce Makefile.in
#
# AM_CPPFLAGS = $(idb_cppflags) AM_CFLAGS = $(idb_cflags) AM_CXXFLAGS = $(idb_cxxflags) AM_LDFLAGS = $(idb_ldflags)
# bin_PROGRAMS = cfread mtread cfread_SOURCES = cfread.cpp cfread_CPPFLAGS = @idb_common_includes@ $(AM_CPPFLAGS)
# cfread_LDFLAGS = @idb_common_ldflags@ -lpthread -lwriteengine -lbrm -lrwlock -lmessageqcpp -ldl -lconfigcpp -lxml2
# -lloggingcpp -lcacheutils -lboost_idb -lpthread -lrt $(AM_LDFLAGS) mtread_SOURCES = mtread.cpp mtread_CPPFLAGS =
# @idb_common_includes@ $(AM_CPPFLAGS) mtread_LDFLAGS = @idb_common_ldflags@ -lpthread -lwriteengine -lbrm -lrwlock
# -lmessageqcpp -ldl -lconfigcpp -lxml2 -lloggingcpp -lcacheutils -lboost_idb -lpthread -lrt $(AM_LDFLAGS)
#
# test:
#
# coverage:
#
# leakcheck:
#
# docs:
#
# bootstrap: install-data-am
#

183
tools/cfread/cfread.cpp
View File

@ -1,183 +0,0 @@
//
#include <iostream>
#include <iomanip>
#include <boost/scoped_array.hpp>
#include <errno.h>
#define NDEBUG
#include <cassert>
#include "writeengine.h"
using namespace std;
void timespec_sub(const struct timespec& tv1, const struct timespec& tv2, struct timespec& diff)
{
if (tv2.tv_nsec < tv1.tv_nsec)
{
diff.tv_sec = tv2.tv_sec - tv1.tv_sec - 1;
diff.tv_nsec = tv1.tv_nsec - tv2.tv_nsec;
}
else
{
diff.tv_sec = tv2.tv_sec - tv1.tv_sec;
diff.tv_nsec = tv2.tv_nsec - tv1.tv_nsec;
}
}
// TODO:
// add getopt for CLAs
// add threads for reading a file
// add threads for reading multiple files
// main()
//
int main(int argc, char** argv)
{
uint64_t acc = 0;
uint64_t readBlocks = 0; // read size ib blocks
uint64_t readSize = 0; // read size ib bytes
uint64_t readBufferSz = 0;
const uint64_t blockSize = 8192;
char* alignedbuff = 0;
boost::scoped_array<char> realbuff;
const unsigned pageSize = 4096; // getpagesize();
WriteEngine::FileOp fFileOp;
BRM::OID_t oid;
char fname[256];
struct timespec tm;
struct timespec tm2;
struct timespec tm3;
struct timespec starttm;
struct timespec endtm;
struct timespec tottm;
bool odirect = true;
int fd = 0;
char response = 'Y';
if (argc <= 1)
{
cerr << "usage: testread <oid> <buffer size in blocks>" << endl;
return -1;
}
oid = atoi(argv[1]);
if (oid <= 0)
exit(-1);
if (argc >= 2)
{
readBlocks = atoi(argv[2]);
if (readBlocks <= 0)
readBlocks = 8;
}
if (argc >= 4)
{
odirect = false;
}
readSize = readBlocks * blockSize;
readBufferSz = readSize + pageSize;
realbuff.reset(new char[readBufferSz]);
if (realbuff.get() == 0)
{
cerr << "thr_popper: Can't allocate space for a whole extent in memory" << endl;
return 0;
}
if (fFileOp.getFileName(oid, fname) != WriteEngine::NO_ERROR)
{
fname[0] = 0;
throw std::runtime_error("fileOp.getFileName failed");
}
else
{
cout << "Reading oid: " << oid << " od: " << odirect << " file: " << fname << endl;
}
#if __LP64__
alignedbuff = (char*)((((ptrdiff_t)realbuff.get() >> 12) << 12) + pageSize);
#else
alignedbuff = (char*)(((((ptrdiff_t)realbuff.get() >> 12) << 12) & 0xffffffff) + pageSize);
#endif
idbassert(((ptrdiff_t)alignedbuff - (ptrdiff_t)realbuff.get()) < (ptrdiff_t)pageSize);
idbassert(((ptrdiff_t)alignedbuff % pageSize) == 0);
if (odirect)
fd = open(fname, O_RDONLY | O_DIRECT | O_LARGEFILE | O_NOATIME);
else
fd = open(fname, O_RDONLY | O_LARGEFILE | O_NOATIME);
if (fd < 0)
{
cerr << "Open failed" << endl;
perror("open");
throw runtime_error("Error opening file");
}
while (toupper(response) != 'N')
{
uint64_t i = 1;
uint64_t rCnt = 0;
clock_gettime(CLOCK_REALTIME, &starttm);
while (i != 0)
{
// clock_gettime(CLOCK_REALTIME, &tm);
i = pread(fd, alignedbuff, readSize, acc);
// clock_gettime(CLOCK_REALTIME, &tm2);
idbassert(i == 0 || i == readSize);
idbassert(i % pageSize == 0);
idbassert(acc % pageSize == 0);
if (i < 0 && errno == EINTR)
{
timespec_sub(tm, tm2, tm3);
cout << "* " << i << " " << right << setw(2) << setfill(' ') << tm3.tv_sec << "." << right << setw(9)
<< setfill('0') << tm3.tv_nsec << endl;
continue;
}
else if (i < 0)
{
timespec_sub(tm, tm2, tm3);
cout << "* i: " << i << " sz: " << readSize << " acc: " << acc << right << setw(2) << setfill(' ')
<< tm3.tv_sec << " " << right << tm3.tv_nsec << endl;
perror("pread");
// make loop exit
i = 0;
}
acc += i;
if (i > 0)
rCnt++;
// timespec_sub(tm, tm2, tm3);
// cout
// << i << " "
// << right << setw(2) << setfill(' ') << tm3.tv_sec << " "
// << right << tm3.tv_nsec
// << endl;
} // while(acc...
clock_gettime(CLOCK_REALTIME, &endtm);
timespec_sub(starttm, endtm, tottm);
cout << "Total reads: " << rCnt << " sz: " << acc / (1024 * 1024) << "MB"
<< " tm: " << tottm.tv_sec << "secs " << tottm.tv_nsec << "ns" << endl;
cout << "Repeat the last scan[Y,N]?" << endl;
cin >> response;
acc = 0;
} // while response...
close(fd);
return 0;
} // main

269
tools/cfread/mtread.cpp
View File

@ -1,269 +0,0 @@
//
#include <iostream>
#include <iomanip>
#include <getopt.h>
#include <errno.h>
#include <boost/thread.hpp>
#define NDEBUG
#include <cassert>
#include "writeengine.h"
using namespace std;
void timespec_sub(const struct timespec& tv1, const struct timespec& tv2, struct timespec& diff)
{
if (tv2.tv_nsec < tv1.tv_nsec)
{
diff.tv_sec = tv2.tv_sec - tv1.tv_sec - 1;
diff.tv_nsec = tv1.tv_nsec - tv2.tv_nsec;
}
else
{
diff.tv_sec = tv2.tv_sec - tv1.tv_sec;
diff.tv_nsec = tv2.tv_nsec - tv1.tv_nsec;
}
}
// TODO:
// add threads for reading a file
// add threads for reading multiple files
struct readThr
{
public:
readThr(const int oid, const int rSize)
{
fblockSize = 8192;
fpageSize = getpagesize();
freadBlocks = rSize;
freadSize = rSize * fblockSize; // read size ib bytes
freadBufferSz = freadSize + fpageSize;
falignedbuff = 0;
foid = oid;
facc = 0;
memset(fname, 0, sizeof(fname));
memset((char*)&ftm, 0, sizeof(ftm));
memset((char*)&ftm2, 0, sizeof(ftm2));
memset((char*)&ftm3, 0, sizeof(ftm3));
memset((char*)&fstarttm, 0, sizeof(fstarttm));
memset((char*)&fendtm, 0, sizeof(fendtm));
memset((char*)&ftottm, 0, sizeof(ftottm));
fodirect = true;
fd = 0;
cout << "o: " << foid << " r: " << freadBlocks << " b: " << freadBufferSz << endl;
}
void operator()()
{
WriteEngine::FileOp fFileOp;
char frealbuff[freadBufferSz];
memset(frealbuff, 0, freadBufferSz);
if (frealbuff == 0)
{
cerr << "thr_popper: Can't allocate space for a whole extent in memory" << endl;
return;
}
if (fFileOp.getFileName(foid, fname) != WriteEngine::NO_ERROR)
{
fname[0] = 0;
throw std::runtime_error("fileOp.getFileName failed");
}
else
{
cout << "Reading oid: " << foid << " od: " << fodirect << " file: " << fname << endl;
}
#if __LP64__
falignedbuff = (char*)((((ptrdiff_t)frealbuff >> 12) << 12) + fpageSize);
#else
falignedbuff = (char*)(((((ptrdiff_t)frealbuff >> 12) << 12) & 0xffffffff) + fpageSize);
#endif
idbassert(((ptrdiff_t)falignedbuff - (ptrdiff_t)frealbuff) < (ptrdiff_t)fpageSize);
idbassert(((ptrdiff_t)falignedbuff % fpageSize) == 0);
if (fodirect)
fd = open(fname, O_RDONLY | O_DIRECT | O_LARGEFILE | O_NOATIME);
else
fd = open(fname, O_RDONLY | O_LARGEFILE | O_NOATIME);
if (fd < 0)
{
cerr << "Open failed" << endl;
perror("open");
throw runtime_error("Error opening file");
}
uint64_t i = 1;
uint64_t rCnt = 0;
clock_gettime(CLOCK_REALTIME, &fstarttm);
while (i > 0)
{
clock_gettime(CLOCK_REALTIME, &ftm);
i = pread(fd, falignedbuff, freadSize, facc);
clock_gettime(CLOCK_REALTIME, &ftm2);
idbassert(i == 0 || i == freadSize);
idbassert(i % fpageSize == 0);
idbassert(facc % fpageSize == 0);
if (i < 0 && errno == EINTR)
{
timespec_sub(ftm, ftm2, ftm3);
cout << "* " << i << " " << right << setw(2) << setfill(' ') << ftm3.tv_sec << "." << right << setw(9)
<< setfill('0') << ftm3.tv_nsec << endl;
continue;
}
else if (i < 0)
{
timespec_sub(ftm, ftm2, ftm3);
cout << "* i: " << i << " sz: " << freadSize << " acc: " << facc << right << setw(2) << setfill(' ')
<< ftm3.tv_sec << " " << right << ftm3.tv_nsec << endl;
perror("pread");
}
facc += i;
if (i > 0)
rCnt++;
/**
timespec_sub(ftm, ftm2, ftm3);
cout
<< rCnt << " " << facc/(1024*1024)
<< right << setw(2) << setfill(' ') << ftm3.tv_sec << "."
<< right << ftm3.tv_nsec << " i: " << i/(1024*1024)
<< endl;
**/
} // while(acc...
clock_gettime(CLOCK_REALTIME, &fendtm);
timespec_sub(fstarttm, fendtm, ftottm);
cout << "Total reads: " << rCnt << " sz: " << facc / (1024 * 1024) << "MB"
<< " tm: " << ftottm.tv_sec << "secs " << ftottm.tv_nsec << "ns" << endl;
facc = 0;
close(fd);
} // operator()
public:
uint64_t facc;
uint64_t freadBlocks; // read size ib blocks
uint64_t freadSize; // read size ib bytes
uint64_t freadBufferSz;
uint64_t fblockSize;
char* falignedbuff;
unsigned fpageSize;
BRM::OID_t foid;
char fname[256];
struct timespec ftm;
struct timespec ftm2;
struct timespec ftm3;
struct timespec fstarttm;
struct timespec fendtm;
struct timespec ftottm;
bool fodirect;
int fd;
}; // struct readThr
//
void usage()
{
cerr << "usage: mtread -o <oid> -s <read size in blocks>" << endl;
}
// main()
//
int main(int argc, char** argv)
{
int ch = 0;
int readBlocks = 0;
BRM::OID_t oid = 0;
std::vector<BRM::OID_t> oidList;
enum CLA_ENUM
{
OID = (int)0,
READSIZE
};
// longopt struct
// struct option {
// const char *name;
// int has_arg;
// int *flag;
// int val;
//};
static struct option long_options[] = {//{const char *name, int has_arg, int *flag, int val},
{"oid", required_argument, NULL, OID},
{"rsize", required_argument, NULL, READSIZE},
{0, 0, 0, 0}};
if (argc <= 1)
{
return -1;
}
// process command line arguments
while ((ch = getopt_long_only(argc, argv, "o:s:", long_options, NULL)) != -1)
{
// pid_t pidId = getpid();
switch (ch)
{
case OID:
case 'o':
oid = atoi(optarg);
oidList.push_back(oid);
cout << "oid: " << optarg << endl;
break;
case READSIZE:
case 's':
readBlocks = atoi(optarg);
cout << "read size: " << optarg << endl;
if (readBlocks <= 0)
readBlocks = 1;
break;
case '?':
default:
cout << "optarg " << optarg << endl;
usage();
break;
} // switch
} // while...
uint32_t idx = 0;
std::vector<boost::thread*> thrList;
while (idx < oidList.size())
{
struct readThr rdr(oidList[idx++], readBlocks);
boost::thread* thr = new boost::thread(rdr);
thrList.push_back(thr);
}
idx = 0;
while (idx < thrList.size())
{
boost::thread* thr = thrList[idx++];
thr->join();
delete thr;
}
thrList.clear();
} // main

10
tools/clearShm/CMakeLists.txt
View File

@ -4,12 +4,6 @@ include_directories(${ENGINE_COMMON_INCLUDES})
set(clearShm_SRCS main.cpp)
add_executable(clearShm ${clearShm_SRCS})
columnstore_executable(clearShm ${clearShm_SRCS})
target_link_libraries(clearShm ${ENGINE_LDFLAGS} ${ENGINE_EXEC_LIBS})
install(
TARGETS clearShm
DESTINATION ${ENGINE_BINDIR}
COMPONENT columnstore-engine
)
columnstore_link(clearShm ${ENGINE_LDFLAGS} ${ENGINE_EXEC_LIBS})

10
tools/cleartablelock/CMakeLists.txt
View File

@ -4,12 +4,6 @@ include_directories(${ENGINE_COMMON_INCLUDES})
set(cleartablelock_SRCS cleartablelock.cpp cleartablelockthread.cpp)
add_executable(cleartablelock ${cleartablelock_SRCS})
columnstore_executable(cleartablelock ${cleartablelock_SRCS})
target_link_libraries(cleartablelock ${ENGINE_LDFLAGS} ${NETSNMP_LIBRARIES} ${ENGINE_WRITE_LIBS})
install(
TARGETS cleartablelock
DESTINATION ${ENGINE_BINDIR}
COMPONENT columnstore-engine
)
columnstore_link(cleartablelock ${ENGINE_LDFLAGS} ${NETSNMP_LIBRARIES} ${ENGINE_WRITE_LIBS})

5
tools/configMgt/CMakeLists.txt
View File

@ -5,7 +5,4 @@ include_directories(${ENGINE_COMMON_INCLUDES})
set(autoConfigure_SRCS autoConfigure.cpp)
add_executable(autoConfigure ${autoConfigure_SRCS})
target_link_libraries(autoConfigure ${ENGINE_LDFLAGS} ${NETSNMP_LIBRARIES} ${ENGINE_EXEC_LIBS})
# install(TARGETS autoConfigure DESTINATION ${ENGINE_BINDIR} COMPONENT columnstore-engine)
columnstore_link(autoConfigure ${ENGINE_LDFLAGS} ${NETSNMP_LIBRARIES} ${ENGINE_EXEC_LIBS})

10
tools/cplogger/CMakeLists.txt
View File

@ -4,12 +4,6 @@ include_directories(${ENGINE_COMMON_INCLUDES})
set(cplogger_SRCS main.cpp)
add_executable(cplogger ${cplogger_SRCS})
columnstore_executable(cplogger ${cplogger_SRCS})
target_link_libraries(cplogger ${ENGINE_LDFLAGS} ${ENGINE_EXEC_LIBS})
install(
TARGETS cplogger
DESTINATION ${ENGINE_BINDIR}
COMPONENT columnstore-engine
)
columnstore_link(cplogger ${ENGINE_LDFLAGS} ${ENGINE_EXEC_LIBS})

10
tools/dbbuilder/CMakeLists.txt
View File

@ -4,12 +4,6 @@ include_directories(${ENGINE_COMMON_INCLUDES})
set(dbbuilder_SRCS dbbuilder.cpp systemcatalog.cpp)
add_executable(dbbuilder ${dbbuilder_SRCS})
columnstore_executable(dbbuilder ${dbbuilder_SRCS})
target_link_libraries(dbbuilder ${ENGINE_LDFLAGS} ${NETSNMP_LIBRARIES} ${ENGINE_WRITE_LIBS})
install(
TARGETS dbbuilder
DESTINATION ${ENGINE_BINDIR}
COMPONENT columnstore-engine
)
columnstore_link(dbbuilder ${ENGINE_LDFLAGS} ${NETSNMP_LIBRARIES} ${ENGINE_WRITE_LIBS})

7
tools/dbbuilder/dbbuilder.cpp
View File

@ -26,6 +26,7 @@
#include <stdexcept>
using namespace std;
#include "basic/string_utils.h"
#include <boost/algorithm/string.hpp>
#include "mcsconfig.h"
@ -202,7 +203,11 @@ int main(int argc, char* argv[])
(iter->second).second = true;
isUpgrade = true;
}
messageHandler("", std::string("Upgrade flag is ") + std::to_string(isUpgrade) + std::string(" after checking upgrade candidate OID ") + oam.itoa(iter->first) + std::string(" "), false);
messageHandler("",
std::string("Upgrade flag is ") + std::to_string(isUpgrade) +
std::string(" after checking upgrade candidate OID ") + oam.itoa(iter->first) +
std::string(" "),
false);
}
if (!isUpgrade)

14
tools/dbloadxml/CMakeLists.txt
View File

@ -4,20 +4,14 @@ include_directories(${ENGINE_COMMON_INCLUDES})
set(dbload_STAT_SRCS inputmgr.cpp)
add_library(dbload STATIC ${dbload_STAT_SRCS})
columnstore_static_library(dbload ${dbload_STAT_SRCS})
add_dependencies(dbload loggingcpp)
columnstore_link(dbload loggingcpp)
# ########## next target ###############
set(colxml_SRCS colxml.cpp)
add_executable(colxml ${colxml_SRCS})
columnstore_executable(colxml ${colxml_SRCS})
target_link_libraries(colxml ${ENGINE_LDFLAGS} dbload ${ENGINE_WRITE_LIBS})
install(
TARGETS colxml
DESTINATION ${ENGINE_BINDIR}
COMPONENT columnstore-engine
)
columnstore_link(colxml ${ENGINE_LDFLAGS} dbload ${ENGINE_WRITE_LIBS})

11
tools/ddlcleanup/CMakeLists.txt
View File

@ -4,12 +4,5 @@ include_directories(${ENGINE_COMMON_INCLUDES})
set(ddlcleanup_SRCS ddlcleanup.cpp)
add_executable(ddlcleanup ${ddlcleanup_SRCS})
target_link_libraries(ddlcleanup ${ENGINE_LDFLAGS} ${NETSNMP_LIBRARIES} ${ENGINE_WRITE_LIBS} ddlcleanuputil)
install(
TARGETS ddlcleanup
DESTINATION ${ENGINE_BINDIR}
COMPONENT columnstore-engine
)
columnstore_executable(ddlcleanup ${ddlcleanup_SRCS})
columnstore_link(ddlcleanup ${ENGINE_LDFLAGS} ${NETSNMP_LIBRARIES} ${ENGINE_WRITE_LIBS} ddlcleanuputil)

43
tools/ddldriver/CMakeLists.txt
View File

@ -1,43 +0,0 @@
include_directories(${KDE4_INCLUDES} ${KDE4_INCLUDE_DIR} ${QT_INCLUDES})
# ########## next target ###############
set(ddldriver_SRCS ddldriver.cpp)
kde4_add_executable(ddldriver ${ddldriver_SRCS})
target_link_libraries(ddldriver ${KDE4_KDECORE_LIBS})
install(TARGETS ddldriver ${INSTALL_TARGETS_DEFAULT_ARGS})
# ########## install files ###############
# original Makefile.am contents follow:
# Copyright (C) 2014 InfiniDB, Inc.
#
# This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public
# License as published by the Free Software Foundation; version 2 of the License.
#
# This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied
# warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License along with this program; if not, write to the Free
# Software Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
#
# $Id: Makefile.am 333 2009-04-03 20:35:04Z rdempsey $ Process this file with automake to produce Makefile.in
#
# AM_CPPFLAGS = $(idb_cppflags) AM_CFLAGS = $(idb_cflags) AM_CXXFLAGS = $(idb_cxxflags) AM_LDFLAGS = $(idb_ldflags)
# bin_PROGRAMS = ddldriver ddldriver_SOURCES = ddldriver.cpp ddldriver_CPPFLAGS = @idb_common_includes@ $(AM_CPPFLAGS)
# ddldriver_LDFLAGS = @idb_common_ldflags@ @idb_write_libs@ @idb_common_libs@ @netsnmp_libs@ $(AM_LDFLAGS)
#
# test:
#
# coverage:
#
# leakcheck:
#
# docs:
#
# bootstrap: install-data-am
#

133
tools/ddldriver/ddldriver.cpp
View File

@ -1,133 +0,0 @@
/* Copyright (C) 2014 InfiniDB, Inc.
This program is free software; you can redistribute it and/or
modify it under the terms of the GNU General Public License
as published by the Free Software Foundation; version 2 of
the License.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program; if not, write to the Free Software
Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
MA 02110-1301, USA. */
// $Id: ddldriver.cpp 2101 2013-01-21 14:12:52Z rdempsey $
#include <unistd.h>
#include <cstdio>
#include <iostream>
#include <cctype>
#include <sstream>
using namespace std;
#include "ddlpkg.h"
#include "sqlparser.h"
using namespace ddlpackage;
#include "bytestream.h"
#include "messagequeue.h"
using namespace messageqcpp;
namespace
{
void usage()
{
cout << "usage: ddlriver [-h] schema sql_text" << endl;
}
const string toupper_(const string& in)
{
string::const_iterator iter = in.begin();
string::const_iterator end = in.end();
ostringstream oss;
while (iter != end)
{
oss << static_cast<char>(toupper(*iter));
++iter;
}
return oss.str();
}
} // namespace
int main(int argc, char** argv)
{
int c;
opterr = 0;
while ((c = getopt(argc, argv, "h")) != EOF)
switch (c)
{
case 'h':
case '?':
default:
usage();
return (c == 'h' ? 0 : 1);
break;
}
if (argc - optind < 2)
{
usage();
return 1;
}
string owner(toupper_(argv[optind++]));
SqlParser parser;
parser.setDefaultSchema(owner);
string stmtStr(toupper_(argv[optind++]));
parser.Parse(stmtStr.c_str());
if (!parser.Good())
{
cerr << "Failed to parse statement: " << stmtStr << endl;
return 1;
}
const ParseTree& ptree = parser.GetParseTree();
SqlStatement& stmt = *ptree.fList[0];
stmt.fSessionID = 1;
stmt.fSql = stmtStr;
stmt.fOwner = owner;
ByteStream bytestream;
bytestream << stmt.fSessionID;
stmt.serialize(bytestream);
MessageQueueClient mq("DDLProc");
ByteStream::byte b;
string errorMsg;
try
{
mq.write(bytestream);
bytestream = mq.read();
bytestream >> b;
bytestream >> errorMsg;
}
catch (runtime_error& rex)
{
cerr << "runtime_error in engine: " << rex.what() << endl;
return 1;
}
catch (...)
{
cerr << "uknown error in engine" << endl;
return 1;
}
if (b != 0)
{
cerr << "DDLProc error: " << errorMsg << endl;
return 1;
}
return 0;
}

44
tools/dmldriver/CMakeLists.txt
View File

@ -1,44 +0,0 @@
include_directories(${KDE4_INCLUDES} ${KDE4_INCLUDE_DIR} ${QT_INCLUDES})
# ########## next target ###############
set(dmldriver_SRCS dmldriver.cpp tpchrf2.cpp dmlif.cpp)
kde4_add_executable(dmldriver ${dmldriver_SRCS})
target_link_libraries(dmldriver ${KDE4_KDECORE_LIBS})
install(TARGETS dmldriver ${INSTALL_TARGETS_DEFAULT_ARGS})
# ########## install files ###############
# original Makefile.am contents follow:
# Copyright (C) 2014 InfiniDB, Inc.
#
# This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public
# License as published by the Free Software Foundation; version 2 of the License.
#
# This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied
# warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License along with this program; if not, write to the Free
# Software Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
#
# $Id: Makefile.am 333 2009-04-03 20:35:04Z rdempsey $ Process this file with automake to produce Makefile.in
#
# AM_CPPFLAGS = $(idb_cppflags) AM_CFLAGS = $(idb_cflags) AM_CXXFLAGS = $(idb_cxxflags) AM_LDFLAGS = $(idb_ldflags)
# bin_PROGRAMS = dmldriver dmldriver_SOURCES = dmldriver.cpp tpchrf2.cpp dmlif.cpp dmldriver_CPPFLAGS =
# @idb_common_includes@ $(AM_CPPFLAGS) dmldriver_LDFLAGS = @idb_common_ldflags@ @idb_write_libs@ @idb_common_libs@
# @netsnmp_libs@ $(AM_LDFLAGS)
#
# test:
#
# coverage:
#
# leakcheck:
#
# docs:
#
# bootstrap: install-data-am
#

205
tools/dmldriver/dmldriver.cpp
View File

@ -1,205 +0,0 @@
/* Copyright (C) 2014 InfiniDB, Inc.
This program is free software; you can redistribute it and/or
modify it under the terms of the GNU General Public License
as published by the Free Software Foundation; version 2 of
the License.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program; if not, write to the Free Software
Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
MA 02110-1301, USA. */
// $Id: dmldriver.cpp 2101 2013-01-21 14:12:52Z rdempsey $
//#define NDEBUG
#include <cassert>
#include <iostream>
#include <string>
#include <unistd.h>
#include <cctype>
#include <sstream>
#include <fstream>
#include <limits>
using namespace std;
#include <boost/scoped_array.hpp>
#include <boost/tokenizer.hpp>
using namespace boost;
#include "tpchrf2.h"
#include "dmlif.h"
namespace
{
bool vflg;
bool dflg;
void usage()
{
cout << "usage: dmldriver [-vhd] [-c intvl] [-f file] [-s sid] [-t flgs] [-r file] [-p cnt] [sql_text]"
<< endl;
cout << " -c intvl \tcommit every intvl statements" << endl;
cout << " -f file \tread statements from file (max 15KB/stmt)" << endl;
cout << " -s sid \tset sid as session id" << endl;
cout << " -t flgs \tset trace flags" << endl;
cout << " -v \tdisplay affected row count(s)" << endl;
cout << " -d \tdisplay debug info" << endl;
cout << " -r file \tread orderkeys from file for TPC-H RF2" << endl;
cout << " -p cnt \tpack cnt orderkeys into each delete stmt (only w/ -r)" << endl;
cout << " -e schema\tset the schema name (only w/ -r)" << endl;
cout << " -h \tdisplay this help text" << endl;
}
} // namespace
int main(int argc, char** argv)
{
int c;
opterr = 0;
vflg = false;
dflg = false;
bool fflg = false;
string infilename;
int cIntvl = numeric_limits<int>::max();
uint32_t sessionID = time(0) & 0x7fffffff;
bool rflg = false;
int packCnt = 1;
uint32_t tflg = 0;
string schema;
while ((c = getopt(argc, argv, "e:t:s:c:f:r:p:vhd")) != EOF)
switch (c)
{
case 'v': vflg = true; break;
case 'd': dflg = true; break;
case 'f':
fflg = true;
infilename = optarg;
break;
case 'c': cIntvl = static_cast<int>(strtol(optarg, 0, 0)); break;
case 's': sessionID = static_cast<uint32_t>(strtoul(optarg, 0, 0)); break;
case 't': tflg = static_cast<uint32_t>(strtoul(optarg, 0, 0)); break;
case 'r':
rflg = true;
infilename = optarg;
break;
case 'p': packCnt = static_cast<int>(strtol(optarg, 0, 0)); break;
case 'e': schema = optarg; break;
case 'h':
case '?':
default:
usage();
return (c == 'h' ? 0 : 1);
break;
}
if (!fflg && !rflg && ((argc - optind) < 1))
{
usage();
return 1;
}
if (fflg && rflg)
{
cout << "-f and -r are mutually exclusive!" << endl << endl;
usage();
return 1;
}
if (!schema.empty() && !rflg)
{
cout << "-e requires -r!" << endl << endl;
usage();
return 1;
}
string stmtStr;
if (!fflg && !rflg)
stmtStr = argv[optind++];
int rc = 0;
dmlif::DMLIF dmlif(sessionID, tflg, dflg, vflg);
if (fflg)
{
ifstream ifs(infilename.c_str());
if (!ifs.good())
{
cerr << "Error accessing file " << infilename << endl;
return 1;
}
const streamsize ilinelen = 15 * 1024;
scoped_array<char> iline(new char[ilinelen]);
int cnt = 0;
for (;;)
{
ifs.getline(iline.get(), ilinelen);
if (ifs.eof())
break;
rc = dmlif.sendOne(iline.get());
if (rc != 0)
break;
cnt++;
if ((cnt % cIntvl) == 0)
dmlif.sendOne("COMMIT;");
}
}
else if (rflg)
{
ifstream ifs(infilename.c_str());
if (!ifs.good())
{
cerr << "Error accessing file " << infilename << endl;
return 1;
}
if (schema.empty())
schema = "tpch";
tpch::RF2 rf2(schema, sessionID, tflg, cIntvl, packCnt, dflg, vflg);
rc = rf2.run(ifs);
}
else
{
rc = dmlif.sendOne(stmtStr);
}
if (rc == 0)
dmlif.sendOne("COMMIT;");
return 0;
}

384
tools/dmldriver/dmlif.cpp
View File

@ -1,384 +0,0 @@
/* Copyright (C) 2014 InfiniDB, Inc.
This program is free software; you can redistribute it and/or
modify it under the terms of the GNU General Public License
as published by the Free Software Foundation; version 2 of
the License.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program; if not, write to the Free Software
Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
MA 02110-1301, USA. */
// $Id: dmlif.cpp 2101 2013-01-21 14:12:52Z rdempsey $
//#define NDEBUG
#include <cassert>
#include <string>
#include <sstream>
using namespace std;
#include <boost/tokenizer.hpp>
using namespace boost;
#include "vendordmlstatement.h"
#include "calpontdmlpackage.h"
#include "calpontdmlfactory.h"
using namespace dmlpackage;
#include "bytestream.h"
#include "messagequeue.h"
using namespace messageqcpp;
#include "simplecolumn.h"
#include "calpontselectexecutionplan.h"
#include "sessionmanager.h"
#include "simplefilter.h"
#include "constantcolumn.h"
#include "constantfilter.h"
using namespace execplan;
#include "brmtypes.h"
#include "dmlif.h"
using namespace dmlif;
namespace dmlif
{
DMLIF::DMLIF(uint32_t sessionid, uint32_t tflg, bool dflg, bool vflg)
: fSessionID(sessionid), fTflg(tflg), fDflg(dflg), fVflg(vflg), fOPt(0), fLPt(0)
{
fMqp.reset(new MessageQueueClient("DMLProc"));
}
DMLIF::~DMLIF()
{
}
int DMLIF::sendOne(const string& stmt)
{
int rc;
string tStmt(stmt);
if (*tStmt.rbegin() != ';')
tStmt += ";";
VendorDMLStatement dmlStmt(tStmt, fSessionID);
CalpontDMLPackage* pDMLPackage = CalpontDMLFactory::makeCalpontDMLPackage(dmlStmt);
if (pDMLPackage == 0)
{
cerr << "Failed to parse statement: " << tStmt << endl;
return -1;
}
string queryString = pDMLPackage->get_QueryString();
if (fDflg)
cout << "qs: >" << queryString << '<' << endl;
string dmlStatement = pDMLPackage->get_DMLStatement();
if (fDflg)
cout << "DML: " << dmlStatement << endl;
bool isDML = true;
if (dmlStatement == "COMMIT" || dmlStatement == "ROLLBACK")
{
isDML = false;
}
if (isDML)
{
char_separator<char> sep(" ");
tokenizer<char_separator<char> > tok(queryString, sep);
tokenizer<char_separator<char> >::iterator iter = tok.begin();
idbassert(iter != tok.end());
string where = *iter;
++iter;
idbassert(iter != tok.end());
string col1 = *iter;
++iter;
idbassert(iter != tok.end());
string op = *iter;
++iter;
idbassert(iter != tok.end());
string col2 = *iter;
++iter;
idbassert(iter == tok.end());
if (fDflg)
cout << "SQL: " << pDMLPackage->get_SQLStatement() << endl;
if (fDflg)
cout << "hf: " << pDMLPackage->HasFilter() << endl;
DMLTable* tp = pDMLPackage->get_Table();
if (fDflg)
cout << "sn: " << tp->get_SchemaName() << " tn: " << tp->get_TableName() << endl;
if (fDflg)
cout << "row count: " << tp->get_RowList().size() << endl;
SRCP srcp(new SimpleColumn(tp->get_SchemaName(), tp->get_TableName(), col1, fSessionID));
CalpontSelectExecutionPlan::ColumnMap cm;
cm.insert(make_pair(col1, srcp));
pDMLPackage->get_ExecutionPlan()->columnMap(cm);
CalpontSelectExecutionPlan::ReturnedColumnList rcl;
rcl.push_back(srcp);
pDMLPackage->get_ExecutionPlan()->returnedCols(rcl);
pDMLPackage->get_ExecutionPlan()->sessionID(fSessionID);
pDMLPackage->get_ExecutionPlan()->traceFlags(fTflg);
SessionManager sm;
BRM::TxnID txnid = sm.getTxnID(fSessionID);
if (!txnid.valid)
txnid = sm.newTxnID(fSessionID);
pDMLPackage->get_ExecutionPlan()->txnID(txnid.id);
pDMLPackage->get_ExecutionPlan()->verID(sm.verID());
ParseTree* pt = new ParseTree();
ReturnedColumn* rc1 = srcp->clone();
ReturnedColumn* rc2 = new ConstantColumn(col2, ConstantColumn::NUM);
SOP sop(new Operator(op));
SimpleFilter* sf = new SimpleFilter(sop, rc1, rc2);
pt->data(sf);
pDMLPackage->get_ExecutionPlan()->filters(pt);
if (fDflg)
cout << "ep: " << *pDMLPackage->get_ExecutionPlan() << endl;
}
ByteStream bytestream;
pDMLPackage->write(bytestream);
delete pDMLPackage;
ByteStream::octbyte rows;
rc = DMLSend(bytestream, rows);
if (isDML && fVflg)
cout << rows << " rows affected" << endl;
return rc;
}
int DMLIF::DMLSend(ByteStream& bytestream, ByteStream::octbyte& rows)
{
ByteStream::byte b;
string errorMsg;
try
{
fMqp->connect();
fMqp->write(bytestream);
bytestream = fMqp->read();
fMqp->shutdown();
if (fDflg)
cout << "read " << bytestream.length() << " bytes from DMLProc" << endl;
bytestream >> b;
if (fDflg)
cout << "b = " << (int)b << endl;
bytestream >> rows;
if (fDflg)
cout << "rows = " << rows << endl;
bytestream >> errorMsg;
if (fDflg)
cout << "errorMsg = " << errorMsg << endl;
}
catch (runtime_error& rex)
{
cerr << "runtime_error in engine: " << rex.what() << endl;
return -1;
}
catch (...)
{
cerr << "uknown error in engine" << endl;
return -1;
}
if (b != 0)
{
cerr << "DMLProc error: " << errorMsg << endl;
return -1;
}
return 0;
}
void DMLIF::rf2Start(const string& sn)
{
fSchema = sn;
fOFilterStr = "";
fLFilterStr = "";
fOPt = 0;
fLPt = 0;
}
void DMLIF::rf2Add(int64_t okey)
{
ostringstream oss;
oss << okey;
string okeyStr(oss.str());
if (fOFilterStr.empty())
{
fOFilterStr = "o_orderkey=" + okeyStr;
ReturnedColumn* rc1 = new SimpleColumn(fSchema, "orders", "o_orderkey", fSessionID);
ReturnedColumn* rc2 = new ConstantColumn(okeyStr, ConstantColumn::NUM);
SOP sop(new Operator("="));
ConstantFilter* cf = new ConstantFilter(sop, rc1, rc2);
sop.reset(new Operator("or"));
cf->op(sop);
fOPt = new ParseTree(cf);
}
else
{
fOFilterStr += " or o_orderkey=" + okeyStr;
ReturnedColumn* rc1 = new SimpleColumn(fSchema, "orders", "o_orderkey", fSessionID);
ReturnedColumn* rc2 = new ConstantColumn(okeyStr, ConstantColumn::NUM);
SOP sop(new Operator("="));
ConstantFilter* cf = dynamic_cast<ConstantFilter*>(fOPt->data());
cf->pushFilter(new SimpleFilter(sop, rc1, rc2));
}
if (fLFilterStr.empty())
{
fLFilterStr = "l_orderkey=" + okeyStr;
ReturnedColumn* rc1 = new SimpleColumn(fSchema, "lineitem", "l_orderkey", fSessionID);
ReturnedColumn* rc2 = new ConstantColumn(okeyStr, ConstantColumn::NUM);
SOP sop(new Operator("="));
ConstantFilter* cf = new ConstantFilter(sop, rc1, rc2);
sop.reset(new Operator("or"));
cf->op(sop);
fLPt = new ParseTree(cf);
}
else
{
fLFilterStr += " or l_orderkey=" + okeyStr;
ReturnedColumn* rc1 = new SimpleColumn(fSchema, "lineitem", "l_orderkey", fSessionID);
ReturnedColumn* rc2 = new ConstantColumn(okeyStr, ConstantColumn::NUM);
SOP sop(new Operator("="));
ConstantFilter* cf = dynamic_cast<ConstantFilter*>(fLPt->data());
cf->pushFilter(new SimpleFilter(sop, rc1, rc2));
}
}
int DMLIF::rf2Send()
{
if (fOFilterStr.empty())
return -1;
int rc = 0;
string dmlstr;
dmlstr = "delete from " + fSchema + ".orders where " + fOFilterStr + ';';
if (fDflg)
cout << dmlstr << endl;
VendorDMLStatement dmlStmt(dmlstr, fSessionID);
CalpontDMLPackage* pDMLPackage = CalpontDMLFactory::makeCalpontDMLPackage(dmlStmt);
if (pDMLPackage == 0)
{
cerr << "Failed to parse statement: " << dmlstr << endl;
return -1;
}
SRCP srcp(new SimpleColumn(fSchema, "orders", "o_orderkey", fSessionID));
CalpontSelectExecutionPlan::ColumnMap cm;
cm.insert(make_pair("o_orderkey", srcp));
pDMLPackage->get_ExecutionPlan()->columnMap(cm);
CalpontSelectExecutionPlan::ReturnedColumnList rcl;
rcl.push_back(srcp);
pDMLPackage->get_ExecutionPlan()->returnedCols(rcl);
pDMLPackage->get_ExecutionPlan()->sessionID(fSessionID);
pDMLPackage->get_ExecutionPlan()->traceFlags(fTflg);
SessionManager sm;
BRM::TxnID txnid = sm.getTxnID(fSessionID);
if (!txnid.valid)
txnid = sm.newTxnID(fSessionID);
pDMLPackage->get_ExecutionPlan()->txnID(txnid.id);
pDMLPackage->get_ExecutionPlan()->verID(sm.verID());
pDMLPackage->get_ExecutionPlan()->filters(fOPt);
if (fDflg)
cout << "ep: " << *pDMLPackage->get_ExecutionPlan() << endl;
ByteStream bytestream;
pDMLPackage->write(bytestream);
delete pDMLPackage;
pDMLPackage = 0;
ByteStream::octbyte rows = 0;
rc = DMLSend(bytestream, rows);
if (fVflg)
cout << rows << " rows affected" << endl;
dmlstr = "delete from " + fSchema + ".lineitem where " + fLFilterStr + ';';
if (fDflg)
cout << dmlstr << endl;
VendorDMLStatement dmlStmt1(dmlstr, fSessionID);
pDMLPackage = CalpontDMLFactory::makeCalpontDMLPackage(dmlStmt1);
if (pDMLPackage == 0)
{
cerr << "Failed to parse statement: " << dmlstr << endl;
return -1;
}
srcp.reset(new SimpleColumn(fSchema, "lineitem", "l_orderkey", fSessionID));
cm.clear();
cm.insert(make_pair("l_orderkey", srcp));
pDMLPackage->get_ExecutionPlan()->columnMap(cm);
rcl.clear();
rcl.push_back(srcp);
pDMLPackage->get_ExecutionPlan()->returnedCols(rcl);
pDMLPackage->get_ExecutionPlan()->sessionID(fSessionID);
pDMLPackage->get_ExecutionPlan()->traceFlags(fTflg);
txnid = sm.getTxnID(fSessionID);
if (!txnid.valid)
txnid = sm.newTxnID(fSessionID);
pDMLPackage->get_ExecutionPlan()->txnID(txnid.id);
pDMLPackage->get_ExecutionPlan()->verID(sm.verID());
pDMLPackage->get_ExecutionPlan()->filters(fLPt);
if (fDflg)
cout << "ep: " << *pDMLPackage->get_ExecutionPlan() << endl;
bytestream.reset();
pDMLPackage->write(bytestream);
delete pDMLPackage;
pDMLPackage = 0;
rows = 0;
rc = DMLSend(bytestream, rows);
if (fVflg)
cout << rows << " rows affected" << endl;
return 0;
}
} // namespace dmlif

65
tools/dmldriver/dmlif.h
View File

@ -1,65 +0,0 @@
/* Copyright (C) 2014 InfiniDB, Inc.
This program is free software; you can redistribute it and/or
modify it under the terms of the GNU General Public License
as published by the Free Software Foundation; version 2 of
the License.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program; if not, write to the Free Software
Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
MA 02110-1301, USA. */
// $Id: dmlif.h 2101 2013-01-21 14:12:52Z rdempsey $
#pragma once
#include <string>
#include <cstddef>
#include <boost/scoped_ptr.hpp>
#include "messagequeue.h"
#include "expressionparser.h"
namespace dmlif
{
class DMLIF
{
public:
DMLIF(uint32_t sessionid, uint32_t tflg = 0, bool dflg = false, bool vflg = false);
~DMLIF();
int sendOne(const std::string& stmt);
void rf2Start(const std::string& sn);
void rf2Add(int64_t okey);
int rf2Send();
protected:
int DMLSend(messageqcpp::ByteStream& bytestream, messageqcpp::ByteStream::octbyte& rows);
private:
// DMLIF(const DMLIF& rhs);
// DMLIF& operator=(const DMLIF& rhs);
uint32_t fSessionID;
uint32_t fTflg;
bool fDflg;
bool fVflg;
boost::scoped_ptr<messageqcpp::MessageQueueClient> fMqp;
std::string fSchema;
std::string fOFilterStr;
std::string fLFilterStr;
execplan::ParseTree* fOPt;
execplan::ParseTree* fLPt;
};
} // namespace dmlif

86
tools/dmldriver/tpchrf2.cpp
View File

@ -1,86 +0,0 @@
/* Copyright (C) 2014 InfiniDB, Inc.
This program is free software; you can redistribute it and/or
modify it under the terms of the GNU General Public License
as published by the Free Software Foundation; version 2 of
the License.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program; if not, write to the Free Software
Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
MA 02110-1301, USA. */
// $Id: tpchrf2.cpp 2101 2013-01-21 14:12:52Z rdempsey $
#include <istream>
using namespace std;
#include <boost/scoped_array.hpp>
#include <boost/tokenizer.hpp>
using namespace boost;
#include "tpchrf2.h"
#include "dmlif.h"
namespace tpch
{
RF2::RF2(const string& sn, uint32_t sid, uint32_t tflg, int c, int p, bool d, bool v)
: fSchema(sn), fSessionID(sid), fTflg(tflg), fIntvl(c), fPack(p), fDflg(d), fVflg(v)
{
}
RF2::~RF2()
{
}
int RF2::run(istream& in)
{
const streamsize ilinelen = 1024;
scoped_array<char> iline(new char[ilinelen]);
int cnt = 0;
dmlif::DMLIF dmlif(fSessionID, fTflg, fDflg, fVflg);
for (;;)
{
dmlif.rf2Start(fSchema);
for (int i = 0; i < fPack; i++)
{
in.getline(iline.get(), ilinelen);
if (in.eof())
break;
typedef char_separator<char> cs;
typedef tokenizer<cs> tk;
cs sep("|");
tk tok(string(iline.get()), sep);
tk::iterator iter = tok.begin();
idbassert(iter != tok.end());
string keystr = *iter;
++iter;
// idbassert(iter == tok.end());
int64_t okey = strtol(keystr.c_str(), 0, 0);
dmlif.rf2Add(okey);
}
dmlif.rf2Send();
cnt++;
if ((cnt % fIntvl) == 0)
dmlif.sendOne("COMMIT;");
if (in.eof())
break;
}
dmlif.sendOne("COMMIT;");
return 0;
}
} // namespace tpch

52
tools/dmldriver/tpchrf2.h
View File

@ -1,52 +0,0 @@
/* Copyright (C) 2014 InfiniDB, Inc.
This program is free software; you can redistribute it and/or
modify it under the terms of the GNU General Public License
as published by the Free Software Foundation; version 2 of
the License.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program; if not, write to the Free Software
Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
MA 02110-1301, USA. */
// $Id: tpchrf2.h 2101 2013-01-21 14:12:52Z rdempsey $
#pragma once
#include <string>
#include <istream>
#include <cstddef>
#include <limits>
#include <stdint.h>
namespace tpch
{
class RF2
{
public:
RF2(const std::string& sn, uint32_t sid, uint32_t tflg = 0, int c = std::numeric_limits<int>::max(),
int p = 1, bool d = false, bool v = false);
~RF2();
int run(std::istream& in);
private:
// RF2(const RF2& rhs);
// RF2& operator=(const RF2& rhs);
std::string fSchema;
uint32_t fSessionID;
uint32_t fTflg;
int fIntvl;
int fPack;
bool fDflg;
bool fVflg;
};
} // namespace tpch

11
tools/editem/CMakeLists.txt
View File

@ -4,12 +4,5 @@ include_directories(${ENGINE_COMMON_INCLUDES})
set(editem_SRCS editem.cpp)
add_executable(editem ${editem_SRCS})
target_link_libraries(editem ${ENGINE_LDFLAGS} ${NETSNMP_LIBRARIES} ${ENGINE_EXEC_LIBS})
install(
TARGETS editem
DESTINATION ${ENGINE_BINDIR}
COMPONENT columnstore-engine
)
columnstore_executable(editem ${editem_SRCS})
columnstore_link(editem ${ENGINE_LDFLAGS} ${NETSNMP_LIBRARIES} ${ENGINE_EXEC_LIBS})

173
tools/fixdate/fixdate.cpp
View File

@ -1,173 +0,0 @@
/* Copyright (C) 2014 InfiniDB, Inc.
This program is free software; you can redistribute it and/or
modify it under the terms of the GNU General Public License
as published by the Free Software Foundation; version 2 of
the License.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program; if not, write to the Free Software
Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
MA 02110-1301, USA. */
/*
* $Id: fixdate.cpp 2101 2013-01-21 14:12:52Z rdempsey $
*
*/
#include <iostream>
#include <fstream>
#include <cassert>
#include <vector>
#include <algorithm>
#include <iterator>
using namespace std;
#include <unistd.h>
#include <boost/format.hpp>
#include <boost/scoped_array.hpp>
using namespace boost;
#include "bytestream.h"
using namespace messageqcpp;
#include "dmlpackageprocessor.h"
using namespace dmlpackageprocessor;
namespace
{
const streamsize blkSz = 8192;
void usage()
{
}
u_int64_t fixDate(ByteStream& bs, ostream& out)
{
ByteStream fixed;
ByteStream::quadbyte o;
u_int64_t cnt = 0;
#if 0
DMLPackageProcessor::Date minDate;
DMLPackageProcessor::Date maxDate;
minDate.year = 1992;
minDate.month = 1;
minDate.day = 2;
maxDate.year = 1998;
maxDate.month = 12;
maxDate.day = 25;
ByteStream::quadbyte mxd;
ByteStream::quadbyte mnd;
mxd = *(reinterpret_cast<ByteStream::quadbyte*>(&maxDate));
mnd = *(reinterpret_cast<ByteStream::quadbyte*>(&minDate));
#endif
DMLPackageProcessor::Date fixDate;
fixDate.spare = 0;
ByteStream::quadbyte f;
while (bs.length() > 0)
{
bs >> o;
if (o >= 0xfffffffe)
{
fixed << o;
continue;
}
f = o & 0xffff;
fixDate.year = f;
o >>= 16;
f = o & 0xf;
fixDate.month = f;
o >>= 4;
f = o & 0x3f;
fixDate.day = f;
// o >>= 6;
o = *(reinterpret_cast<ByteStream::quadbyte*>(&fixDate));
fixed << o;
#if 0
cout << DMLPackageProcessor::dateToString(o) << endl;
idbassert(o >= mnd && o <= mxd);
cnt++;
#endif
}
out << fixed;
return cnt;
}
} // namespace
int main(int argc, char* argv[])
{
int c;
opterr = 0;
while ((c = getopt(argc, argv, "h")) != EOF)
switch (c)
{
case 'h':
usage();
return 0;
break;
default:
usage();
return 1;
break;
}
if ((argc - optind) < 1)
{
usage();
return 1;
}
ByteStream bs;
ifstream ifs(argv[optind + 0]);
ByteStream::byte inbuf[blkSz];
streampos fLen;
u_int64_t blkNo = 0;
ifs.seekg(0, ios_base::end);
fLen = ifs.tellg();
ifs.seekg(0, ios_base::beg);
idbassert((fLen % blkSz) == 0);
u_int64_t numBlks = fLen / blkSz;
cout << numBlks << " blocks to fix..." << endl;
ofstream ofs("fixdate.cdf");
cout << "pct done: " << setw(3);
for (;;)
{
ifs.read(reinterpret_cast<char*>(inbuf), blkSz);
if (ifs.eof())
break;
bs.load(inbuf, blkSz);
fixDate(bs, ofs);
cout << "\b\b\b" << setw(3) << (u_int64_t)(blkNo * 100 / numBlks);
// cout << setw(3) << (u_int64_t)(blkNo * 100 / numBlks) << endl;
blkNo++;
}
cout << "\b\b\b" << setw(3) << 100 << endl;
// cout << setw(3) << 100 << endl;
return 0;
}

10
tools/getConfig/CMakeLists.txt
View File

@ -4,12 +4,6 @@ include_directories(${ENGINE_COMMON_INCLUDES})
set(getConfig_SRCS main.cpp)
add_executable(mcsGetConfig ${getConfig_SRCS})
columnstore_executable(mcsGetConfig ${getConfig_SRCS})
target_link_libraries(mcsGetConfig ${ENGINE_LDFLAGS} ${ENGINE_EXEC_LIBS})
install(
TARGETS mcsGetConfig
DESTINATION ${ENGINE_BINDIR}
COMPONENT columnstore-engine
)
columnstore_link(mcsGetConfig ${ENGINE_LDFLAGS} ${ENGINE_EXEC_LIBS})

265
tools/idb_comp/main.cpp
View File

@ -1,265 +0,0 @@
#include <unistd.h>
#include <sys/types.h>
#include <sys/stat.h>
#include <fcntl.h>
#include <cerrno>
//#define NDEBUG
#include <cassert>
#include <cstring>
#include <lzo/lzoconf.h>
#include <lzo/lzo1x.h>
#include <iostream>
#include <iomanip>
#include <sstream>
#include <inttypes.h>
#include <cstdlib>
using namespace std;
#include <boost/scoped_array.hpp>
namespace
{
size_t exp_buf_size;
unsigned vflg;
const string lzo_rctos(int r)
{
switch (r)
{
case LZO_E_INPUT_NOT_CONSUMED: return "LZO_E_INPUT_NOT_CONSUMED";
default: break;
}
return "Unknown Error!";
}
void usage()
{
cout << "usage: idb_comp [-e size][-h] file(s)" << endl
<< "\t-e size size (in KB) of expansion buffer" << endl
<< "\t-h display this help" << endl;
}
int doit(const string& infile)
{
int rc = 0;
int ifd = open(infile.c_str(), O_RDONLY | O_LARGEFILE | O_DIRECT);
if (ifd < 0)
{
cerr << infile << ": open: " << strerror(errno) << endl;
return 1;
}
struct stat istatbuf;
fstat(ifd, &istatbuf);
string outname(infile);
string::size_type ptr;
ptr = outname.find_last_of('.');
if (ptr != string::npos)
outname.erase(ptr);
ptr = outname.find_last_of('/');
if (ptr != string::npos)
outname.erase(0, ptr + 1);
outname = "./" + outname + ".cmp";
int ofd = open(outname.c_str(), O_WRONLY | O_CREAT | O_TRUNC | O_LARGEFILE | O_DIRECT, 0644);
if (ofd < 0)
{
cerr << outname << ": open: " << strerror(errno) << endl;
close(ifd);
return 1;
}
lzo_init();
ssize_t nread = -1;
ssize_t nwritten = -1;
lzo_bytep ibuf;
lzo_bytep tibuf;
lzo_bytep cbuf;
lzo_bytep tcbuf;
lzo_bytep wkmem;
lzo_uint32 ibuf_len = 0;
lzo_uint cbuf_len = 0;
ibuf_len = 512 * 1024 * 8;
tibuf = new lzo_byte[ibuf_len + 4095];
if ((ptrdiff_t)tibuf & 0xfffULL)
ibuf = (lzo_bytep)((ptrdiff_t)tibuf & 0xfffffffffffff000ULL) + 4096;
else
ibuf = tibuf;
cbuf_len = 512 * 1024 * 8;
tcbuf = new lzo_byte[cbuf_len + 4095 + exp_buf_size * 1024];
if ((ptrdiff_t)tcbuf & 0xfff)
cbuf = (lzo_bytep)((ptrdiff_t)tcbuf & 0xfffffffffffff000ULL) + 4096;
else
cbuf = tcbuf;
wkmem = new lzo_byte[LZO1X_1_15_MEM_COMPRESS];
int r = LZO_E_OK;
const int TOTAL_HDR_LEN = 4096 * 2;
char* hdrbuf = new char[TOTAL_HDR_LEN + 4095];
memset(hdrbuf, 0, TOTAL_HDR_LEN + 4095);
char* hdrbufp = 0;
if ((ptrdiff_t)hdrbuf & 0xfff)
hdrbufp = (char*)((ptrdiff_t)hdrbuf & 0xfffffffffffff000ULL) + 4096;
else
hdrbufp = hdrbuf;
struct compHdr
{
uint64_t ptrs[512];
};
idbassert(sizeof(compHdr) <= 4096);
compHdr* hdrptr1 = (compHdr*)hdrbufp;
compHdr* hdrptr = hdrptr1 + 1; // advance to 2nd hdr to store compression ptrs
lseek(ofd, TOTAL_HDR_LEN, SEEK_SET);
nread = read(ifd, ibuf, ibuf_len);
int idx = 0;
off_t cmpoff = TOTAL_HDR_LEN;
while (nread > 0)
{
cbuf_len = 512 * 1024 * 8;
memset(cbuf, 0, cbuf_len);
r = lzo1x_1_15_compress(ibuf, nread, cbuf, &cbuf_len, wkmem);
if (r != LZO_E_OK)
{
cerr << "compression failed!: " << r << endl;
rc = 1;
goto out;
}
if (cbuf_len > (unsigned)nread)
{
cerr << "WARNING: expansion detected! (output grew by " << (cbuf_len - nread) << " bytes)" << endl;
idbassert((cbuf_len - nread) <= exp_buf_size * 1024);
}
if (cbuf_len & 0xfff)
cbuf_len = (cbuf_len & 0xfffffffffffff000ULL) + 4096;
// cbuf_len = 512 * 1024 * 8;
nwritten = write(ofd, cbuf, cbuf_len);
if (nwritten < 0 || (unsigned)nwritten != cbuf_len)
{
cerr << outname << ": write: " << strerror(errno) << " (" << nwritten << ')' << endl;
rc = 1;
goto out;
}
if (vflg > 0)
{
lzo_bytep tbuf;
lzo_uint tbuflen = 4 * 1024 * 1024 + 4;
boost::scoped_array<lzo_byte> tbuf_sa(new lzo_byte[tbuflen]);
tbuf = tbuf_sa.get();
cout << "idx: " << idx << " off: " << cmpoff << " size: " << cbuf_len;
r = lzo1x_decompress(cbuf, cbuf_len, tbuf, &tbuflen, 0);
cout << " r: " << lzo_rctos(r) << " size: " << tbuflen << endl;
}
hdrptr->ptrs[idx] = cmpoff;
idx++;
cmpoff += cbuf_len;
nread = read(ifd, ibuf, ibuf_len);
}
if (nread < 0)
{
cerr << infile << ": read: " << strerror(errno) << endl;
rc = 1;
goto out;
}
hdrptr->ptrs[idx] = cmpoff;
idbassert(idx <= 64);
// Fill in meta-data information in first header
hdrptr1->ptrs[0] = 0xfdc119a384d0778eULL;
hdrptr1->ptrs[1] = 1;
hdrptr1->ptrs[2] = 1;
nwritten = pwrite(ofd, hdrbufp, TOTAL_HDR_LEN, 0);
idbassert(nwritten == TOTAL_HDR_LEN);
out:
delete[] wkmem;
delete[] tcbuf;
delete[] tibuf;
fsync(ofd);
struct stat ostatbuf;
fstat(ofd, &ostatbuf);
idbassert(ostatbuf.st_size == (signed)hdrptr->ptrs[idx]);
delete[] hdrbuf;
cout << infile << ": Input Size: " << istatbuf.st_size << " Output size: " << ostatbuf.st_size
<< " Compression: " << (100LL - (ostatbuf.st_size * 100LL / istatbuf.st_size)) << '%' << endl;
close(ofd);
close(ifd);
return rc;
}
} // namespace
int main(int argc, char** argv)
{
opterr = 0;
int c;
exp_buf_size = 128;
vflg = 0;
while ((c = getopt(argc, argv, "e:vh")) != -1)
switch (c)
{
case 'e': exp_buf_size = atoi(optarg); break;
case 'v': vflg++; break;
case 'h':
default:
usage();
return (c == 'h' ? 0 : 1);
break;
}
if ((argc - optind) < 1)
{
usage();
return 1;
}
int rc = 0;
for (int i = optind; i < argc; i++)
if (doit(argv[i]))
rc = 1;
return rc;
}

11
tools/idbmeminfo/CMakeLists.txt
View File

@ -4,12 +4,5 @@ include_directories(${ENGINE_COMMON_INCLUDES})
set(idbmeminfo_SRCS idbmeminfo.cpp)
add_executable(idbmeminfo ${idbmeminfo_SRCS})
target_link_libraries(idbmeminfo ${ENGINE_LDFLAGS})
install(
TARGETS idbmeminfo
DESTINATION ${ENGINE_BINDIR}
COMPONENT columnstore-engine
)
columnstore_executable(idbmeminfo ${idbmeminfo_SRCS})
columnstore_link(idbmeminfo ${ENGINE_LDFLAGS})

19
tools/passwd/CMakeLists.txt
View File

@ -4,21 +4,10 @@ include_directories(${ENGINE_COMMON_INCLUDES} ${ENGINE_UTILS_COMMON_INCLUDE})
set(cspasswd_SRCS cspasswd.cpp secrets.cpp)
set(cskeys_SRCS cskeys.cpp secrets.cpp)
add_executable(cspasswd ${cspasswd_SRCS})
add_executable(cskeys ${cskeys_SRCS})
columnstore_executable(cspasswd ${cspasswd_SRCS})
columnstore_executable(cskeys ${cskeys_SRCS})
target_include_directories(cspasswd BEFORE PUBLIC ${OPENSSL_INCLUDE_DIR})
target_include_directories(cskeys BEFORE PUBLIC ${OPENSSL_INCLUDE_DIR})
target_link_libraries(cspasswd ${ENGINE_LDFLAGS} ${ENGINE_EXEC_LIBS} ${SSL_LIBRARIES})
target_link_libraries(cskeys ${ENGINE_LDFLAGS} ${ENGINE_EXEC_LIBS} ${SSL_LIBRARIES})
install(
TARGETS cspasswd
DESTINATION ${ENGINE_BINDIR}
COMPONENT columnstore-engine
)
install(
TARGETS cskeys
DESTINATION ${ENGINE_BINDIR}
COMPONENT columnstore-engine
)
columnstore_link(cspasswd ${ENGINE_LDFLAGS} ${ENGINE_EXEC_LIBS} ${SSL_LIBRARIES})
columnstore_link(cskeys ${ENGINE_LDFLAGS} ${ENGINE_EXEC_LIBS} ${SSL_LIBRARIES})

2357
tools/pingproc/pingproc.cpp
View File

File diff suppressed because it is too large Load Diff

55
tools/qfe/CMakeLists.txt
View File

@ -1,55 +0,0 @@
include_directories(${KDE4_INCLUDES} ${KDE4_INCLUDE_DIR} ${QT_INCLUDES})
# ########## next target ###############
set(QFE_SRCS
cseputils.cpp
ddlstmts.cpp
parsequery.cpp
returnedrows.cpp
sendcsep.cpp
server.cpp
socketio.cpp
qfelexer.lpp
qfeparser.ypp
)
kde4_add_executable(QFE ${QFE_SRCS})
target_link_libraries(QFE ${KDE4_KDECORE_LIBS})
install(TARGETS QFE ${INSTALL_TARGETS_DEFAULT_ARGS})
# ########## install files ###############
# original Makefile.am contents follow:
# Copyright (C) 2014 InfiniDB, Inc.
#
# This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public
# License as published by the Free Software Foundation; version 2 of the License.
#
# This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied
# warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License along with this program; if not, write to the Free
# Software Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
#
# $Id$ Process this file with automake to produce Makefile.in
#
# AM_CPPFLAGS = $(idb_cppflags) AM_CFLAGS = $(idb_cflags) AM_CXXFLAGS = $(idb_cxxflags) AM_LDFLAGS = $(idb_ldflags)
# AM_YFLAGS = -d -p qfe AM_LFLAGS = -i -Pqfe -olex.yy.c bin_PROGRAMS = QFE QFE_SOURCES = \ cseputils.cpp \ ddlstmts.cpp
# \ parsequery.cpp \ returnedrows.cpp \ sendcsep.cpp \ server.cpp \ socketio.cpp \ qfelexer.lpp \ qfeparser.ypp
# QFE_CPPFLAGS = @idb_common_includes@ $(AM_CPPFLAGS) QFE_LDFLAGS = @idb_common_ldflags@ @idb_write_libs@
# @idb_exec_libs@ $(AM_LDFLAGS) BUILT_SOURCES = qfeparser.cpp qfelexer.cpp qfeparser.h
#
# test:
#
# coverage:
#
# leakcheck:
#
# docs:
#
# bootstrap: install-data-am
#

148
tools/qfe/cli.cpp
View File

@ -1,148 +0,0 @@
#include <poll.h>
#include <sys/socket.h>
#include <netinet/in.h>
#include <arpa/inet.h>
#include <netdb.h>
#include <iostream>
#include <string>
#include <sstream>
#include <iomanip>
#include <sys/types.h>
#include <sys/stat.h>
#include <fcntl.h>
#include <unistd.h>
#include <cerrno>
#include <cstring>
#include <cstdlib>
using namespace std;
#include "socktype.h"
#include "socketio.h"
using namespace qfe;
namespace
{
void usage()
{
cout << "usage: cli [-h] [-s serverip] [-p port] [-c schema] <query>" << endl;
cout << '\t' << "-s serverip connect to serverip instead of 127.0.0.1" << endl;
cout << '\t' << "-p port connect on port instead of 9198" << endl;
cout << '\t' << "-c schema use schema as default instead of tpch1" << endl;
cout << '\t' << "-h display this help" << endl;
}
} // namespace
int main(int argc, char** argv)
{
opterr = 0;
int c;
string serverip("127.0.0.1");
string schema("tpch1");
short PortNo = 9198;
while ((c = getopt(argc, argv, "s:c:p:h")) != -1)
switch (c)
{
case 's': serverip = optarg; break;
case 'c': schema = optarg; break;
case 'p':
PortNo = atoi(optarg);
if (PortNo == 0)
{
usage();
return 1;
}
break;
case 'h':
case '?':
default:
usage();
return 0;
break;
}
if (argc - optind < 1)
{
usage();
return 1;
}
string query(argv[optind + 0]);
SockType fd = -1;
fd = ::socket(PF_INET, SOCK_STREAM, IPPROTO_TCP);
if (fd < 0)
{
cerr << "socket create error: " << strerror(errno) << endl;
return 1;
}
int rc = 0;
struct sockaddr_in serv_addr;
struct addrinfo hints;
struct addrinfo* res = 0;
memset(&hints, 0, sizeof(struct addrinfo));
hints.ai_family = PF_INET;
hints.ai_socktype = SOCK_STREAM;
hints.ai_protocol = IPPROTO_TCP;
rc = getaddrinfo(serverip.c_str(), 0, &hints, &res);
if (rc != 0)
{
cerr << "Error resolving '" << serverip << "': " << gai_strerror(rc) << endl;
return 1;
}
sockaddr_in* sain = 0;
sain = reinterpret_cast<sockaddr_in*>(res->ai_addr);
memset(&serv_addr, 0, sizeof(serv_addr));
serv_addr.sin_family = PF_INET;
serv_addr.sin_addr = sain->sin_addr;
serv_addr.sin_port = htons(PortNo);
freeaddrinfo(res);
rc = ::connect(fd, (sockaddr*)&serv_addr, sizeof(serv_addr));
if (rc < 0)
{
cerr << "socket connect error: " << strerror(errno) << endl;
return 1;
}
socketio::writeString(fd, schema);
socketio::writeString(fd, query);
uint32_t flag = 0;
string row;
row = socketio::readString(fd);
if (row != "OK")
{
cerr << "query failed: " << row << endl;
goto bailout;
}
row = socketio::readString(fd);
while (!row.empty())
{
cout << row << endl;
row = socketio::readString(fd);
}
flag = 0;
SockWriteFcn(fd, &flag, 4);
bailout:
::shutdown(fd, SHUT_RDWR);
::close(fd);
fd = -1;
return 0;
}

141
tools/qfe/cseputils.cpp
View File

@ -1,141 +0,0 @@
#include <unistd.h>
#include <string>
using namespace std;
#include "constantcolumn.h"
#include "simplefilter.h"
#include "calpontsystemcatalog.h"
#include "parsetree.h"
#include "simplecolumn.h"
#include "calpontselectexecutionplan.h"
using namespace execplan;
#include "cseputils.h"
namespace qfe
{
extern string DefaultSchema;
namespace utils
{
ConstantColumn* createConstCol(const string& valstr)
{
ConstantColumn* cc = new ConstantColumn(valstr);
cc->alias(valstr);
return cc;
}
template <typename T>
ConstantColumn* createConstCol(const string& valstr, T val)
{
ConstantColumn* cc = new ConstantColumn(valstr, val);
cc->alias(valstr);
return cc;
}
SimpleFilter* createSimpleFilter(boost::shared_ptr<CalpontSystemCatalog>& csc,
const CalpontSystemCatalog::TableColName& tcn, const string& opstr,
ConstantColumn* cc)
{
SimpleFilter* lsf = new SimpleFilter();
Operator* op = new Operator();
op->data(opstr);
CalpontSystemCatalog::ColType ccct;
ccct = op->resultType();
ccct.colDataType = cc->resultType().colDataType;
op->operationType(ccct);
SOP sop(op);
lsf->op(sop);
CalpontSystemCatalog::OID oid = csc->lookupOID(tcn);
CalpontSystemCatalog::ColType ct = csc->colType(oid);
SimpleColumn* sc = new SimpleColumn();
sc->schemaName(tcn.schema);
sc->tableName(tcn.table);
sc->tableAlias(tcn.table);
sc->columnName(tcn.column);
sc->oid(oid);
sc->resultType(ct);
sc->alias(tcn.toString());
lsf->lhs(sc);
lsf->rhs(cc);
return lsf;
}
void appendSimpleFilter(ParseTree*& ptree, SimpleFilter* filter)
{
if (ptree->data() == 0)
{
// degenerate case, this filter goes at this node
ptree->data(filter);
}
else if (ptree->right() == 0 && ptree->left() == 0)
{
// this will be the case when there is a single node in the tree
// that contains a filter. Here we want to make the root node an
// 'and' operator, push the existing down to the lhs and make a
// new node for the new filter
ParseTree* newLhs = new ParseTree(ptree->data());
ParseTree* newRhs = new ParseTree(filter);
Operator* op = new Operator();
op->data("and");
ptree->data(op);
ptree->left(newLhs);
ptree->right(newRhs);
}
else
{
// this will be the case once we have a tree with an 'and' at the
// root node, a filter in the lhs, and an arbitrary height tree
// with the same properties on the rhs. Because all operators
// are guaranteed to be and for now we simply insert a new rhs
// node and "push down" the existing tree
Operator* op = new Operator();
op->data("and");
ParseTree* newRhs = new ParseTree(op);
newRhs->left(new ParseTree(filter));
newRhs->right(ptree->right());
ptree->right(newRhs);
}
}
void updateParseTree(boost::shared_ptr<execplan::CalpontSystemCatalog>& csc,
execplan::CalpontSelectExecutionPlan*& csep, execplan::SimpleColumn* sc,
const std::string& relop, pair<int, string> cval)
{
execplan::ConstantColumn* cc = 0;
if (cval.first == 0)
cc = createConstCol(cval.second, static_cast<int64_t>(atoll(cval.second.c_str())));
else
cc = createConstCol(cval.second);
if (sc->schemaName() == "infinidb_unknown" && !DefaultSchema.empty())
sc->schemaName(DefaultSchema);
execplan::SimpleFilter* sf = 0;
sf = createSimpleFilter(csc, execplan::make_tcn(sc->schemaName(), sc->tableName(), sc->columnName()), relop,
cc);
execplan::ParseTree* ptp = 0;
ptp = csep->filters();
if (ptp == 0)
ptp = new execplan::ParseTree();
appendSimpleFilter(ptp, sf);
csep->filters(ptp);
}
// template instantiations
template ConstantColumn* createConstCol<int64_t>(const string& valstr, int64_t val);
} // namespace utils
} // namespace qfe

34
tools/qfe/cseputils.h
View File

@ -1,34 +0,0 @@
#pragma once
#include <string>
namespace execplan
{
class ConstantColumn;
class SimpleFilter;
class ParseTree;
class SimpleColumn;
} // namespace execplan
#include "calpontsystemcatalog.h"
namespace qfe
{
namespace utils
{
execplan::ConstantColumn* createConstCol(const std::string& valstr);
template <typename T>
execplan::ConstantColumn* createConstCol(const std::string& valstr, T val);
execplan::SimpleFilter* createSimpleFilter(boost::shared_ptr<execplan::CalpontSystemCatalog>& csc,
const execplan::CalpontSystemCatalog::TableColName& tcn,
const std::string& opstr, execplan::ConstantColumn* cc);
void appendSimpleFilter(execplan::ParseTree*& ptree, execplan::SimpleFilter* filter);
void updateParseTree(boost::shared_ptr<execplan::CalpontSystemCatalog>&,
execplan::CalpontSelectExecutionPlan*&, execplan::SimpleColumn*, const std::string&,
pair<int, string>);
} // namespace utils
} // namespace qfe

80
tools/qfe/ddlstmts.cpp
View File

@ -1,80 +0,0 @@
#include <unistd.h>
#include <string>
#include <stdexcept>
using namespace std;
#include <boost/scoped_ptr.hpp>
using namespace boost;
#include "ddlpkg.h"
#include "sqlparser.h"
using namespace ddlpackage;
#include "bytestream.h"
#include "messagequeue.h"
using namespace messageqcpp;
#include "ddlstmts.h"
namespace qfe
{
extern string DefaultSchema;
}
namespace
{
using namespace qfe;
int processDDLStmt(const string& stmt, uint32_t sid)
{
MessageQueueClient* mq = 0;
SqlParser parser;
parser.setDefaultSchema(DefaultSchema);
parser.Parse(stmt.c_str());
if (parser.Good())
{
const ddlpackage::ParseTree& ptree = parser.GetParseTree();
SqlStatement& ddlstmt = *ptree.fList[0];
ddlstmt.fSessionID = sid;
ddlstmt.fSql = stmt;
ddlstmt.fOwner = DefaultSchema;
ByteStream bytestream;
bytestream << ddlstmt.fSessionID;
ddlstmt.serialize(bytestream);
mq = new MessageQueueClient("DDLProc");
scoped_ptr<MessageQueueClient> smq(mq);
ByteStream::byte b = 0;
mq->write(bytestream);
bytestream = mq->read();
bytestream >> b;
string emsg;
bytestream >> emsg;
if (b != 0)
throw runtime_error(emsg);
}
else
{
throw runtime_error("syntax error");
}
return 0;
}
} // namespace
namespace qfe
{
void processCreateStmt(const string& stmt, uint32_t sid)
{
processDDLStmt(stmt, sid);
}
void processDropStmt(const string& stmt, uint32_t sid)
{
processDDLStmt(stmt, sid);
}
} // namespace qfe

10
tools/qfe/ddlstmts.h
View File

@ -1,10 +0,0 @@
#pragma once
#include <unistd.h>
#include <string>
namespace qfe
{
void processCreateStmt(const std::string&, uint32_t);
void processDropStmt(const std::string&, uint32_t);
} // namespace qfe

97
tools/qfe/parsequery.cpp
View File

@ -1,97 +0,0 @@
#include <unistd.h>
#include <string>
#include <memory>
using namespace std;
#include <boost/thread/mutex.hpp>
#include <boost/scoped_ptr.hpp>
using namespace boost;
#include "cseputils.h"
#include "parsequery.h"
#include "calpontselectexecutionplan.h"
#include "simplecolumn.h"
#include "simplefilter.h"
#include "constantcolumn.h"
#include "calpontsystemcatalog.h"
#include "sessionmanager.h"
using namespace execplan;
#include "brmtypes.h"
using namespace BRM;
extern int qfeparse();
extern int qfedebug;
struct yy_buffer_state;
extern yy_buffer_state* qfe_scan_string(const char*);
extern void qfe_delete_buffer(yy_buffer_state*);
extern CalpontSelectExecutionPlan* ParserCSEP;
extern boost::shared_ptr<CalpontSystemCatalog> ParserCSC;
namespace
{
mutex ParserMutex;
} // namespace
namespace qfe
{
extern string DefaultSchema;
CalpontSelectExecutionPlan* parseQuery(const string& query, const uint32_t sid)
{
// We're going to make parsing the query single-threaded for now. This makes it a lot
// easier to interface with the parser and doesn;t materially affect overall query
// performance (I think)
mutex::scoped_lock lk(ParserMutex);
boost::shared_ptr<CalpontSystemCatalog> csc = CalpontSystemCatalog::makeCalpontSystemCatalog(sid);
CalpontSelectExecutionPlan* csep = 0;
csep = new CalpontSelectExecutionPlan();
// we use an auto_ptr here with some trepidation. We only want auto delete on an execption.
// If the parseing and plan build succeed, we want the ptr to stay around. boost::scoped_ptr<>
// doesn't have an API to release ownership, so we use auto_ptr...
auto_ptr<CalpontSelectExecutionPlan> scsep(csep);
yy_buffer_state* ybs = 0;
ybs = qfe_scan_string(query.c_str());
if (ybs != 0)
{
ParserCSEP = csep;
ParserCSC = csc;
if (qfeparse() != 0)
throw runtime_error("syntax error");
qfe_delete_buffer(ybs);
}
else
throw runtime_error("Internal parser memory error");
csep->data(query);
SessionManager sm;
TxnID txnID;
txnID = sm.getTxnID(sid);
if (!txnID.valid)
{
txnID.id = 0;
txnID.valid = true;
}
QueryContext verID;
verID = sm.verID();
csep->txnID(txnID.id);
csep->verID(verID);
csep->sessionID(sid);
// cout << *csep << endl;
scsep.release();
return csep;
}
} // namespace qfe

12
tools/qfe/parsequery.h
View File

@ -1,12 +0,0 @@
#pragma once
#include <unistd.h>
#include <string>
#include "calpontselectexecutionplan.h"
namespace qfe
{
execplan::CalpontSelectExecutionPlan* parseQuery(const std::string&, uint32_t);
}

51
tools/qfe/qfe-cli.py
View File

@ -1,51 +0,0 @@
#!/usr/bin/python
'''
Created on May 1, 2013
@author: rtw
'''
import struct
import binascii
import socket
import sys
QFE_HOST = 'localhost'
#QFE_HOST = 'winsvr2008r2.calpont.com'
QFE_SOCK = 9198
def do_query(query):
client_socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
client_socket.connect((QFE_HOST, QFE_SOCK))
defschem = "tpch1";
lenstruct = struct.pack("I", len(defschem))
client_socket.send(lenstruct)
client_socket.send(defschem)
lenstruct = struct.pack("I", len(query))
client_socket.send(lenstruct)
client_socket.send(query)
rowlen = 1
while rowlen:
rowlen = client_socket.recv(4)
if len(rowlen) < 4:
break
rowval = struct.unpack("I",rowlen)[0]
if rowval == 0:
lenstruct = struct.pack("I", 0);
client_socket.send(lenstruct)
break
row = client_socket.recv(rowval)
print row
client_socket.close()
if __name__ == '__main__':
if len(sys.argv) < 2:
query = raw_input ( "QFE-cli > " )
else:
query = sys.argv[1]
sys.exit(do_query(query))

65
tools/qfe/qfelexer.lpp
View File

@ -1,65 +0,0 @@
%{
#include <utility>
#include <string>
#include "returnedcolumn.h"
#ifndef _MSC_VER
#include "qfeparser.h"
#else
#include "bison-win.h"
#endif
extern int qfeerror(const char *s);
extern std::string* newstr(const char* cp);
%}
%option noyywrap
%option yylineno
%option nounput
%x CCONST
digit [0-9]
int_const [+-]?{digit}+
letter [a-zA-Z_]
objname {letter}({letter}|{digit})*
%%
"SELECT" { return QFEP_SELECT; }
"FROM" { return QFEP_FROM; }
"WHERE" { return QFEP_WHERE; }
"GROUP BY" { return GROUPBY; }
"AND" { qfelval.cp = newstr(qfetext); return LOGICOP; }
"OR" { qfelval.cp = newstr(qfetext); return LOGICOP; }
"LIMIT" { return LIMIT; }
"ORDER BY" { return ORDERBY; }
"ASC" { return ASC; }
"DESC" { return DESC; }
"AS" { return AS; }
"SUM" { qfelval.cp = newstr(qfetext); return FUNC; }
"MIN" { qfelval.cp = newstr(qfetext); return FUNC; }
"MAX" { qfelval.cp = newstr(qfetext); return FUNC; }
"AVG" { qfelval.cp = newstr(qfetext); return FUNC; }
{objname} { qfelval.cp = newstr(qfetext); return OBJNAME; }
"'" { BEGIN(CCONST); }
<CCONST>[^']* { qfelval.cp = newstr(qfetext); return CHAR_CONST; }
<CCONST>"'" { BEGIN(INITIAL); }
{int_const} { qfelval.cp = newstr(qfetext); return INT_CONST; }
">" { qfelval.cp = newstr(qfetext); return RELOP; }
">=" { qfelval.cp = newstr(qfetext); return RELOP; }
"<" { qfelval.cp = newstr(qfetext); return RELOP; }
"<=" { qfelval.cp = newstr(qfetext); return RELOP; }
"<>" { qfelval.cp = newstr(qfetext); return RELOP; }
"=" { qfelval.cp = newstr(qfetext); return RELOP; }
[ \t]* {}
[\n] { qfelineno++; }
. { return qfetext[0]; }

306
tools/qfe/qfeparser.ypp
View File

@ -1,306 +0,0 @@
%{
#include <iostream>
#include <string>
#include <utility>
using namespace std;
namespace qfe
{
extern string DefaultSchema;
}
int qfeerror(const char *s);
int qfelex(void);
#include "calpontselectexecutionplan.h"
#include "simplecolumn.h"
#include "calpontsystemcatalog.h"
#include "aggregatecolumn.h"
execplan::CalpontSelectExecutionPlan* ParserCSEP;
execplan::CalpontSystemCatalog* ParserCSC;
namespace execplan
{
class ReturnedColumn;
}
#include "cseputils.h"
%}
%union
{
execplan::ReturnedColumn* rcp;
std::string* cp;
std::pair<int, std::string>* cvp;
}
//Some of these defines cause conflicts in calpontselectexecutionplan.h, so we prepend QFEP_
%token CHAR_CONST RELOP LOGICOP QFEP_SELECT QFEP_FROM QFEP_WHERE GROUPBY OBJNAME INT_CONST LIMIT ORDERBY ASC
%token DESC AS FUNC
%type <cp> select_column_list table_name filter_predicate_list obj_name_aux opt_alias opt_as
%type <cp> opt_limit_clause
%type <cp> OBJNAME CHAR_CONST INT_CONST RELOP LOGICOP FUNC AS
%type <rcp> aliased_column_name select_column_name obj_name func_col groupby_item
%type <cvp> constant
%destructor { delete $$; } OBJNAME CHAR_CONST INT_CONST RELOP LOGICOP FUNC AS
%destructor { delete $$; } aliased_column_name select_column_name obj_name func_col groupby_item
%destructor { delete $$; } constant
%%
statement: QFEP_SELECT select_column_list
QFEP_FROM table_spec
opt_where_clause
opt_groupby_clause
opt_orderby_clause
opt_limit_clause ';'
;
select_column_list: aliased_column_name
{
execplan::SRCP rcp($1);
ParserCSEP->returnedCols().push_back(rcp);
$$ = 0;
}
| select_column_list ',' aliased_column_name
{
execplan::SRCP rcp($3);
ParserCSEP->returnedCols().push_back(rcp);
$$ = 0;
}
;
select_column_name: obj_name
{
execplan::SimpleColumn* sc = dynamic_cast<execplan::SimpleColumn*>($1);
if (sc->schemaName() == "infinidb_unknown" && !qfe::DefaultSchema.empty())
sc->schemaName(qfe::DefaultSchema);
sc->setOID();
sc->tableAlias(sc->tableName());
sc->alias(execplan::make_tcn(sc->schemaName(), sc->tableName(), sc->columnName()).toString());
$$ = $1;
}
| func_col
;
func_col: FUNC '(' select_column_name ')'
{
execplan::AggregateColumn* ag = new execplan::AggregateColumn(execplan::AggregateColumn::agname2num(*($1)), $3);
delete $1;
$$ = ag;
}
;
aliased_column_name: select_column_name opt_alias
;
opt_alias: /* empty */
{ $$ = 0; }
| opt_as OBJNAME
{
delete $2;
$$ = $1;
}
| opt_as CHAR_CONST
{
delete $2;
$$ = $1;
}
;
opt_as: /* empty */
{ $$ = 0; }
| AS
;
table_spec: aliased_table_name
;
/* this is too loose for a table name, but it's easier to parse */
table_name: obj_name
{
execplan::SimpleColumn* sc=0;
sc = dynamic_cast<execplan::SimpleColumn*>($1);
execplan::CalpontSystemCatalog::TableAliasName tan;
tan.schema = sc->tableName();
if (tan.schema == "infinidb_unknown" && !qfe::DefaultSchema.empty())
tan.schema = qfe::DefaultSchema;
if (qfe::DefaultSchema.empty())
qfe::DefaultSchema = tan.schema;
tan.table = sc->columnName();
tan.alias = tan.table;
execplan::CalpontSelectExecutionPlan::TableList tl;
tl.push_back(tan);
ParserCSEP->tableList(tl);
delete sc;
$$ = 0;
}
;
aliased_table_name: table_name opt_alias
;
opt_where_clause: /* empty */
| QFEP_WHERE filter_predicate_list
;
filter_predicate_list: obj_name RELOP constant
{
pair<int, string> cval = *($3);
delete $3;
execplan::SimpleColumn* sc=0;
sc = dynamic_cast<execplan::SimpleColumn*>($1);
qfe::utils::updateParseTree(ParserCSC, ParserCSEP,
sc, *($2), cval);
delete sc;
delete $2;
$$ = 0;
}
| filter_predicate_list LOGICOP obj_name RELOP constant
{
//string logicop = *($2);
delete $2;
pair<int, string> cval = *($5);
delete $5;
execplan::SimpleColumn* sc=0;
sc = dynamic_cast<execplan::SimpleColumn*>($3);
qfe::utils::updateParseTree(ParserCSC, ParserCSEP,
sc, *($4), cval);
delete sc;
delete $4;
$$ = 0;
}
;
obj_name: obj_name_aux
{
//This is possibly a table name, but we shove it into a SimpleColumn. We'll
// fix this in the table_spec production
execplan::SimpleColumn* sc = new execplan::SimpleColumn("infinidb_unknown", "infinidb_unknown", *($1));
sc->tableAlias("infinidb_unknown");
sc->alias(execplan::make_tcn("infinidb_unknown", "infinidb_unknown", *($1)).toString());
//cerr << "inside parser: " << *sc << endl;
delete $1;
$$ = sc;
}
| obj_name_aux '.' obj_name_aux
{
//This is possibly a table name, but we shove it into a SimpleColumn. We'll
// fix this in the table_spec production
execplan::SimpleColumn* sc = new execplan::SimpleColumn("infinidb_unknown", *($1), *($3));
sc->tableAlias(*($1));
sc->alias(execplan::make_tcn("infinidb_unknown", *($1), *($3)).toString());
//cerr << "inside parser: " << *sc << endl;
delete $1;
delete $3;
$$ = sc;
}
| obj_name_aux '.' obj_name_aux '.' obj_name_aux
{
if (qfe::DefaultSchema.empty())
qfe::DefaultSchema = *($1);
execplan::SimpleColumn* sc = new execplan::SimpleColumn(*($1), *($3), *($5));
sc->tableAlias(*($3));
sc->alias(execplan::make_tcn(*($1), *($3), *($5)).toString());
//cerr << "inside parser: " << *sc << endl;
delete $1;
delete $3;
delete $5;
$$ = sc;
}
;
obj_name_aux: OBJNAME
| '`' OBJNAME '`'
{
$$ = $2;
}
;
constant: CHAR_CONST
{
pair<int, string>* p = new pair<int, string>(1, *($1));
delete $1;
$$ = p;
}
| INT_CONST
{
pair<int, string>* p = new pair<int, string>(0, *($1));
delete $1;
$$ = p;
}
;
opt_groupby_clause: /* empty */
| GROUPBY groupby_items_list
{ YYERROR; }
;
groupby_items_list: groupby_item
| groupby_items_list ',' groupby_item
{ delete $3; }
;
groupby_item: obj_name
| INT_CONST
{
delete $1;
$$ = 0;
}
;
opt_orderby_clause: /* empty */
| ORDERBY groupby_items_list opt_direction
{ YYERROR; }
;
opt_direction: /* empty */
| ASC
| DESC
;
opt_limit_clause: /* empty */
{ $$ = 0; }
| LIMIT INT_CONST
{
delete $2;
YYERROR;
$$ = 0;
}
| LIMIT INT_CONST ',' INT_CONST
{
delete $2;
delete $4;
YYERROR;
$$ = 0;
}
;
%%
int qfeerror(const string& s)
{
extern int qfelineno; // defined and maintained in lex.c
extern char *qfetext; // defined and maintained in lex.c
cerr << "ERROR: " << s << " at symbol \"" << qfetext;
cerr << "\" on line " << qfelineno << endl;
return -1;
}
int qfeerror(const char *s)
{
return qfeerror(string(s));
}
string* newstr(const char* cp)
{
string* strp;
strp = new string(cp);
return strp;
}

95
tools/qfe/returnedrows.cpp
View File

@ -1,95 +0,0 @@
#include <unistd.h>
#include <iostream>
#include <string>
#include <cerrno>
using namespace std;
#include <boost/scoped_ptr.hpp>
#include <boost/scoped_array.hpp>
using namespace boost;
#include "mcsconfig.h"
#include "socktype.h"
#include "exceptclasses.h"
#include "socketio.h"
#include "bytestream.h"
#include "messagequeue.h"
using namespace messageqcpp;
#include "rowgroup.h"
using namespace rowgroup;
namespace qfe
{
void processReturnedRows(MessageQueueClient* mqc, SockType fd)
{
scoped_ptr<MessageQueueClient> cleaner(mqc);
SBS sbs;
sbs = mqc->read();
// cerr << "got a bs of " << sbs->length() << " bytes" << endl;
RowGroup rg;
rg.deserialize(*sbs);
// cerr << "got a base rowgroup with rows of " << rg.getRowSize() << " bytes" << endl;
// cerr << rg.toString() << endl;
ByteStream bs;
ByteStream::quadbyte tableOID = 100;
bs.reset();
bs << tableOID;
mqc->write(bs);
sbs = mqc->read();
// cerr << "got a bs of " << sbs->length() << " bytes" << endl;
RGData rgd;
rgd.deserialize(*sbs, true);
rg.setData(&rgd);
// cerr << "got a rowgroup with: " << rg.getRowCount() << " rows" << endl;
socketio::writeString(fd, "OK");
Row r;
while (rg.getRowCount() > 0)
{
rg.initRow(&r);
rg.getRow(0, &r);
string csv;
bs.reset();
for (unsigned i = 0; i < rg.getRowCount(); i++)
{
csv = r.toCSV();
bs << csv;
r.nextRow();
}
// cerr << "writing " << bs.length() << " bytes back to client" << endl;
SockWriteFcn(fd, bs.buf(), bs.length());
bs.reset();
bs << tableOID;
mqc->write(bs);
sbs = mqc->read();
// cerr << "got a bs of " << sbs->length() << " bytes" << endl;
rgd.deserialize(*sbs, true);
rg.setData(&rgd);
// cerr << "got a rowgroup with: " << rg.getRowCount() << " rows" << endl;
}
tableOID = 0;
bs.reset();
bs << tableOID;
mqc->write(bs);
// sync with the client on end-of-results
SockWriteFcn(fd, &tableOID, 4);
SockReadFcn(fd, &tableOID, 4);
}
} // namespace qfe

10
tools/qfe/returnedrows.h
View File

@ -1,10 +0,0 @@
#pragma once
#include "socktype.h"
#include "messagequeue.h"
namespace qfe
{
void processReturnedRows(messageqcpp::MessageQueueClient*, SockType);
}

53
tools/qfe/sendcsep.cpp
View File

@ -1,53 +0,0 @@
#include <unistd.h>
#include <stdexcept>
#include <memory>
using namespace std;
#include <boost/scoped_ptr.hpp>
using namespace boost;
#include "calpontselectexecutionplan.h"
using namespace execplan;
#include "bytestream.h"
#include "messagequeue.h"
using namespace messageqcpp;
namespace qfe
{
MessageQueueClient* sendCSEP(CalpontSelectExecutionPlan* csep)
{
scoped_ptr<CalpontSelectExecutionPlan> cleaner(csep);
ByteStream bs;
MessageQueueClient* mqc = 0;
mqc = new MessageQueueClient("ExeMgr1");
auto_ptr<MessageQueueClient> smqc(mqc);
bs.reset();
ByteStream::quadbyte wantTuples = 4;
bs << wantTuples;
mqc->write(bs);
bs.reset();
csep->serialize(bs);
mqc->write(bs);
SBS sbs;
sbs = mqc->read();
*sbs >> wantTuples;
// cerr << "got flag: " << wantTuples << endl;
string msg;
sbs = mqc->read();
*sbs >> msg;
// cerr << "got msg: " << msg << endl;
if (wantTuples != 0)
throw runtime_error(msg);
smqc.release();
return mqc;
}
} // namespace qfe

12
tools/qfe/sendcsep.h
View File

@ -1,12 +0,0 @@
#pragma once
#include "messagequeue.h"
#include "calpontselectexecutionplan.h"
namespace qfe
{
// Takes ownership of the alloc'd ptr
// Returns an alloc'd mqc
messageqcpp::MessageQueueClient* sendCSEP(execplan::CalpontSelectExecutionPlan*);
} // namespace qfe

429
tools/qfe/server.cpp
View File

@ -1,429 +0,0 @@
/* Copyright (C) 2014 InfiniDB, Inc.
This program is free software; you can redistribute it and/or
modify it under the terms of the GNU General Public License
as published by the Free Software Foundation; version 2 of
the License.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program; if not, write to the Free Software
Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
MA 02110-1301, USA. */
/*
Protocol definition:
*/
#include <poll.h>
#include <sys/socket.h>
#include <netinet/in.h>
#include <arpa/inet.h>
#include <iostream>
#include <string>
#include <stdexcept>
#include <sstream>
#include <iomanip>
#include <sys/types.h>
#include <sys/stat.h>
#include <fcntl.h>
#include <unistd.h>
#include <signal.h>
#include <cerrno>
using namespace std;
#include <boost/thread.hpp>
#include <boost/scoped_ptr.hpp>
#include <boost/scoped_array.hpp>
#include <boost/tokenizer.hpp>
#include <boost/algorithm/string.hpp>
using namespace boost;
#include "mcsconfig.h"
#include "socktype.h"
#include "parsequery.h"
#include "sendcsep.h"
#include "returnedrows.h"
#include "socketio.h"
#include "ddlstmts.h"
#include "exceptclasses.h" //brings in idbassert_s macro
#include "messagequeue.h"
using namespace messageqcpp;
#include "atomicops.h"
namespace execplan
{
class CalpontSelectExecutionPlan;
}
#define SINGLE_THREADED
namespace qfe
{
string DefaultSchema;
}
namespace
{
using namespace qfe;
enum StmtType
{
UNKNOWN,
QUERY,
CREATE,
DROP,
SHOW,
};
volatile uint32_t SystemSID;
void log(const string& s)
{
cerr << s << endl;
}
struct QueryMessage
{
QueryMessage() : isValid(false)
{
}
~QueryMessage()
{
}
string toString() const;
bool isValid;
string queryText;
string defaultSchema;
};
string QueryMessage::toString() const
{
ostringstream oss;
oss << "valid: " << boolalpha << isValid << ", "
<< "queryText: " << queryText << ", "
<< "defaultSchema: " << defaultSchema;
return oss.str();
}
ostream& operator<<(ostream& os, const QueryMessage& rhs)
{
os << rhs.toString();
return os;
}
class ThreadFunc
{
public:
ThreadFunc(SockType fd) : fFd(fd)
{
}
~ThreadFunc()
{
}
void run();
void operator()()
{
run();
}
private:
ThreadFunc(const ThreadFunc& rhs);
ThreadFunc& operator=(const ThreadFunc& rhs);
SockType fFd;
};
bool serverInit()
{
setsid();
// Handle certain signals (we want these to return EINTR so we can throw)
// SIGPIPE
// I don't think we'll get any of these from init (except possibly HUP, but that's an indication
// of bad things anyway)
// SIGHUP?
// SIGUSR1?
// SIGUSR2?
// SIGPOLL?
struct sigaction sa;
memset(&sa, 0, sizeof(struct sigaction));
sa.sa_handler = SIG_IGN;
sigaction(SIGPIPE, &sa, 0);
sigaction(SIGHUP, &sa, 0);
sigaction(SIGUSR1, &sa, 0);
sigaction(SIGUSR2, &sa, 0);
#ifndef __FreeBSD__
sigaction(SIGPOLL, &sa, 0);
#endif
#if 0
int fd;
close(2);
fd = open("/tmp/qfe.err", O_CREAT | O_TRUNC | O_WRONLY, 0644);
if (fd >= 0 && fd != 2)
{
dup2(fd, 2);
close(fd);
}
#endif
return true;
}
SockType initListenSock(short portNo)
{
SockType listenSock = -1;
listenSock = socket(PF_INET, SOCK_STREAM, IPPROTO_TCP);
idbassert_s(listenSock >= 0, string("socket create error: ") + strerror(errno));
// if (listenSock < 0) throw runtime_error(string("socket create error: ") + strerror(errno));
int optval = 1;
setsockopt(listenSock, SOL_SOCKET, SO_REUSEADDR, reinterpret_cast<const char*>(&optval), sizeof(optval));
int rc = 0;
struct sockaddr_in serv_addr;
memset(&serv_addr, 0, sizeof(serv_addr));
serv_addr.sin_family = AF_INET;
serv_addr.sin_addr.s_addr = INADDR_ANY;
serv_addr.sin_port = htons(portNo);
const int MaxTries = 5 * 60 / 10;
int tries = 0;
again:
rc = ::bind(listenSock, (sockaddr*)&serv_addr, sizeof(serv_addr));
if (rc < 0)
{
if (errno == EADDRINUSE)
{
// cerr << "Addr in use..." << endl;
if (++tries >= MaxTries)
{
log("Waited too long for socket to bind...giving up");
// cerr << "Waited too long for socket to bind...giving up" << endl;
exit(1);
}
sleep(10);
goto again;
}
idbassert_s(0, string("socket bind error: ") + strerror(errno));
// throw runtime_error(string("socket bind error: ") + strerror(errno));
}
rc = listen(listenSock, 16);
idbassert_s(rc >= 0, string("socket listen error") + strerror(errno));
// if (rc < 0) throw runtime_error(string("socket listen error") + strerror(errno));
return listenSock;
}
QueryMessage getNextMsg(SockType fd)
{
QueryMessage msg;
try
{
msg.defaultSchema = socketio::readString(fd);
msg.queryText = socketio::readString(fd);
msg.isValid = true;
}
catch (runtime_error& rex)
{
cerr << "re reading ctl msg: " << rex.what() << endl;
msg.queryText = "";
}
catch (...)
{
cerr << "ex reading ctl msg" << endl;
msg.queryText = "";
}
return msg;
}
StmtType guessStatementType(const string& stmt)
{
typedef boost::tokenizer<boost::char_separator<char> > tokenizer;
char_separator<char> sep;
tokenizer tokens(stmt, sep);
tokenizer::iterator tok_iter = tokens.begin();
string first_word;
first_word = *tok_iter;
algorithm::to_lower(first_word);
if (first_word == "select")
return QUERY;
if (first_word == "create")
return CREATE;
if (first_word == "drop")
return DROP;
if (first_word == "show")
return SHOW;
return UNKNOWN;
}
struct ScopedCleaner
{
ScopedCleaner(SockType fd = -1) : fFd(fd)
{
}
~ScopedCleaner()
{
if (fFd >= 0)
shutdown(fFd, SHUT_RDWR);
close(fFd);
}
SockType fFd;
};
void ThreadFunc::run()
{
QueryMessage m;
execplan::CalpontSelectExecutionPlan* csep = 0;
MessageQueueClient* msgqcl;
ScopedCleaner cleaner(fFd);
uint32_t sid = 1;
sid = atomicops::atomicInc(&SystemSID);
try
{
m = getNextMsg(fFd);
if (m.isValid)
{
DefaultSchema = m.defaultSchema;
StmtType st = guessStatementType(m.queryText);
switch (st)
{
case QUERY:
csep = parseQuery(m.queryText, sid);
// sendCSEP takes ownership of the ptr from parseQuery
msgqcl = sendCSEP(csep);
// processReturnedRows takes ownership of the ptr from sendCSEP
processReturnedRows(msgqcl, fFd);
break;
case CREATE: processCreateStmt(m.queryText, sid); break;
case DROP: processDropStmt(m.queryText, sid); break;
case SHOW:
{
ostringstream oss;
oss << "select calpontsys.systable.tablename from calpontsys.systable where "
"calpontsys.systable.schema='"
<< m.defaultSchema << "';";
csep = parseQuery(oss.str(), sid);
msgqcl = sendCSEP(csep);
processReturnedRows(msgqcl, fFd);
break;
}
default: throw runtime_error("couldn't guess the statement type"); break;
}
}
}
catch (std::exception& ex)
{
socketio::writeString(fFd, ex.what());
throw; // in a multi-threaded server this will simply cause this thread to exit
}
catch (...)
{
socketio::writeString(fFd, "internal query processing error");
throw;
}
}
} // namespace
int main(int argc, char** argv)
{
int c;
SockType listenSock;
short portNo;
portNo = 0;
char* p = getenv("IDB_QFE_PORT");
if (p && *p)
portNo = atoi(p);
if (portNo <= 0)
portNo = 9198;
listenSock = -1;
opterr = 0;
while ((c = getopt(argc, argv, "p:")) != -1)
switch (c)
{
case 'p': portNo = atoi(optarg); break;
case '?':
default: break;
}
if (!serverInit())
{
log("Could not initialize the QFE Server!");
cerr << "Could not initialize the QFE Server!" << endl;
return 1;
}
listenSock = initListenSock(portNo);
SystemSID = 0;
for (;;)
{
int querySock = -1;
querySock = accept(listenSock, 0, 0);
idbassert_s(querySock >= 0, string("socket accept error: ") + strerror(errno));
// ThreadFunc now owns querySock and is responsible for cleaning it up
ThreadFunc tf(querySock);
#ifdef SINGLE_THREADED
try
{
tf.run();
}
catch (std::exception& ex)
{
cerr << "ThreadFunc run threw an exception: " << ex.what() << endl;
}
catch (...)
{
cerr << "ThreadFunc run threw an exception" << endl;
}
#else
thread t(tf);
#endif
}
return 0;
}

133
tools/qfe/socketio.cpp
View File

@ -1,133 +0,0 @@
#include <unistd.h>
#include <poll.h>
#include <sys/socket.h>
#include <netinet/in.h>
#include <arpa/inet.h>
#include <cerrno>
#include <sstream>
using namespace std;
#include <boost/scoped_array.hpp>
using namespace boost;
#include "mcsconfig.h"
#include "exceptclasses.h"
#include "socketio.h"
namespace qfe
{
namespace socketio
{
void readn(int fd, void* buf, const size_t wanted)
{
size_t needed = wanted;
size_t sofar = 0;
char* p = static_cast<char*>(buf);
ssize_t rrc = -1;
pollfd fds[1];
int en = 0;
fds[0].fd = fd;
fds[0].events = POLLIN;
while (wanted > sofar)
{
fds[0].revents = 0;
poll(fds, 1, -1);
errno = 0;
rrc = read(fd, (p + sofar), needed);
en = errno;
if (rrc < 0)
{
if (en == EAGAIN || en == EINTR || en == 512)
continue;
ostringstream oss;
oss << "qfe: readn: read() returned " << rrc << " (" << strerror(en) << ")";
idbassert_s(0, oss.str());
}
needed -= rrc;
sofar += rrc;
}
}
size_t writen(int fd, const void* data, const size_t nbytes)
{
size_t nleft;
ssize_t nwritten;
const char* bufp = static_cast<const char*>(data);
nleft = nbytes;
while (nleft > 0)
{
// the O_NONBLOCK flag is not set, this is a blocking I/O.
if ((nwritten = ::write(fd, bufp, nleft)) < 0)
{
if (errno == EINTR)
nwritten = 0;
else
{
// save the error no first
int e = errno;
string errorMsg = "qfe: writen: write() error: ";
scoped_array<char> buf(new char[80]);
#if STRERROR_R_CHAR_P
const char* p;
if ((p = strerror_r(e, buf.get(), 80)) != 0)
errorMsg += p;
#else
int p;
if ((p = strerror_r(e, buf.get(), 80)) == 0)
errorMsg += buf.get();
#endif
idbassert_s(0, errorMsg);
}
}
nleft -= nwritten;
bufp += nwritten;
}
return nbytes;
}
uint32_t readNumber32(SockType fd)
{
uint32_t np;
SockReadFcn(fd, &np, 4);
return np;
}
string readString(SockType fd)
{
string s;
uint32_t len = readNumber32(fd);
if (len > 0)
{
scoped_array<char> buf(new char[len + 1]);
SockReadFcn(fd, buf.get(), len);
buf[len] = 0;
s = buf.get();
}
return s;
}
void writeString(SockType fd, const string& data)
{
uint32_t len = data.length();
SockWriteFcn(fd, &len, 4);
SockWriteFcn(fd, data.c_str(), len);
}
} // namespace socketio
} // namespace qfe

20
tools/qfe/socketio.h
View File

@ -1,20 +0,0 @@
#pragma once
#include <stdint.h>
#include <unistd.h>
#include <string>
#include "socktype.h"
namespace qfe
{
namespace socketio
{
void readn(int fd, void* buf, const size_t wanted);
size_t writen(int fd, const void* data, const size_t nbytes);
uint32_t readNumber32(SockType);
std::string readString(SockType);
void writeString(SockType, const std::string&);
} // namespace socketio
} // namespace qfe

6
tools/qfe/socktype.h
View File

@ -1,6 +0,0 @@
#pragma once
typedef int SockType;
#define SockReadFcn qfe::socketio::readn
#define SockWriteFcn qfe::socketio::writen

9
tools/rebuildEM/CMakeLists.txt
View File

@ -1,10 +1,5 @@
include_directories(${ENGINE_COMMON_INCLUDES})
set(rebuildEM_SRCS main.cpp rebuildEM.cpp)
add_executable(mcsRebuildEM ${rebuildEM_SRCS})
target_link_libraries(mcsRebuildEM ${ENGINE_LDFLAGS} ${ENGINE_WRITE_LIBS} boost_system boost_filesystem)
install(
TARGETS mcsRebuildEM
DESTINATION ${ENGINE_BINDIR}
COMPONENT columnstore-engine
)
columnstore_executable(mcsRebuildEM ${rebuildEM_SRCS})
columnstore_link(mcsRebuildEM ${ENGINE_LDFLAGS} ${ENGINE_WRITE_LIBS} boost_system boost_filesystem)

10
tools/rgprint/CMakeLists.txt
View File

@ -4,12 +4,6 @@ include_directories(${ENGINE_COMMON_INCLUDES})
set(rgprint_SRCS rgprint.cpp)
add_executable(rgprint ${rgprint_SRCS})
columnstore_executable(rgprint ${rgprint_SRCS})
target_link_libraries(rgprint ${ENGINE_LDFLAGS} ${NETSNMP_LIBRARIES} ${ENGINE_WRITE_LIBS})
install(
TARGETS rgprint
DESTINATION ${ENGINE_BINDIR}
COMPONENT columnstore-engine
)
columnstore_link(rgprint ${ENGINE_LDFLAGS} ${NETSNMP_LIBRARIES} ${ENGINE_WRITE_LIBS})

32
tools/sendPlan/CMakeLists.txt
View File

@ -1,32 +0,0 @@
include_directories(${KDE4_INCLUDES} ${KDE4_INCLUDE_DIR} ${QT_INCLUDES})
# ########## next target ###############
set(sendPlan_SRCS sendplan.cpp)
kde4_add_executable(sendPlan ${sendPlan_SRCS})
target_link_libraries(sendPlan ${KDE4_KDECORE_LIBS})
install(TARGETS sendPlan ${INSTALL_TARGETS_DEFAULT_ARGS})
# ########## install files ###############
# original Makefile.am contents follow:
# $Id: Makefile.am 333 2009-04-03 20:35:04Z rdempsey $ Process this file with automake to produce Makefile.in
#
# AM_CPPFLAGS = $(idb_cppflags) AM_CFLAGS = $(idb_cflags) AM_CXXFLAGS = $(idb_cxxflags) AM_LDFLAGS = $(idb_ldflags)
# bin_PROGRAMS = sendPlan sendPlan_SOURCES = sendplan.cpp sendPlan_CPPFLAGS = @idb_common_includes@ $(AM_CPPFLAGS)
# sendPlan_LDFLAGS = @idb_common_ldflags@ @idb_exec_libs@ $(AM_LDFLAGS)
#
# test:
#
# coverage:
#
# leakcheck:
#
# docs:
#
# bootstrap: install-data-am
#

392
tools/sendPlan/sendplan.cpp
View File

@ -1,392 +0,0 @@
// $Id: sendplan.cpp 1739 2012-03-22 12:57:59Z pleblanc $
#include <iostream>
#include <fstream>
#include <sstream>
#include <unistd.h>
#include <sys/time.h>
#include <iomanip>
using namespace std;
#include "bytestream.h"
using namespace messageqcpp;
#include "calpontselectexecutionplan.h"
#include "sessionmanager.h"
using namespace execplan;
#include "calpontsystemcatalog.h"
using namespace execplan;
#include "tableband.h"
#include "joblist.h"
#include "joblistfactory.h"
using namespace joblist;
#include "configcpp.h"
#include "errorcodes.h"
#include "rowgroup.h"
using namespace rowgroup;
typedef CalpontSystemCatalog::OID OID;
namespace
{
bool vflg;
void usage()
{
cout << "usage: sendPlan [-v|h|d|B] [-t lvl] [-s sid] plan_file ..." << endl;
cout << "-v verbose output" << endl;
cout << "-t lvl set trace level to lvl" << endl;
cout << "-s sid set session id to sid" << endl;
cout << "-d display the query, but don't run it" << endl;
cout << "-B Bob's personal preferred output format" << endl;
cout << "-h display this help" << endl;
}
double tm_diff(const struct timeval* st, const struct timeval* et)
{
double sd = (double)st->tv_sec + (double)st->tv_usec / 1000000.0;
double ed = (double)et->tv_sec + (double)et->tv_usec / 1000000.0;
return (ed - sd);
}
//...Extract the stats string to be printed and the runtime start time from
//...the full stats string we receive from ExeMgr.
void parseStatsString(const string& fullStatsString, string& printStatsString,
struct timeval& queryRunStartTime)
{
string::size_type delimPos = fullStatsString.find('|');
printStatsString = fullStatsString.substr(0, delimPos);
istringstream startTimeString(fullStatsString.substr(delimPos + 1));
startTimeString >> queryRunStartTime.tv_sec >> queryRunStartTime.tv_usec;
}
} // namespace
int main(int argc, char** argv)
{
vflg = false;
uint32_t tlvl = 0;
bool dflg = false;
int c;
int32_t sid = -1;
bool Bflg = false;
opterr = 0;
while ((c = getopt(argc, argv, "vt:ds:Bh")) != EOF)
switch (c)
{
case 't': tlvl = static_cast<uint32_t>(strtoul(optarg, 0, 0)); break;
case 'v': vflg = true; break;
case 'd': dflg = true; break;
case 's': sid = static_cast<int32_t>(strtol(optarg, 0, 0)); break;
case 'B': Bflg = true; break;
case 'h':
case '?':
default:
usage();
return (c == 'h' ? 0 : 1);
break;
}
if (dflg)
vflg = true;
if ((argc - optind) < 1)
{
usage();
return 1;
}
ifstream inputf;
ByteStream bs;
ByteStream dbs;
ByteStream eoq;
ByteStream tbs;
ByteStream statsStream;
ByteStream::quadbyte q = 0;
eoq << q;
uint32_t sessionid;
time_t t;
SJLP jl;
DeliveredTableMap tm;
DeliveredTableMap::iterator iter;
DeliveredTableMap::iterator end;
CalpontSelectExecutionPlan csep;
struct timeval start_time;
struct timeval end_time;
MessageQueueClient* mqc = 0;
if (!dflg)
mqc = new MessageQueueClient("ExeMgr1");
if (sid == -1)
{
time(&t);
sessionid = static_cast<uint32_t>(t);
}
else
{
sessionid = static_cast<uint32_t>(sid);
}
sessionid &= 0x7fffffff;
logging::ErrorCodes errorCodes;
for (; optind < argc; optind++)
{
inputf.open(argv[optind]);
if (!inputf.good())
{
cerr << "error opening plan stream " << argv[optind] << endl;
return 1;
}
bs.reset();
inputf >> bs;
inputf.close();
csep.unserialize(bs);
csep.sessionID(sessionid);
SessionManager sm;
csep.verID(sm.verID());
csep.traceFlags(0);
ResourceManager* rm = ResourceManager::instance();
jl = JobListFactory::makeJobList(&csep, rm);
csep.traceFlags(tlvl);
if (vflg)
{
if (dflg)
cout << endl << "Query:" << endl;
else
{
cout << endl << "Session: " << sessionid << ", Sending Query";
if (Bflg)
cout << " (" << argv[optind] << ')';
cout << ':' << endl;
}
if (!Bflg)
cout << csep.data() << endl << endl;
}
if (dflg)
continue;
try
{
dbs.reset();
csep.serialize(dbs);
gettimeofday(&start_time, 0);
// try tuples first, but expect the worst...
bool expectTuples = false;
ByteStream tbs;
ByteStream::quadbyte tqb = 4;
tbs << tqb;
mqc->write(tbs);
// send the CSEP
mqc->write(dbs);
// read the response to the tuple request
tbs = mqc->read();
idbassert(tbs.length() == 4);
tbs >> tqb;
if (tqb == 4)
expectTuples = true;
if (!expectTuples)
cout << "Using TableBand I/F" << endl;
else
cout << "Using tuple I/F" << endl;
tm = jl->deliveredTables();
iter = tm.begin();
end = tm.end();
OID toid;
uint64_t rowTot;
bool reported = false;
bool needRGCtor = true;
while (iter != end)
{
toid = iter->first;
q = static_cast<ByteStream::quadbyte>(toid);
tbs.reset();
tbs << q;
mqc->write(tbs);
ByteStream tbbs;
TableBand tb;
RowGroup rg;
rowTot = 0;
uint16_t status = 0;
TableBand::VBA::size_type rc;
ofstream out;
for (;;)
{
tbbs = mqc->read();
#if 0
cout << tbbs.length() << endl;
out.open("bs1.dat");
idbassert(out.good());
out << tbbs;
out.close();
tbbs = mqc->read();
cout << tbbs.length() << endl;
out.open("bs2.dat");
idbassert(out.good());
out << tbbs;
out.close();
tbbs = mqc->read();
cout << tbbs.length() << endl;
out.open("bs3.dat");
idbassert(out.good());
out << tbbs;
out.close();
#endif
if (tbbs.length())
{
if (!expectTuples)
tb.unserialize(tbbs);
else
{
if (needRGCtor)
{
rg.deserialize(tbbs);
needRGCtor = false;
tbbs = mqc->read();
}
rg.setData((uint8_t*)tbbs.buf());
}
}
else
{
//@bug 1346
if (!status)
status = logging::makeJobListErr;
break;
}
if (!expectTuples)
{
rc = tb.getRowCount();
status = tb.getStatus();
}
else
{
rc = rg.getRowCount();
status = rg.getStatus();
if (rc == 0)
status = 0;
}
if (rc == 0)
break;
rowTot += rc;
}
BatchPrimitive* step = dynamic_cast<BatchPrimitive*>(iter->second.get());
if (vflg && step)
{
cout << "For table " << step->tableName();
if (!Bflg)
cout << " " << toid;
cout << ": read " << rowTot << " rows" << endl;
}
if (status && !reported)
{
cout << "### Query failed: " << errorCodes.errorString(status) << " Check crit.log\n";
reported = true;
}
if (!step && !reported)
{
cout << "### Query failed: Did not return project BatchPrimitive. Check crit.log\n";
reported = true;
}
++iter;
}
if (vflg)
{
gettimeofday(&end_time, 0);
cout << "Query time: " << fixed << setprecision(1) << tm_diff(&start_time, &end_time) << " secs"
<< endl;
//...Ask for query stats through special table id of 3
const OID TABLE_ID_TO_GET_QUERY_STATS = 3;
if (!Bflg)
cout << "Retrieving stats..." << endl;
toid = TABLE_ID_TO_GET_QUERY_STATS;
q = static_cast<ByteStream::quadbyte>(toid);
statsStream.reset();
statsStream << q;
mqc->write(statsStream);
ByteStream bs_statsString;
bs_statsString = mqc->read();
string statsString;
bs_statsString >> statsString;
string printStatsString;
struct timeval startRunTime;
parseStatsString(statsString, printStatsString, startRunTime);
cout << printStatsString << "; QuerySetupTime-" << tm_diff(&start_time, &startRunTime) << "secs"
<< endl;
}
//...Close this query/session
mqc->write(eoq);
jl.reset();
}
catch (const exception& ex)
{
cout << "### SendPlan caught an exception: " << ex.what() << endl;
}
}
// jl.reset();
CalpontSystemCatalog::removeCalpontSystemCatalog(sessionid);
config::Config::deleteInstanceMap();
delete mqc;
return 0;
}

11
tools/setConfig/CMakeLists.txt
View File

@ -4,12 +4,5 @@ include_directories(${ENGINE_COMMON_INCLUDES})
set(setConfig_SRCS main.cpp)
add_executable(mcsSetConfig ${setConfig_SRCS})
target_link_libraries(mcsSetConfig ${ENGINE_LDFLAGS} ${NETSNMP_LIBRARIES} ${ENGINE_EXEC_LIBS})
install(
TARGETS mcsSetConfig
DESTINATION ${ENGINE_BINDIR}
COMPONENT columnstore-engine
)
columnstore_executable(mcsSetConfig ${setConfig_SRCS})
columnstore_link(mcsSetConfig ${ENGINE_LDFLAGS} ${NETSNMP_LIBRARIES} ${ENGINE_EXEC_LIBS})

132
tools/snconvert/snconvert.pl
View File

@ -1,132 +0,0 @@
#!/usr/bin/perl -w -s
###################################
# Control parameters
###################################
# location of the data files to process
$srcDir="";
$srcFileName="";
if ( $#ARGV>-1 )
{
$srcDir = $ARGV[0];
$srcFileName= $ARGV[1];
print "\n\nCLA $srcDir $srcFileName\n\n";
}
else
{
$srcDir = ".";
$srcFileName = "FILE000.cdf";
}
# full path of the data file
$srcFile = $srcDir . $srcFileName;
# name of the new file/extent file
$newFileName = $srcFileName . '_';
# get current directory
$currDir = `pwd`;
chomp $currDir;
# number of file system files
$fsFileCnt=8;
# once file data has been reorganized they will be moved
# to these locations.
@pmDest = (
"root\@srvswdev2\:\/tmp",
"root\@srvswdev2\:\/tmp",
"root\@srvswdev2\:\/tmp",
"root\@srvswdev2\:\/tmp"
);
# split column file into seperate extent files
@sysCmd = ("split", "-d", "-b", "8m", "$srcFile", "$newFileName");
print "@sysCmd\n";
print "Starting split . . . .\n";
system(@sysCmd);
print "Split complete.\n";
# get names of extent files
@files = <$newFileName*>;
$extentCnt = @files;
$i=0;
# list of extent files to place into a file system file
@extList = ("");
#list of file system files
@fsFiles = ("");
#init file lists
while ( $i<$fsFileCnt )
{
$fsFiles[$i]="fsFile" . $i;
$extList[$i]="";
$i++;
}
# organize extents into files per file systems
#in this case there are 8 file systems
$i=0;
$j=0;
while ( $i<$extentCnt )
{
$j=0;
while ( $j<$fsFileCnt && $i<$extentCnt )
{
$extList[$j] = $extList[$j] . " " . $files[$i];
$i++;
$j++;
}
}
$i=0;
while ( $i < $fsFileCnt )
{
print "List $i: $extList[$i]\n";
$Cmd="cat $extList[$i] >> $fsFiles[$i]";
print "concat $i: $Cmd\n";
`$Cmd`;
$i++;
}
# copy the file system files to the appropriate PM
# in this case there are 4 PMs
$pmCnt=4;
$i=0;
$j=0;
@pmFiles =("");
while ( $i<$fsFileCnt )
{
$j=0;
while ( $j < $pmCnt && $i < $fsFileCnt)
{
# $pmFiles[$j] = $pmFiles[$j] . " " . $fsFiles[$i++];
$Cmd = "./remote_scp_put.sh $fsFiles[$i++] $pmDest[$j++] Calpont1";
print "scp command: $Cmd\n";
`$Cmd`;
}
}
#$j=0;
#while ( $j < $pmCnt )
#{
# print "pmFileList $j : $pmFiles[$j]\n";
# $j++;
#}
#
#$j=0;
#while ( $j < $pmCnt )
#{
# $Cmd = "scp $pmFiles[$j] $pmDest[$j]";
# print "scp command: $Cmd\n";
# system($Cmd);
# $j++;
#}

58
tools/updateGenii/listRepos.pl
View File

@ -1,58 +0,0 @@
#!/usr/bin/perl -w
#
# $Id: listRepos.pl 192 2007-05-14 14:42:49Z rdempsey $
#
use File::Basename;
$reporoot = '/Calpont/repos/genii/*';
$opt_a = 0;
if (defined($ENV{'QUERY_STRING'}))
{
$opt_a = 1 if ($ENV{'QUERY_STRING'} eq "All");
}
sub findRepos
{
my ($root) = @_;
my @repolist = ();
my @repodirs = glob($root);
my $dir;
for $dir (@repodirs)
{
$dir = basename($dir);
if ($opt_a == 0)
{
next if ($dir eq "net-snmp");
# deprecated projects
next if ($dir eq "altera");
next if ($dir eq "diskmgr");
next if ($dir eq "emulator");
next if ($dir eq "message");
next if ($dir eq "sqlengine");
}
next if ($dir eq "demo");
next if ($dir eq "doc");
push(@repolist, $dir);
}
return @repolist;
}
@repos = findRepos($reporoot);
print 'content-type: text/plain', "\r\n";
print "\r\n";
foreach $repo (@repos)
{
print $repo, "\r\n";
}

14
tools/updateGenii/test.dat
View File

@ -1,14 +0,0 @@
A OID3046.dat
? brmtest/brmtest
? dumpcol/dumpcol
? fixdate/fixdate.cdf
? fixdate/f1.cdf
? fixdate/fixdate
? evalcol/evalcol
? editem/editem
? dbbuilder/dbbuilder
? oid2file/oid2file
M Makefile
G updateGenii
C updateGenii/updateGenii.pl
G updateGenii/listRepos.pl

186
tools/updateGenii/updateGenii.pl
View File

@ -1,186 +0,0 @@
#!/usr/bin/perl -w
#
# $Id: updateGenii.pl 1265 2010-09-17 12:25:42Z rdempsey $
#
use Getopt::Std;
sub usage
{
my $msg = <<'EOD'
usage: updateGenii.pl [-h] [-b|-t branch/tag] [-r root] [-a]
-h Display this help
-b branch Update a branch wc
-t tag Update a tag wc
-r root Use root as the starting point rather than $HOME
-a Update wc for all svn projects, even deprecated ones
EOD
;
print "$msg\n";
}
sub findRepos
{
my @repolist = ();
my $all = '';
$all = '?All' if ($opt_a);
open(FH, 'curl -s http://srvengcm1.calpont.com/cgi-bin/listRepos.pl' . $all . ' |');
while (<FH>)
{
$_ =~ s/[\r\n]*$//;
push(@repolist, $_);
}
close(FH);
return @repolist;
}
if (!getopts('hab:r:t:'))
{
usage();
exit 1;
}
if (defined($opt_h))
{
$opt_h = 1;
usage();
exit 0;
}
$opt_a = 1 if (defined($opt_a));
$prjroot = 'genii';
$prjroot = $opt_r if (defined($opt_r));
$prjroot = $opt_b if (defined($opt_b));
$prjroot = $opt_t if (defined($opt_t));
if (!defined($opt_b) && !defined($opt_t) && $#ARGV >= 0)
{
usage();
exit 1;
}
if (defined($opt_b) || defined($opt_t))
{
$cmd = 'svn list http://srvengcm1.calpont.com/svn/genii/dbcon/';
if (defined($opt_b))
{
$cmd = $cmd . 'branches';
}
else
{
$cmd = $cmd . 'tags';
}
$listcmd = $cmd;
$tmp = $prjroot;
$tmp =~ s/\./\\./g;
$cmd = $cmd . " 2>/dev/null | egrep '^" . $tmp . "/' | wc -l";
$brcnt = `$cmd`;
if ($brcnt != 1)
{
print "branch/tag $prjroot not found. Choices are:\n";
$listcmd = $listcmd . ' | sed s,/\$,,';
system($listcmd);
exit 1;
}
}
chdir $ENV{'HOME'} or die;
$destDir = $prjroot;
if ((defined($opt_b) || defined($opt_t)) && defined($opt_r))
{
$destDir = $opt_r;
}
mkdir $destDir if (! -d $destDir);
chdir $destDir or die;
@repos = findRepos();
@mergelines = ();
@conflictlines = ();
foreach $repo (@repos)
{
#print $repo, "\n"; next;
if (-d $repo . '/.svn')
{
print $repo, ": \n";
chdir $repo or die;
open FH, 'svn update |';
@svnlines = <FH>;
close FH;
foreach $line (@svnlines)
{
print "$line";
if ($line =~ /^C/)
{
push(@conflictlines, $repo . ": " . $line);
}
elsif ($line =~ /^G/)
{
push(@mergelines, $repo . ": " . $line);
}
}
chdir '..' or die;
}
else
{
print $repo, ": \n";
if (defined($opt_b))
{
$svncmd = 'svn checkout http://srvengcm1.calpont.com/svn/genii/' . $repo . '/branches/' .
$prjroot . ' ' . $repo;
}
elsif (defined($opt_t))
{
$svncmd = 'svn checkout http://srvengcm1.calpont.com/svn/genii/' . $repo . '/tags/' .
$prjroot . ' ' . $repo;
}
else
{
$svncmd = 'svn checkout http://srvengcm1.calpont.com/svn/genii/' . $repo . '/trunk/' . ' ' .
$repo;
}
system $svncmd;
}
}
unlink 'merges.txt';
unlink 'conflicts.txt';
if ($#mergelines >= 0)
{
print "\n** Warning! merges:\n";
foreach $merge (@mergelines)
{
print "$merge";
}
open FH, ">merges.txt";
print FH @mergelines;
close FH;
print "This information has been saved in merges.txt\n";
}
if ($#conflictlines >= 0)
{
print "\n** Warning! conflicts:\n";
foreach $conflict (@conflictlines)
{
print "$conflict";
}
open FH, ">conflicts.txt";
print FH @conflictlines;
close FH;
print "This information has been saved in conflicts.txt\n";
}
if (! -e 'Makefile')
{
system './build/bootstrap';
}

48
tools/valprint/valprint.cpp
View File

@ -1,48 +0,0 @@
/* Copyright (C) 2014 InfiniDB, Inc.
This program is free software; you can redistribute it and/or
modify it under the terms of the GNU General Public License
as published by the Free Software Foundation; version 2 of
the License.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program; if not, write to the Free Software
Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
MA 02110-1301, USA. */
// $Id: valprint.cpp 2101 2013-01-21 14:12:52Z rdempsey $
#include <iostream>
#include <cstdlib>
using namespace std;
#include "configcpp.h"
using namespace config;
int main(int argc, char** argv)
{
int c;
opterr = 0;
while ((c = getopt(argc, argv, "")) != EOF)
switch (c)
{
case '?':
default: break;
}
string s;
for (int i = optind; i < argc; i++)
{
s = argv[i];
cout << s << " = " << Config::fromText(s) << endl;
}
return 0;
}

11
tools/viewtablelock/CMakeLists.txt
View File

@ -4,12 +4,5 @@ include_directories(${ENGINE_COMMON_INCLUDES})
set(viewtablelock_SRCS viewtablelock.cpp)
add_executable(viewtablelock ${viewtablelock_SRCS})
target_link_libraries(viewtablelock ${ENGINE_LDFLAGS} ${ENGINE_EXEC_LIBS})
install(
TARGETS viewtablelock
DESTINATION ${ENGINE_BINDIR}
COMPONENT columnstore-engine
)
columnstore_executable(viewtablelock ${viewtablelock_SRCS})
columnstore_link(viewtablelock ${ENGINE_LDFLAGS} ${ENGINE_EXEC_LIBS})

15
utils/basic/string_utils.h Normal file
View File

@ -0,0 +1,15 @@
#pragma once
#include <algorithm>
#include <string>
namespace datatypes
{
inline bool ASCIIStringCaseInsensetiveEquals(const std::string& left, const std::string& right)
{
auto asciiHigher = [](char c) { return (c >= 'a' && c <= 'z') ? c - 'a' + 'A' : c; };
return left.size() == right.size() &&
std::equal(left.begin(), left.end(), right.begin(),
[&asciiHigher](char l, char r) { return asciiHigher(l) == asciiHigher(r); });
}
} // namespace datatypes

10
utils/batchloader/CMakeLists.txt
View File

@ -6,12 +6,4 @@ set(batchloader_LIB_SRCS batchloader.cpp)
columnstore_library(batchloader ${batchloader_LIB_SRCS})
add_dependencies(batchloader loggingcpp)
target_link_libraries(batchloader ${NETSNMP_LIBRARIES})
install(
TARGETS batchloader
DESTINATION ${ENGINE_LIBDIR}
COMPONENT columnstore-engine
)
columnstore_link(batchloader ${NETSNMP_LIBRARIES} loggingcpp)

9
utils/cacheutils/CMakeLists.txt
View File

@ -5,11 +5,4 @@ include_directories(${ENGINE_COMMON_INCLUDES})
set(cacheutils_LIB_SRCS cacheutils.cpp)
columnstore_library(cacheutils ${cacheutils_LIB_SRCS})
add_dependencies(cacheutils loggingcpp)
install(
TARGETS cacheutils
DESTINATION ${ENGINE_LIBDIR}
COMPONENT columnstore-engine
)
columnstore_link(cacheutils messageqcpp)

17
utils/cloudio/CMakeLists.txt
View File

@ -14,22 +14,15 @@ columnstore_library(cloudio ${cloudio_LIB_SRCS})
# IDBDataFile currently depends on cloudio, which is backward. Once cloudio has been turned into a proper plugin for
# idbdatafile, we should be able to reverse the dependency like so:
target_link_libraries(cloudio idbdatafile messageqcpp loggingcpp)
install(
TARGETS cloudio
DESTINATION ${ENGINE_LIBDIR}
COMPONENT columnstore-engine
)
columnstore_link(cloudio idbdatafile messageqcpp)
add_executable(cloudio_component_test component_test.cpp)
add_executable(end_to_end_test end_to_end_test.cpp)
add_executable(cloudio_end_to_end_test end_to_end_test.cpp)
# see the comment above and change this dependency to cloudio. Hm our lib dependencies seem not to be declared. Punting
# on that, maybe in the future we can have some poor unfortunate intern untangle all of that and declare lib
# dependencies properly. For now I'm going to do like the other executables, which means nearly everything AFAICT.
target_link_libraries(cloudio_component_test ${ENGINE_LDFLAGS} ${ENGINE_EXEC_LIBS} cloudio)
target_link_libraries(end_to_end_test ${ENGINE_LDFLAGS} ${ENGINE_EXEC_LIBS} cloudio)
# target_link_libraries(cloudio_component_test cloudio) Copy end_to_end_test dataFile to binary directory
columnstore_link(cloudio_component_test ${ENGINE_LDFLAGS} ${ENGINE_EXEC_LIBS} cloudio)
columnstore_link(cloudio_end_to_end_test ${ENGINE_LDFLAGS} ${ENGINE_EXEC_LIBS} cloudio)
# columnstore_link(cloudio_component_test cloudio) Copy end_to_end_test dataFile to binary directory
file(COPY testData DESTINATION ${CMAKE_CURRENT_BINARY_DIR}/)

11
utils/common/CMakeLists.txt
View File

@ -15,13 +15,4 @@ set(common_LIB_SRCS
)
columnstore_library(common ${common_LIB_SRCS})
target_link_libraries(common boost_filesystem)
add_dependencies(common loggingcpp)
install(
TARGETS common
DESTINATION ${ENGINE_LIBDIR}
COMPONENT columnstore-engine
)
columnstore_link(common boost_filesystem configcpp loggingcpp messageqcpp ${MARIADB_STRING_LIBS})

8
utils/common/collation.h
View File

@ -275,12 +275,4 @@ class CollationAwareComparator : public Charset
}
};
inline bool ASCIIStringCaseInsensetiveEquals(const std::string& left, const std::string& right)
{
auto asciiHigher = [](char c) { return (c >= 'a' && c <= 'z') ? c - 'a' + 'A' : c; };
return left.size() == right.size() &&
std::equal(left.begin(), left.end(), right.begin(),
[&asciiHigher](char l, char r) { return asciiHigher(l) == asciiHigher(r); });
}
} // end of namespace datatypes

13
utils/compress/CMakeLists.txt
View File

@ -11,19 +11,10 @@ set_property(DIRECTORY PROPERTY INCLUDE_DIRECTORIES "${dirs}")
set(compress_LIB_SRCS idbcompress.cpp)
add_definitions(-DNDEBUG)
columnstore_library(compress ${compress_LIB_SRCS})
add_dependencies(compress loggingcpp external_boost)
columnstore_link(compress PRIVATE loggingcpp boost_filesystem ${SNAPPY_LIBRARIES})
target_link_libraries(compress ${SNAPPY_LIBRARIES})
if(HAVE_LZ4)
message_once(STATUS "LINK WITH LZ4")
target_link_libraries(compress ${LZ4_LIBRARIES})
columnstore_link(compress PRIVATE ${LZ4_LIBRARIES})
endif()
install(
TARGETS compress
DESTINATION ${ENGINE_LIBDIR}
COMPONENT columnstore-engine
)

16
utils/configcpp/CMakeLists.txt
View File

@ -1,16 +1,4 @@
set(configcpp_LIB_SRCS configcpp.cpp xmlparser.cpp)
include_directories(${ENGINE_COMMON_INCLUDES})
# ########## next target ###############
set(configcpp_LIB_SRCS configcpp.cpp xmlparser.cpp configstream.cpp)
columnstore_library(configcpp ${configcpp_LIB_SRCS})
add_dependencies(configcpp loggingcpp)
target_compile_definitions(configcpp PUBLIC BOOST_NO_CXX11_SCOPED_ENUMS)
install(
TARGETS configcpp
DESTINATION ${ENGINE_LIBDIR}
COMPONENT columnstore-engine
)
columnstore_link(configcpp PRIVATE LibXml2::LibXml2 boost_filesystem)

12
utils/configcpp/Calpont.xml
View File

@ -1,12 +0,0 @@
<?xml version="1.0" encoding="UTF-8"?>
<Calpont Version="V1.0.0">
<Message>
<Name>Message</Name>
<Dir>.</Dir>
<Length>875</Length>
<Version>V1.0</Version>
</Message>
<SystemConfig>
<SystemVersion>V1.0</SystemVersion>
</SystemConfig>
</Calpont>

Some files were not shown because too many files have changed in this diff Show More