From aa7e0fb9b459f903c518a4c4d518ec5a7ae9d9b8 Mon Sep 17 00:00:00 2001 From: Leonid Fedorov <79837786+mariadb-LeonidFedorov@users.noreply.github.com> Date: Fri, 30 May 2025 14:05:21 +0400 Subject: [PATCH] Deep build refactoring phase 1 (#3562) * configcpp refactored * logging and datatypes refactored * more dataconvert * chore(build): massive removals, auto add files to debian install file * chore(codemanagement): nodeps headers, potentioal library * chore(build): configure before autobake * chore(build): use custom cmake commands for components, mariadb-plugin-columnstore.install generated * chore(build): install deps as separate step for build-packages * more deps * check debian/mariadb-plugin-columnstore.install automatically * chore(build): add option for multibracnh compilation * Fix warning --- .drone.jsonnet | 5 +- .gitignore | 2 +- build/bootstrap_mcs.sh | 61 +- build/debian_install_file_compare.py | 81 + cmake/ColumnstoreLibrary.cmake | 103 +- cmake/compiler_flags.cmake | 5 +- cmake/configureEngine.cmake | 15 +- cmake/packages.cmake | 1 - datatypes/CMakeLists.txt | 9 +- dbcon/ddlpackage/CMakeLists.txt | 8 +- dbcon/ddlpackageproc/CMakeLists.txt | 11 +- dbcon/dmlpackage/CMakeLists.txt | 8 +- dbcon/dmlpackageproc/CMakeLists.txt | 11 +- dbcon/execplan/CMakeLists.txt | 11 +- dbcon/execplan/pseudocolumn.cpp | 7 +- dbcon/execplan/windowfunctioncolumn.cpp | 32 +- dbcon/joblist/CMakeLists.txt | 21 +- dbcon/mysql/CMakeLists.txt | 42 +- dbcon/mysql/ha_mcs_client_udfs.cpp | 1 + dbcon/mysql/ha_mcs_impl_if.h | 10 +- ddlproc/CMakeLists.txt | 12 +- dmlproc/CMakeLists.txt | 13 +- oam/etc/CMakeLists.txt | 6 +- oam/install_scripts/CMakeLists.txt | 70 +- oam/oamcpp/CMakeLists.txt | 11 +- oamapps/columnstoreDB/CMakeLists.txt | 10 +- oamapps/columnstoreSupport/CMakeLists.txt | 23 +- oamapps/postConfigure/CMakeLists.txt | 12 +- oamapps/postConfigure/mycnfUpgrade.cpp | 3 +- primitives/blockcache/CMakeLists.txt | 12 +- primitives/linux-port/CMakeLists.txt | 9 +- primitives/primproc/CMakeLists.txt | 12 +- storage-manager/CMakeLists.txt | 57 +- tests/CMakeLists.txt | 34 +- tools/bincvt/li2bin.cpp | 258 -- tools/cfread/CMakeLists.txt | 77 - tools/cfread/cfread.cpp | 183 - tools/cfread/mtread.cpp | 269 -- tools/clearShm/CMakeLists.txt | 10 +- tools/cleartablelock/CMakeLists.txt | 10 +- tools/configMgt/CMakeLists.txt | 5 +- tools/cplogger/CMakeLists.txt | 10 +- tools/dbbuilder/CMakeLists.txt | 10 +- tools/dbbuilder/dbbuilder.cpp | 7 +- tools/dbloadxml/CMakeLists.txt | 14 +- tools/ddlcleanup/CMakeLists.txt | 11 +- tools/ddldriver/CMakeLists.txt | 43 - tools/ddldriver/ddldriver.cpp | 133 - tools/dmldriver/CMakeLists.txt | 44 - tools/dmldriver/dmldriver.cpp | 205 -- tools/dmldriver/dmlif.cpp | 384 --- tools/dmldriver/dmlif.h | 65 - tools/dmldriver/tpchrf2.cpp | 86 - tools/dmldriver/tpchrf2.h | 52 - tools/editem/CMakeLists.txt | 11 +- tools/fixdate/fixdate.cpp | 173 - tools/getConfig/CMakeLists.txt | 10 +- tools/idb_comp/main.cpp | 265 -- tools/idbmeminfo/CMakeLists.txt | 11 +- tools/passwd/CMakeLists.txt | 19 +- tools/pingproc/pingproc.cpp | 2357 ------------- tools/qfe/CMakeLists.txt | 55 - tools/qfe/cli.cpp | 148 - tools/qfe/cseputils.cpp | 141 - tools/qfe/cseputils.h | 34 - tools/qfe/ddlstmts.cpp | 80 - tools/qfe/ddlstmts.h | 10 - tools/qfe/parsequery.cpp | 97 - tools/qfe/parsequery.h | 12 - tools/qfe/qfe-cli.py | 51 - tools/qfe/qfelexer.lpp | 65 - tools/qfe/qfeparser.ypp | 306 -- tools/qfe/returnedrows.cpp | 95 - tools/qfe/returnedrows.h | 10 - tools/qfe/sendcsep.cpp | 53 - tools/qfe/sendcsep.h | 12 - tools/qfe/server.cpp | 429 --- tools/qfe/socketio.cpp | 133 - tools/qfe/socketio.h | 20 - tools/qfe/socktype.h | 6 - tools/rebuildEM/CMakeLists.txt | 9 +- tools/rgprint/CMakeLists.txt | 10 +- tools/sendPlan/CMakeLists.txt | 32 - tools/sendPlan/sendplan.cpp | 392 --- tools/setConfig/CMakeLists.txt | 11 +- tools/snconvert/snconvert.pl | 132 - tools/updateGenii/listRepos.pl | 58 - tools/updateGenii/test.dat | 14 - tools/updateGenii/updateGenii.pl | 186 - tools/valprint/valprint.cpp | 48 - tools/viewtablelock/CMakeLists.txt | 11 +- utils/basic/string_utils.h | 15 + utils/batchloader/CMakeLists.txt | 10 +- utils/cacheutils/CMakeLists.txt | 9 +- utils/cloudio/CMakeLists.txt | 17 +- utils/common/CMakeLists.txt | 11 +- utils/common/collation.h | 8 - utils/compress/CMakeLists.txt | 13 +- utils/configcpp/CMakeLists.txt | 16 +- utils/configcpp/Calpont.xml | 12 - utils/configcpp/XCalpont.xml | 12 - utils/configcpp/configcpp.cpp | 71 +- utils/configcpp/configcpp.h | 17 +- utils/configcpp/configstream.cpp | 66 - utils/configcpp/configstream.h | 63 - utils/configcpp/resource.h | 14 - utils/configcpp/tdriver.cpp | 371 -- utils/configcpp/tdriver2.cpp | 123 - utils/configcpp/xmlparser.cpp | 5 +- utils/dataconvert/CMakeLists.txt | 9 +- utils/dataconvert/dataconvert.cpp | 2 + utils/dataconvert/tdriver.cpp | 1375 -------- utils/ddlcleanup/CMakeLists.txt | 14 +- utils/funcexp/CMakeLists.txt | 18 +- utils/idbdatafile/CMakeLists.txt | 11 +- utils/joiner/CMakeLists.txt | 9 +- utils/libmarias3/CMakeLists.txt | 6 +- utils/libmysql_client/CMakeLists.txt | 13 +- utils/loggingcpp/CMakeLists.txt | 14 +- utils/loggingcpp/message.cpp | 7 - utils/messageqcpp/CMakeLists.txt | 21 +- utils/multicast/CMakeLists.txt | 33 - utils/multicast/impl.cpp | 3820 --------------------- utils/multicast/impl.h | 63 - utils/multicast/mc_fifo.h | 33 - utils/multicast/multicast.cpp | 95 - utils/multicast/multicast.h | 128 - utils/multicast/participants.h | 28 - utils/multicast/produconsum.h | 25 - utils/multicast/socklib.h | 169 - utils/multicast/statistics.h | 36 - utils/multicast/threads.h | 141 - utils/multicast/udp-sender.h | 100 - utils/multicast/udpc-protoc.h | 204 -- utils/multicast/udpcast.h | 157 - utils/multicast/util.h | 30 - utils/pron/CMakeLists.txt | 7 +- utils/querystats/CMakeLists.txt | 12 +- utils/querytele/CMakeLists.txt | 13 +- utils/regr/CMakeLists.txt | 17 +- utils/rowgroup/CMakeLists.txt | 14 +- utils/rwlock/CMakeLists.txt | 9 +- utils/startup/CMakeLists.txt | 7 +- utils/testbc/blockcacheclient.cpp | 97 - utils/testbc/blockcacheclient.h | 104 - utils/testbc/blockrequestprocessor.cpp | 227 -- utils/testbc/blockrequestprocessor.h | 131 - utils/testbc/fileblockrequestqueue.cpp | 92 - utils/testbc/fileblockrequestqueue.h | 108 - utils/testbc/filebuffer.cpp | 85 - utils/testbc/filebuffer.h | 182 - utils/testbc/filebuffermgr.cpp | 299 -- utils/testbc/filebuffermgr.h | 197 -- utils/testbc/filerequest.cpp | 98 - utils/testbc/filerequest.h | 252 -- utils/testbc/iomanager.cpp | 327 -- utils/testbc/iomanager.h | 100 - utils/testbc/logger.cpp | 63 - utils/testbc/logger.h | 62 - utils/testbc/stats.cpp | 273 -- utils/testbc/stats.h | 63 - utils/testbc/testbc.cpp | 361 -- utils/testbc/testbc2.cpp | 354 -- utils/testbc/testbc3.cpp | 394 --- utils/threadpool/CMakeLists.txt | 12 +- utils/udfsdk/CMakeLists.txt | 22 +- utils/udfsdk/mcsv1_udaf.h | 9 +- utils/windowfunction/CMakeLists.txt | 8 +- versioning/BRM/CMakeLists.txt | 127 +- writeengine/bulk/CMakeLists.txt | 16 +- writeengine/client/CMakeLists.txt | 8 +- writeengine/redistribute/CMakeLists.txt | 8 +- writeengine/server/CMakeLists.txt | 10 +- writeengine/splitter/CMakeLists.txt | 10 +- writeengine/wrapper/CMakeLists.txt | 8 +- writeengine/xml/CMakeLists.txt | 2 - 176 files changed, 557 insertions(+), 19155 deletions(-) create mode 100644 build/debian_install_file_compare.py delete mode 100644 tools/bincvt/li2bin.cpp delete mode 100644 tools/cfread/CMakeLists.txt delete mode 100644 tools/cfread/cfread.cpp delete mode 100644 tools/cfread/mtread.cpp delete mode 100644 tools/ddldriver/CMakeLists.txt delete mode 100644 tools/ddldriver/ddldriver.cpp delete mode 100644 tools/dmldriver/CMakeLists.txt delete mode 100644 tools/dmldriver/dmldriver.cpp delete mode 100644 tools/dmldriver/dmlif.cpp delete mode 100644 tools/dmldriver/dmlif.h delete mode 100644 tools/dmldriver/tpchrf2.cpp delete mode 100644 tools/dmldriver/tpchrf2.h delete mode 100644 tools/fixdate/fixdate.cpp delete mode 100644 tools/idb_comp/main.cpp delete mode 100644 tools/pingproc/pingproc.cpp delete mode 100644 tools/qfe/CMakeLists.txt delete mode 100644 tools/qfe/cli.cpp delete mode 100644 tools/qfe/cseputils.cpp delete mode 100644 tools/qfe/cseputils.h delete mode 100644 tools/qfe/ddlstmts.cpp delete mode 100644 tools/qfe/ddlstmts.h delete mode 100644 tools/qfe/parsequery.cpp delete mode 100644 tools/qfe/parsequery.h delete mode 100755 tools/qfe/qfe-cli.py delete mode 100644 tools/qfe/qfelexer.lpp delete mode 100644 tools/qfe/qfeparser.ypp delete mode 100644 tools/qfe/returnedrows.cpp delete mode 100644 tools/qfe/returnedrows.h delete mode 100644 tools/qfe/sendcsep.cpp delete mode 100644 tools/qfe/sendcsep.h delete mode 100644 tools/qfe/server.cpp delete mode 100644 tools/qfe/socketio.cpp delete mode 100644 tools/qfe/socketio.h delete mode 100644 tools/qfe/socktype.h delete mode 100644 tools/sendPlan/CMakeLists.txt delete mode 100644 tools/sendPlan/sendplan.cpp delete mode 100755 tools/snconvert/snconvert.pl delete mode 100755 tools/updateGenii/listRepos.pl delete mode 100644 tools/updateGenii/test.dat delete mode 100755 tools/updateGenii/updateGenii.pl delete mode 100644 tools/valprint/valprint.cpp create mode 100644 utils/basic/string_utils.h delete mode 100644 utils/configcpp/Calpont.xml delete mode 100644 utils/configcpp/XCalpont.xml delete mode 100644 utils/configcpp/configstream.cpp delete mode 100644 utils/configcpp/configstream.h delete mode 100644 utils/configcpp/resource.h delete mode 100644 utils/configcpp/tdriver.cpp delete mode 100644 utils/configcpp/tdriver2.cpp delete mode 100644 utils/dataconvert/tdriver.cpp delete mode 100644 utils/multicast/CMakeLists.txt delete mode 100644 utils/multicast/impl.cpp delete mode 100644 utils/multicast/impl.h delete mode 100644 utils/multicast/mc_fifo.h delete mode 100644 utils/multicast/multicast.cpp delete mode 100644 utils/multicast/multicast.h delete mode 100644 utils/multicast/participants.h delete mode 100644 utils/multicast/produconsum.h delete mode 100644 utils/multicast/socklib.h delete mode 100644 utils/multicast/statistics.h delete mode 100644 utils/multicast/threads.h delete mode 100644 utils/multicast/udp-sender.h delete mode 100644 utils/multicast/udpc-protoc.h delete mode 100644 utils/multicast/udpcast.h delete mode 100644 utils/multicast/util.h delete mode 100644 utils/testbc/blockcacheclient.cpp delete mode 100644 utils/testbc/blockcacheclient.h delete mode 100644 utils/testbc/blockrequestprocessor.cpp delete mode 100644 utils/testbc/blockrequestprocessor.h delete mode 100644 utils/testbc/fileblockrequestqueue.cpp delete mode 100644 utils/testbc/fileblockrequestqueue.h delete mode 100644 utils/testbc/filebuffer.cpp delete mode 100644 utils/testbc/filebuffer.h delete mode 100644 utils/testbc/filebuffermgr.cpp delete mode 100644 utils/testbc/filebuffermgr.h delete mode 100644 utils/testbc/filerequest.cpp delete mode 100644 utils/testbc/filerequest.h delete mode 100644 utils/testbc/iomanager.cpp delete mode 100644 utils/testbc/iomanager.h delete mode 100644 utils/testbc/logger.cpp delete mode 100644 utils/testbc/logger.h delete mode 100644 utils/testbc/stats.cpp delete mode 100644 utils/testbc/stats.h delete mode 100644 utils/testbc/testbc.cpp delete mode 100644 utils/testbc/testbc2.cpp delete mode 100644 utils/testbc/testbc3.cpp diff --git a/.drone.jsonnet b/.drone.jsonnet index 2d9b09c1a..64cfe43e0 100644 --- a/.drone.jsonnet +++ b/.drone.jsonnet @@ -631,9 +631,6 @@ local Pipeline(branch, platform, event, arch="amd64", server="10.6-enterprise", volumes: [pipeline._volumes.mdb], environment: { DEBIAN_FRONTEND: "noninteractive", - DEB_BUILD_OPTIONS: "parallel=4", - DH_BUILD_DDEBS: "1", - BUILDPACKAGE_FLAGS: "-b", // Save time and produce only binary packages, not source AWS_ACCESS_KEY_ID: { from_secret: "aws_access_key_id", }, @@ -656,7 +653,7 @@ local Pipeline(branch, platform, event, arch="amd64", server="10.6-enterprise", 'bash -c "set -o pipefail && bash /mdb/' + builddir + "/storage/columnstore/columnstore/build/bootstrap_mcs.sh " + "--build-type RelWithDebInfo " + "--distro " + platform + " " + - "--build-packages --sccache " + + "--build-packages --install-deps --sccache " + " " + customBootstrapParams + " " + customBootstrapParamsForExisitingPipelines(platform) + " | " + "/mdb/" + builddir + "/storage/columnstore/columnstore/build/ansi2txt.sh " + diff --git a/.gitignore b/.gitignore index 05a42f3ff..60385d18c 100644 --- a/.gitignore +++ b/.gitignore @@ -153,6 +153,7 @@ oam/install_scripts/mcs-workernode.service oam/install_scripts/mcs-writeengineserver.service oam/install_scripts/mcs-loadbrm.py oam/install_scripts/mcs-savebrm.py +oam/install_scripts/mcssavebrm.py oam/install_scripts/mcs-storagemanager.service oam/install_scripts/mcs_module_installer.sh oam/install_scripts/slave-rep-columnstore.sh @@ -186,4 +187,3 @@ versioning/BRM/shmkeys.cpp obj/ build/build - diff --git a/build/bootstrap_mcs.sh b/build/bootstrap_mcs.sh index 550d3df6a..722e69679 100755 --- a/build/bootstrap_mcs.sh +++ b/build/bootstrap_mcs.sh @@ -19,6 +19,9 @@ CONFIG_DIR=$RPM_CONFIG_DIR SCRIPT_LOCATION=$(dirname "$0") MDB_SOURCE_PATH=$(realpath "$SCRIPT_LOCATION"/../../../..) +COLUMSNTORE_SOURCE_PATH=$(realpath "$SCRIPT_LOCATION"/../) + +DEFAULT_MARIA_BUILD_PATH=$(realpath "$MDB_SOURCE_PATH"/../BuildOf_$(basename "$MDB_SOURCE_PATH")) BUILD_TYPE_OPTIONS=("Debug" "RelWithDebInfo") DISTRO_OPTIONS=("ubuntu:20.04" "ubuntu:22.04" "ubuntu:24.04" "debian:11" "debian:12" "rockylinux:8" "rockylinux:9") @@ -28,21 +31,10 @@ MDB_CMAKE_FLAGS=() source "$SCRIPT_LOCATION"/utils.sh -if [ "$EUID" -ne 0 ]; then - error "Please run script as root to install MariaDb to system paths" - exit 1 -fi - -cd $SCRIPT_LOCATION -CURRENT_BRANCH=$(git rev-parse --abbrev-ref HEAD) -message "Columnstore will be built from $color_yellow$CURRENT_BRANCH$color_cyan branch" -cd - >/dev/null - echo "Arguments received: $@" -message "Building Mariadb Server from $color_yellow$MDB_SOURCE_PATH$color_normal" optparse.define short=A long=asan desc="Build with ASAN" variable=ASAN default=false value=true -optparse.define short=a long=build-path variable=MARIA_BUILD_PATH default=$(realpath "$MDB_SOURCE_PATH"/../BuildOf_$(basename "$MDB_SOURCE_PATH")) +optparse.define short=a long=build-path desc="Path for build output" variable=MARIA_BUILD_PATH default=$DEFAULT_MARIA_BUILD_PATH optparse.define short=B long=run-microbench desc="Compile and run microbenchmarks " variable=RUN_BENCHMARKS default=false value=true optparse.define short=c long=cloud desc="Enable cloud storage" variable=CLOUD_STORAGE_ENABLED default=false value=true optparse.define short=C long=force-cmake-reconfig desc="Force cmake reconfigure" variable=FORCE_CMAKE_CONFIG default=false value=true @@ -68,10 +60,26 @@ optparse.define short=T long=tsan desc="Build with TSAN" variable=TSAN default=f optparse.define short=u long=skip-unit-tests desc="Skip UnitTests" variable=SKIP_UNIT_TESTS default=false value=true optparse.define short=U long=ubsan desc="Build with UBSAN" variable=UBSAN default=false value=true optparse.define short=v long=verbose desc="Verbose makefile commands" variable=MAKEFILE_VERBOSE default=false value=true +optparse.define short=V long=add-branch-name-to-outdir desc="Add branch name to build output directory" variable=BRANCH_NAME_TO_OUTDIR default=false value=true optparse.define short=W long=without-core-dumps desc="Do not produce core dumps" variable=WITHOUT_COREDUMPS default=false value=true source $(optparse.build) +message "Building MariaDB Server from $color_yellow$MDB_SOURCE_PATH$color_normal" + +cd $COLUMSNTORE_SOURCE_PATH +COLUMNSTORE_BRANCH=$(git rev-parse --abbrev-ref HEAD) +message "Columnstore will be built from $color_yellow$COLUMNSTORE_BRANCH$color_cyan branch" + +cd $MDB_SOURCE_PATH +MARIADB_BRANCH=$(git rev-parse --abbrev-ref HEAD) +message "MariaDB will be built from $color_yellow$MARIADB_BRANCH$color_cyan branch" +cd - >/dev/null + +if [[ ${BRANCH_NAME_TO_OUTDIR} = true ]]; then + MARIA_BUILD_PATH="${MARIA_BUILD_PATH}_${MARIADB_BRANCH}_${COLUMNSTORE_BRANCH}" +fi + if [[ ! " ${BUILD_TYPE_OPTIONS[*]} " =~ " ${MCS_BUILD_TYPE} " ]]; then getChoice -q "Select your Build Type" -o BUILD_TYPE_OPTIONS MCS_BUILD_TYPE=$selectedChoice @@ -104,7 +112,7 @@ install_deps() { cppunit-devel cmake3 libxcrypt-devel xz-devel zlib-devel libzstd-devel glibc-devel" DEB_BUILD_DEPS="apt-get -y update && apt-get -y install build-essential automake libboost-all-dev \ - bison cmake libncurses5-dev libaio-dev libsystemd-dev libpcre2-dev libperl-dev libssl-dev libxml2-dev \ + bison cmake libncurses5-dev python3 libaio-dev libsystemd-dev libpcre2-dev libperl-dev libssl-dev libxml2-dev \ libkrb5-dev flex libpam-dev git libsnappy-dev libcurl4-openssl-dev libgtest-dev libcppunit-dev googletest \ libjemalloc-dev liblz-dev liblzo2-dev liblzma-dev liblz4-dev libbz2-dev libbenchmark-dev libdistro-info-perl \ graphviz devscripts ccache equivs eatmydata curl" @@ -259,8 +267,9 @@ construct_cmake_flags() { -DWITH_WSREP=NO ) - if [[ MAINTAINER_MODE = true ]]; then + if [[ $MAINTAINER_MODE = true ]]; then MDB_CMAKE_FLAGS+=(-DCOLUMNSTORE_MAINTAINER=YES) + message "Columnstore mainteiner mode on" else warn "Maintainer mode is disabled, be careful, alien" fi @@ -406,9 +415,15 @@ generate_svgs() { } build_package() { + cd $MDB_SOURCE_PATH + if [[ $pkg_format == "rpm" ]]; then command="cmake ${MDB_CMAKE_FLAGS[@]} && make -j\$(nproc) package" else + export DEBIAN_FRONTEND="noninteractive" + export DEB_BUILD_OPTIONS="parallel=$(nproc)" + export DH_BUILD_DDEBS="1" + export BUILDPACKAGE_FLAGS="-b" command="mk-build-deps debian/control -t 'apt-get -y -o Debug::pkgProblemResolver=yes --no-install-recommends' -r -i && \ CMAKEFLAGS=\"${MDB_CMAKE_FLAGS[@]}\" debian/autobake-deb.sh" fi @@ -420,6 +435,14 @@ build_package() { check_errorcode } +check_debian_install_file() { + message "checking debian/mariadb-plugin-columnstore.install" + message_split + python3 $COLUMSNTORE_SOURCE_PATH/build/debian_install_file_compare.py \ + ${COLUMSNTORE_SOURCE_PATH}/debian/mariadb-plugin-columnstore.install \ + $MARIA_BUILD_PATH/mariadb-plugin-columnstore.install.generated +} + build_binary() { MARIA_BUILD_PATH=$(realpath "$MARIA_BUILD_PATH") message_split @@ -438,7 +461,7 @@ build_binary() { message "Configuring cmake silently" ${CMAKE_BIN_NAME} "${MDB_CMAKE_FLAGS[@]}" -S"$MDB_SOURCE_PATH" -B"$MARIA_BUILD_PATH" | spinner message_split - + check_debian_install_file generate_svgs ${CMAKE_BIN_NAME} --build "$MARIA_BUILD_PATH" -j "$CPUS" | onelinearizator && @@ -553,6 +576,12 @@ make_dir() { install() { if [[ $RECOMPILE_ONLY = false ]]; then + + if [ "$EUID" -ne 0 ]; then + error "Please run script as root to install MariaDb to system paths" + exit 1 + fi + message_split message "Installing MariaDB" disable_plugins_for_bootstrap @@ -636,7 +665,7 @@ smoke() { fi } -if [[ $INSTALL_DEPS = true || $BUILD_PACKAGES = true ]]; then +if [[ $INSTALL_DEPS = true ]]; then install_deps fi diff --git a/build/debian_install_file_compare.py b/build/debian_install_file_compare.py new file mode 100644 index 000000000..eeeb8e415 --- /dev/null +++ b/build/debian_install_file_compare.py @@ -0,0 +1,81 @@ +#!/usr/bin/env python3 + +import argparse +import fnmatch +import sys + +def read_patterns(filename): + with open(filename, 'r') as f: + return [line.strip() for line in f if line.strip()] + +def read_paths_with_comments(filename): + paths = [] + with open(filename, 'r') as f: + for line in f: + raw = line.rstrip('\n') + if not raw.strip(): + continue + # Split into path and comment + if '#' in raw: + path, comment = raw.split('#', 1) + path = path.strip() + comment = comment.strip() + else: + path = raw.strip() + comment = '' + if path: + paths.append((raw, path, comment)) + return paths + +def print_red(msg): + if sys.stdout.isatty(): + print("\033[31m" + msg + "\033[0m") + else: + print(msg) + +def main(): + parser = argparse.ArgumentParser(description="Compare wildcard patterns to file paths.") + parser.add_argument('patterns_file', help='File with wildcard patterns (one per line)') + parser.add_argument('paths_file', help='File with file paths (one per line)') + args = parser.parse_args() + + patterns = read_patterns(args.patterns_file) + paths_with_comments = read_paths_with_comments(args.paths_file) + + # Track which paths are matched (by index) + path_matched = [False] * len(paths_with_comments) + + # Collect patterns that don't match any path + unmatched_patterns = [] + for pattern in patterns: + matched = False + for i, (raw, path, comment) in enumerate(paths_with_comments): + if fnmatch.fnmatch(path, pattern): + matched = True + path_matched[i] = True + if not matched: + unmatched_patterns.append(pattern) + + if unmatched_patterns: + print_red("The files declared in debian/mariadb-plugin-columnstore.install " + " are not added to CMakeLists via columnstore_* statements see cmake/ColumnstoreLibrary.cmake") + for pattern in unmatched_patterns: + print(f"- {pattern}") + + # Collect paths that weren't matched by any pattern + unmatched_paths = [] + for (raw, path, comment), matched in zip(paths_with_comments, path_matched): + if not matched: + unmatched_paths.append((path, comment)) + + if unmatched_paths: + print_red("The files added via columnstore_* statements from cmake/ColumnstoreLibrary.cmake " + "are missing in debian/mariadb-plugin-columnstore.install file") + for path, comment in unmatched_paths: + if comment: + print(f"- {path} {comment}") + else: + print(f"- {path}") + +if __name__ == "__main__": + main() diff --git a/cmake/ColumnstoreLibrary.cmake b/cmake/ColumnstoreLibrary.cmake index 503b4c540..36dbc6813 100644 --- a/cmake/ColumnstoreLibrary.cmake +++ b/cmake/ColumnstoreLibrary.cmake @@ -1,8 +1,105 @@ -# Function to create either a static or shared library based on COLUMNSTORE_STATIC_LIBRARIES +set(DEBIAN_INSTALL_FILE "${CMAKE_BINARY_DIR}/mariadb-plugin-columnstore.install.generated") +file(WRITE ${DEBIAN_INSTALL_FILE} "#File is generated by ColumnstoreLibrary.cmake, do not edit\n") + +macro(add_to_debian_install_file file_path) + string(SUBSTRING "${file_path}" 1 -1 BINARY_ENTRY) + file(RELATIVE_PATH CMAKEFILE "${CMAKE_SOURCE_DIR}/storage/columnstore/columnstore" ${CMAKE_CURRENT_LIST_FILE}) + string(STRIP "${BINARY_ENTRY}" BINARY_ENTRY) + + file(APPEND ${DEBIAN_INSTALL_FILE} "${BINARY_ENTRY} # added in ${CMAKEFILE}\n") +endmacro() + +function(get_target_output_filename TARGET_NAME OUTPUT_VAR) + # 1. Get the target's OUTPUT_NAME (falls back to TARGET_NAME) + get_target_property(OUTPUT_NAME ${TARGET_NAME} OUTPUT_NAME) + if(NOT OUTPUT_NAME) + set(OUTPUT_NAME "${TARGET_NAME}") + endif() + + # 1. Get the correct suffix based on target type + get_target_property(TARGET_TYPE ${TARGET_NAME} TYPE) + if(TARGET_TYPE STREQUAL "EXECUTABLE") + set(SUFFIX "${CMAKE_EXECUTABLE_SUFFIX}") + set(PREFIX "") # No prefix for executables + elseif(TARGET_TYPE STREQUAL "SHARED_LIBRARY") + set(SUFFIX "${CMAKE_SHARED_LIBRARY_SUFFIX}") + set(PREFIX "lib") + elseif(TARGET_TYPE STREQUAL "STATIC_LIBRARY") + set(SUFFIX "${CMAKE_STATIC_LIBRARY_SUFFIX}") + set(PREFIX "lib") + else() + message(WARNING "Unknown target type for ${TARGET_NAME}") + set(SUFFIX "") + endif() + + # 1. Combine into the final filename + set(${OUTPUT_VAR} + "${PREFIX}${OUTPUT_NAME}${SUFFIX}" + PARENT_SCOPE + ) +endfunction() + +macro(columnstore_install_target target destination) + install( + TARGETS ${target} + DESTINATION ${destination} + COMPONENT columnstore-engine + ) + get_target_output_filename(${target} OUTPUT_FILENAME) + add_to_debian_install_file("${destination}/${OUTPUT_FILENAME}") +endmacro() + +macro(columnstore_install_file file destination) + install( + FILES ${file} + DESTINATION ${destination} + COMPONENT columnstore-engine + ) + get_filename_component(FILENAME ${file} NAME) + add_to_debian_install_file("${destination}/${FILENAME}") +endmacro() + +macro(columnstore_install_program file destination) + install( + PROGRAMS ${file} + DESTINATION ${destination} + COMPONENT columnstore-engine + ) + get_filename_component(FILENAME ${file} NAME) + add_to_debian_install_file( + "${destination}/${FILENAME} + " + ) +endmacro() + +macro(columnstore_static_library libname) + add_definitions(-fPIC -DPIC) + add_library(${libname} STATIC ${ARGN}) +endmacro() + +macro(columnstore_shared_library libname) + add_library(${libname} SHARED ${ARGN}) + columnstore_install_target(${libname} ${ENGINE_LIBDIR}) +endmacro() + macro(columnstore_library libname) if(COLUMNSTORE_STATIC_LIBRARIES) - add_library(${libname} STATIC ${ARGN}) + columnstore_static_library(${libname} ${ARGN}) else() - add_library(${libname} SHARED ${ARGN}) + columnstore_shared_library(${libname} ${ARGN}) endif() endmacro() + +macro(columnstore_mysql_plugin_library libname) + add_library(${libname} SHARED ${ARGN}) + columnstore_install_target(${libname} ${MARIADB_PLUGINDIR}) +endmacro() + +macro(columnstore_link libname) + target_link_libraries(${libname} ${ARGN}) +endmacro() + +macro(columnstore_executable executable_name) + add_executable(${executable_name} ${ARGN}) + columnstore_install_target(${executable_name} ${ENGINE_BINDIR}) +endmacro() diff --git a/cmake/compiler_flags.cmake b/cmake/compiler_flags.cmake index 5a3192ed1..6a0b7e66f 100644 --- a/cmake/compiler_flags.cmake +++ b/cmake/compiler_flags.cmake @@ -51,6 +51,7 @@ set(COLUMNSTORE_MAINTAINER_FLAGS ${WERROR_FLAG}) set(FLAGS_ALL -Wall -Wextra + -ggdb3 -fno-omit-frame-pointer -fno-strict-aliasing -fsigned-char @@ -59,9 +60,9 @@ set(FLAGS_ALL -DBOOST_BIND_GLOBAL_PLACEHOLDERS ) -set(FLAGS_RELEASE -g -O3 -DDBUG_OFF) +set(FLAGS_RELEASE -O3 -DDBUG_OFF) -set(FLAGS_DEBUG -ggdb3 -O0 -D_DEBUG) +set(FLAGS_DEBUG -O0 -D_DEBUG) # } end Release, Debug and common flags # linker flags { diff --git a/cmake/configureEngine.cmake b/cmake/configureEngine.cmake index eabf4c24e..cd8bff26a 100644 --- a/cmake/configureEngine.cmake +++ b/cmake/configureEngine.cmake @@ -773,17 +773,10 @@ endif() # releasenum is used by external scripts for various tasks. Leave it alone. configure_file(${CMAKE_CURRENT_SOURCE_DIR}/build/releasenum.in ${CMAKE_CURRENT_BINARY_DIR}/build/releasenum IMMEDIATE) -install( - FILES ${CMAKE_CURRENT_BINARY_DIR}/build/releasenum - DESTINATION ${ENGINE_SUPPORTDIR} - COMPONENT columnstore-engine -) +columnstore_install_file(${CMAKE_CURRENT_BINARY_DIR}/build/releasenum ${ENGINE_SUPPORTDIR}) + configure_file(${CMAKE_CURRENT_SOURCE_DIR}/columnstoreversion.h.in ${CMAKE_CURRENT_SOURCE_DIR}/columnstoreversion.h) configure_file(${CMAKE_CURRENT_SOURCE_DIR}/mcsconfig.h.in ${CMAKE_CURRENT_BINARY_DIR}/mcsconfig.h) - configure_file(${CMAKE_CURRENT_SOURCE_DIR}/gitversionEngine.in ${CMAKE_CURRENT_BINARY_DIR}/gitversionEngine IMMEDIATE) -install( - FILES ${CMAKE_CURRENT_BINARY_DIR}/gitversionEngine - DESTINATION ${ENGINE_SUPPORTDIR} - COMPONENT columnstore-engine -) + +columnstore_install_file(${CMAKE_CURRENT_BINARY_DIR}/gitversionEngine ${ENGINE_SUPPORTDIR}) diff --git a/cmake/packages.cmake b/cmake/packages.cmake index 70b001604..3fee24098 100644 --- a/cmake/packages.cmake +++ b/cmake/packages.cmake @@ -14,7 +14,6 @@ if(NOT LEX_EXECUTABLE) endif() find_package(LibXml2) - if(NOT LIBXML2_FOUND) message_once(CS_NO_LIBXML "Could not find a usable libxml2 development environment!") return() diff --git a/datatypes/CMakeLists.txt b/datatypes/CMakeLists.txt index f2b4bbe97..4cc2c0e65 100644 --- a/datatypes/CMakeLists.txt +++ b/datatypes/CMakeLists.txt @@ -2,10 +2,5 @@ include_directories(${ENGINE_COMMON_INCLUDES}) set(datatypes_LIB_SRCS mcs_int128.cpp mcs_decimal.cpp) columnstore_library(datatypes ${datatypes_LIB_SRCS}) -add_dependencies(datatypes loggingcpp external_boost) - -install( - TARGETS datatypes - DESTINATION ${ENGINE_LIBDIR} - COMPONENT columnstore-engine -) +columnstore_link(datatypes PRIVATE loggingcpp) +add_dependencies(datatypes external_boost) diff --git a/dbcon/ddlpackage/CMakeLists.txt b/dbcon/ddlpackage/CMakeLists.txt index 975655070..4ed7d69cc 100644 --- a/dbcon/ddlpackage/CMakeLists.txt +++ b/dbcon/ddlpackage/CMakeLists.txt @@ -42,10 +42,4 @@ columnstore_library( ${FLEX_ddl_scan_OUTPUTS} ) -add_dependencies(ddlpackage loggingcpp) - -install( - TARGETS ddlpackage - DESTINATION ${ENGINE_LIBDIR} - COMPONENT columnstore-engine -) +columnstore_link(ddlpackage loggingcpp) diff --git a/dbcon/ddlpackageproc/CMakeLists.txt b/dbcon/ddlpackageproc/CMakeLists.txt index 159b695c3..3eed93356 100644 --- a/dbcon/ddlpackageproc/CMakeLists.txt +++ b/dbcon/ddlpackageproc/CMakeLists.txt @@ -13,13 +13,4 @@ set(ddlpackageproc_LIB_SRCS ) columnstore_library(ddlpackageproc ${ddlpackageproc_LIB_SRCS}) - -add_dependencies(ddlpackageproc loggingcpp) - -target_link_libraries(ddlpackageproc ${NETSNMP_LIBRARIES}) - -install( - TARGETS ddlpackageproc - DESTINATION ${ENGINE_LIBDIR} - COMPONENT columnstore-engine -) +columnstore_link(ddlpackageproc ${NETSNMP_LIBRARIES} loggingcpp) diff --git a/dbcon/dmlpackage/CMakeLists.txt b/dbcon/dmlpackage/CMakeLists.txt index 00bdbc4ba..a0e4ddffa 100644 --- a/dbcon/dmlpackage/CMakeLists.txt +++ b/dbcon/dmlpackage/CMakeLists.txt @@ -41,10 +41,4 @@ columnstore_library( ${FLEX_dml_scan_OUTPUTS} ) -add_dependencies(dmlpackage loggingcpp) - -install( - TARGETS dmlpackage - DESTINATION ${ENGINE_LIBDIR} - COMPONENT columnstore-engine -) +columnstore_link(dmlpackage loggingcpp) diff --git a/dbcon/dmlpackageproc/CMakeLists.txt b/dbcon/dmlpackageproc/CMakeLists.txt index 78559befd..177bb799b 100644 --- a/dbcon/dmlpackageproc/CMakeLists.txt +++ b/dbcon/dmlpackageproc/CMakeLists.txt @@ -13,13 +13,4 @@ set(dmlpackageproc_LIB_SRCS ) columnstore_library(dmlpackageproc ${dmlpackageproc_LIB_SRCS}) - -add_dependencies(dmlpackageproc loggingcpp) - -target_link_libraries(dmlpackageproc ${NETSNMP_LIBRARIES}) - -install( - TARGETS dmlpackageproc - DESTINATION ${ENGINE_LIBDIR} - COMPONENT columnstore-engine -) +columnstore_link(dmlpackageproc ${NETSNMP_LIBRARIES} loggingcpp) diff --git a/dbcon/execplan/CMakeLists.txt b/dbcon/execplan/CMakeLists.txt index bc00e368c..6e1acbb98 100755 --- a/dbcon/execplan/CMakeLists.txt +++ b/dbcon/execplan/CMakeLists.txt @@ -46,13 +46,4 @@ set(execplan_LIB_SRCS ) columnstore_library(execplan ${execplan_LIB_SRCS}) - -add_dependencies(execplan loggingcpp) - -target_link_libraries(execplan messageqcpp ${NETSNMP_LIBRARIES} ${ENGINE_DT_LIB} pron) - -install( - TARGETS execplan - DESTINATION ${ENGINE_LIBDIR} - COMPONENT columnstore-engine -) +columnstore_link(execplan messageqcpp ${NETSNMP_LIBRARIES} ${ENGINE_DT_LIB} pron loggingcpp) diff --git a/dbcon/execplan/pseudocolumn.cpp b/dbcon/execplan/pseudocolumn.cpp index c361037f0..a6055489e 100644 --- a/dbcon/execplan/pseudocolumn.cpp +++ b/dbcon/execplan/pseudocolumn.cpp @@ -25,7 +25,7 @@ #include #include using namespace std; - +#include "basic/string_utils.h" #include "bytestream.h" #include "collation.h" @@ -138,8 +138,9 @@ string PseudoColumn::toCppCode(IncludeSet& includes) const { includes.insert("pseudocolumn.h"); stringstream ss; - ss << "PseudoColumn(" << std::quoted(fSchemaName) << ", " << std::quoted(fTableName) << ", " << fisColumnStore << ", " << std::quoted(fColumnName) - << ", " << fPseudoType << ", " << sessionID() << ")"; + ss << "PseudoColumn(" << std::quoted(fSchemaName) << ", " << std::quoted(fTableName) << ", " + << fisColumnStore << ", " << std::quoted(fColumnName) << ", " << fPseudoType << ", " << sessionID() + << ")"; return ss.str(); } diff --git a/dbcon/execplan/windowfunctioncolumn.cpp b/dbcon/execplan/windowfunctioncolumn.cpp index 47a3bdb3f..3ba88a5f9 100644 --- a/dbcon/execplan/windowfunctioncolumn.cpp +++ b/dbcon/execplan/windowfunctioncolumn.cpp @@ -27,10 +27,13 @@ #include #include using namespace std; +#include "basic/string_utils.h" #include using namespace boost; +#include "basic/string_utils.h" + #include "bytestream.h" #include "windowfunctioncolumn.h" #include "constantcolumn.h" @@ -398,23 +401,32 @@ void WindowFunctionColumn::adjustResultType() { if ((fResultType.colDataType == CalpontSystemCatalog::DECIMAL || fResultType.colDataType == CalpontSystemCatalog::UDECIMAL) && - !datatypes::ASCIIStringCaseInsensetiveEquals(fFunctionName, "COUNT") && !datatypes::ASCIIStringCaseInsensetiveEquals(fFunctionName, "COUNT(*)") && - !datatypes::ASCIIStringCaseInsensetiveEquals(fFunctionName, "ROW_NUMBER") && !datatypes::ASCIIStringCaseInsensetiveEquals(fFunctionName, "RANK") && - !datatypes::ASCIIStringCaseInsensetiveEquals(fFunctionName, "PERCENT_RANK") && !datatypes::ASCIIStringCaseInsensetiveEquals(fFunctionName, "DENSE_RANK") && - !datatypes::ASCIIStringCaseInsensetiveEquals(fFunctionName, "CUME_DIST") && !datatypes::ASCIIStringCaseInsensetiveEquals(fFunctionName, "NTILE") && + !datatypes::ASCIIStringCaseInsensetiveEquals(fFunctionName, "COUNT") && + !datatypes::ASCIIStringCaseInsensetiveEquals(fFunctionName, "COUNT(*)") && + !datatypes::ASCIIStringCaseInsensetiveEquals(fFunctionName, "ROW_NUMBER") && + !datatypes::ASCIIStringCaseInsensetiveEquals(fFunctionName, "RANK") && + !datatypes::ASCIIStringCaseInsensetiveEquals(fFunctionName, "PERCENT_RANK") && + !datatypes::ASCIIStringCaseInsensetiveEquals(fFunctionName, "DENSE_RANK") && + !datatypes::ASCIIStringCaseInsensetiveEquals(fFunctionName, "CUME_DIST") && + !datatypes::ASCIIStringCaseInsensetiveEquals(fFunctionName, "NTILE") && !datatypes::ASCIIStringCaseInsensetiveEquals(fFunctionName, "PERCENTILE") && !fFunctionParms.empty() && fFunctionParms[0]->resultType().colDataType == CalpontSystemCatalog::DOUBLE) fResultType = fFunctionParms[0]->resultType(); - if ((datatypes::ASCIIStringCaseInsensetiveEquals(fFunctionName, "LEAD") || datatypes::ASCIIStringCaseInsensetiveEquals(fFunctionName, "LAG") || - datatypes::ASCIIStringCaseInsensetiveEquals(fFunctionName, "MIN") || datatypes::ASCIIStringCaseInsensetiveEquals(fFunctionName, "MAX") || - datatypes::ASCIIStringCaseInsensetiveEquals(fFunctionName, "FIRST_VALUE") || datatypes::ASCIIStringCaseInsensetiveEquals(fFunctionName, "LAST_VALUE") || + if ((datatypes::ASCIIStringCaseInsensetiveEquals(fFunctionName, "LEAD") || + datatypes::ASCIIStringCaseInsensetiveEquals(fFunctionName, "LAG") || + datatypes::ASCIIStringCaseInsensetiveEquals(fFunctionName, "MIN") || + datatypes::ASCIIStringCaseInsensetiveEquals(fFunctionName, "MAX") || + datatypes::ASCIIStringCaseInsensetiveEquals(fFunctionName, "FIRST_VALUE") || + datatypes::ASCIIStringCaseInsensetiveEquals(fFunctionName, "LAST_VALUE") || datatypes::ASCIIStringCaseInsensetiveEquals(fFunctionName, "NTH_VALUE")) && !fFunctionParms.empty()) fResultType = fFunctionParms[0]->resultType(); - if (datatypes::ASCIIStringCaseInsensetiveEquals(fFunctionName, "SUM") || datatypes::ASCIIStringCaseInsensetiveEquals(fFunctionName, "AVG") || - datatypes::ASCIIStringCaseInsensetiveEquals(fFunctionName, "AVG_DISTINCT") || datatypes::ASCIIStringCaseInsensetiveEquals(fFunctionName, "PERCENTILE")) + if (datatypes::ASCIIStringCaseInsensetiveEquals(fFunctionName, "SUM") || + datatypes::ASCIIStringCaseInsensetiveEquals(fFunctionName, "AVG") || + datatypes::ASCIIStringCaseInsensetiveEquals(fFunctionName, "AVG_DISTINCT") || + datatypes::ASCIIStringCaseInsensetiveEquals(fFunctionName, "PERCENTILE")) { if (fFunctionParms[0]->resultType().colDataType == CalpontSystemCatalog::DECIMAL || fFunctionParms[0]->resultType().colDataType == CalpontSystemCatalog::UDECIMAL) @@ -531,7 +543,7 @@ void WindowFunctionColumn::evaluate(Row& row, bool& isNull) fResult.strVal.assign(str.unsafeStringRef()); // stringColVal is padded with '\0' to colWidth so can't use str.length() - //if (strlen(fResult.strVal.str()) == 0) + // if (strlen(fResult.strVal.str()) == 0) // isNull = true; break; diff --git a/dbcon/joblist/CMakeLists.txt b/dbcon/joblist/CMakeLists.txt index 9342761c0..3783ccf14 100644 --- a/dbcon/joblist/CMakeLists.txt +++ b/dbcon/joblist/CMakeLists.txt @@ -57,8 +57,8 @@ set(joblist_LIB_SRCS # ########## next target ############### -add_library(joblist_server STATIC ${joblist_LIB_SRCS}) -add_dependencies(joblist_server loggingcpp) +columnstore_static_library(joblist_server ${joblist_LIB_SRCS}) +columnstore_link(joblist_server loggingcpp) # ########## next target ############### @@ -68,20 +68,9 @@ columnstore_library(joblist ${joblist_LIB_SRCS}) target_include_directories( joblist BEFORE PUBLIC ${OPENSSL_INCLUDE_DIR} ${LIBMARIADB_BININC_DIR} ${LIBMARIADB_SRCINC_DIR} ) -add_dependencies(joblist loggingcpp) - -install( - TARGETS joblist - DESTINATION ${ENGINE_LIBDIR} - COMPONENT columnstore-engine -) +columnstore_link(joblist loggingcpp) if(WITH_ORDERBY_UT) - add_executable(job_orderby_tests orderby-tests.cpp) - target_link_libraries(job_orderby_tests ${ENGINE_LDFLAGS} ${ENGINE_WRITE_LIBS} ${CPPUNIT_LIBRARIES} cppunit) - install( - TARGETS job_orderby_tests - DESTINATION ${ENGINE_BINDIR} - COMPONENT columnstore-engine - ) + columnstore_executable(job_orderby_tests orderby-tests.cpp) + columnstore_link(job_orderby_tests ${ENGINE_LDFLAGS} ${ENGINE_WRITE_LIBS} ${CPPUNIT_LIBRARIES} cppunit) endif() diff --git a/dbcon/mysql/CMakeLists.txt b/dbcon/mysql/CMakeLists.txt index 7979669bd..7bd6ed78b 100644 --- a/dbcon/mysql/CMakeLists.txt +++ b/dbcon/mysql/CMakeLists.txt @@ -71,16 +71,17 @@ if(COMMAND mysql_add_plugin) CONFIG columnstore.cnf ) + add_to_debian_install_file(${MARIADB_MYCNFDIR}/columnstore.cnf) + add_to_debian_install_file("${MARIADB_PLUGINDIR}/ha_columnstore.so") + else() message("Adding plugin via CMake") - add_library(ha_columnstore SHARED ${libcalmysql_SRCS}) + columnstore_shared_library(ha_columnstore ${libcalmysql_SRCS}) set_target_properties(ha_columnstore PROPERTIES PREFIX "") - add_dependencies(ha_columnstore loggingcpp) - add_dependencies(ha_columnstore marias3) add_definitions(-DMYSQL_DYNAMIC_PLUGIN -DPLUGIN_COLUMNSTORE_VERSION="${PACKAGE_VERSION}") - target_link_libraries( + columnstore_link( ha_columnstore ${S3API_DEPS} ${ENGINE_LDFLAGS} @@ -88,33 +89,20 @@ else() ${NETSNMP_LIBRARIES} ${SERVER_BUILD_DIR}/libservices/libmysqlservices.a threadpool + loggingcpp + marias3 ) - - install( - TARGETS ha_columnstore - DESTINATION ${MARIADB_PLUGINDIR} - COMPONENT columnstore-engine - ) - # define this dummy target for standalone builds (ie, when mysql_add_plugin doesn't exist) add_custom_target(columnstore DEPENDS ha_columnstore) - install( - FILES columnstore.cnf - DESTINATION ${MARIADB_MYCNFDIR} - COMPONENT columnstore-engine - ) + columnstore_install_file(columnstore.cnf ${MARIADB_MYCNFDIR}) endif() if(TARGET columnstore) - install( - FILES syscatalog_mysql.sql dumpcat_mysql.sql calsetuserpriority.sql calremoveuserpriority.sql - calshowprocesslist.sql columnstore_info.sql - DESTINATION ${ENGINE_SUPPORTDIR} - COMPONENT columnstore-engine - ) - install( - PROGRAMS install_mcs_mysql.sh - DESTINATION ${ENGINE_SBINDIR} - COMPONENT columnstore-engine - ) + columnstore_install_file(syscatalog_mysql.sql ${ENGINE_SUPPORTDIR}) + columnstore_install_file(dumpcat_mysql.sql ${ENGINE_SUPPORTDIR}) + columnstore_install_file(calsetuserpriority.sql ${ENGINE_SUPPORTDIR}) + columnstore_install_file(calremoveuserpriority.sql ${ENGINE_SUPPORTDIR}) + columnstore_install_file(calshowprocesslist.sql ${ENGINE_SUPPORTDIR}) + columnstore_install_file(columnstore_info.sql ${ENGINE_SUPPORTDIR}) + columnstore_install_program(install_mcs_mysql.sh ${ENGINE_SBINDIR}) endif() diff --git a/dbcon/mysql/ha_mcs_client_udfs.cpp b/dbcon/mysql/ha_mcs_client_udfs.cpp index 32d8f004e..c6dfc14a1 100644 --- a/dbcon/mysql/ha_mcs_client_udfs.cpp +++ b/dbcon/mysql/ha_mcs_client_udfs.cpp @@ -45,6 +45,7 @@ using namespace logging; #include "columnstoreversion.h" #include "ha_mcs_sysvars.h" +#include "basic/string_utils.h" extern "C" { diff --git a/dbcon/mysql/ha_mcs_impl_if.h b/dbcon/mysql/ha_mcs_impl_if.h index 2cac13f67..5dfa77aa4 100644 --- a/dbcon/mysql/ha_mcs_impl_if.h +++ b/dbcon/mysql/ha_mcs_impl_if.h @@ -26,7 +26,7 @@ #include #include #include - +#include "basic/string_utils.h" #include "idb_mysql.h" #include "ha_mcs_sysvars.h" @@ -36,6 +36,8 @@ struct st_ha_create_information; class ha_columnstore_select_handler; class ha_columnstore_derived_handler; +#include "basic/string_utils.h" + #include "configcpp.h" #include "idberrorinfo.h" #include "calpontselectexecutionplan.h" @@ -117,7 +119,8 @@ struct gp_walk_info std::vector localCols; std::stack rcWorkStack; std::stack ptWorkStack; - boost::shared_ptr scsp; // while defined as SSCP, it is used as SRCP, nothing specific to SimpleColumn is used in use sites. + boost::shared_ptr scsp; // while defined as SSCP, it is used as SRCP, nothing + // specific to SimpleColumn is used in use sites. uint32_t sessionid; bool fatalParseError; std::string parseErrorText; @@ -227,7 +230,6 @@ struct gp_walk_info { } ~gp_walk_info(); - }; struct SubQueryChainHolder; @@ -237,7 +239,7 @@ struct ext_cond_info // circular dependency on header inclusion with ha_subquery.h. boost::shared_ptr chainHolder; gp_walk_info gwi; - ext_cond_info(long timeZone); // needs knowledge on SubQueryChainHolder, will be defined elsewhere + ext_cond_info(long timeZone); // needs knowledge on SubQueryChainHolder, will be defined elsewhere }; struct cal_table_info diff --git a/ddlproc/CMakeLists.txt b/ddlproc/CMakeLists.txt index 0c8c3d8cb..64924fb79 100644 --- a/ddlproc/CMakeLists.txt +++ b/ddlproc/CMakeLists.txt @@ -4,14 +4,6 @@ include_directories(${ENGINE_COMMON_INCLUDES}) set(DDLProc_SRCS ddlproc.cpp ddlprocessor.cpp ../utils/common/crashtrace.cpp) -add_executable(DDLProc ${DDLProc_SRCS}) +columnstore_executable(DDLProc ${DDLProc_SRCS}) -add_dependencies(DDLProc loggingcpp) - -target_link_libraries(DDLProc ${ENGINE_LDFLAGS} ${ENGINE_WRITE_LIBS} ${NETSNMP_LIBRARIES} threadpool) - -install( - TARGETS DDLProc - DESTINATION ${ENGINE_BINDIR} - COMPONENT columnstore-engine -) +columnstore_link(DDLProc ${ENGINE_LDFLAGS} ${ENGINE_WRITE_LIBS} ${NETSNMP_LIBRARIES} threadpool loggingcpp) diff --git a/dmlproc/CMakeLists.txt b/dmlproc/CMakeLists.txt index e551e4e54..af9ed6492 100644 --- a/dmlproc/CMakeLists.txt +++ b/dmlproc/CMakeLists.txt @@ -6,11 +6,9 @@ set(DMLProc_SRCS dmlproc.cpp dmlprocessor.cpp dmlresultbuffer.cpp batchinsertpro ../utils/common/crashtrace.cpp ) -add_executable(DMLProc ${DMLProc_SRCS}) +columnstore_executable(DMLProc ${DMLProc_SRCS}) -add_dependencies(DMLProc loggingcpp) - -target_link_libraries( +columnstore_link( DMLProc ${ENGINE_LDFLAGS} ${ENGINE_WRITE_LIBS} @@ -18,10 +16,5 @@ target_link_libraries( threadpool ddlcleanuputil batchloader -) - -install( - TARGETS DMLProc - DESTINATION ${ENGINE_BINDIR} - COMPONENT columnstore-engine + loggingcpp ) diff --git a/oam/etc/CMakeLists.txt b/oam/etc/CMakeLists.txt index 5fbaaf449..36523d15b 100644 --- a/oam/etc/CMakeLists.txt +++ b/oam/etc/CMakeLists.txt @@ -1,5 +1 @@ -install( - FILES Columnstore.xml - DESTINATION ${ENGINE_SYSCONFDIR}/columnstore - COMPONENT columnstore-engine -) +columnstore_install_file(Columnstore.xml ${ENGINE_SYSCONFDIR}/columnstore) diff --git a/oam/install_scripts/CMakeLists.txt b/oam/install_scripts/CMakeLists.txt index 068dad945..aaabc3559 100644 --- a/oam/install_scripts/CMakeLists.txt +++ b/oam/install_scripts/CMakeLists.txt @@ -131,49 +131,37 @@ configure_file("${CMAKE_CURRENT_SOURCE_DIR}/mcs-savebrm.py.in" "${CMAKE_CURRENT_ configure_file("${CMAKE_CURRENT_SOURCE_DIR}/mcs-savebrm.py.in" "${CMAKE_CURRENT_SOURCE_DIR}/mcssavebrm.py" @ONLY) configure_file("${CMAKE_CURRENT_SOURCE_DIR}/columnstoreSyslog.in" "${CMAKE_CURRENT_SOURCE_DIR}/columnstoreSyslog" @ONLY) -install( - PROGRAMS columnstore-post-install - columnstore-pre-uninstall - columnstore_run.sh - post-mysql-install - post-mysqld-install - columnstoreSyslogSetup.sh - mcs-stop-controllernode.sh - mcs-loadbrm.py - mcs-savebrm.py - mariadb-columnstore-start.sh - mariadb-columnstore-stop.sh - loop_process_starter.sh - DESTINATION ${ENGINE_BINDIR} - COMPONENT columnstore-engine -) +columnstore_install_program(columnstore-post-install ${ENGINE_BINDIR}) +columnstore_install_program(columnstore-pre-uninstall ${ENGINE_BINDIR}) +columnstore_install_program(columnstore_run.sh ${ENGINE_BINDIR}) +columnstore_install_program(post-mysql-install ${ENGINE_BINDIR}) +columnstore_install_program(post-mysqld-install ${ENGINE_BINDIR}) +columnstore_install_program(columnstoreSyslogSetup.sh ${ENGINE_BINDIR}) +columnstore_install_program(mcs-stop-controllernode.sh ${ENGINE_BINDIR}) +columnstore_install_program(mcs-loadbrm.py ${ENGINE_BINDIR}) +columnstore_install_program(mcs-savebrm.py ${ENGINE_BINDIR}) +columnstore_install_program(mariadb-columnstore-start.sh ${ENGINE_BINDIR}) +columnstore_install_program(mariadb-columnstore-stop.sh ${ENGINE_BINDIR}) +columnstore_install_program(loop_process_starter.sh ${ENGINE_BINDIR}) -install( - FILES mariadb-columnstore.service - columnstoreAlias - columnstoreSyslog - columnstoreSyslog7 - columnstoreSyslog-ng - columnstoreLogRotate - myCnf-include-args.text - myCnf-exclude-args.text - mcs-workernode.service - mcs-controllernode.service - mcs-primproc.service - mcs-writeengineserver.service - mcs-dmlproc.service - mcs-ddlproc.service - mcs-loadbrm.service - mcs-storagemanager.service - DESTINATION ${ENGINE_SUPPORTDIR} - COMPONENT columnstore-engine -) +columnstore_install_file(mariadb-columnstore.service ${ENGINE_SUPPORTDIR}) +columnstore_install_file(columnstoreAlias ${ENGINE_SUPPORTDIR}) +columnstore_install_file(columnstoreSyslog ${ENGINE_SUPPORTDIR}) +columnstore_install_file(columnstoreSyslog7 ${ENGINE_SUPPORTDIR}) +columnstore_install_file(columnstoreSyslog-ng ${ENGINE_SUPPORTDIR}) +columnstore_install_file(columnstoreLogRotate ${ENGINE_SUPPORTDIR}) +columnstore_install_file(myCnf-include-args.text ${ENGINE_SUPPORTDIR}) +columnstore_install_file(myCnf-exclude-args.text ${ENGINE_SUPPORTDIR}) +columnstore_install_file(mcs-workernode.service ${ENGINE_SUPPORTDIR}) +columnstore_install_file(mcs-controllernode.service ${ENGINE_SUPPORTDIR}) +columnstore_install_file(mcs-primproc.service ${ENGINE_SUPPORTDIR}) +columnstore_install_file(mcs-writeengineserver.service ${ENGINE_SUPPORTDIR}) +columnstore_install_file(mcs-dmlproc.service ${ENGINE_SUPPORTDIR}) +columnstore_install_file(mcs-ddlproc.service ${ENGINE_SUPPORTDIR}) +columnstore_install_file(mcs-loadbrm.service ${ENGINE_SUPPORTDIR}) +columnstore_install_file(mcs-storagemanager.service ${ENGINE_SUPPORTDIR}) -install( - FILES module - DESTINATION ${ENGINE_DATADIR}/local - COMPONENT columnstore-engine -) +columnstore_install_file(module ${ENGINE_DATADIR}/local) find_package(Python3 COMPONENTS Interpreter) if(Python3_FOUND) diff --git a/oam/oamcpp/CMakeLists.txt b/oam/oamcpp/CMakeLists.txt index 59014f291..d5b4ac307 100644 --- a/oam/oamcpp/CMakeLists.txt +++ b/oam/oamcpp/CMakeLists.txt @@ -5,15 +5,6 @@ include_directories(${ENGINE_COMMON_INCLUDES}) set(oamcpp_LIB_SRCS liboamcpp.cpp oamcache.cpp) columnstore_library(oamcpp ${oamcpp_LIB_SRCS}) - -add_dependencies(oamcpp loggingcpp) - -target_link_libraries(oamcpp) +columnstore_link(oamcpp loggingcpp) target_compile_options(oamcpp PRIVATE -Wno-unused-result) - -install( - TARGETS oamcpp - DESTINATION ${ENGINE_LIBDIR} - COMPONENT columnstore-engine -) diff --git a/oamapps/columnstoreDB/CMakeLists.txt b/oamapps/columnstoreDB/CMakeLists.txt index 17c66c8f9..120432900 100644 --- a/oamapps/columnstoreDB/CMakeLists.txt +++ b/oamapps/columnstoreDB/CMakeLists.txt @@ -4,12 +4,6 @@ include_directories(${ENGINE_COMMON_INCLUDES}) set(columnstoreDBWrite_SRCS columnstoreDB.cpp) -add_executable(columnstoreDBWrite ${columnstoreDBWrite_SRCS}) +columnstore_executable(columnstoreDBWrite ${columnstoreDBWrite_SRCS}) -target_link_libraries(columnstoreDBWrite ${ENGINE_LDFLAGS} ncurses ${ENGINE_EXEC_LIBS}) - -install( - TARGETS columnstoreDBWrite - DESTINATION ${ENGINE_BINDIR} - COMPONENT columnstore-engine -) +columnstore_link(columnstoreDBWrite ${ENGINE_LDFLAGS} ncurses ${ENGINE_EXEC_LIBS}) diff --git a/oamapps/columnstoreSupport/CMakeLists.txt b/oamapps/columnstoreSupport/CMakeLists.txt index 38afc74c1..feaf17aa2 100644 --- a/oamapps/columnstoreSupport/CMakeLists.txt +++ b/oamapps/columnstoreSupport/CMakeLists.txt @@ -4,20 +4,13 @@ include_directories(${ENGINE_COMMON_INCLUDES}) set(columnstoreSupport_SRCS columnstoreSupport.cpp mcsSupportUtil.cpp) -add_executable(columnstoreSupport ${columnstoreSupport_SRCS}) - +columnstore_executable(columnstoreSupport ${columnstoreSupport_SRCS}) target_compile_options(columnstoreSupport PRIVATE -Wno-unused-result) +columnstore_link(columnstoreSupport ${ENGINE_LDFLAGS} ncurses ${ENGINE_EXEC_LIBS}) -target_link_libraries(columnstoreSupport ${ENGINE_LDFLAGS} ncurses ${ENGINE_EXEC_LIBS}) - -install( - TARGETS columnstoreSupport - DESTINATION ${ENGINE_BINDIR} - COMPONENT columnstore-engine -) - -install( - PROGRAMS dbmsReport.sh bulklogReport.sh configReport.sh hardwareReport.sh logReport.sh resourceReport.sh - DESTINATION ${ENGINE_BINDIR} - COMPONENT columnstore-engine -) +columnstore_install_program(dbmsReport.sh, ${ENGINE_BINDIR}) +columnstore_install_program(bulklogReport.sh, ${ENGINE_BINDIR}) +columnstore_install_program(configReport.sh, ${ENGINE_BINDIR}) +columnstore_install_program(hardwareReport.sh, ${ENGINE_BINDIR}) +columnstore_install_program(logReport.sh, ${ENGINE_BINDIR}) +columnstore_install_program(resourceReport.sh, ${ENGINE_BINDIR}) diff --git a/oamapps/postConfigure/CMakeLists.txt b/oamapps/postConfigure/CMakeLists.txt index 50a25abea..dc431de1a 100644 --- a/oamapps/postConfigure/CMakeLists.txt +++ b/oamapps/postConfigure/CMakeLists.txt @@ -4,14 +4,6 @@ include_directories(${ENGINE_COMMON_INCLUDES}) set(mycnfUpgrade_SRCS mycnfUpgrade.cpp) -add_executable(mycnfUpgrade ${mycnfUpgrade_SRCS}) +columnstore_executable(mycnfUpgrade ${mycnfUpgrade_SRCS}) -target_compile_options(mycnfUpgrade PRIVATE -Wno-unused-result) - -target_link_libraries(mycnfUpgrade ${ENGINE_LDFLAGS} ${ENGINE_READLINE_LIBRARY} ncurses ${ENGINE_EXEC_LIBS}) - -install( - TARGETS mycnfUpgrade - DESTINATION ${ENGINE_BINDIR} - COMPONENT columnstore-engine -) +columnstore_link(mycnfUpgrade ${ENGINE_LDFLAGS} ${ENGINE_READLINE_LIBRARY} ncurses ${ENGINE_EXEC_LIBS}) diff --git a/oamapps/postConfigure/mycnfUpgrade.cpp b/oamapps/postConfigure/mycnfUpgrade.cpp index ae79eb1bc..168e58442 100644 --- a/oamapps/postConfigure/mycnfUpgrade.cpp +++ b/oamapps/postConfigure/mycnfUpgrade.cpp @@ -46,6 +46,7 @@ #include #include #include +#include #include #include @@ -282,7 +283,7 @@ int main(int argc, char* argv[]) USER = p; string cmd = "chown " + USER + ":" + USER + " " + mycnfFile; - system(cmd.c_str()); + std::ignore = std::system(cmd.c_str()); exit(0); } diff --git a/primitives/blockcache/CMakeLists.txt b/primitives/blockcache/CMakeLists.txt index b000247b9..9ca0c950f 100644 --- a/primitives/blockcache/CMakeLists.txt +++ b/primitives/blockcache/CMakeLists.txt @@ -13,13 +13,5 @@ set(dbbc_STAT_SRCS stats.cpp fsutils.cpp ) - -# libdbbc_a_CXXFLAGS = $(march_flags) $(AM_CXXFLAGS) - -add_library(dbbc STATIC ${dbbc_STAT_SRCS}) - -add_dependencies(dbbc loggingcpp) - -target_link_libraries(dbbc ${NETSNMP_LIBRARIES}) - -install(TARGETS dbbc DESTINATION ${ENGINE_LIBDIR}) +columnstore_static_library(dbbc ${dbbc_STAT_SRCS}) +columnstore_link(dbbc ${NETSNMP_LIBRARIES} loggingcpp) diff --git a/primitives/linux-port/CMakeLists.txt b/primitives/linux-port/CMakeLists.txt index 9c566dc94..1a0032707 100644 --- a/primitives/linux-port/CMakeLists.txt +++ b/primitives/linux-port/CMakeLists.txt @@ -4,10 +4,5 @@ include_directories(${ENGINE_COMMON_INCLUDES} ../blockcache ../primproc) set(processor_STAT_SRCS primitiveprocessor.cpp dictionary.cpp column.cpp) -add_library(processor STATIC ${processor_STAT_SRCS}) - -add_dependencies(processor loggingcpp) - -target_link_libraries(processor ${NETSNMP_LIBRARIES}) - -install(TARGETS processor DESTINATION ${ENGINE_LIBDIR}) +columnstore_static_library(processor ${processor_STAT_SRCS}) +columnstore_link(processor ${NETSNMP_LIBRARIES} loggingcpp) diff --git a/primitives/primproc/CMakeLists.txt b/primitives/primproc/CMakeLists.txt index 740a43e74..cd1232812 100644 --- a/primitives/primproc/CMakeLists.txt +++ b/primitives/primproc/CMakeLists.txt @@ -25,11 +25,10 @@ set(PrimProc_SRCS ../../utils/common/crashtrace.cpp ) -add_executable(PrimProc ${PrimProc_SRCS}) +columnstore_executable(PrimProc ${PrimProc_SRCS}) -add_dependencies(PrimProc loggingcpp) target_include_directories(PrimProc PRIVATE ${Boost_INCLUDE_DIRS}) -target_link_libraries( +columnstore_link( PrimProc ${ENGINE_LDFLAGS} ${NETSNMP_LIBRARIES} @@ -38,10 +37,5 @@ target_link_libraries( cacheutils dbbc processor -) - -install( - TARGETS PrimProc - DESTINATION ${ENGINE_BINDIR} - COMPONENT columnstore-engine + loggingcpp ) diff --git a/storage-manager/CMakeLists.txt b/storage-manager/CMakeLists.txt index 8c4350122..fa459e2f8 100755 --- a/storage-manager/CMakeLists.txt +++ b/storage-manager/CMakeLists.txt @@ -64,11 +64,11 @@ configure_file( link_directories(${CMAKE_BINARY_DIR}/lib) set(CMAKE_INSTALL_RPATH $ORIGIN $ORIGIN/../lib) -add_library(storagemanager SHARED ${storagemanager_SRCS}) +columnstore_library(storagemanager SHARED ${storagemanager_SRCS}) add_dependencies(storagemanager marias3 external_boost) target_compile_definitions(storagemanager PUBLIC BOOST_NO_CXX11_SCOPED_ENUMS) -target_link_libraries( +columnstore_link( storagemanager boost_chrono boost_system @@ -80,8 +80,8 @@ target_link_libraries( ) target_include_directories(storagemanager PRIVATE ${Boost_INCLUDE_DIRS}) -add_executable(StorageManager src/main.cpp) -target_link_libraries(StorageManager storagemanager) +columnstore_executable(StorageManager src/main.cpp) +columnstore_link(StorageManager storagemanager) set_property(TARGET StorageManager PROPERTY CXX_STANDARD 20) set(TMPDIR ${CMAKE_RUNTIME_OUTPUT_DIRECTORY}) @@ -89,11 +89,11 @@ set(CMAKE_RUNTIME_OUTPUT_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}) add_executable(unit_tests src/unit_tests.cpp) target_compile_definitions(unit_tests PUBLIC BOOST_NO_CXX11_SCOPED_ENUMS) -target_link_libraries(unit_tests storagemanager) +columnstore_link(unit_tests storagemanager) -add_executable(testS3Connection src/testS3Connection.cpp) +columnstore_executable(testS3Connection src/testS3Connection.cpp) target_compile_definitions(testS3Connection PUBLIC BOOST_NO_CXX11_SCOPED_ENUMS) -target_link_libraries(testS3Connection storagemanager) +columnstore_link(testS3Connection storagemanager) set(CMAKE_RUNTIME_OUTPUT_DIRECTORY ${TMPDIR}) @@ -112,42 +112,25 @@ add_custom_command( # The includes and lib linkages required to link against cloudio ... pretty crazy. When lib dependencies are eventually # config'd right, change this to only include and link against cloudio. include_directories(${ENGINE_SRC_DIR}/utils/cloudio ${ENGINE_COMMON_INCLUDES}) -add_executable(smcat src/smcat.cpp) -target_link_libraries(smcat storagemanager cloudio ${ENGINE_LDFLAGS} ${ENGINE_EXEC_LIBS}) +columnstore_executable(smcat src/smcat.cpp) +columnstore_link(smcat storagemanager cloudio ${ENGINE_LDFLAGS} ${ENGINE_EXEC_LIBS}) -add_executable(smput src/smput.cpp) -target_link_libraries(smput storagemanager cloudio ${ENGINE_LDFLAGS} ${ENGINE_EXEC_LIBS}) +columnstore_executable(smput src/smput.cpp) +columnstore_link(smput storagemanager cloudio ${ENGINE_LDFLAGS} ${ENGINE_EXEC_LIBS}) -add_executable(smls src/smls.cpp) -target_link_libraries(smls storagemanager cloudio ${ENGINE_LDFLAGS} ${ENGINE_EXEC_LIBS}) +columnstore_executable(smls src/smls.cpp) +columnstore_link(smls storagemanager cloudio ${ENGINE_LDFLAGS} ${ENGINE_EXEC_LIBS}) -add_executable(smrm src/smrm.cpp) -target_link_libraries(smrm storagemanager cloudio ${ENGINE_LDFLAGS} ${ENGINE_EXEC_LIBS}) +columnstore_executable(smrm src/smrm.cpp) +columnstore_link(smrm storagemanager cloudio ${ENGINE_LDFLAGS} ${ENGINE_EXEC_LIBS}) -add_executable(smps src/smps.cpp) -target_link_libraries(smps storagemanager cloudio ${ENGINE_LDFLAGS} ${ENGINE_EXEC_LIBS}) +columnstore_executable(smps src/smps.cpp) +columnstore_link(smps storagemanager cloudio ${ENGINE_LDFLAGS} ${ENGINE_EXEC_LIBS}) -add_executable(smkill src/smkill.cpp) -target_link_libraries(smkill storagemanager cloudio ${ENGINE_LDFLAGS} ${ENGINE_EXEC_LIBS}) -install(TARGETS storagemanager LIBRARY DESTINATION ${ENGINE_LIBDIR} COMPONENT columnstore-engine) +columnstore_executable(smkill src/smkill.cpp) +columnstore_link(smkill storagemanager cloudio ${ENGINE_LDFLAGS} ${ENGINE_EXEC_LIBS}) -install( - TARGETS StorageManager - smcat - smkill - smps - smput - smls - smrm - testS3Connection - RUNTIME DESTINATION ${ENGINE_BINDIR} COMPONENT columnstore-engine -) - -install( - FILES storagemanager.cnf - DESTINATION ${ENGINE_SYSCONFDIR}/columnstore - COMPONENT columnstore-engine -) +columnstore_install_file(storagemanager.cnf ${ENGINE_SYSCONFDIR}/columnstore) install( FILES storagemanager.cnf diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index caaa047a8..396baf71c 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -30,82 +30,80 @@ if(WITH_UNITTESTS) add_executable(rowgroup_tests rowgroup-tests.cpp) target_compile_options(rowgroup_tests PRIVATE -Wno-sign-compare) add_dependencies(rowgroup_tests googletest) - target_link_libraries(rowgroup_tests ${ENGINE_LDFLAGS} ${GTEST_LIBRARIES} ${ENGINE_EXEC_LIBS}) + columnstore_link(rowgroup_tests ${ENGINE_LDFLAGS} ${GTEST_LIBRARIES} ${ENGINE_EXEC_LIBS}) gtest_add_tests(TARGET rowgroup_tests TEST_PREFIX columnstore:) add_executable(rewritetest rewritetest.cpp) add_dependencies(rewritetest googletest) - target_link_libraries(rewritetest ${ENGINE_LDFLAGS} ${GTEST_LIBRARIES} ${ENGINE_EXEC_LIBS}) + columnstore_link(rewritetest ${ENGINE_LDFLAGS} ${GTEST_LIBRARIES} ${ENGINE_EXEC_LIBS}) gtest_add_tests(TARGET rewritetest TEST_PREFIX columnstore:) add_executable(mcs_decimal_tests mcs_decimal-tests.cpp) add_dependencies(mcs_decimal_tests googletest) - target_link_libraries(mcs_decimal_tests ${ENGINE_LDFLAGS} ${GTEST_LIBRARIES} ${ENGINE_EXEC_LIBS}) + columnstore_link(mcs_decimal_tests ${ENGINE_LDFLAGS} ${GTEST_LIBRARIES} ${ENGINE_EXEC_LIBS}) gtest_add_tests(TARGET mcs_decimal_tests TEST_PREFIX columnstore:) add_executable(dataconvert_tests dataconvert-tests.cpp) add_dependencies(dataconvert_tests googletest) - target_link_libraries(dataconvert_tests ${ENGINE_LDFLAGS} ${GTEST_LIBRARIES} ${ENGINE_EXEC_LIBS}) + columnstore_link(dataconvert_tests ${ENGINE_LDFLAGS} ${GTEST_LIBRARIES} ${ENGINE_EXEC_LIBS}) gtest_add_tests(TARGET dataconvert_tests TEST_PREFIX columnstore:) add_executable(rebuild_em_tests rebuild-em-tests.cpp) add_dependencies(rebuild_em_tests googletest) - target_link_libraries(rebuild_em_tests ${ENGINE_LDFLAGS} ${GTEST_LIBRARIES} ${ENGINE_WRITE_LIBS}) + columnstore_link(rebuild_em_tests ${ENGINE_LDFLAGS} ${GTEST_LIBRARIES} ${ENGINE_WRITE_LIBS}) gtest_add_tests(TARGET rebuild_em_tests TEST_PREFIX columnstore:) add_executable(compression_tests compression-tests.cpp) add_dependencies(compression_tests googletest) - target_link_libraries(compression_tests ${ENGINE_LDFLAGS} ${GTEST_LIBRARIES} ${ENGINE_WRITE_LIBS}) + columnstore_link(compression_tests ${ENGINE_LDFLAGS} ${GTEST_LIBRARIES} ${ENGINE_WRITE_LIBS}) gtest_add_tests(TARGET compression_tests TEST_PREFIX columnstore:) add_executable(column_scan_filter_tests primitives_column_scan_and_filter.cpp) target_compile_options(column_scan_filter_tests PRIVATE -Wno-error -Wno-sign-compare) add_dependencies(column_scan_filter_tests googletest) - target_link_libraries( - column_scan_filter_tests ${ENGINE_LDFLAGS} ${ENGINE_WRITE_LIBS} ${GTEST_LIBRARIES} processor dbbc - ) + columnstore_link(column_scan_filter_tests ${ENGINE_LDFLAGS} ${ENGINE_WRITE_LIBS} ${GTEST_LIBRARIES} processor dbbc) gtest_add_tests(TARGET column_scan_filter_tests TEST_PREFIX columnstore:) add_executable(simd_processors simd_processors.cpp) target_compile_options(simd_processors PRIVATE -Wno-error) add_dependencies(simd_processors googletest) - target_link_libraries(simd_processors ${ENGINE_LDFLAGS} ${ENGINE_WRITE_LIBS} ${GTEST_LIBRARIES} processor dbbc) + columnstore_link(simd_processors ${ENGINE_LDFLAGS} ${ENGINE_WRITE_LIBS} ${GTEST_LIBRARIES} processor dbbc) gtest_add_tests(TARGET simd_processors TEST_PREFIX columnstore:) add_executable(fair_threadpool_test fair_threadpool.cpp) add_dependencies(fair_threadpool_test googletest) - target_link_libraries(fair_threadpool_test ${ENGINE_LDFLAGS} ${ENGINE_WRITE_LIBS} ${GTEST_LIBRARIES} processor dbbc) + columnstore_link(fair_threadpool_test ${ENGINE_LDFLAGS} ${ENGINE_WRITE_LIBS} ${GTEST_LIBRARIES} processor dbbc) gtest_add_tests(TARGET fair_threadpool_test TEST_PREFIX columnstore:) add_executable(counting_allocator counting_allocator.cpp) target_compile_options(counting_allocator PRIVATE -Wno-sign-compare) add_dependencies(counting_allocator googletest) - target_link_libraries(counting_allocator ${ENGINE_LDFLAGS} ${ENGINE_WRITE_LIBS} ${GTEST_LIBRARIES}) + columnstore_link(counting_allocator ${ENGINE_LDFLAGS} ${ENGINE_WRITE_LIBS} ${GTEST_LIBRARIES}) gtest_add_tests(TARGET counting_allocator TEST_PREFIX columnstore:) add_executable(poolallocator poolallocator.cpp) target_compile_options(poolallocator PRIVATE -Wno-sign-compare) add_dependencies(poolallocator googletest) - target_link_libraries(poolallocator ${ENGINE_LDFLAGS} ${ENGINE_WRITE_LIBS} ${GTEST_LIBRARIES}) + columnstore_link(poolallocator ${ENGINE_LDFLAGS} ${ENGINE_WRITE_LIBS} ${GTEST_LIBRARIES}) gtest_add_tests(TARGET poolallocator TEST_PREFIX columnstore:) add_executable(stlpoolallocator stlpoolallocator.cpp) target_compile_options(stlpoolallocator PRIVATE -Wno-sign-compare) add_dependencies(stlpoolallocator googletest) - target_link_libraries(stlpoolallocator ${ENGINE_LDFLAGS} ${ENGINE_WRITE_LIBS} ${GTEST_LIBRARIES}) + columnstore_link(stlpoolallocator ${ENGINE_LDFLAGS} ${ENGINE_WRITE_LIBS} ${GTEST_LIBRARIES}) gtest_add_tests(TARGET stlpoolallocator TEST_PREFIX columnstore:) add_executable(comparators_tests comparators-tests.cpp) - target_link_libraries(comparators_tests ${ENGINE_LDFLAGS} ${ENGINE_WRITE_LIBS} ${CPPUNIT_LIBRARIES} cppunit) + columnstore_link(comparators_tests ${ENGINE_LDFLAGS} ${ENGINE_WRITE_LIBS} ${CPPUNIT_LIBRARIES} cppunit) add_test(NAME columnstore:comparators_tests COMMAND comparators_tests) add_executable(bytestream bytestream.cpp) - target_link_libraries(bytestream ${ENGINE_LDFLAGS} ${ENGINE_WRITE_LIBS} ${CPPUNIT_LIBRARIES} cppunit) + columnstore_link(bytestream ${ENGINE_LDFLAGS} ${ENGINE_WRITE_LIBS} ${CPPUNIT_LIBRARIES} cppunit) add_test(NAME columnstore:bytestream COMMAND bytestream) # standalone EM routines test add_executable(brm_em_standalone brm-em-standalone.cpp) - target_link_libraries(brm_em_standalone ${ENGINE_LDFLAGS} ${ENGINE_WRITE_LIBS} ${CPPUNIT_LIBRARIES} cppunit) + columnstore_link(brm_em_standalone ${ENGINE_LDFLAGS} ${ENGINE_WRITE_LIBS} ${CPPUNIT_LIBRARIES} cppunit) add_test(NAME columnstore:brm_em_standalone COMMAND brm_em_standalone) set_tests_properties(columnstore:brm_em_standalone PROPERTIES DISABLED True) endif() @@ -116,7 +114,7 @@ if(WITH_MICROBENCHMARKS AND (NOT CMAKE_BUILD_TYPE STREQUAL "Debug")) target_include_directories( primitives_scan_bench PUBLIC ${ENGINE_COMMON_INCLUDES} ${ENGINE_BLOCKCACHE_INCLUDE} ${ENGINE_PRIMPROC_INCLUDE} ) - target_link_libraries( + columnstore_link( primitives_scan_bench ${ENGINE_LDFLAGS} ${ENGINE_WRITE_LIBS} diff --git a/tools/bincvt/li2bin.cpp b/tools/bincvt/li2bin.cpp deleted file mode 100644 index 0cf52e33d..000000000 --- a/tools/bincvt/li2bin.cpp +++ /dev/null @@ -1,258 +0,0 @@ -#include -#include -#include -#include -#include -//#define NDEBUG -#include -using namespace std; - -#include -#include -using namespace boost; - - -// 1|155190|7706|1|17|21168.23|0.04|0.02|N|O|1996-03-13|1996-02-12|1996-03-22|DELIVER IN PERSON|TRUCK|egular -// courts above the| - -#pragma pack(1) -struct lineitem_image -{ - int32_t l_orderkey; - int32_t l_partkey; - int32_t l_suppkey; // 12 - int64_t l_linenumber; - int64_t l_quantity; - int64_t l_extendedprice; - int64_t l_discount; - int64_t l_tax; // 40 - char l_returnflag; - char l_linestatus; // 2 - int32_t l_shipdate; - int32_t l_commitdate; - int32_t l_receiptdate; // 12 - char l_shipinstruct[25]; - char l_shipmode[10]; - char l_comment[44]; // 79 -}; -BOOST_STATIC_ASSERT(sizeof(struct lineitem_image) == 145); - -// version of lineitem_image that treats some integer columns as unsigned ints. -// Decimal, date, and date/time columns are still treated as integer. -struct unsigned_lineitem_image -{ - uint32_t l_orderkey; - uint32_t l_partkey; - uint32_t l_suppkey; // 12 - uint64_t l_linenumber; - int64_t l_quantity; - int64_t l_extendedprice; - int64_t l_discount; - int64_t l_tax; // 40 - char l_returnflag; - char l_linestatus; // 2 - int32_t l_shipdate; - int32_t l_commitdate; - int32_t l_receiptdate; // 12 - char l_shipinstruct[25]; - char l_shipmode[10]; - char l_comment[44]; // 79 -}; -BOOST_STATIC_ASSERT(sizeof(struct unsigned_lineitem_image) == 145); - -struct Date -{ - unsigned spare : 6; - unsigned day : 6; - unsigned month : 4; - unsigned year : 16; -}; -BOOST_STATIC_ASSERT(sizeof(struct Date) == 4); - -union date_image -{ - struct Date d; - int32_t i; -}; - -namespace -{ -typedef boost::tokenizer > tokenizer; - -int64_t idbbigdec2(const string& str) -{ - int64_t val = 0; - string::size_type idx = string::npos; - string tmp(str); - idx = tmp.find('.'); - - if (idx != string::npos) - tmp.erase(idx, 1); - else - tmp.append("00"); - - val = atoll(tmp.c_str()); - return val; -} - -int32_t idbdate(const string& str) -{ - date_image di; - di.i = 0; - boost::char_separator sep("-"); - tokenizer tokens(str, sep); - tokenizer::iterator tok_iter = tokens.begin(); - - // Note that the spare bits must be set, instead of leaving them - // initialized to 0; to be compatible with internal date format. - if (distance(tok_iter, tokens.end()) == 3) - { - di.d.spare = 0x3E; - di.d.year = atoi(tok_iter->c_str()); - ++tok_iter; - di.d.month = atoi(tok_iter->c_str()); - ++tok_iter; - di.d.day = atoi(tok_iter->c_str()); - ++tok_iter; - } - - return di.i; -} - -int parseinto(lineitem_image& img, const string& buf) -{ - memset(&img, 0, sizeof(img)); - - boost::char_separator sep("|"); - tokenizer tokens(buf, sep); - tokenizer::iterator tok_iter = tokens.begin(); - - if (distance(tok_iter, tokens.end()) < 16) - return -1; - - img.l_orderkey = atoi(tok_iter->c_str()); - ++tok_iter; - img.l_partkey = atoi(tok_iter->c_str()); - ++tok_iter; - img.l_suppkey = atoi(tok_iter->c_str()); - ++tok_iter; - img.l_linenumber = atoll(tok_iter->c_str()); - ++tok_iter; - img.l_quantity = idbbigdec2(tok_iter->c_str()); - ++tok_iter; - img.l_extendedprice = idbbigdec2(tok_iter->c_str()); - ++tok_iter; - img.l_discount = idbbigdec2(tok_iter->c_str()); - ++tok_iter; - img.l_tax = idbbigdec2(tok_iter->c_str()); - ++tok_iter; - img.l_returnflag = tok_iter->at(0); - ++tok_iter; - img.l_linestatus = tok_iter->at(0); - ++tok_iter; - img.l_shipdate = idbdate(tok_iter->c_str()); - ++tok_iter; - img.l_commitdate = idbdate(tok_iter->c_str()); - ++tok_iter; - img.l_receiptdate = idbdate(tok_iter->c_str()); - ++tok_iter; - memcpy(&img.l_shipinstruct[0], tok_iter->c_str(), tok_iter->size()); - ++tok_iter; - memcpy(&img.l_shipmode[0], tok_iter->c_str(), tok_iter->size()); - ++tok_iter; - memcpy(&img.l_comment[0], tok_iter->c_str(), tok_iter->size()); - ++tok_iter; - - return 0; -} - -int unsigned_parseinto(unsigned_lineitem_image& img, const string& buf) -{ - memset(&img, 0, sizeof(img)); - - boost::char_separator sep("|"); - tokenizer tokens(buf, sep); - tokenizer::iterator tok_iter = tokens.begin(); - - if (distance(tok_iter, tokens.end()) < 16) - return -1; - - img.l_orderkey = strtoul(tok_iter->c_str(), 0, 10); - ++tok_iter; - img.l_partkey = strtoul(tok_iter->c_str(), 0, 10); - ++tok_iter; - img.l_suppkey = strtoul(tok_iter->c_str(), 0, 10); - ++tok_iter; - img.l_linenumber = strtoull(tok_iter->c_str(), 0, 10); - ++tok_iter; - img.l_quantity = idbbigdec2(tok_iter->c_str()); - ++tok_iter; - img.l_extendedprice = idbbigdec2(tok_iter->c_str()); - ++tok_iter; - img.l_discount = idbbigdec2(tok_iter->c_str()); - ++tok_iter; - img.l_tax = idbbigdec2(tok_iter->c_str()); - ++tok_iter; - img.l_returnflag = tok_iter->at(0); - ++tok_iter; - img.l_linestatus = tok_iter->at(0); - ++tok_iter; - img.l_shipdate = idbdate(tok_iter->c_str()); - ++tok_iter; - img.l_commitdate = idbdate(tok_iter->c_str()); - ++tok_iter; - img.l_receiptdate = idbdate(tok_iter->c_str()); - ++tok_iter; - memcpy(&img.l_shipinstruct[0], tok_iter->c_str(), tok_iter->size()); - ++tok_iter; - memcpy(&img.l_shipmode[0], tok_iter->c_str(), tok_iter->size()); - ++tok_iter; - memcpy(&img.l_comment[0], tok_iter->c_str(), tok_iter->size()); - ++tok_iter; - - return 0; -} - -} // namespace - -int main(int argc, char** argv) -{ - if ((argc > 1) && (strcmp(argv[1], "-h") == 0)) - { - std::cerr << "li2bin [-u]" << std::endl; - std::cerr << " -u Create first 4 fields as unsigned integers" << std::endl; - return 0; - } - - string input; - - - getline(cin, input); - - if ((argc > 1) && (strcmp(argv[1], "-u") == 0)) - { - unsigned_lineitem_image i; - - while (!cin.eof()) - { - if (unsigned_parseinto(i, input) == 0) - cout.write(reinterpret_cast(&i), sizeof(i)); - - getline(cin, input); - } - } - else - { - lineitem_image i; - - while (!cin.eof()) - { - if (parseinto(i, input) == 0) - cout.write(reinterpret_cast(&i), sizeof(i)); - - getline(cin, input); - } - } - - return 0; -} diff --git a/tools/cfread/CMakeLists.txt b/tools/cfread/CMakeLists.txt deleted file mode 100644 index ba724302e..000000000 --- a/tools/cfread/CMakeLists.txt +++ /dev/null @@ -1,77 +0,0 @@ -include_directories(${KDE4_INCLUDES} ${KDE4_INCLUDE_DIR} ${QT_INCLUDES}) - -# ########## next target ############### - -set(cfread_SRCS cfread.cpp) - -kde4_add_executable(cfread ${cfread_SRCS}) - -target_link_libraries( - cfread - ${KDE4_KDECORE_LIBS} - pthread - writeengine - brm - rwlock - messageqcpp - dl - configcpp - xml2 - loggingcpp - cacheutils - boost_idb - pthread - rt -) - -install(TARGETS cfread ${INSTALL_TARGETS_DEFAULT_ARGS}) - -# ########## next target ############### - -set(mtread_SRCS mtread.cpp) - -kde4_add_executable(mtread ${mtread_SRCS}) - -target_link_libraries( - mtread - ${KDE4_KDECORE_LIBS} - pthread - writeengine - brm - rwlock - messageqcpp - dl - configcpp - xml2 - loggingcpp - cacheutils - boost_idb - pthread - rt -) - -install(TARGETS mtread ${INSTALL_TARGETS_DEFAULT_ARGS}) - -# ########## install files ############### - -# original Makefile.am contents follow: - -# $Id: Makefile.am 333 2009-04-03 20:35:04Z rdempsey $ Process this file with automake to produce Makefile.in -# -# AM_CPPFLAGS = $(idb_cppflags) AM_CFLAGS = $(idb_cflags) AM_CXXFLAGS = $(idb_cxxflags) AM_LDFLAGS = $(idb_ldflags) -# bin_PROGRAMS = cfread mtread cfread_SOURCES = cfread.cpp cfread_CPPFLAGS = @idb_common_includes@ $(AM_CPPFLAGS) -# cfread_LDFLAGS = @idb_common_ldflags@ -lpthread -lwriteengine -lbrm -lrwlock -lmessageqcpp -ldl -lconfigcpp -lxml2 -# -lloggingcpp -lcacheutils -lboost_idb -lpthread -lrt $(AM_LDFLAGS) mtread_SOURCES = mtread.cpp mtread_CPPFLAGS = -# @idb_common_includes@ $(AM_CPPFLAGS) mtread_LDFLAGS = @idb_common_ldflags@ -lpthread -lwriteengine -lbrm -lrwlock -# -lmessageqcpp -ldl -lconfigcpp -lxml2 -lloggingcpp -lcacheutils -lboost_idb -lpthread -lrt $(AM_LDFLAGS) -# -# test: -# -# coverage: -# -# leakcheck: -# -# docs: -# -# bootstrap: install-data-am -# diff --git a/tools/cfread/cfread.cpp b/tools/cfread/cfread.cpp deleted file mode 100644 index e6fe14c2a..000000000 --- a/tools/cfread/cfread.cpp +++ /dev/null @@ -1,183 +0,0 @@ -// - -#include -#include -#include -#include -#define NDEBUG -#include -#include "writeengine.h" - -using namespace std; - -void timespec_sub(const struct timespec& tv1, const struct timespec& tv2, struct timespec& diff) -{ - if (tv2.tv_nsec < tv1.tv_nsec) - { - diff.tv_sec = tv2.tv_sec - tv1.tv_sec - 1; - diff.tv_nsec = tv1.tv_nsec - tv2.tv_nsec; - } - else - { - diff.tv_sec = tv2.tv_sec - tv1.tv_sec; - diff.tv_nsec = tv2.tv_nsec - tv1.tv_nsec; - } -} - -// TODO: -// add getopt for CLAs -// add threads for reading a file -// add threads for reading multiple files - -// main() -// -int main(int argc, char** argv) -{ - uint64_t acc = 0; - uint64_t readBlocks = 0; // read size ib blocks - uint64_t readSize = 0; // read size ib bytes - uint64_t readBufferSz = 0; - const uint64_t blockSize = 8192; - char* alignedbuff = 0; - boost::scoped_array realbuff; - const unsigned pageSize = 4096; // getpagesize(); - WriteEngine::FileOp fFileOp; - BRM::OID_t oid; - char fname[256]; - struct timespec tm; - struct timespec tm2; - struct timespec tm3; - struct timespec starttm; - struct timespec endtm; - struct timespec tottm; - bool odirect = true; - int fd = 0; - char response = 'Y'; - - if (argc <= 1) - { - cerr << "usage: testread " << endl; - return -1; - } - - oid = atoi(argv[1]); - - if (oid <= 0) - exit(-1); - - if (argc >= 2) - { - readBlocks = atoi(argv[2]); - - if (readBlocks <= 0) - readBlocks = 8; - } - - if (argc >= 4) - { - odirect = false; - } - - readSize = readBlocks * blockSize; - readBufferSz = readSize + pageSize; - - realbuff.reset(new char[readBufferSz]); - - if (realbuff.get() == 0) - { - cerr << "thr_popper: Can't allocate space for a whole extent in memory" << endl; - return 0; - } - - if (fFileOp.getFileName(oid, fname) != WriteEngine::NO_ERROR) - { - fname[0] = 0; - throw std::runtime_error("fileOp.getFileName failed"); - } - else - { - cout << "Reading oid: " << oid << " od: " << odirect << " file: " << fname << endl; - } - -#if __LP64__ - alignedbuff = (char*)((((ptrdiff_t)realbuff.get() >> 12) << 12) + pageSize); -#else - alignedbuff = (char*)(((((ptrdiff_t)realbuff.get() >> 12) << 12) & 0xffffffff) + pageSize); -#endif - idbassert(((ptrdiff_t)alignedbuff - (ptrdiff_t)realbuff.get()) < (ptrdiff_t)pageSize); - idbassert(((ptrdiff_t)alignedbuff % pageSize) == 0); - - if (odirect) - fd = open(fname, O_RDONLY | O_DIRECT | O_LARGEFILE | O_NOATIME); - else - fd = open(fname, O_RDONLY | O_LARGEFILE | O_NOATIME); - - if (fd < 0) - { - cerr << "Open failed" << endl; - perror("open"); - throw runtime_error("Error opening file"); - } - - while (toupper(response) != 'N') - { - uint64_t i = 1; - uint64_t rCnt = 0; - clock_gettime(CLOCK_REALTIME, &starttm); - - while (i != 0) - { - // clock_gettime(CLOCK_REALTIME, &tm); - i = pread(fd, alignedbuff, readSize, acc); - // clock_gettime(CLOCK_REALTIME, &tm2); - idbassert(i == 0 || i == readSize); - idbassert(i % pageSize == 0); - idbassert(acc % pageSize == 0); - - if (i < 0 && errno == EINTR) - { - timespec_sub(tm, tm2, tm3); - cout << "* " << i << " " << right << setw(2) << setfill(' ') << tm3.tv_sec << "." << right << setw(9) - << setfill('0') << tm3.tv_nsec << endl; - continue; - } - else if (i < 0) - { - timespec_sub(tm, tm2, tm3); - cout << "* i: " << i << " sz: " << readSize << " acc: " << acc << right << setw(2) << setfill(' ') - << tm3.tv_sec << " " << right << tm3.tv_nsec << endl; - perror("pread"); - // make loop exit - i = 0; - } - - acc += i; - - if (i > 0) - rCnt++; - - // timespec_sub(tm, tm2, tm3); - // cout - // << i << " " - // << right << setw(2) << setfill(' ') << tm3.tv_sec << " " - // << right << tm3.tv_nsec - // << endl; - - } // while(acc... - - clock_gettime(CLOCK_REALTIME, &endtm); - timespec_sub(starttm, endtm, tottm); - - cout << "Total reads: " << rCnt << " sz: " << acc / (1024 * 1024) << "MB" - << " tm: " << tottm.tv_sec << "secs " << tottm.tv_nsec << "ns" << endl; - - cout << "Repeat the last scan[Y,N]?" << endl; - cin >> response; - acc = 0; - - } // while response... - - close(fd); - return 0; - -} // main diff --git a/tools/cfread/mtread.cpp b/tools/cfread/mtread.cpp deleted file mode 100644 index d51c990cb..000000000 --- a/tools/cfread/mtread.cpp +++ /dev/null @@ -1,269 +0,0 @@ -// - -#include -#include -#include -#include -#include -#define NDEBUG -#include -#include "writeengine.h" - -using namespace std; - -void timespec_sub(const struct timespec& tv1, const struct timespec& tv2, struct timespec& diff) -{ - if (tv2.tv_nsec < tv1.tv_nsec) - { - diff.tv_sec = tv2.tv_sec - tv1.tv_sec - 1; - diff.tv_nsec = tv1.tv_nsec - tv2.tv_nsec; - } - else - { - diff.tv_sec = tv2.tv_sec - tv1.tv_sec; - diff.tv_nsec = tv2.tv_nsec - tv1.tv_nsec; - } -} - -// TODO: -// add threads for reading a file -// add threads for reading multiple files - -struct readThr -{ - public: - readThr(const int oid, const int rSize) - { - fblockSize = 8192; - fpageSize = getpagesize(); - freadBlocks = rSize; - freadSize = rSize * fblockSize; // read size ib bytes - freadBufferSz = freadSize + fpageSize; - falignedbuff = 0; - foid = oid; - facc = 0; - memset(fname, 0, sizeof(fname)); - memset((char*)&ftm, 0, sizeof(ftm)); - memset((char*)&ftm2, 0, sizeof(ftm2)); - memset((char*)&ftm3, 0, sizeof(ftm3)); - memset((char*)&fstarttm, 0, sizeof(fstarttm)); - memset((char*)&fendtm, 0, sizeof(fendtm)); - memset((char*)&ftottm, 0, sizeof(ftottm)); - fodirect = true; - fd = 0; - cout << "o: " << foid << " r: " << freadBlocks << " b: " << freadBufferSz << endl; - } - - void operator()() - { - WriteEngine::FileOp fFileOp; - char frealbuff[freadBufferSz]; - memset(frealbuff, 0, freadBufferSz); - - if (frealbuff == 0) - { - cerr << "thr_popper: Can't allocate space for a whole extent in memory" << endl; - return; - } - - if (fFileOp.getFileName(foid, fname) != WriteEngine::NO_ERROR) - { - fname[0] = 0; - throw std::runtime_error("fileOp.getFileName failed"); - } - else - { - cout << "Reading oid: " << foid << " od: " << fodirect << " file: " << fname << endl; - } - -#if __LP64__ - falignedbuff = (char*)((((ptrdiff_t)frealbuff >> 12) << 12) + fpageSize); -#else - falignedbuff = (char*)(((((ptrdiff_t)frealbuff >> 12) << 12) & 0xffffffff) + fpageSize); -#endif - idbassert(((ptrdiff_t)falignedbuff - (ptrdiff_t)frealbuff) < (ptrdiff_t)fpageSize); - idbassert(((ptrdiff_t)falignedbuff % fpageSize) == 0); - - if (fodirect) - fd = open(fname, O_RDONLY | O_DIRECT | O_LARGEFILE | O_NOATIME); - else - fd = open(fname, O_RDONLY | O_LARGEFILE | O_NOATIME); - - if (fd < 0) - { - cerr << "Open failed" << endl; - perror("open"); - throw runtime_error("Error opening file"); - } - - uint64_t i = 1; - uint64_t rCnt = 0; - - clock_gettime(CLOCK_REALTIME, &fstarttm); - - while (i > 0) - { - clock_gettime(CLOCK_REALTIME, &ftm); - i = pread(fd, falignedbuff, freadSize, facc); - clock_gettime(CLOCK_REALTIME, &ftm2); - - idbassert(i == 0 || i == freadSize); - idbassert(i % fpageSize == 0); - idbassert(facc % fpageSize == 0); - - if (i < 0 && errno == EINTR) - { - timespec_sub(ftm, ftm2, ftm3); - cout << "* " << i << " " << right << setw(2) << setfill(' ') << ftm3.tv_sec << "." << right << setw(9) - << setfill('0') << ftm3.tv_nsec << endl; - continue; - } - else if (i < 0) - { - timespec_sub(ftm, ftm2, ftm3); - cout << "* i: " << i << " sz: " << freadSize << " acc: " << facc << right << setw(2) << setfill(' ') - << ftm3.tv_sec << " " << right << ftm3.tv_nsec << endl; - perror("pread"); - } - - facc += i; - - if (i > 0) - rCnt++; - - /** - timespec_sub(ftm, ftm2, ftm3); - cout - << rCnt << " " << facc/(1024*1024) - << right << setw(2) << setfill(' ') << ftm3.tv_sec << "." - << right << ftm3.tv_nsec << " i: " << i/(1024*1024) - << endl; - **/ - - } // while(acc... - - clock_gettime(CLOCK_REALTIME, &fendtm); - timespec_sub(fstarttm, fendtm, ftottm); - - cout << "Total reads: " << rCnt << " sz: " << facc / (1024 * 1024) << "MB" - << " tm: " << ftottm.tv_sec << "secs " << ftottm.tv_nsec << "ns" << endl; - - facc = 0; - close(fd); - } // operator() - - public: - uint64_t facc; - uint64_t freadBlocks; // read size ib blocks - uint64_t freadSize; // read size ib bytes - uint64_t freadBufferSz; - uint64_t fblockSize; - char* falignedbuff; - unsigned fpageSize; - BRM::OID_t foid; - char fname[256]; - struct timespec ftm; - struct timespec ftm2; - struct timespec ftm3; - struct timespec fstarttm; - struct timespec fendtm; - struct timespec ftottm; - bool fodirect; - int fd; - -}; // struct readThr - -// -void usage() -{ - cerr << "usage: mtread -o -s " << endl; -} - -// main() -// -int main(int argc, char** argv) -{ - int ch = 0; - int readBlocks = 0; - BRM::OID_t oid = 0; - std::vector oidList; - - enum CLA_ENUM - { - OID = (int)0, - READSIZE - }; - - // longopt struct - // struct option { - // const char *name; - // int has_arg; - // int *flag; - // int val; - //}; - - static struct option long_options[] = {//{const char *name, int has_arg, int *flag, int val}, - {"oid", required_argument, NULL, OID}, - {"rsize", required_argument, NULL, READSIZE}, - {0, 0, 0, 0}}; - - if (argc <= 1) - { - return -1; - } - - // process command line arguments - while ((ch = getopt_long_only(argc, argv, "o:s:", long_options, NULL)) != -1) - { - // pid_t pidId = getpid(); - switch (ch) - { - case OID: - case 'o': - oid = atoi(optarg); - oidList.push_back(oid); - cout << "oid: " << optarg << endl; - break; - - case READSIZE: - case 's': - readBlocks = atoi(optarg); - cout << "read size: " << optarg << endl; - - if (readBlocks <= 0) - readBlocks = 1; - - break; - - case '?': - default: - cout << "optarg " << optarg << endl; - usage(); - break; - - } // switch - - } // while... - - uint32_t idx = 0; - std::vector thrList; - - while (idx < oidList.size()) - { - struct readThr rdr(oidList[idx++], readBlocks); - boost::thread* thr = new boost::thread(rdr); - thrList.push_back(thr); - } - - idx = 0; - - while (idx < thrList.size()) - { - boost::thread* thr = thrList[idx++]; - thr->join(); - delete thr; - } - - thrList.clear(); - -} // main diff --git a/tools/clearShm/CMakeLists.txt b/tools/clearShm/CMakeLists.txt index b36acb03c..788b24c45 100644 --- a/tools/clearShm/CMakeLists.txt +++ b/tools/clearShm/CMakeLists.txt @@ -4,12 +4,6 @@ include_directories(${ENGINE_COMMON_INCLUDES}) set(clearShm_SRCS main.cpp) -add_executable(clearShm ${clearShm_SRCS}) +columnstore_executable(clearShm ${clearShm_SRCS}) -target_link_libraries(clearShm ${ENGINE_LDFLAGS} ${ENGINE_EXEC_LIBS}) - -install( - TARGETS clearShm - DESTINATION ${ENGINE_BINDIR} - COMPONENT columnstore-engine -) +columnstore_link(clearShm ${ENGINE_LDFLAGS} ${ENGINE_EXEC_LIBS}) diff --git a/tools/cleartablelock/CMakeLists.txt b/tools/cleartablelock/CMakeLists.txt index 02f93d42a..30dc54edd 100644 --- a/tools/cleartablelock/CMakeLists.txt +++ b/tools/cleartablelock/CMakeLists.txt @@ -4,12 +4,6 @@ include_directories(${ENGINE_COMMON_INCLUDES}) set(cleartablelock_SRCS cleartablelock.cpp cleartablelockthread.cpp) -add_executable(cleartablelock ${cleartablelock_SRCS}) +columnstore_executable(cleartablelock ${cleartablelock_SRCS}) -target_link_libraries(cleartablelock ${ENGINE_LDFLAGS} ${NETSNMP_LIBRARIES} ${ENGINE_WRITE_LIBS}) - -install( - TARGETS cleartablelock - DESTINATION ${ENGINE_BINDIR} - COMPONENT columnstore-engine -) +columnstore_link(cleartablelock ${ENGINE_LDFLAGS} ${NETSNMP_LIBRARIES} ${ENGINE_WRITE_LIBS}) diff --git a/tools/configMgt/CMakeLists.txt b/tools/configMgt/CMakeLists.txt index 9bf076855..9e6b3f3e9 100644 --- a/tools/configMgt/CMakeLists.txt +++ b/tools/configMgt/CMakeLists.txt @@ -5,7 +5,4 @@ include_directories(${ENGINE_COMMON_INCLUDES}) set(autoConfigure_SRCS autoConfigure.cpp) add_executable(autoConfigure ${autoConfigure_SRCS}) - -target_link_libraries(autoConfigure ${ENGINE_LDFLAGS} ${NETSNMP_LIBRARIES} ${ENGINE_EXEC_LIBS}) - -# install(TARGETS autoConfigure DESTINATION ${ENGINE_BINDIR} COMPONENT columnstore-engine) +columnstore_link(autoConfigure ${ENGINE_LDFLAGS} ${NETSNMP_LIBRARIES} ${ENGINE_EXEC_LIBS}) diff --git a/tools/cplogger/CMakeLists.txt b/tools/cplogger/CMakeLists.txt index 2d6c50233..bff674a17 100644 --- a/tools/cplogger/CMakeLists.txt +++ b/tools/cplogger/CMakeLists.txt @@ -4,12 +4,6 @@ include_directories(${ENGINE_COMMON_INCLUDES}) set(cplogger_SRCS main.cpp) -add_executable(cplogger ${cplogger_SRCS}) +columnstore_executable(cplogger ${cplogger_SRCS}) -target_link_libraries(cplogger ${ENGINE_LDFLAGS} ${ENGINE_EXEC_LIBS}) - -install( - TARGETS cplogger - DESTINATION ${ENGINE_BINDIR} - COMPONENT columnstore-engine -) +columnstore_link(cplogger ${ENGINE_LDFLAGS} ${ENGINE_EXEC_LIBS}) diff --git a/tools/dbbuilder/CMakeLists.txt b/tools/dbbuilder/CMakeLists.txt index 284240d98..9d687f912 100644 --- a/tools/dbbuilder/CMakeLists.txt +++ b/tools/dbbuilder/CMakeLists.txt @@ -4,12 +4,6 @@ include_directories(${ENGINE_COMMON_INCLUDES}) set(dbbuilder_SRCS dbbuilder.cpp systemcatalog.cpp) -add_executable(dbbuilder ${dbbuilder_SRCS}) +columnstore_executable(dbbuilder ${dbbuilder_SRCS}) -target_link_libraries(dbbuilder ${ENGINE_LDFLAGS} ${NETSNMP_LIBRARIES} ${ENGINE_WRITE_LIBS}) - -install( - TARGETS dbbuilder - DESTINATION ${ENGINE_BINDIR} - COMPONENT columnstore-engine -) +columnstore_link(dbbuilder ${ENGINE_LDFLAGS} ${NETSNMP_LIBRARIES} ${ENGINE_WRITE_LIBS}) diff --git a/tools/dbbuilder/dbbuilder.cpp b/tools/dbbuilder/dbbuilder.cpp index b5bb02efd..61df348a8 100644 --- a/tools/dbbuilder/dbbuilder.cpp +++ b/tools/dbbuilder/dbbuilder.cpp @@ -26,6 +26,7 @@ #include using namespace std; +#include "basic/string_utils.h" #include #include "mcsconfig.h" @@ -202,7 +203,11 @@ int main(int argc, char* argv[]) (iter->second).second = true; isUpgrade = true; } - messageHandler("", std::string("Upgrade flag is ") + std::to_string(isUpgrade) + std::string(" after checking upgrade candidate OID ") + oam.itoa(iter->first) + std::string(" "), false); + messageHandler("", + std::string("Upgrade flag is ") + std::to_string(isUpgrade) + + std::string(" after checking upgrade candidate OID ") + oam.itoa(iter->first) + + std::string(" "), + false); } if (!isUpgrade) diff --git a/tools/dbloadxml/CMakeLists.txt b/tools/dbloadxml/CMakeLists.txt index f94641a53..1e3b9d958 100644 --- a/tools/dbloadxml/CMakeLists.txt +++ b/tools/dbloadxml/CMakeLists.txt @@ -4,20 +4,14 @@ include_directories(${ENGINE_COMMON_INCLUDES}) set(dbload_STAT_SRCS inputmgr.cpp) -add_library(dbload STATIC ${dbload_STAT_SRCS}) +columnstore_static_library(dbload ${dbload_STAT_SRCS}) -add_dependencies(dbload loggingcpp) +columnstore_link(dbload loggingcpp) # ########## next target ############### set(colxml_SRCS colxml.cpp) -add_executable(colxml ${colxml_SRCS}) +columnstore_executable(colxml ${colxml_SRCS}) -target_link_libraries(colxml ${ENGINE_LDFLAGS} dbload ${ENGINE_WRITE_LIBS}) - -install( - TARGETS colxml - DESTINATION ${ENGINE_BINDIR} - COMPONENT columnstore-engine -) +columnstore_link(colxml ${ENGINE_LDFLAGS} dbload ${ENGINE_WRITE_LIBS}) diff --git a/tools/ddlcleanup/CMakeLists.txt b/tools/ddlcleanup/CMakeLists.txt index 6e67a6de1..9ba55097e 100644 --- a/tools/ddlcleanup/CMakeLists.txt +++ b/tools/ddlcleanup/CMakeLists.txt @@ -4,12 +4,5 @@ include_directories(${ENGINE_COMMON_INCLUDES}) set(ddlcleanup_SRCS ddlcleanup.cpp) -add_executable(ddlcleanup ${ddlcleanup_SRCS}) - -target_link_libraries(ddlcleanup ${ENGINE_LDFLAGS} ${NETSNMP_LIBRARIES} ${ENGINE_WRITE_LIBS} ddlcleanuputil) - -install( - TARGETS ddlcleanup - DESTINATION ${ENGINE_BINDIR} - COMPONENT columnstore-engine -) +columnstore_executable(ddlcleanup ${ddlcleanup_SRCS}) +columnstore_link(ddlcleanup ${ENGINE_LDFLAGS} ${NETSNMP_LIBRARIES} ${ENGINE_WRITE_LIBS} ddlcleanuputil) diff --git a/tools/ddldriver/CMakeLists.txt b/tools/ddldriver/CMakeLists.txt deleted file mode 100644 index b75b7ef45..000000000 --- a/tools/ddldriver/CMakeLists.txt +++ /dev/null @@ -1,43 +0,0 @@ -include_directories(${KDE4_INCLUDES} ${KDE4_INCLUDE_DIR} ${QT_INCLUDES}) - -# ########## next target ############### - -set(ddldriver_SRCS ddldriver.cpp) - -kde4_add_executable(ddldriver ${ddldriver_SRCS}) - -target_link_libraries(ddldriver ${KDE4_KDECORE_LIBS}) - -install(TARGETS ddldriver ${INSTALL_TARGETS_DEFAULT_ARGS}) - -# ########## install files ############### - -# original Makefile.am contents follow: - -# Copyright (C) 2014 InfiniDB, Inc. -# -# This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public -# License as published by the Free Software Foundation; version 2 of the License. -# -# This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied -# warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License along with this program; if not, write to the Free -# Software Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. -# -# $Id: Makefile.am 333 2009-04-03 20:35:04Z rdempsey $ Process this file with automake to produce Makefile.in -# -# AM_CPPFLAGS = $(idb_cppflags) AM_CFLAGS = $(idb_cflags) AM_CXXFLAGS = $(idb_cxxflags) AM_LDFLAGS = $(idb_ldflags) -# bin_PROGRAMS = ddldriver ddldriver_SOURCES = ddldriver.cpp ddldriver_CPPFLAGS = @idb_common_includes@ $(AM_CPPFLAGS) -# ddldriver_LDFLAGS = @idb_common_ldflags@ @idb_write_libs@ @idb_common_libs@ @netsnmp_libs@ $(AM_LDFLAGS) -# -# test: -# -# coverage: -# -# leakcheck: -# -# docs: -# -# bootstrap: install-data-am -# diff --git a/tools/ddldriver/ddldriver.cpp b/tools/ddldriver/ddldriver.cpp deleted file mode 100644 index 0bf1fa727..000000000 --- a/tools/ddldriver/ddldriver.cpp +++ /dev/null @@ -1,133 +0,0 @@ -/* Copyright (C) 2014 InfiniDB, Inc. - - This program is free software; you can redistribute it and/or - modify it under the terms of the GNU General Public License - as published by the Free Software Foundation; version 2 of - the License. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU General Public License for more details. - - You should have received a copy of the GNU General Public License - along with this program; if not, write to the Free Software - Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, - MA 02110-1301, USA. */ - -// $Id: ddldriver.cpp 2101 2013-01-21 14:12:52Z rdempsey $ - -#include -#include -#include -#include -#include -using namespace std; - -#include "ddlpkg.h" -#include "sqlparser.h" -using namespace ddlpackage; - -#include "bytestream.h" -#include "messagequeue.h" -using namespace messageqcpp; - -namespace -{ -void usage() -{ - cout << "usage: ddlriver [-h] schema sql_text" << endl; -} -const string toupper_(const string& in) -{ - string::const_iterator iter = in.begin(); - string::const_iterator end = in.end(); - ostringstream oss; - - while (iter != end) - { - oss << static_cast(toupper(*iter)); - ++iter; - } - - return oss.str(); -} -} // namespace - -int main(int argc, char** argv) -{ - int c; - - opterr = 0; - - while ((c = getopt(argc, argv, "h")) != EOF) - switch (c) - { - case 'h': - case '?': - default: - usage(); - return (c == 'h' ? 0 : 1); - break; - } - - if (argc - optind < 2) - { - usage(); - return 1; - } - - string owner(toupper_(argv[optind++])); - - SqlParser parser; - parser.setDefaultSchema(owner); - - string stmtStr(toupper_(argv[optind++])); - parser.Parse(stmtStr.c_str()); - - if (!parser.Good()) - { - cerr << "Failed to parse statement: " << stmtStr << endl; - return 1; - } - - const ParseTree& ptree = parser.GetParseTree(); - SqlStatement& stmt = *ptree.fList[0]; - - stmt.fSessionID = 1; - stmt.fSql = stmtStr; - stmt.fOwner = owner; - - ByteStream bytestream; - bytestream << stmt.fSessionID; - stmt.serialize(bytestream); - MessageQueueClient mq("DDLProc"); - ByteStream::byte b; - string errorMsg; - - try - { - mq.write(bytestream); - bytestream = mq.read(); - bytestream >> b; - bytestream >> errorMsg; - } - catch (runtime_error& rex) - { - cerr << "runtime_error in engine: " << rex.what() << endl; - return 1; - } - catch (...) - { - cerr << "uknown error in engine" << endl; - return 1; - } - - if (b != 0) - { - cerr << "DDLProc error: " << errorMsg << endl; - return 1; - } - - return 0; -} diff --git a/tools/dmldriver/CMakeLists.txt b/tools/dmldriver/CMakeLists.txt deleted file mode 100644 index a98dd62b1..000000000 --- a/tools/dmldriver/CMakeLists.txt +++ /dev/null @@ -1,44 +0,0 @@ -include_directories(${KDE4_INCLUDES} ${KDE4_INCLUDE_DIR} ${QT_INCLUDES}) - -# ########## next target ############### - -set(dmldriver_SRCS dmldriver.cpp tpchrf2.cpp dmlif.cpp) - -kde4_add_executable(dmldriver ${dmldriver_SRCS}) - -target_link_libraries(dmldriver ${KDE4_KDECORE_LIBS}) - -install(TARGETS dmldriver ${INSTALL_TARGETS_DEFAULT_ARGS}) - -# ########## install files ############### - -# original Makefile.am contents follow: - -# Copyright (C) 2014 InfiniDB, Inc. -# -# This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public -# License as published by the Free Software Foundation; version 2 of the License. -# -# This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied -# warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License along with this program; if not, write to the Free -# Software Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. -# -# $Id: Makefile.am 333 2009-04-03 20:35:04Z rdempsey $ Process this file with automake to produce Makefile.in -# -# AM_CPPFLAGS = $(idb_cppflags) AM_CFLAGS = $(idb_cflags) AM_CXXFLAGS = $(idb_cxxflags) AM_LDFLAGS = $(idb_ldflags) -# bin_PROGRAMS = dmldriver dmldriver_SOURCES = dmldriver.cpp tpchrf2.cpp dmlif.cpp dmldriver_CPPFLAGS = -# @idb_common_includes@ $(AM_CPPFLAGS) dmldriver_LDFLAGS = @idb_common_ldflags@ @idb_write_libs@ @idb_common_libs@ -# @netsnmp_libs@ $(AM_LDFLAGS) -# -# test: -# -# coverage: -# -# leakcheck: -# -# docs: -# -# bootstrap: install-data-am -# diff --git a/tools/dmldriver/dmldriver.cpp b/tools/dmldriver/dmldriver.cpp deleted file mode 100644 index 506a6c745..000000000 --- a/tools/dmldriver/dmldriver.cpp +++ /dev/null @@ -1,205 +0,0 @@ -/* Copyright (C) 2014 InfiniDB, Inc. - - This program is free software; you can redistribute it and/or - modify it under the terms of the GNU General Public License - as published by the Free Software Foundation; version 2 of - the License. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU General Public License for more details. - - You should have received a copy of the GNU General Public License - along with this program; if not, write to the Free Software - Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, - MA 02110-1301, USA. */ - -// $Id: dmldriver.cpp 2101 2013-01-21 14:12:52Z rdempsey $ - -//#define NDEBUG -#include -#include -#include -#include -#include -#include -#include -#include -using namespace std; - -#include -#include -using namespace boost; - -#include "tpchrf2.h" -#include "dmlif.h" - -namespace -{ -bool vflg; -bool dflg; - -void usage() -{ - cout << "usage: dmldriver [-vhd] [-c intvl] [-f file] [-s sid] [-t flgs] [-r file] [-p cnt] [sql_text]" - << endl; - cout << " -c intvl \tcommit every intvl statements" << endl; - cout << " -f file \tread statements from file (max 15KB/stmt)" << endl; - cout << " -s sid \tset sid as session id" << endl; - cout << " -t flgs \tset trace flags" << endl; - cout << " -v \tdisplay affected row count(s)" << endl; - cout << " -d \tdisplay debug info" << endl; - cout << " -r file \tread orderkeys from file for TPC-H RF2" << endl; - cout << " -p cnt \tpack cnt orderkeys into each delete stmt (only w/ -r)" << endl; - cout << " -e schema\tset the schema name (only w/ -r)" << endl; - cout << " -h \tdisplay this help text" << endl; -} -} // namespace - -int main(int argc, char** argv) -{ - int c; - - opterr = 0; - - vflg = false; - dflg = false; - - bool fflg = false; - string infilename; - - int cIntvl = numeric_limits::max(); - - uint32_t sessionID = time(0) & 0x7fffffff; - - bool rflg = false; - - int packCnt = 1; - - uint32_t tflg = 0; - - string schema; - - while ((c = getopt(argc, argv, "e:t:s:c:f:r:p:vhd")) != EOF) - switch (c) - { - case 'v': vflg = true; break; - - case 'd': dflg = true; break; - - case 'f': - fflg = true; - infilename = optarg; - break; - - case 'c': cIntvl = static_cast(strtol(optarg, 0, 0)); break; - - case 's': sessionID = static_cast(strtoul(optarg, 0, 0)); break; - - case 't': tflg = static_cast(strtoul(optarg, 0, 0)); break; - - case 'r': - rflg = true; - infilename = optarg; - break; - - case 'p': packCnt = static_cast(strtol(optarg, 0, 0)); break; - - case 'e': schema = optarg; break; - - case 'h': - case '?': - default: - usage(); - return (c == 'h' ? 0 : 1); - break; - } - - if (!fflg && !rflg && ((argc - optind) < 1)) - { - usage(); - return 1; - } - - if (fflg && rflg) - { - cout << "-f and -r are mutually exclusive!" << endl << endl; - usage(); - return 1; - } - - if (!schema.empty() && !rflg) - { - cout << "-e requires -r!" << endl << endl; - usage(); - return 1; - } - - string stmtStr; - - if (!fflg && !rflg) - stmtStr = argv[optind++]; - - int rc = 0; - - dmlif::DMLIF dmlif(sessionID, tflg, dflg, vflg); - - if (fflg) - { - ifstream ifs(infilename.c_str()); - - if (!ifs.good()) - { - cerr << "Error accessing file " << infilename << endl; - return 1; - } - - const streamsize ilinelen = 15 * 1024; - scoped_array iline(new char[ilinelen]); - int cnt = 0; - - for (;;) - { - ifs.getline(iline.get(), ilinelen); - - if (ifs.eof()) - break; - - rc = dmlif.sendOne(iline.get()); - - if (rc != 0) - break; - - cnt++; - - if ((cnt % cIntvl) == 0) - dmlif.sendOne("COMMIT;"); - } - } - else if (rflg) - { - ifstream ifs(infilename.c_str()); - - if (!ifs.good()) - { - cerr << "Error accessing file " << infilename << endl; - return 1; - } - - if (schema.empty()) - schema = "tpch"; - - tpch::RF2 rf2(schema, sessionID, tflg, cIntvl, packCnt, dflg, vflg); - rc = rf2.run(ifs); - } - else - { - rc = dmlif.sendOne(stmtStr); - } - - if (rc == 0) - dmlif.sendOne("COMMIT;"); - - return 0; -} diff --git a/tools/dmldriver/dmlif.cpp b/tools/dmldriver/dmlif.cpp deleted file mode 100644 index 2887f17d8..000000000 --- a/tools/dmldriver/dmlif.cpp +++ /dev/null @@ -1,384 +0,0 @@ -/* Copyright (C) 2014 InfiniDB, Inc. - - This program is free software; you can redistribute it and/or - modify it under the terms of the GNU General Public License - as published by the Free Software Foundation; version 2 of - the License. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU General Public License for more details. - - You should have received a copy of the GNU General Public License - along with this program; if not, write to the Free Software - Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, - MA 02110-1301, USA. */ - -// $Id: dmlif.cpp 2101 2013-01-21 14:12:52Z rdempsey $ - -//#define NDEBUG -#include -#include -#include -using namespace std; - -#include -using namespace boost; - -#include "vendordmlstatement.h" -#include "calpontdmlpackage.h" -#include "calpontdmlfactory.h" -using namespace dmlpackage; - -#include "bytestream.h" -#include "messagequeue.h" -using namespace messageqcpp; - -#include "simplecolumn.h" -#include "calpontselectexecutionplan.h" -#include "sessionmanager.h" -#include "simplefilter.h" -#include "constantcolumn.h" -#include "constantfilter.h" -using namespace execplan; - -#include "brmtypes.h" - -#include "dmlif.h" -using namespace dmlif; - -namespace dmlif -{ -DMLIF::DMLIF(uint32_t sessionid, uint32_t tflg, bool dflg, bool vflg) - : fSessionID(sessionid), fTflg(tflg), fDflg(dflg), fVflg(vflg), fOPt(0), fLPt(0) -{ - fMqp.reset(new MessageQueueClient("DMLProc")); -} - -DMLIF::~DMLIF() -{ -} - -int DMLIF::sendOne(const string& stmt) -{ - int rc; - - string tStmt(stmt); - - if (*tStmt.rbegin() != ';') - tStmt += ";"; - - VendorDMLStatement dmlStmt(tStmt, fSessionID); - CalpontDMLPackage* pDMLPackage = CalpontDMLFactory::makeCalpontDMLPackage(dmlStmt); - - if (pDMLPackage == 0) - { - cerr << "Failed to parse statement: " << tStmt << endl; - return -1; - } - - string queryString = pDMLPackage->get_QueryString(); - - if (fDflg) - cout << "qs: >" << queryString << '<' << endl; - - string dmlStatement = pDMLPackage->get_DMLStatement(); - - if (fDflg) - cout << "DML: " << dmlStatement << endl; - - bool isDML = true; - - if (dmlStatement == "COMMIT" || dmlStatement == "ROLLBACK") - { - isDML = false; - } - - if (isDML) - { - char_separator sep(" "); - tokenizer > tok(queryString, sep); - tokenizer >::iterator iter = tok.begin(); - idbassert(iter != tok.end()); - string where = *iter; - ++iter; - idbassert(iter != tok.end()); - string col1 = *iter; - ++iter; - idbassert(iter != tok.end()); - string op = *iter; - ++iter; - idbassert(iter != tok.end()); - string col2 = *iter; - ++iter; - idbassert(iter == tok.end()); - - if (fDflg) - cout << "SQL: " << pDMLPackage->get_SQLStatement() << endl; - - if (fDflg) - cout << "hf: " << pDMLPackage->HasFilter() << endl; - - DMLTable* tp = pDMLPackage->get_Table(); - - if (fDflg) - cout << "sn: " << tp->get_SchemaName() << " tn: " << tp->get_TableName() << endl; - - if (fDflg) - cout << "row count: " << tp->get_RowList().size() << endl; - - SRCP srcp(new SimpleColumn(tp->get_SchemaName(), tp->get_TableName(), col1, fSessionID)); - CalpontSelectExecutionPlan::ColumnMap cm; - cm.insert(make_pair(col1, srcp)); - pDMLPackage->get_ExecutionPlan()->columnMap(cm); - CalpontSelectExecutionPlan::ReturnedColumnList rcl; - rcl.push_back(srcp); - pDMLPackage->get_ExecutionPlan()->returnedCols(rcl); - pDMLPackage->get_ExecutionPlan()->sessionID(fSessionID); - pDMLPackage->get_ExecutionPlan()->traceFlags(fTflg); - SessionManager sm; - BRM::TxnID txnid = sm.getTxnID(fSessionID); - - if (!txnid.valid) - txnid = sm.newTxnID(fSessionID); - - pDMLPackage->get_ExecutionPlan()->txnID(txnid.id); - pDMLPackage->get_ExecutionPlan()->verID(sm.verID()); - ParseTree* pt = new ParseTree(); - ReturnedColumn* rc1 = srcp->clone(); - ReturnedColumn* rc2 = new ConstantColumn(col2, ConstantColumn::NUM); - SOP sop(new Operator(op)); - SimpleFilter* sf = new SimpleFilter(sop, rc1, rc2); - pt->data(sf); - pDMLPackage->get_ExecutionPlan()->filters(pt); - - if (fDflg) - cout << "ep: " << *pDMLPackage->get_ExecutionPlan() << endl; - } - - ByteStream bytestream; - pDMLPackage->write(bytestream); - delete pDMLPackage; - ByteStream::octbyte rows; - - rc = DMLSend(bytestream, rows); - - if (isDML && fVflg) - cout << rows << " rows affected" << endl; - - return rc; -} - -int DMLIF::DMLSend(ByteStream& bytestream, ByteStream::octbyte& rows) -{ - ByteStream::byte b; - string errorMsg; - - try - { - fMqp->connect(); - fMqp->write(bytestream); - bytestream = fMqp->read(); - fMqp->shutdown(); - - if (fDflg) - cout << "read " << bytestream.length() << " bytes from DMLProc" << endl; - - bytestream >> b; - - if (fDflg) - cout << "b = " << (int)b << endl; - - bytestream >> rows; - - if (fDflg) - cout << "rows = " << rows << endl; - - bytestream >> errorMsg; - - if (fDflg) - cout << "errorMsg = " << errorMsg << endl; - } - catch (runtime_error& rex) - { - cerr << "runtime_error in engine: " << rex.what() << endl; - return -1; - } - catch (...) - { - cerr << "uknown error in engine" << endl; - return -1; - } - - if (b != 0) - { - cerr << "DMLProc error: " << errorMsg << endl; - return -1; - } - - return 0; -} - -void DMLIF::rf2Start(const string& sn) -{ - fSchema = sn; - fOFilterStr = ""; - fLFilterStr = ""; - fOPt = 0; - fLPt = 0; -} - -void DMLIF::rf2Add(int64_t okey) -{ - ostringstream oss; - oss << okey; - string okeyStr(oss.str()); - - if (fOFilterStr.empty()) - { - fOFilterStr = "o_orderkey=" + okeyStr; - ReturnedColumn* rc1 = new SimpleColumn(fSchema, "orders", "o_orderkey", fSessionID); - ReturnedColumn* rc2 = new ConstantColumn(okeyStr, ConstantColumn::NUM); - SOP sop(new Operator("=")); - ConstantFilter* cf = new ConstantFilter(sop, rc1, rc2); - sop.reset(new Operator("or")); - cf->op(sop); - fOPt = new ParseTree(cf); - } - else - { - fOFilterStr += " or o_orderkey=" + okeyStr; - ReturnedColumn* rc1 = new SimpleColumn(fSchema, "orders", "o_orderkey", fSessionID); - ReturnedColumn* rc2 = new ConstantColumn(okeyStr, ConstantColumn::NUM); - SOP sop(new Operator("=")); - ConstantFilter* cf = dynamic_cast(fOPt->data()); - cf->pushFilter(new SimpleFilter(sop, rc1, rc2)); - } - - if (fLFilterStr.empty()) - { - fLFilterStr = "l_orderkey=" + okeyStr; - ReturnedColumn* rc1 = new SimpleColumn(fSchema, "lineitem", "l_orderkey", fSessionID); - ReturnedColumn* rc2 = new ConstantColumn(okeyStr, ConstantColumn::NUM); - SOP sop(new Operator("=")); - ConstantFilter* cf = new ConstantFilter(sop, rc1, rc2); - sop.reset(new Operator("or")); - cf->op(sop); - fLPt = new ParseTree(cf); - } - else - { - fLFilterStr += " or l_orderkey=" + okeyStr; - ReturnedColumn* rc1 = new SimpleColumn(fSchema, "lineitem", "l_orderkey", fSessionID); - ReturnedColumn* rc2 = new ConstantColumn(okeyStr, ConstantColumn::NUM); - SOP sop(new Operator("=")); - ConstantFilter* cf = dynamic_cast(fLPt->data()); - cf->pushFilter(new SimpleFilter(sop, rc1, rc2)); - } -} - -int DMLIF::rf2Send() -{ - if (fOFilterStr.empty()) - return -1; - - int rc = 0; - string dmlstr; - dmlstr = "delete from " + fSchema + ".orders where " + fOFilterStr + ';'; - - if (fDflg) - cout << dmlstr << endl; - - VendorDMLStatement dmlStmt(dmlstr, fSessionID); - CalpontDMLPackage* pDMLPackage = CalpontDMLFactory::makeCalpontDMLPackage(dmlStmt); - - if (pDMLPackage == 0) - { - cerr << "Failed to parse statement: " << dmlstr << endl; - return -1; - } - - SRCP srcp(new SimpleColumn(fSchema, "orders", "o_orderkey", fSessionID)); - CalpontSelectExecutionPlan::ColumnMap cm; - cm.insert(make_pair("o_orderkey", srcp)); - pDMLPackage->get_ExecutionPlan()->columnMap(cm); - CalpontSelectExecutionPlan::ReturnedColumnList rcl; - rcl.push_back(srcp); - pDMLPackage->get_ExecutionPlan()->returnedCols(rcl); - pDMLPackage->get_ExecutionPlan()->sessionID(fSessionID); - pDMLPackage->get_ExecutionPlan()->traceFlags(fTflg); - SessionManager sm; - BRM::TxnID txnid = sm.getTxnID(fSessionID); - - if (!txnid.valid) - txnid = sm.newTxnID(fSessionID); - - pDMLPackage->get_ExecutionPlan()->txnID(txnid.id); - pDMLPackage->get_ExecutionPlan()->verID(sm.verID()); - pDMLPackage->get_ExecutionPlan()->filters(fOPt); - - if (fDflg) - cout << "ep: " << *pDMLPackage->get_ExecutionPlan() << endl; - - ByteStream bytestream; - pDMLPackage->write(bytestream); - delete pDMLPackage; - pDMLPackage = 0; - ByteStream::octbyte rows = 0; - - rc = DMLSend(bytestream, rows); - - if (fVflg) - cout << rows << " rows affected" << endl; - - dmlstr = "delete from " + fSchema + ".lineitem where " + fLFilterStr + ';'; - - if (fDflg) - cout << dmlstr << endl; - - VendorDMLStatement dmlStmt1(dmlstr, fSessionID); - pDMLPackage = CalpontDMLFactory::makeCalpontDMLPackage(dmlStmt1); - - if (pDMLPackage == 0) - { - cerr << "Failed to parse statement: " << dmlstr << endl; - return -1; - } - - srcp.reset(new SimpleColumn(fSchema, "lineitem", "l_orderkey", fSessionID)); - cm.clear(); - cm.insert(make_pair("l_orderkey", srcp)); - pDMLPackage->get_ExecutionPlan()->columnMap(cm); - rcl.clear(); - rcl.push_back(srcp); - pDMLPackage->get_ExecutionPlan()->returnedCols(rcl); - pDMLPackage->get_ExecutionPlan()->sessionID(fSessionID); - pDMLPackage->get_ExecutionPlan()->traceFlags(fTflg); - txnid = sm.getTxnID(fSessionID); - - if (!txnid.valid) - txnid = sm.newTxnID(fSessionID); - - pDMLPackage->get_ExecutionPlan()->txnID(txnid.id); - pDMLPackage->get_ExecutionPlan()->verID(sm.verID()); - pDMLPackage->get_ExecutionPlan()->filters(fLPt); - - if (fDflg) - cout << "ep: " << *pDMLPackage->get_ExecutionPlan() << endl; - - bytestream.reset(); - pDMLPackage->write(bytestream); - delete pDMLPackage; - pDMLPackage = 0; - rows = 0; - - rc = DMLSend(bytestream, rows); - - if (fVflg) - cout << rows << " rows affected" << endl; - - return 0; -} - -} // namespace dmlif diff --git a/tools/dmldriver/dmlif.h b/tools/dmldriver/dmlif.h deleted file mode 100644 index abc1b8d82..000000000 --- a/tools/dmldriver/dmlif.h +++ /dev/null @@ -1,65 +0,0 @@ -/* Copyright (C) 2014 InfiniDB, Inc. - - This program is free software; you can redistribute it and/or - modify it under the terms of the GNU General Public License - as published by the Free Software Foundation; version 2 of - the License. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU General Public License for more details. - - You should have received a copy of the GNU General Public License - along with this program; if not, write to the Free Software - Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, - MA 02110-1301, USA. */ - -// $Id: dmlif.h 2101 2013-01-21 14:12:52Z rdempsey $ - -#pragma once - -#include -#include -#include - -#include "messagequeue.h" -#include "expressionparser.h" - -namespace dmlif -{ -class DMLIF -{ - public: - DMLIF(uint32_t sessionid, uint32_t tflg = 0, bool dflg = false, bool vflg = false); - ~DMLIF(); - - int sendOne(const std::string& stmt); - - void rf2Start(const std::string& sn); - void rf2Add(int64_t okey); - int rf2Send(); - - protected: - int DMLSend(messageqcpp::ByteStream& bytestream, messageqcpp::ByteStream::octbyte& rows); - - private: - // DMLIF(const DMLIF& rhs); - // DMLIF& operator=(const DMLIF& rhs); - - uint32_t fSessionID; - uint32_t fTflg; - bool fDflg; - bool fVflg; - - boost::scoped_ptr fMqp; - - std::string fSchema; - std::string fOFilterStr; - std::string fLFilterStr; - - execplan::ParseTree* fOPt; - execplan::ParseTree* fLPt; -}; - -} // namespace dmlif diff --git a/tools/dmldriver/tpchrf2.cpp b/tools/dmldriver/tpchrf2.cpp deleted file mode 100644 index 03ef3742d..000000000 --- a/tools/dmldriver/tpchrf2.cpp +++ /dev/null @@ -1,86 +0,0 @@ -/* Copyright (C) 2014 InfiniDB, Inc. - - This program is free software; you can redistribute it and/or - modify it under the terms of the GNU General Public License - as published by the Free Software Foundation; version 2 of - the License. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU General Public License for more details. - - You should have received a copy of the GNU General Public License - along with this program; if not, write to the Free Software - Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, - MA 02110-1301, USA. */ - -// $Id: tpchrf2.cpp 2101 2013-01-21 14:12:52Z rdempsey $ - -#include -using namespace std; - -#include -#include -using namespace boost; - -#include "tpchrf2.h" -#include "dmlif.h" - -namespace tpch -{ -RF2::RF2(const string& sn, uint32_t sid, uint32_t tflg, int c, int p, bool d, bool v) - : fSchema(sn), fSessionID(sid), fTflg(tflg), fIntvl(c), fPack(p), fDflg(d), fVflg(v) -{ -} - -RF2::~RF2() -{ -} - -int RF2::run(istream& in) -{ - const streamsize ilinelen = 1024; - scoped_array iline(new char[ilinelen]); - int cnt = 0; - dmlif::DMLIF dmlif(fSessionID, fTflg, fDflg, fVflg); - - for (;;) - { - dmlif.rf2Start(fSchema); - - for (int i = 0; i < fPack; i++) - { - in.getline(iline.get(), ilinelen); - - if (in.eof()) - break; - - typedef char_separator cs; - typedef tokenizer tk; - cs sep("|"); - tk tok(string(iline.get()), sep); - tk::iterator iter = tok.begin(); - idbassert(iter != tok.end()); - string keystr = *iter; - ++iter; - // idbassert(iter == tok.end()); - int64_t okey = strtol(keystr.c_str(), 0, 0); - dmlif.rf2Add(okey); - } - - dmlif.rf2Send(); - cnt++; - - if ((cnt % fIntvl) == 0) - dmlif.sendOne("COMMIT;"); - - if (in.eof()) - break; - } - - dmlif.sendOne("COMMIT;"); - return 0; -} - -} // namespace tpch diff --git a/tools/dmldriver/tpchrf2.h b/tools/dmldriver/tpchrf2.h deleted file mode 100644 index 4124eaf2a..000000000 --- a/tools/dmldriver/tpchrf2.h +++ /dev/null @@ -1,52 +0,0 @@ -/* Copyright (C) 2014 InfiniDB, Inc. - - This program is free software; you can redistribute it and/or - modify it under the terms of the GNU General Public License - as published by the Free Software Foundation; version 2 of - the License. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU General Public License for more details. - - You should have received a copy of the GNU General Public License - along with this program; if not, write to the Free Software - Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, - MA 02110-1301, USA. */ - -// $Id: tpchrf2.h 2101 2013-01-21 14:12:52Z rdempsey $ - -#pragma once - -#include -#include -#include -#include -#include - -namespace tpch -{ -class RF2 -{ - public: - RF2(const std::string& sn, uint32_t sid, uint32_t tflg = 0, int c = std::numeric_limits::max(), - int p = 1, bool d = false, bool v = false); - ~RF2(); - - int run(std::istream& in); - - private: - // RF2(const RF2& rhs); - // RF2& operator=(const RF2& rhs); - - std::string fSchema; - uint32_t fSessionID; - uint32_t fTflg; - int fIntvl; - int fPack; - bool fDflg; - bool fVflg; -}; - -} // namespace tpch diff --git a/tools/editem/CMakeLists.txt b/tools/editem/CMakeLists.txt index 8e49c91e1..b0dc993c6 100644 --- a/tools/editem/CMakeLists.txt +++ b/tools/editem/CMakeLists.txt @@ -4,12 +4,5 @@ include_directories(${ENGINE_COMMON_INCLUDES}) set(editem_SRCS editem.cpp) -add_executable(editem ${editem_SRCS}) - -target_link_libraries(editem ${ENGINE_LDFLAGS} ${NETSNMP_LIBRARIES} ${ENGINE_EXEC_LIBS}) - -install( - TARGETS editem - DESTINATION ${ENGINE_BINDIR} - COMPONENT columnstore-engine -) +columnstore_executable(editem ${editem_SRCS}) +columnstore_link(editem ${ENGINE_LDFLAGS} ${NETSNMP_LIBRARIES} ${ENGINE_EXEC_LIBS}) diff --git a/tools/fixdate/fixdate.cpp b/tools/fixdate/fixdate.cpp deleted file mode 100644 index 73b1fa643..000000000 --- a/tools/fixdate/fixdate.cpp +++ /dev/null @@ -1,173 +0,0 @@ -/* Copyright (C) 2014 InfiniDB, Inc. - - This program is free software; you can redistribute it and/or - modify it under the terms of the GNU General Public License - as published by the Free Software Foundation; version 2 of - the License. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU General Public License for more details. - - You should have received a copy of the GNU General Public License - along with this program; if not, write to the Free Software - Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, - MA 02110-1301, USA. */ - -/* - * $Id: fixdate.cpp 2101 2013-01-21 14:12:52Z rdempsey $ - * - */ - -#include -#include -#include -#include -#include -#include -using namespace std; - -#include - -#include -#include -using namespace boost; - -#include "bytestream.h" -using namespace messageqcpp; - -#include "dmlpackageprocessor.h" -using namespace dmlpackageprocessor; - -namespace -{ -const streamsize blkSz = 8192; - -void usage() -{ -} - -u_int64_t fixDate(ByteStream& bs, ostream& out) -{ - ByteStream fixed; - ByteStream::quadbyte o; - u_int64_t cnt = 0; -#if 0 - DMLPackageProcessor::Date minDate; - DMLPackageProcessor::Date maxDate; - minDate.year = 1992; - minDate.month = 1; - minDate.day = 2; - maxDate.year = 1998; - maxDate.month = 12; - maxDate.day = 25; - ByteStream::quadbyte mxd; - ByteStream::quadbyte mnd; - mxd = *(reinterpret_cast(&maxDate)); - mnd = *(reinterpret_cast(&minDate)); -#endif - DMLPackageProcessor::Date fixDate; - fixDate.spare = 0; - ByteStream::quadbyte f; - - while (bs.length() > 0) - { - bs >> o; - - if (o >= 0xfffffffe) - { - fixed << o; - continue; - } - - f = o & 0xffff; - fixDate.year = f; - o >>= 16; - f = o & 0xf; - fixDate.month = f; - o >>= 4; - f = o & 0x3f; - fixDate.day = f; - // o >>= 6; - o = *(reinterpret_cast(&fixDate)); - fixed << o; -#if 0 - cout << DMLPackageProcessor::dateToString(o) << endl; - idbassert(o >= mnd && o <= mxd); - cnt++; -#endif - } - - out << fixed; - return cnt; -} - -} // namespace - -int main(int argc, char* argv[]) -{ - int c; - - opterr = 0; - - while ((c = getopt(argc, argv, "h")) != EOF) - switch (c) - { - case 'h': - usage(); - return 0; - break; - - default: - usage(); - return 1; - break; - } - - if ((argc - optind) < 1) - { - usage(); - return 1; - } - - ByteStream bs; - - ifstream ifs(argv[optind + 0]); - ByteStream::byte inbuf[blkSz]; - streampos fLen; - u_int64_t blkNo = 0; - - ifs.seekg(0, ios_base::end); - fLen = ifs.tellg(); - ifs.seekg(0, ios_base::beg); - - idbassert((fLen % blkSz) == 0); - u_int64_t numBlks = fLen / blkSz; - cout << numBlks << " blocks to fix..." << endl; - - ofstream ofs("fixdate.cdf"); - - cout << "pct done: " << setw(3); - - for (;;) - { - ifs.read(reinterpret_cast(inbuf), blkSz); - - if (ifs.eof()) - break; - - bs.load(inbuf, blkSz); - - fixDate(bs, ofs); - cout << "\b\b\b" << setw(3) << (u_int64_t)(blkNo * 100 / numBlks); - // cout << setw(3) << (u_int64_t)(blkNo * 100 / numBlks) << endl; - - blkNo++; - } - - cout << "\b\b\b" << setw(3) << 100 << endl; - // cout << setw(3) << 100 << endl; - - return 0; -} diff --git a/tools/getConfig/CMakeLists.txt b/tools/getConfig/CMakeLists.txt index 182c9455f..05fa9c37e 100644 --- a/tools/getConfig/CMakeLists.txt +++ b/tools/getConfig/CMakeLists.txt @@ -4,12 +4,6 @@ include_directories(${ENGINE_COMMON_INCLUDES}) set(getConfig_SRCS main.cpp) -add_executable(mcsGetConfig ${getConfig_SRCS}) +columnstore_executable(mcsGetConfig ${getConfig_SRCS}) -target_link_libraries(mcsGetConfig ${ENGINE_LDFLAGS} ${ENGINE_EXEC_LIBS}) - -install( - TARGETS mcsGetConfig - DESTINATION ${ENGINE_BINDIR} - COMPONENT columnstore-engine -) +columnstore_link(mcsGetConfig ${ENGINE_LDFLAGS} ${ENGINE_EXEC_LIBS}) diff --git a/tools/idb_comp/main.cpp b/tools/idb_comp/main.cpp deleted file mode 100644 index 474591f4e..000000000 --- a/tools/idb_comp/main.cpp +++ /dev/null @@ -1,265 +0,0 @@ -#include -#include -#include -#include -#include -//#define NDEBUG -#include -#include - -#include -#include - -#include -#include -#include -#include -#include -using namespace std; - -#include - -namespace -{ -size_t exp_buf_size; -unsigned vflg; - -const string lzo_rctos(int r) -{ - switch (r) - { - case LZO_E_INPUT_NOT_CONSUMED: return "LZO_E_INPUT_NOT_CONSUMED"; - - default: break; - } - - return "Unknown Error!"; -} - -void usage() -{ - cout << "usage: idb_comp [-e size][-h] file(s)" << endl - << "\t-e size size (in KB) of expansion buffer" << endl - << "\t-h display this help" << endl; -} - -int doit(const string& infile) -{ - int rc = 0; - int ifd = open(infile.c_str(), O_RDONLY | O_LARGEFILE | O_DIRECT); - - if (ifd < 0) - { - cerr << infile << ": open: " << strerror(errno) << endl; - return 1; - } - - struct stat istatbuf; - - fstat(ifd, &istatbuf); - - string outname(infile); - - string::size_type ptr; - - ptr = outname.find_last_of('.'); - - if (ptr != string::npos) - outname.erase(ptr); - - ptr = outname.find_last_of('/'); - - if (ptr != string::npos) - outname.erase(0, ptr + 1); - - outname = "./" + outname + ".cmp"; - - int ofd = open(outname.c_str(), O_WRONLY | O_CREAT | O_TRUNC | O_LARGEFILE | O_DIRECT, 0644); - - if (ofd < 0) - { - cerr << outname << ": open: " << strerror(errno) << endl; - close(ifd); - return 1; - } - - lzo_init(); - - ssize_t nread = -1; - ssize_t nwritten = -1; - lzo_bytep ibuf; - lzo_bytep tibuf; - lzo_bytep cbuf; - lzo_bytep tcbuf; - lzo_bytep wkmem; - lzo_uint32 ibuf_len = 0; - lzo_uint cbuf_len = 0; - - ibuf_len = 512 * 1024 * 8; - tibuf = new lzo_byte[ibuf_len + 4095]; - - if ((ptrdiff_t)tibuf & 0xfffULL) - ibuf = (lzo_bytep)((ptrdiff_t)tibuf & 0xfffffffffffff000ULL) + 4096; - else - ibuf = tibuf; - - cbuf_len = 512 * 1024 * 8; - tcbuf = new lzo_byte[cbuf_len + 4095 + exp_buf_size * 1024]; - - if ((ptrdiff_t)tcbuf & 0xfff) - cbuf = (lzo_bytep)((ptrdiff_t)tcbuf & 0xfffffffffffff000ULL) + 4096; - else - cbuf = tcbuf; - - wkmem = new lzo_byte[LZO1X_1_15_MEM_COMPRESS]; - - int r = LZO_E_OK; - - const int TOTAL_HDR_LEN = 4096 * 2; - char* hdrbuf = new char[TOTAL_HDR_LEN + 4095]; - memset(hdrbuf, 0, TOTAL_HDR_LEN + 4095); - char* hdrbufp = 0; - - if ((ptrdiff_t)hdrbuf & 0xfff) - hdrbufp = (char*)((ptrdiff_t)hdrbuf & 0xfffffffffffff000ULL) + 4096; - else - hdrbufp = hdrbuf; - - struct compHdr - { - uint64_t ptrs[512]; - }; - - idbassert(sizeof(compHdr) <= 4096); - - compHdr* hdrptr1 = (compHdr*)hdrbufp; - compHdr* hdrptr = hdrptr1 + 1; // advance to 2nd hdr to store compression ptrs - lseek(ofd, TOTAL_HDR_LEN, SEEK_SET); - - nread = read(ifd, ibuf, ibuf_len); - - int idx = 0; - off_t cmpoff = TOTAL_HDR_LEN; - - while (nread > 0) - { - cbuf_len = 512 * 1024 * 8; - memset(cbuf, 0, cbuf_len); - r = lzo1x_1_15_compress(ibuf, nread, cbuf, &cbuf_len, wkmem); - - if (r != LZO_E_OK) - { - cerr << "compression failed!: " << r << endl; - rc = 1; - goto out; - } - - if (cbuf_len > (unsigned)nread) - { - cerr << "WARNING: expansion detected! (output grew by " << (cbuf_len - nread) << " bytes)" << endl; - idbassert((cbuf_len - nread) <= exp_buf_size * 1024); - } - - if (cbuf_len & 0xfff) - cbuf_len = (cbuf_len & 0xfffffffffffff000ULL) + 4096; - - // cbuf_len = 512 * 1024 * 8; - nwritten = write(ofd, cbuf, cbuf_len); - - if (nwritten < 0 || (unsigned)nwritten != cbuf_len) - { - cerr << outname << ": write: " << strerror(errno) << " (" << nwritten << ')' << endl; - rc = 1; - goto out; - } - - if (vflg > 0) - { - lzo_bytep tbuf; - lzo_uint tbuflen = 4 * 1024 * 1024 + 4; - boost::scoped_array tbuf_sa(new lzo_byte[tbuflen]); - tbuf = tbuf_sa.get(); - cout << "idx: " << idx << " off: " << cmpoff << " size: " << cbuf_len; - r = lzo1x_decompress(cbuf, cbuf_len, tbuf, &tbuflen, 0); - cout << " r: " << lzo_rctos(r) << " size: " << tbuflen << endl; - } - - hdrptr->ptrs[idx] = cmpoff; - idx++; - cmpoff += cbuf_len; - - nread = read(ifd, ibuf, ibuf_len); - } - - if (nread < 0) - { - cerr << infile << ": read: " << strerror(errno) << endl; - rc = 1; - goto out; - } - - hdrptr->ptrs[idx] = cmpoff; - idbassert(idx <= 64); - - // Fill in meta-data information in first header - hdrptr1->ptrs[0] = 0xfdc119a384d0778eULL; - hdrptr1->ptrs[1] = 1; - hdrptr1->ptrs[2] = 1; - - nwritten = pwrite(ofd, hdrbufp, TOTAL_HDR_LEN, 0); - idbassert(nwritten == TOTAL_HDR_LEN); - -out: - delete[] wkmem; - delete[] tcbuf; - delete[] tibuf; - fsync(ofd); - struct stat ostatbuf; - fstat(ofd, &ostatbuf); - idbassert(ostatbuf.st_size == (signed)hdrptr->ptrs[idx]); - delete[] hdrbuf; - cout << infile << ": Input Size: " << istatbuf.st_size << " Output size: " << ostatbuf.st_size - << " Compression: " << (100LL - (ostatbuf.st_size * 100LL / istatbuf.st_size)) << '%' << endl; - close(ofd); - close(ifd); - - return rc; -} - -} // namespace - -int main(int argc, char** argv) -{ - opterr = 0; - int c; - exp_buf_size = 128; - vflg = 0; - - while ((c = getopt(argc, argv, "e:vh")) != -1) - switch (c) - { - case 'e': exp_buf_size = atoi(optarg); break; - - case 'v': vflg++; break; - - case 'h': - default: - usage(); - return (c == 'h' ? 0 : 1); - break; - } - - if ((argc - optind) < 1) - { - usage(); - return 1; - } - - int rc = 0; - - for (int i = optind; i < argc; i++) - if (doit(argv[i])) - rc = 1; - - return rc; -} diff --git a/tools/idbmeminfo/CMakeLists.txt b/tools/idbmeminfo/CMakeLists.txt index c49eb8fa2..90081a17d 100644 --- a/tools/idbmeminfo/CMakeLists.txt +++ b/tools/idbmeminfo/CMakeLists.txt @@ -4,12 +4,5 @@ include_directories(${ENGINE_COMMON_INCLUDES}) set(idbmeminfo_SRCS idbmeminfo.cpp) -add_executable(idbmeminfo ${idbmeminfo_SRCS}) - -target_link_libraries(idbmeminfo ${ENGINE_LDFLAGS}) - -install( - TARGETS idbmeminfo - DESTINATION ${ENGINE_BINDIR} - COMPONENT columnstore-engine -) +columnstore_executable(idbmeminfo ${idbmeminfo_SRCS}) +columnstore_link(idbmeminfo ${ENGINE_LDFLAGS}) diff --git a/tools/passwd/CMakeLists.txt b/tools/passwd/CMakeLists.txt index 94f7eec4b..0af71771a 100644 --- a/tools/passwd/CMakeLists.txt +++ b/tools/passwd/CMakeLists.txt @@ -4,21 +4,10 @@ include_directories(${ENGINE_COMMON_INCLUDES} ${ENGINE_UTILS_COMMON_INCLUDE}) set(cspasswd_SRCS cspasswd.cpp secrets.cpp) set(cskeys_SRCS cskeys.cpp secrets.cpp) -add_executable(cspasswd ${cspasswd_SRCS}) -add_executable(cskeys ${cskeys_SRCS}) +columnstore_executable(cspasswd ${cspasswd_SRCS}) +columnstore_executable(cskeys ${cskeys_SRCS}) target_include_directories(cspasswd BEFORE PUBLIC ${OPENSSL_INCLUDE_DIR}) target_include_directories(cskeys BEFORE PUBLIC ${OPENSSL_INCLUDE_DIR}) -target_link_libraries(cspasswd ${ENGINE_LDFLAGS} ${ENGINE_EXEC_LIBS} ${SSL_LIBRARIES}) -target_link_libraries(cskeys ${ENGINE_LDFLAGS} ${ENGINE_EXEC_LIBS} ${SSL_LIBRARIES}) - -install( - TARGETS cspasswd - DESTINATION ${ENGINE_BINDIR} - COMPONENT columnstore-engine -) -install( - TARGETS cskeys - DESTINATION ${ENGINE_BINDIR} - COMPONENT columnstore-engine -) +columnstore_link(cspasswd ${ENGINE_LDFLAGS} ${ENGINE_EXEC_LIBS} ${SSL_LIBRARIES}) +columnstore_link(cskeys ${ENGINE_LDFLAGS} ${ENGINE_EXEC_LIBS} ${SSL_LIBRARIES}) diff --git a/tools/pingproc/pingproc.cpp b/tools/pingproc/pingproc.cpp deleted file mode 100644 index 618420f1f..000000000 --- a/tools/pingproc/pingproc.cpp +++ /dev/null @@ -1,2357 +0,0 @@ -/* Copyright (C) 2014 InfiniDB, Inc. - - This program is free software; you can redistribute it and/or - modify it under the terms of the GNU General Public License - as published by the Free Software Foundation; version 2 of - the License. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU General Public License for more details. - - You should have received a copy of the GNU General Public License - along with this program; if not, write to the Free Software - Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, - MA 02110-1301, USA. */ - -// $Id: pingproc.cpp 2101 2013-01-21 14:12:52Z rdempsey $ -#include -#include -#include -#include -#include -#include -#include -using namespace std; - -#include -using namespace boost; - -#include "bytestream.h" -using namespace messageqcpp; - -#include "distributedenginecomm.h" -#include "primitivemsg.h" -#include "jobstep.h" -#include "batchprimitiveprocessor-jl.h" -using namespace joblist; - -#include "calpontsystemcatalog.h" -using namespace execplan; - -#include "brm.h" -using namespace BRM; - -// Global vars -bool debug; -bool thdFcnFailure; - -// -// TODO: Why is this namespace here? - -namespace -{ -void timespec_sub(const struct timespec& tv1, const struct timespec& tv2, struct timespec& diff) -{ - if (tv2.tv_nsec < tv1.tv_nsec) - { - diff.tv_sec = tv2.tv_sec - tv1.tv_sec - 1; - diff.tv_nsec = tv2.tv_nsec - tv1.tv_nsec + 1000000000; - } - else - { - diff.tv_sec = tv2.tv_sec - tv1.tv_sec; - diff.tv_nsec = tv2.tv_nsec - tv1.tv_nsec; - } -} // timespec_sub - -// -// -class OidOperation -{ - public: - enum OpType_t - { - SCAN = 0, - BLOCK = 1, - LOOPBACK = 2, - NONE = 3, - BATCHSCAN = 4, - BATCHSTEP = 5, - BATCHFILT = 6 - }; - - OidOperation(const OID_t oid, const OpType_t opType, const uint32_t sessionId = 0); - ~OidOperation(){}; - - void addFilter(const int8_t COP, const int64_t value); - - const OID_t OID() const - { - return fOid; - } - const OpType_t OpType() const - { - return fOpType; - } - - const CalpontSystemCatalog::ColType& ColumnType() - { - return fColType; - } - const uint32_t ColumnWidth() const - { - return fColType.colWidth; - } - - const uint32_t FilterCount() const - { - return fFilterCount; - } - const ByteStream& FilterString() - { - return fFilterList; - } - const uint32_t SessionId() const - { - return fSessionId; - } - const uint32_t DataType() const - { - return fColType.colDataType; - } - const uint32_t BOP() const - { - return fBOP; - } - void BOP(const uint32_t bop) - { - if (FilterCount() >= 2) - fBOP = bop; - } - const uint32_t COP1() const - { - return fCOP1; - } - void COP1(const uint32_t cop) - { - fCOP1 = cop; - } - const uint32_t COP2() const - { - return fCOP2; - } - void COP2(const uint32_t cop) - { - fCOP2 = cop; - } - bool isIntegralDataType(); - void setLbidTraceOn(); - void setPMProfileOn(); - bool LbidTrace() - { - return fLbidTrace; - } - bool PMProfile() - { - return fPMProfile; - } - - void deSerializeFilter(int8_t& COP, int64_t& value); - - // private: - OidOperation(){}; - OID_t fOid; - - ByteStream fFilterList; - uint32_t fFilterCount; - OpType_t fOpType; - CalpontSystemCatalog::ColType fColType; - uint32_t fSessionId; - uint32_t fBOP; - uint32_t fCOP1; - uint32_t fCOP2; - bool fLbidTrace; - bool fPMProfile; -}; // class OidOperation - -// -// -OidOperation::OidOperation(const OID_t oid, const OpType_t opType, const uint32_t sessionId) - : fOid(oid) - , fFilterList() - , fFilterCount(0) - , fOpType(opType) - , fSessionId(sessionId) - , fBOP(BOP_NONE) - , fCOP1(COMPARE_NIL) - , fCOP2(COMPARE_NIL) - , fLbidTrace(false) - , fPMProfile(false) -{ - boost::shared_ptr cat = - execplan::CalpontSystemCatalog::makeCalpontSystemCatalog(getpid()); - fColType = cat->colType(oid); - fFilterList.reset(); -} - -void OidOperation::setLbidTraceOn() -{ - fLbidTrace = true; -} - -void OidOperation::setPMProfileOn() -{ - fPMProfile = true; -} - -void OidOperation::addFilter(const int8_t COP, const int64_t value) -{ - if (fFilterCount == 2) - return; - - fFilterList << (uint8_t)COP; - - // converts to a type of the appropriate width, then bitwise - // copies into the filter ByteStream - switch (ColumnWidth()) - { - case 1: - int8_t tmp8; - tmp8 = value; - fFilterList << *((uint8_t*)&tmp8); - break; - - case 2: - int16_t tmp16; - tmp16 = value; - fFilterList << *((uint16_t*)&tmp16); - break; - - case 4: - int32_t tmp32; - tmp32 = value; - fFilterList << *((uint32_t*)&tmp32); - break; - - case 8: fFilterList << *((uint64_t*)&value); break; - - default: - ostringstream o; - - o << "addFilter: colType says OID " - << " has a width of " << ColumnWidth(); - throw runtime_error(o.str()); - } - - fFilterCount++; -} - -void OidOperation::deSerializeFilter(int8_t& COP, int64_t& value) -{ - if (fFilterCount == 0) - { - COP = COMPARE_NIL; - return; - } - - fFilterList >> *(uint8_t*)&COP; - - switch (ColumnWidth()) - { - case 1: - int8_t tmp8; - tmp8 = value; - fFilterList >> *((uint8_t*)&tmp8); - value = tmp8; - break; - - case 2: - int16_t tmp16; - fFilterList >> *((uint16_t*)&tmp16); - value = tmp16; - break; - - case 4: - int32_t tmp32; - fFilterList >> *((uint32_t*)&tmp32); - value = tmp32; - break; - - case 8: fFilterList >> *((uint64_t*)&value); break; - - default: - ostringstream o; - - o << "deSerializeFilter: colType says OID " - << " has a width of " << ColumnWidth(); - throw runtime_error(o.str()); - } - - fFilterCount--; -} - -typedef vector OperationList; - -// Only process these column types -// -bool OidOperation::isIntegralDataType() -{ - if (DataType() == CalpontSystemCatalog::BIT || DataType() == CalpontSystemCatalog::TINYINT || - DataType() == CalpontSystemCatalog::SMALLINT || DataType() == CalpontSystemCatalog::MEDINT || - DataType() == CalpontSystemCatalog::INT || DataType() == CalpontSystemCatalog::DATE || - DataType() == CalpontSystemCatalog::BIGINT || DataType() == CalpontSystemCatalog::DATETIME || - DataType() == CalpontSystemCatalog::TIMESTAMP || DataType() == CalpontSystemCatalog::TIME || - DataType() == CalpontSystemCatalog::UTINYINT || DataType() == CalpontSystemCatalog::USMALLINT || - DataType() == CalpontSystemCatalog::UMEDINT || DataType() == CalpontSystemCatalog::UINT || - DataType() == CalpontSystemCatalog::UBIGINT) - return true; - - if (DataType() == CalpontSystemCatalog::CHAR && 1 == fColType.colWidth) - return true; - - return false; -} - -const ByteStream formatLoopBackMsg(const uint32_t sessionId, uint32_t uniqueId) -{ - ByteStream primMsg; - ISMPacketHeader ism; - memset(&ism, 0, sizeof(ism)); - ism.Command = COL_LOOPBACK; - ism.Size = sizeof(ism) + sizeof(ColLoopback); - ism.Type = 2; - primMsg.load((const uint8_t*)&ism, sizeof(ism)); - struct ColLoopback lb; - - memset(&lb, 0, sizeof(lb)); - - lb.Hdr.SessionID = sessionId; - lb.Hdr.StatementID = 0; - lb.Hdr.TransactionID = sessionId; - lb.Hdr.VerID = 0; - lb.Hdr.StepID = sessionId; - lb.Hdr.UniqueID = uniqueId; - primMsg.append((const uint8_t*)&lb, sizeof(lb)); - - return primMsg; -} // formatLoopBackMsg - -const ByteStream formatDictionaryMsg(const uint64_t lbid, ByteStream& ridList, const uint16_t ridCount, - OidOperation& oidOp) -{ - ByteStream primMsg; - DictSignatureRequestHeader hdr; - ISMPacketHeader ism; - - ism.Flags = 0; // planFlagsToPrimFlags(fTraceFlags); - ism.Command = DICT_SIGNATURE; - ism.Size = sizeof(DictSignatureRequestHeader) + ridList.length(); - ism.Type = 2; - - hdr.Hdr.SessionID = oidOp.SessionId(); - hdr.Hdr.StatementID = 0; - hdr.Hdr.TransactionID = oidOp.SessionId(); - hdr.Hdr.VerID = 0; - hdr.Hdr.StepID = 0; - - hdr.LBID = lbid; - hdr.PBID = 0; - idbassert(ridCount <= 8000); - hdr.NVALS = ridCount; - - primMsg.load((const uint8_t*)&ism, sizeof(ism)); - primMsg.append((const uint8_t*)&hdr, sizeof(DictSignatureRequestHeader)); - primMsg += ridList; - - return primMsg; -} - -const ByteStream formatColStepMsg(const uint64_t lbid, ByteStream& ridList, const uint16_t ridCount, - OidOperation& oidOp, uint32_t uniqueId) -{ - ByteStream primMsg; - NewColRequestHeader hdr; - - memset(&hdr, 0, sizeof(hdr)); - - hdr.ism.Reserve = 0; - hdr.ism.Flags = 0; - - if (oidOp.LbidTrace() == true) - hdr.ism.Flags |= PF_LBID_TRACE; - - if (oidOp.PMProfile() == true) - hdr.ism.Flags |= PF_PM_PROF; - - hdr.ism.Command = COL_BY_SCAN; - hdr.ism.Size = sizeof(NewColRequestHeader) + oidOp.FilterString().length() + ridList.length(); - hdr.ism.Type = 2; - - hdr.hdr.SessionID = oidOp.SessionId(); - hdr.hdr.StatementID = 0; - hdr.hdr.TransactionID = oidOp.SessionId(); - hdr.hdr.VerID = 0; - hdr.hdr.StepID = oidOp.SessionId(); - hdr.hdr.UniqueID = uniqueId; - - hdr.LBID = lbid; - idbassert(hdr.LBID > 0); - hdr.PBID = 0; - hdr.DataSize = oidOp.ColumnWidth(); - hdr.DataType = oidOp.DataType(); - hdr.OutputType = OT_BOTH; - hdr.BOP = BOP_NONE; - // hdr.InputFlags = 0; - hdr.NOPS = oidOp.FilterCount(); - hdr.NVALS = ridCount; - hdr.sort = 0; - - primMsg.load((const uint8_t*)&hdr, sizeof(NewColRequestHeader)); - - if (oidOp.FilterCount() > 0) - primMsg += oidOp.FilterString(); - - if (ridCount > 0) - primMsg += ridList; - - return primMsg; - -} // formatColStepMsg - -const ByteStream formatDictionaryScanMsg(const uint64_t lbid, const uint16_t count, OidOperation& oidOp) -{ - ByteStream primMsg; - DictTokenByScanRequestHeader hdr; - - hdr.ism.Reserve = 0; - hdr.ism.Flags = 0; - hdr.ism.Command = DICT_TOKEN_BY_SCAN_COMPARE; - hdr.ism.Size = sizeof(DictTokenByScanRequestHeader) + oidOp.FilterString().length(); - hdr.ism.Type = 2; - - hdr.Hdr.SessionID = oidOp.SessionId(); - hdr.Hdr.StatementID = 0; - hdr.Hdr.TransactionID = 0; - hdr.Hdr.VerID = 0; - hdr.Hdr.StepID = oidOp.SessionId(); - - hdr.LBID = lbid; - idbassert(hdr.LBID >= 0); - hdr.PBID = 0; - hdr.OutputType = OT_TOKEN; - hdr.BOP = oidOp.BOP(); - hdr.COP1 = oidOp.COP1(); - hdr.COP2 = oidOp.COP2(); - hdr.NVALS = oidOp.FilterCount(); - hdr.Count = count; - hdr.charsetNumber = oidOp.ColumnType().charsetNumber; - idbassert(hdr.Count > 0); - - primMsg.load((const uint8_t*)&hdr.ism, sizeof(ISMPacketHeader)); - primMsg.append((const uint8_t*)&hdr, sizeof(DictTokenByScanRequestHeader)); - primMsg += oidOp.FilterString(); - - return primMsg; -} - -const ByteStream formatColScanMsg(const uint64_t lbid, const uint16_t count, OidOperation& oidOp, - uint32_t uniqueId) -{ - ByteStream primMsg; - ISMPacketHeader ism; - ColByScanRangeRequestHeader fMsgHeader; - - memset(&fMsgHeader, 0, sizeof(fMsgHeader)); - memset(&ism, 0, sizeof(ism)); - - ism.Reserve = 0; - ism.Flags = 0; - - if (oidOp.LbidTrace() == true) - ism.Flags |= PF_LBID_TRACE; - - if (oidOp.PMProfile() == true) - ism.Flags |= PF_PM_PROF; - - ism.Command = COL_BY_SCAN_RANGE; - ism.Size = sizeof(fMsgHeader) + sizeof(ism) + oidOp.FilterString().length(); - ism.Type = 2; - - primMsg.load((const uint8_t*)&ism, sizeof(ism)); - - fMsgHeader.LBID = lbid; - idbassert(fMsgHeader.LBID >= 0); - fMsgHeader.PBID = 0; - fMsgHeader.DataSize = oidOp.ColumnWidth(); - fMsgHeader.DataType = oidOp.DataType(); - fMsgHeader.OutputType = OT_BOTH; - fMsgHeader.BOP = oidOp.BOP(); - fMsgHeader.NOPS = oidOp.FilterCount(); - fMsgHeader.NVALS = 0; - fMsgHeader.Count = count; //(hwm > (fbo + (*it).size - 1) ? (*it).size : hwm - fbo + 1); - idbassert(fMsgHeader.Count > 0); - fMsgHeader.Hdr.SessionID = oidOp.SessionId(); - fMsgHeader.Hdr.StatementID = 0; - fMsgHeader.Hdr.TransactionID = oidOp.SessionId(); - fMsgHeader.Hdr.VerID = 0; - fMsgHeader.Hdr.StepID = oidOp.SessionId(); - fMsgHeader.Hdr.UniqueID = uniqueId; - - primMsg.append((const uint8_t*)&fMsgHeader, sizeof(fMsgHeader)); - - if (oidOp.FilterCount() > 0) - primMsg += oidOp.FilterString(); - - return primMsg; - -} // formatColScanMsg - -// -void doBatchOp_scan(OidOperation& OidOp); -void doBatchOp_step(OidOperation& OidOp); -void doBatchOp_filt(OidOperation& OidOp); -void doBatchQueryOp(OperationList& OidOps); -void doColScan(OidOperation& OidOp); -void doColStep(OidOperation& OidOp); -void doDictScan(OidOperation& OidOp); -void doDictStep(OidOperation& OidOp); - -// receive the responses from PrimProc -// -struct ThdFcn -{ - void operator()() - { - uint64_t totalBytes = 0; - - try - { - ByteStream ibs; - - if (debug) - cout << "Waiting on " << fNumMsgs << " messages." << endl; - - for (uint32_t k = 0; k < fNumMsgs; k++) - { - // cout << "reading msg #" << k << "...\n"; - ibs = fDec->read(uniqueID); - - // cout << "got msg #" << k << endl; - if (debug) - if (k % 10240 == 0) - cout << "ThdFcn: read " << fSessionid << " " << k << "/" << fNumMsgs << " " << ibs.length() << "/" - << totalBytes << endl; - - if (ibs.length() == 0) - break; - - totalBytes += ibs.length(); - } - } - catch (exception& e) - { - cerr << "read exception: " << e.what() << endl; - thdFcnFailure = true; - } - - if (debug) - cout << totalBytes << " bytes read in " << fNumMsgs << " messages" << endl; - - } // void operator() - - uint32_t fSessionid; - uint32_t uniqueID; - DistributedEngineComm* fDec; - unsigned fNumMsgs; - -}; // struct ThdFcn - -struct QryThdFcn -{ - void operator()() - { - uint64_t totalBytes = 0; - int64_t min; - int64_t max; - uint64_t lbid; - uint32_t cachedIO; - uint32_t physIO; - uint32_t touchedBlocks; - bool validCPData; - - try - { - ByteStream ibs; - ByteStream obs; - - if (debug) - cout << "Waiting on " << fNumMsgs << " messages." << endl; - - for (uint32_t k = 0; k < fNumMsgs; k++) - { - // cout << "reading msg #" << k << "...\n"; - ibs = fDec->read(uniqueID); - - // cout << "got msg #" << k << endl; - if (debug) - if (k % 10240 == 0) - cout << "QryThdFcn: read " << fSessionid << " " << k << "/" << fNumMsgs << " " << ibs.length() - << "/" << totalBytes << " rows: " << fRows << endl; - - if (ibs.length() == 0) - break; - - totalBytes += ibs.length(); - fRows += - fBpp.getTableBand(ibs, &obs, &validCPData, &lbid, &min, &max, &cachedIO, &physIO, &touchedBlocks); - fBlockTouched += touchedBlocks; - } - } - catch (exception& e) - { - cerr << "read exception: " << e.what() << endl; - thdFcnFailure = true; - } - - if (debug) - cout << totalBytes << " bytes read in " << fNumMsgs << " messages for " << fRows << " rows and " - << fBlockTouched << " blocks\n"; - - } // void operator() - - QryThdFcn(BatchPrimitiveProcessorJL& bpp, uint64_t& rows, uint32_t& blk) - : fBpp(bpp), fNumMsgs(0), fRows(rows), fBlockTouched(blk) - { - } - BatchPrimitiveProcessorJL& fBpp; - uint32_t fSessionid; - DistributedEngineComm* fDec; - unsigned fNumMsgs; - uint64_t& fRows; - uint32_t& fBlockTouched; - uint32_t uniqueID; - -}; // struct ThdFcn - -struct BatchScanThr -{ - BatchScanThr(OidOperation& oidOp) : fOidOp(oidOp) - { - } - - void operator()() - { - doBatchOp_scan(fOidOp); - } // void operator() - - OidOperation& fOidOp; - -}; // struct BatchScanThr - -struct BatchStepThr -{ - BatchStepThr(OidOperation& oidOp) : fOidOp(oidOp) - { - } - - void operator()() - { - doBatchOp_step(fOidOp); - } // void operator() - - OidOperation& fOidOp; - -}; // struct BatchStepThr - -struct BatchFiltThr -{ - BatchFiltThr(OidOperation& oidOp) : fOidOp(oidOp) - { - } - - void operator()() - { - doBatchOp_filt(fOidOp); - } // void operator() - - OidOperation& fOidOp; - -}; // struct BatchFiltThr - -struct BatchQueryThr -{ - BatchQueryThr(OperationList& oidOps) : fOidOps(oidOps) - { - } - - void operator()() - { - doBatchQueryOp(fOidOps); - } // void operator() - - OperationList& fOidOps; - -}; // struct - -struct ColStepThr -{ - ColStepThr(OidOperation& oidOp) : fOidOp(oidOp) - { - } - - void operator()() - { - doColStep(fOidOp); - } // void operator() - - OidOperation& fOidOp; - -}; // struct ColStepThr - -struct ColScanThr -{ - ColScanThr(OidOperation& oidOp) : fOidOp(oidOp) - { - } - - void operator()() - { - doColScan(fOidOp); - } // void operator() - - OidOperation& fOidOp; - -}; // struct ColStepThr - -struct DictSigThr -{ -}; // DictSigThr - -struct DictScanThr -{ -}; // DictScanThr - -// doColScan -void doDictionaryScan(OidOperation& OidOp) -{ -} - -// doColScan -void doColScan(OidOperation& OidOp) -{ - if (debug) - cout << "beginning doColScan\n"; - - BRM::LBIDRange_v lbidRanges; - HWM_t hwm = 0; - ResourceManager* rm = ResourceManager::instance(); - DistributedEngineComm* dec = DistributedEngineComm::instance(rm); - struct timespec ts1; - struct timespec ts2; - struct timespec diff; - uint32_t uniqueID; - - uint32_t sessionid = getpid(); - uint32_t totalBlks = 0; - // dec->addSession(sessionid); - // dec->addStep(sessionid, sessionid); - DBRM dbrm; - - uniqueID = dbrm.getUnique32(); - dec->addQueue(uniqueID); - - int err = dbrm.lookup(OidOp.OID(), lbidRanges); - - if (err) - throw runtime_error("doAColScan: BRM LBID range lookup failure (1)"); - - err = dbrm.getHWM(OidOp.OID(), hwm); - - if (err) - throw runtime_error("doAColScan: BRM HWM lookup failure (3)"); - - LBIDRange_v::iterator it; - OID_t tmp; - uint32_t fbo; - - ThdFcn f1; - f1.fSessionid = sessionid; - f1.uniqueID = uniqueID; - f1.fDec = dec; - f1.fNumMsgs = 0; - thdFcnFailure = false; - uint32_t rangeSize = 0; - ByteStream obs; - - // calculate the expected number of messages - for (it = lbidRanges.begin(); it != lbidRanges.end(); it++) - { - BRM::LBID_t lbid = (*it).start; - - if (dbrm.lookup(lbid, 0, false, tmp, fbo)) - { - cerr << "pColScanStep::sendPrimitiveMessages: dbrm.lookup failed for lbid " << lbid << endl; - abort(); - } - - if (hwm < fbo) - continue; - - rangeSize = (hwm > (fbo + (*it).size - 1) ? (*it).size : hwm - fbo + 1); - totalBlks += rangeSize; - } // for - - f1.fNumMsgs = (totalBlks / OidOp.ColumnWidth()); - - if (0 < totalBlks % OidOp.ColumnWidth()) - ++f1.fNumMsgs; - - idbassert(f1.fNumMsgs); - - if (debug) - cout << "Scanning OID " << OidOp.OID() << " " << f1.fNumMsgs << " msgs" << endl; - - thread t1(f1); - clock_gettime(CLOCK_REALTIME, &ts1); - // send the primitive requests - int rCount = 0; - totalBlks = 0; - - for (it = lbidRanges.begin(); it != lbidRanges.end(); it++) - { - try - { - BRM::LBID_t lbid = (*it).start; - - if (dbrm.lookup(lbid, 0, false, tmp, fbo)) - { - cerr << "doAColScan dbrm.lookup failed for lbid " << lbid << endl; - abort(); - } - - if (hwm < fbo) - continue; - - rangeSize = (hwm > (fbo + (*it).size - 1) ? (*it).size : hwm - fbo + 1); - obs = formatColScanMsg(lbid, rangeSize, OidOp, uniqueID); - - if (obs.length() > 0) - { - dec->write(obs); - rCount++; - - if (debug) - cout << "colScan: " << rCount << "/" << lbidRanges.size() << " sending " << obs.length() - << " bytes " - << " lbid " << lbid << " sz " << rangeSize << endl; - } - } - catch (exception& e) - { - cerr << "catch " << e.what() << endl; - } - - totalBlks += rangeSize; - - } // for (lbidRanges ... - - t1.join(); - clock_gettime(CLOCK_REALTIME, &ts2); - // dec->removeSession(sessionid); - dec->removeQueue(uniqueID); - - timespec_sub(ts1, ts2, diff); - - cout << "ColScan stats OID: " << OidOp.OID() << "\tFilter: " << (int)OidOp.FilterCount() - << "\tBlocks: " << (int)totalBlks << "\tElapse: " << diff.tv_sec + (diff.tv_nsec / 1000000000.0) - << "s"; - - float rate = 0; - rate = totalBlks / (diff.tv_sec + (diff.tv_nsec / 1000000000.0)); - cout << " Blks/sec : " << rate << endl; - - if (thdFcnFailure) - cout << "There was a failure in the read thread." << endl; - - cout << endl; - -} // doAScan - -// -void usage() -{ - cerr << "PingProc operation [filter] [operation] [filter] [reporting]" << endl - << "PingProc -c -s -gt 0 -t -d " << endl - << "\t---- operation flags ----" << endl - << "\t--scan -s " << endl - << "\t--block -t " << endl - << "\t--BatchPrimitiveScan -B " << endl - << "\t--BatchPrimitiveStep -Z " << endl - << "\t--concurrent -c perform each operation in its own thread" << endl - << "\t--lbid-trace -lb set lbid trace flag in request" << endl - << "\t---- filter flags ----" << endl - << "\t--equal -eq equivalency test of values in a block>" << endl - << "\t--greater-than -gt - greater than (>) test of values in a block" << endl - << "\t--greater-than-equal -ge greater than or equal to (>=) test of values in a block" << endl - << "\t--less-than -lt less than (<) test of values in a block" << endl - << "\t--less-than-equal -le less than or equal to (<=) test of values in a block" << endl - << "\t--not-equal -ne not equal to (!=) test of values in a block" << endl - << "\t--bop <1 or 0> binary operator when 2 comparison filters are present" << endl - << "\t---- reporting flags ----" << endl - << "\t--debug -d turn debug output on>" << endl - << "\t--list -list -l print out all oids and their ranges" << endl - << "\t--loopback -p send count loopback requests" << endl - << "\t--query -q run batch query for all the oids (enter with -B or -Z)" << endl; - -} // usage() - -const int64_t getInt(string s) -{ - if (s.length() <= 0) - return -1; - - // if (atoll(s.data()) < 0) - // return -1; - - return atoll(s.data()); - -} // getInt - -// dictionary -void doDictionarySig(OidOperation& OidOp) -{ -} // doDictionarySig - -// col step -void doColStep(OidOperation& OidOp) -{ - struct timespec ts1; - struct timespec ts2; - struct timespec diff; - DBRM dbrm; - BRM::LBIDRange_v lbidRanges; - HWM_t hwm = 0; - LBIDRange_v::iterator it; - OID_t tmp; - uint32_t fbo; - uint32_t totalBlks = 0; - ResourceManager* rm = ResourceManager::instance(); - DistributedEngineComm* dec = DistributedEngineComm::instance(rm); - ThdFcn f1; - - // dec->addSession(OidOp.SessionId()); - // dec->addStep(OidOp.SessionId(), OidOp.SessionId()); - uint32_t uniqueID = dbrm.getUnique32(); - dec->addQueue(uniqueID); - - f1.fSessionid = OidOp.SessionId(); - f1.uniqueID = uniqueID; - f1.fDec = dec; - f1.fNumMsgs = 0; - thdFcnFailure = false; - ByteStream ridlist; - uint16_t ridCount = 0; // BLOCK_SIZE/OidOp.ColumnWidth(); - - for (uint16_t i = 0; i < ridCount; i++) - ridlist << i; - - int err = dbrm.lookup(OidOp.OID(), lbidRanges); - - if (err) - throw runtime_error("doAColStep: BRM LBID range lookup failure (1)"); - - err = dbrm.getHWM(OidOp.OID(), hwm); - - if (err) - throw runtime_error("doAColStep: BRM HWM lookup failure (3)"); - - uint32_t rangeSize = 0; - - for (it = lbidRanges.begin(); it != lbidRanges.end(); it++) - { - BRM::LBID_t lbid = (*it).start; - - if (dbrm.lookup(lbid, 0, false, tmp, fbo)) - { - cerr << "pColStep::sendPrimitiveMessages: dbrm.lookup failed for lbid " << lbid << endl; - abort(); - } - - if (hwm < fbo) - break; // continue; - - rangeSize = (hwm > (fbo + (*it).size - 1) ? (*it).size : hwm - fbo + 1); - totalBlks += rangeSize; - } // for - - uint32_t colwidth = OidOp.ColumnWidth(); - f1.fNumMsgs = totalBlks / (colwidth); - - if (0 < totalBlks % colwidth) - ++f1.fNumMsgs; - - idbassert(f1.fNumMsgs); - thread t1(f1); - ByteStream obs; - totalBlks = 0; - clock_gettime(CLOCK_REALTIME, &ts1); - - for (it = lbidRanges.begin(); it != lbidRanges.end(); it++) - { - BRM::LBID_t lbid = (*it).start; - - if (dbrm.lookup(lbid, 0, false, tmp, fbo)) - { - if (debug) - cerr << "pColScanStep::sendPrimitiveMessages: dbrm.lookup failed for lbid " << lbid << endl; - - abort(); - } - - if (hwm < fbo) - break; // continue; - - rangeSize = (hwm > (fbo + (*it).size - 1) ? (*it).size : hwm - fbo + 1); - - for (unsigned i = 0; i < rangeSize; i++) - { - obs += formatColStepMsg(lbid + i, ridlist, ridCount, OidOp, uniqueID); - - if (0 == (i + 1) % colwidth) - { - dec->write(obs); - - if (debug && i + 1 == rangeSize) - cout << "colStep: " << i << "/" << rangeSize << " " << obs.length() << " lbid " << lbid + i << endl; - - obs.restart(); - } - } - - totalBlks += rangeSize; - } // for - - if (obs.length()) - { - dec->write(obs); - - if (debug) - cout << "colStep: last" - << "/" << rangeSize << " " << obs.length() << endl; - } - - obs.reset(); - - t1.join(); //@bug 849 moved join here and changed output to be like pColScan. - clock_gettime(CLOCK_REALTIME, &ts2); - timespec_sub(ts1, ts2, diff); - // t1.join(); - cout << "ColStep stats OID: " << OidOp.OID() << "\tFilter: " << (int)OidOp.FilterCount() - << "\tBlocks: " << (int)totalBlks << "\tElapse: " << diff.tv_sec + (diff.tv_nsec / 1000000000.0) - << "s"; - - float rate = 0; - rate = totalBlks / (diff.tv_sec + (diff.tv_nsec / 1000000000.0)); - cout << "\tBlks/sec " << rate << endl; - - if (thdFcnFailure) - cerr << "There was a failure in the read thread." << endl; - - cout << endl; - -} // doColStep - -void doBatchOp_scan(OidOperation& OidOp) -{ - struct timespec ts1, ts2, diff; - JobStepAssociation injs, outjs; - ResourceManager* rm = ResourceManager::instance(); - DistributedEngineComm* dec = DistributedEngineComm::instance(rm); - ThdFcn f1; - boost::shared_ptr sysCat = - execplan::CalpontSystemCatalog::makeCalpontSystemCatalog(getpid()); - - pColScanStep scan(injs, outjs, dec, sysCat, OidOp.fOid, OidOp.fOid, OidOp.fSessionId, 0, OidOp.fSessionId, - OidOp.fSessionId, OidOp.fSessionId, rm); - - int32_t filters = OidOp.FilterCount(); - - while (OidOp.FilterCount() > 0) - { - int8_t cop; - int64_t value; - OidOp.deSerializeFilter(cop, value); - scan.addFilter(cop, value); - } - - BatchPrimitiveProcessorJL bpp; - ByteStream bs; - DBRM dbrm; - BRM::LBIDRange_v lbidRanges; - HWM_t hwm = 0; - LBIDRange_v::iterator it; - OID_t tmp; - uint32_t fbo; - - uint32_t uniqueID = dbrm.getUnique32(); - bpp.setUniqueID(uniqueID); - - bpp.setSessionID(OidOp.SessionId()); - bpp.setStepID(OidOp.SessionId()); - bpp.addFilterStep(scan); - - cout << "session number = " << OidOp.SessionId() << endl; - // dec->addSession(OidOp.SessionId()); - // dec->addStep(OidOp.SessionId(), OidOp.SessionId()); - dec->addQueue(uniqueID); - f1.fSessionid = OidOp.SessionId(); - f1.uniqueID = uniqueID; - f1.fDec = dec; - thdFcnFailure = false; - int err = dbrm.lookup(OidOp.OID(), lbidRanges); - - if (err) - { - cerr << "doAColScan: BRM LBID range lookup failure (1)\n"; - throw runtime_error("doAColScan: BRM LBID range lookup failure (1)"); - } - - err = dbrm.getHWM(OidOp.OID(), hwm); - - if (err) - { - cerr << "doAColScan: BRM HWM lookup failure (3)" << endl; - throw runtime_error("doAColScan: BRM HWM lookup failure (3)"); - } - - f1.fNumMsgs = hwm / OidOp.fColType.colWidth + (hwm % OidOp.fColType.colWidth ? 1 : 0); - - thread t1(f1); - - bpp.createBPP(bs); - dec->write(bs); - bs.restart(); - - uint32_t rangeSize = 0, totalBlks = 0; - clock_gettime(CLOCK_REALTIME, &ts1); - - // cout << "BPP scaning\n"; - for (it = lbidRanges.begin(); it != lbidRanges.end(); it++) - { - BRM::LBID_t lbid = (*it).start; - - if (dbrm.lookup(lbid, 0, false, tmp, fbo)) - { - cerr << "pColScanStep::sendPrimitiveMessages: dbrm.lookup failed for lbid " << lbid << endl; - abort(); - } - - if (hwm < fbo) - continue; - - rangeSize = (hwm > (fbo + (*it).size - 1) ? (*it).size : hwm - fbo + 1); - bpp.setLBID(lbid); - bpp.setCount(rangeSize / OidOp.fColType.colWidth + (rangeSize % OidOp.fColType.colWidth ? 1 : 0)); - bpp.runBPP(bs); - dec->write(bs); - // cout << "sending the BPP\n"; - bpp.reset(); - bs.restart(); - totalBlks += rangeSize; - } - - t1.join(); - clock_gettime(CLOCK_REALTIME, &ts2); - timespec_sub(ts1, ts2, diff); - float rate = 0; - cout << "ColStep stats OID: " << OidOp.OID() << " " << (diff.tv_sec + (diff.tv_nsec / 1000000000.0)) << "s" - << "\tFilters: " << filters << "\tBlocks : " << (int)totalBlks; - - rate = totalBlks / (diff.tv_sec + (diff.tv_nsec / 1000000000.0)); - cout << "\tBlks/sec " << rate << endl; - - if (thdFcnFailure) - cerr << "There was a failure in the read thread." << endl; - - bpp.destroyBPP(bs); - dec->write(bs); - cout << endl; -} - -void doBatchOp_filt(OidOperation& OidOp) -{ - struct timespec ts1, ts2, diff; - JobStepAssociation injs, outjs; - ResourceManager* rm = ResourceManager::instance(); - DistributedEngineComm* dec = DistributedEngineComm::instance(rm); - ThdFcn f1; - boost::shared_ptr sysCat = CalpontSystemCatalog::makeCalpontSystemCatalog(getpid()); - - ByteStream bs; - DBRM dbrm; - BRM::LBIDRange_v lbidRanges; - HWM_t hwm = 0; - LBIDRange_v::iterator it; - OID_t tmp; - uint32_t fbo; - uint32_t uniqueID; - - BatchPrimitiveProcessorJL bpp; - uniqueID = dbrm.getUnique32(); - bpp.setUniqueID(uniqueID); - bpp.setSessionID(OidOp.SessionId()); - bpp.setStepID(OidOp.SessionId()); - - pColScanStep scan(injs, outjs, dec, sysCat, OidOp.fOid, OidOp.fOid, OidOp.fSessionId, 0, OidOp.fSessionId, - OidOp.fSessionId, OidOp.fSessionId, rm); - - while (OidOp.FilterCount() > 0) - { - int8_t cop; - int64_t value; - OidOp.deSerializeFilter(cop, value); - scan.addFilter(cop, value); - } - - bpp.addFilterStep(scan); - - pColStep step(injs, outjs, dec, sysCat, OidOp.fOid + 1, OidOp.fOid + 1, OidOp.fSessionId, 0, - OidOp.fSessionId, OidOp.fSessionId, OidOp.fSessionId, rm); - - while (OidOp.FilterCount() > 0) - { - int8_t cop; - int64_t value; - OidOp.deSerializeFilter(cop, value); - step.addFilter(cop, value); - } - - bpp.addFilterStep(step); - - execplan::CalpontSystemCatalog::ColType colType; - FilterStep filt(OidOp.fSessionId, OidOp.fSessionId, OidOp.fSessionId, colType); - filt.setBOP(OidOp.BOP()); - bpp.addFilterStep(filt); - - cout << "session number = " << OidOp.SessionId() << endl; - // dec->addSession(OidOp.SessionId()); - // dec->addStep(OidOp.SessionId(), OidOp.SessionId()); - dec->addQueue(uniqueID); - f1.fSessionid = OidOp.SessionId(); - f1.uniqueID = uniqueID; - f1.fDec = dec; - thdFcnFailure = false; - int err = dbrm.lookup(OidOp.OID(), lbidRanges); - - if (err) - { - cerr << "doBatchOp_filt: BRM LBID range lookup failure (1)\n"; - throw runtime_error("doAColScan: BRM LBID range lookup failure (1)"); - } - - err = dbrm.getHWM(OidOp.OID(), hwm); - - if (err) - { - cerr << "doBatchOp_filt: BRM HWM lookup failure (2)" << endl; - throw runtime_error("doBatchOp_filt: BRM HWM lookup failure (2)"); - } - - f1.fNumMsgs = hwm / OidOp.fColType.colWidth + (hwm % OidOp.fColType.colWidth ? 1 : 0); - - thread t1(f1); - - bpp.createBPP(bs); - dec->write(bs); - bs.restart(); - - uint32_t rangeSize = 0, totalBlks = 0; - clock_gettime(CLOCK_REALTIME, &ts1); - - // cout << "BPP scaning\n"; - for (it = lbidRanges.begin(); it != lbidRanges.end(); it++) - { - BRM::LBID_t lbid = (*it).start; - - if (dbrm.lookup(lbid, 0, false, tmp, fbo)) - { - cerr << "doBatchOp_filt: dbrm.lookup failed for lbid (3)" << lbid << endl; - abort(); - } - - if (hwm < fbo) - continue; - - rangeSize = (hwm > (fbo + (*it).size - 1) ? (*it).size : hwm - fbo + 1); - bpp.setLBID(lbid); - bpp.setCount(rangeSize / OidOp.fColType.colWidth + (rangeSize % OidOp.fColType.colWidth ? 1 : 0)); - bpp.runBPP(bs); - dec->write(bs); - // cout << "sending the BPP\n"; - bpp.reset(); - bs.restart(); - totalBlks += rangeSize; - } - - t1.join(); - clock_gettime(CLOCK_REALTIME, &ts2); - timespec_sub(ts1, ts2, diff); - float rate = 0; - cout << "doBatchOp_filt stats OID: " << OidOp.OID() << " " << (diff.tv_sec + (diff.tv_nsec / 1000000000.0)) - << "s" - << "\tBlocks : " << (int)totalBlks; - - rate = totalBlks / (diff.tv_sec + (diff.tv_nsec / 1000000000.0)); - cout << "\tBlks/sec " << rate << endl; - - if (thdFcnFailure) - cerr << "There was a failure in the read thread." << endl; - - bpp.destroyBPP(bs); - dec->write(bs); - cout << endl; -} - -void doBatchQueryOp(OperationList& OidOps) -{ - struct timespec ts1, ts2, diff; - - JobStepAssociation injs, outjs; - BatchPrimitiveProcessorJL bpp; - uint64_t rows = 0; - uint32_t blockTouched = 0; - DBRM dbrm; - - QryThdFcn f1(bpp, rows, blockTouched); - - OperationList::iterator filterOp = OidOps.begin(); - uint32_t sessionId = (*filterOp)->SessionId(); - uint32_t uniqueID = dbrm.getUnique32(); - bpp.setUniqueID(uniqueID); - bpp.setSessionID(sessionId); - bpp.setStepID(sessionId); - cout << "session number = " << sessionId << endl; - - f1.fSessionid = sessionId; - ResourceManager* rm = ResourceManager::instance(); - DistributedEngineComm* dec = DistributedEngineComm::instance(rm); - // dec->addSession(sessionId); - // dec->addStep(sessionId, sessionId); - dec->addQueue(uniqueID); - - f1.fDec = dec; - f1.uniqueID = uniqueID; - // boost::shared_ptr sysCat = - //execplan::CalpontSystemCatalog::makeCalpontSystemCatalog(getpid()); - - // first column is made into the first scan filter step including filters - OID_t scanOid = (*filterOp)->fOid; - uint32_t scanWidth = (*filterOp)->ColumnWidth(); - uint32_t maxWidth = scanWidth; - - uint32_t pid = getpid(); - pColScanStep scan(injs, outjs, dec, execplan::CalpontSystemCatalog::makeCalpontSystemCatalog(pid), scanOid, - scanOid, sessionId, 0, sessionId, sessionId, sessionId, rm); - - uint32_t filterCount = (*filterOp)->FilterCount(); - - while ((*filterOp)->FilterCount() > 0) - { - int8_t cop; - int64_t value; - (*filterOp)->deSerializeFilter(cop, value); - scan.addFilter(cop, value); - } - - bpp.addFilterStep(scan); - - // Any other columns that are batch scans are added as filter steps, the rest as project steps. - // The last filter step is added as a passthru step into the project list. - - OperationList::iterator listend = OidOps.end(); - - for (OperationList::iterator op = OidOps.begin() + 1; op != listend; ++op) - { - pColStep step(injs, outjs, dec, execplan::CalpontSystemCatalog::makeCalpontSystemCatalog(pid), - (*op)->fOid, (*op)->fOid, sessionId, 0, sessionId, sessionId, sessionId, rm); - - if ((*op)->OpType() == OidOperation::BATCHSCAN) - { - filterCount += (*op)->FilterCount(); - - while ((*op)->FilterCount() > 0) - { - int8_t cop; - int64_t value; - (*op)->deSerializeFilter(cop, value); - step.addFilter(cop, value); - } - - filterOp = op; - bpp.addFilterStep(step); - } - else - { - bpp.addProjectStep(step); - } - - if ((*op)->ColumnWidth() > maxWidth) - maxWidth = (*op)->ColumnWidth(); - } - - PassThruStep pass(injs, outjs, dec, (*filterOp)->ColumnType(), (*filterOp)->fOid, (*filterOp)->fOid, - sessionId, 0, sessionId, sessionId, sessionId, false, rm); - bpp.addProjectStep(pass); - - ByteStream bs; - BRM::LBIDRange_v lbidRanges; - HWM_t hwm = 0; - LBIDRange_v::iterator it; - OID_t tmp; - uint32_t fbo; - - thdFcnFailure = false; - int err = dbrm.lookup(scanOid, lbidRanges); - - if (err) - { - cerr << "doQueryScan: BRM LBID range lookup failure (1)\n"; - throw runtime_error("doQueryScan: BRM LBID range lookup failure (1)"); - } - - err = dbrm.getHWM(scanOid, hwm); - - if (err) - { - cerr << "doQueryScan: BRM HWM lookup failure (3)" << endl; - throw runtime_error("doQueryScan: BRM HWM lookup failure (3)"); - } - - f1.fNumMsgs = hwm / scanWidth + (hwm % scanWidth ? 1 : 0); - - thread t1(f1); - uint32_t cnt = dbrm.getExtentSize() / maxWidth; - - bpp.createBPP(bs); - dec->write(bs); - bs.restart(); - uint32_t rangeSize = 0, totalBlks = 0; - clock_gettime(CLOCK_REALTIME, &ts1); - - // cout << "BPP scaning\n"; - for (it = lbidRanges.begin(); it != lbidRanges.end(); it++) - { - BRM::LBID_t lbid = (*it).start; - - if (dbrm.lookup(lbid, 0, false, tmp, fbo)) - { - cerr << "doBatchQuery dbrm.lookup failed for lbid " << lbid << endl; - abort(); - } - - if (hwm < fbo) - continue; - - rangeSize = (hwm > (fbo + (*it).size - 1) ? (*it).size : hwm - fbo + 1); - uint32_t totallbid = rangeSize / scanWidth + (0 < rangeSize % scanWidth ? 1 : 0); - - while (0 < totallbid) - { - if (totallbid < cnt) - cnt = totallbid; - - bpp.setLBID(lbid); - bpp.setCount(cnt); - bpp.runBPP(bs); - dec->write(bs); - // cout << "sending the BPP with range cnt " << cnt << " lbid " << lbid << "\n"; - bpp.reset(); - bs.restart(); - lbid += cnt * scanWidth; - totallbid -= cnt; - } - - for (OperationList::iterator op = OidOps.begin(); op != OidOps.end(); ++op) - { - totalBlks += (uint32_t)(rangeSize * (double)((double)(*op)->ColumnWidth() / scanWidth)); - } - } - - t1.join(); - clock_gettime(CLOCK_REALTIME, &ts2); - timespec_sub(ts1, ts2, diff); - - float rate = 0; - cout << "QueryScan stats - " << bpp.toString() - << "\tElapsed: " << (diff.tv_sec + (diff.tv_nsec / 1000000000.0)) << "s" - << "\tFilters: " << filterCount << "\tBlocks : " << (int)totalBlks; - - rate = totalBlks / (diff.tv_sec + (diff.tv_nsec / 1000000000.0)); - cout << "\tBlks/sec " << rate << endl; - cout << "\tTouched Blocks: " << blockTouched; - rate = blockTouched / (diff.tv_sec + (diff.tv_nsec / 1000000000.0)); - cout << "\tTouched Blks/sec " << rate << endl; - - cout << "\tRows: " << rows; - rate = rows / (diff.tv_sec + (diff.tv_nsec / 1000000000.0)); - cout << "\t\tRows/sec " << fixed << setprecision(2) << rate << endl; - - if (thdFcnFailure) - cerr << "There was a failure in the read thread." << endl; - - bpp.destroyBPP(bs); - dec->write(bs); - cout << endl; -} - -void doBatchOp_step(OidOperation& OidOp) -{ - struct timespec ts1, ts2, diff; - DBRM dbrm; - BRM::LBIDRange_v lbidRanges; - HWM_t hwm = 0; - LBIDRange_v::iterator it; - OID_t tmp; - uint32_t fbo; - uint32_t totalBlks = 0; - ResourceManager* rm = ResourceManager::instance(); - DistributedEngineComm* dec = DistributedEngineComm::instance(rm); - ThdFcn f1; - JobStepAssociation injs, outjs; - - boost::shared_ptr sysCat = - execplan::CalpontSystemCatalog::makeCalpontSystemCatalog(getpid()); - - pColStep step(injs, outjs, dec, sysCat, OidOp.fOid, OidOp.fOid, OidOp.fSessionId, 0, OidOp.fSessionId, - OidOp.fSessionId, OidOp.fSessionId, rm); - - int32_t filters = OidOp.FilterCount(); - - while (OidOp.FilterCount() > 0) - { - int8_t cop; - int64_t value; - OidOp.deSerializeFilter(cop, value); - step.addFilter(cop, value); - } - - BatchPrimitiveProcessorJL bpp; - ElementType et; - ByteStream obs; - uint32_t uniqueID = dbrm.getUnique32(); - bpp.setUniqueID(uniqueID); - bpp.setSessionID(OidOp.SessionId()); - bpp.setStepID(OidOp.SessionId()); - bpp.addFilterStep(step); - - // dec->addSession(OidOp.SessionId()); - // dec->addStep(OidOp.SessionId(), OidOp.SessionId()); - dec->addQueue(uniqueID); - f1.fSessionid = OidOp.SessionId(); - f1.uniqueID = uniqueID; - f1.fDec = dec; - f1.fNumMsgs = 0; - thdFcnFailure = false; - ByteStream ridlist; - uint16_t ridCount = 0; // BLOCK_SIZE/OidOp.ColumnWidth(); - - for (uint16_t i = 0; i < ridCount; i++) - ridlist << i; - - int err = dbrm.lookup(OidOp.OID(), lbidRanges); - - if (err) - throw runtime_error("doAColScan: BRM LBID range lookup failure (1)"); - - err = dbrm.getHWM(OidOp.OID(), hwm); - - if (err) - throw runtime_error("doAColScan: BRM HWM lookup failure (3)"); - - uint32_t rangeSize = 0; - f1.fNumMsgs = hwm / OidOp.fColType.colWidth + (hwm % OidOp.fColType.colWidth ? 1 : 0); - thread t1(f1); - - totalBlks = 0; - bpp.createBPP(obs); - dec->write(obs); - obs.restart(); - - clock_gettime(CLOCK_REALTIME, &ts1); - - for (it = lbidRanges.begin(); it != lbidRanges.end(); it++) - { - BRM::LBID_t lbid = (*it).start; - - if (dbrm.lookup(lbid, 0, false, tmp, fbo)) - { - if (debug) - cerr << "pColScanStep::sendPrimitiveMessages: dbrm.lookup failed for lbid " << lbid << endl; - - abort(); - } - - if (hwm < fbo) - continue; - - rangeSize = (hwm > (fbo + (*it).size - 1) ? (*it).size : hwm - fbo + 1); - - for (unsigned i = 0; i < rangeSize; i++) - { - /* insert all rids for this LBID */ - for (uint32_t j = 0; j < BLOCK_SIZE / OidOp.fColType.colWidth; ++j) - { - et.first = ((fbo + i) * BLOCK_SIZE / OidOp.fColType.colWidth) + j; - et.second = j; - bpp.addElementType(et); - } - - /* If on a logical block boundary, send the primitive */ - if (i % OidOp.fColType.colWidth == (unsigned)OidOp.fColType.colWidth - 1) - { - // cout << "serializing at extent offset " << i << endl; - bpp.runBPP(obs); - dec->write(obs); - bpp.reset(); - obs.restart(); - } - - if (debug && i + 1 == rangeSize) - cout << "colStep: " << i + 1 << "/" << rangeSize << " " << obs.length() << " lbid " << lbid + i - << endl; - } - - if (rangeSize % OidOp.fColType.colWidth) - { - // cout << "serializing last msg\n"; - bpp.runBPP(obs); - dec->write(obs); - bpp.reset(); - obs.restart(); - } - - totalBlks += rangeSize; - } // for - - t1.join(); - clock_gettime(CLOCK_REALTIME, &ts2); - timespec_sub(ts1, ts2, diff); - float rate = 0; - cout << "ColStep stats OID: " << OidOp.OID() << " " << (diff.tv_sec + (diff.tv_nsec / 1000000000.0)) << "s" - << "\tFilters: " << filters << "\tBlocks : " << (int)totalBlks; - - rate = totalBlks / (diff.tv_sec + (diff.tv_nsec / 1000000000.0)); - cout << "\tBlks/sec " << rate << endl; - - if (thdFcnFailure) - cerr << "There was a failure in the read thread." << endl; - - bpp.destroyBPP(obs); - dec->write(obs); - - cout << endl; -} - -// -// -void doListOp(const OID_t o = 0) -{ - DBRM dbrm; - BRM::LBIDRange_v lbidRanges; - LBIDRange_v::iterator it; - OID_t oid = 3000; - HWM_t hwm = 0; - - if (o != 0) - { - int err = dbrm.lookup(o, lbidRanges); - - if (err) - throw runtime_error("doAColScan: BRM LBID range lookup failure (1)"); - - err = dbrm.getHWM(o, hwm); - - if (err) - throw runtime_error("doAColScan: BRM HWM lookup failure (3)"); - - cout << "Object ID: " << o << " HWM: " << hwm << endl; - - for (it = lbidRanges.begin(); it != lbidRanges.end(); it++) - cout << "\tStart: " << (*it).start << " sz: " << (*it).size << endl; - } - else - { - for (; oid < 100000; oid++) - { - int err = dbrm.lookup(oid, lbidRanges); - - if (lbidRanges.size() == 0) - continue; - - if (err) - throw runtime_error("doAColScan: BRM LBID range lookup failure (1)"); - - err = dbrm.getHWM(oid, hwm); - - if (err) - throw runtime_error("doAColScan: BRM HWM lookup failure (3)"); - - cout << "Object ID: " << oid << " HWM: " << hwm << endl; - - for (it = lbidRanges.begin(); it != lbidRanges.end(); it++) - cout << "\tStart: " << (*it).start << " sz: " << (*it).size << endl; - - hwm = 0; - lbidRanges.clear(); - } // for (; oid... - } // else - -} // doListOp - -// -// do LoopBackOp - -void doLoopBack(const uint64_t loopcount) -{ - ByteStream lbMsg; - struct timespec ts1; - struct timespec ts2; - struct timespec diff; - uint32_t sessionid = getpid(); - DBRM dbrm; - ResourceManager* rm = ResourceManager::instance(); - DistributedEngineComm* dec = DistributedEngineComm::instance(rm); - ThdFcn f1; - - // dec->addSession(sessionid); - // dec->addStep(sessionid, sessionid); - uint32_t uniqueID = dbrm.getUnique32(); - dec->addQueue(uniqueID); - f1.fSessionid = sessionid; - f1.uniqueID = uniqueID; - f1.fDec = dec; - f1.fNumMsgs = loopcount; - thdFcnFailure = false; - thread t1(f1); - lbMsg = formatLoopBackMsg(sessionid, uniqueID); - - cout << "Sending " << loopcount << " LOOPBACK requests" << endl; - clock_gettime(CLOCK_REALTIME, &ts1); - - for (uint64_t i = 0; i < loopcount; i++) - { - lbMsg = formatLoopBackMsg(sessionid, uniqueID); - dec->write(lbMsg); - } - - clock_gettime(CLOCK_REALTIME, &ts2); - cout << loopcount << " LOOPBACK msgs sent" << endl; - - t1.join(); - timespec_sub(ts1, ts2, diff); - cout << "\ttotal runtime: " << (diff.tv_sec + (diff.tv_nsec / 1000000000.0)) << "s" << endl; - - float rate = 0; - rate = loopcount / (diff.tv_sec + (diff.tv_nsec / 1000000000.0)); - cout << "\t" << rate << " rqsts/s" << endl; - dec->removeQueue(uniqueID); - -} // doLoopBack - -} // namespace - -/*--------------------------------------------------------------------------- -//Command line parameter definition -// -// -o -o -o . . . -// -s -// -t -// -d -// -eq -equal < == test of values in a block> -// -gt -greater-than < greater than (>) test of values in a block> -// -ge -greater-than-equal < greater than or equal to (>=) test of values in a block> -// -lt -less-than < less than (<) test of values in a block> -// -le -less-than-equal < less than or equal to (<=) test of values in a block> -// -ne -not-equal < not equal to (!=) test of values in a block> -// -bop <1 or 0> <1 AND 0 OR binary operator when 2 comparison filters are present> -// --list -list -l -// --loopback -loopback send loopback requests -// --concurrent run all jobs currently if set to true. defaults to false -// --lbid-trace -lb turn on lbid tracing -// --pm-profile -pmp turn on pm profiling ------------------------------------------------------------------------------*/ - -int main(int argc, char** argv) -{ - int64_t eq_val = 0; - int64_t gt_val = 0; - int64_t ge_val = 0; - int64_t lt_val = 0; - int64_t le_val = 0; - int64_t ne_val = 0; - int64_t bop_val = 0; - int64_t loopback_count = 100000; - int64_t scanOid = 0; - int64_t stepOid = 0; - bool list = false; - // OID_t listOid=0; - string oidString; - vector oidv; - int ch = 0; - bool concurrent_flag = false; - bool lbidtrace_flag = false; - bool pmprofile_flag = false; - bool query_flag = false; - - enum CLA_ENUM - { - OID = (int)0, - SCANOP = (int)1, - BLOCKOP = (int)2, - DEBUG = (int)3, - EQFILTER = (int)4, - GTFILTER = (int)5, - GEFILTER = (int)6, - LTFILTER = (int)7, - LEFILTER = (int)8, - NEFILTER = (int)9, - BOP = (int)10, - LISTOP = (int)11, - LOOPBACKOP = (int)12, - CONCURRENT = (int)13, - LBIDTRACE = (int)14, - PMPROFILE = (int)15, - INVALIDOP = (int)16, - BATCHSCANOP = (int)17, - BATCHSTEPOP = (int)18, - BATCHFILTOP = (int)19, - QUERYOP = (int)20 - - }; - - /** - // longopt struct - struct option { - const char *name; - int has_arg; - int *flag; - int val; - }; - **/ - - static struct option long_options[] = { - // {const char *name, int has_arg, int *flag, int val}, - {"scan", required_argument, NULL, SCANOP}, - {"block", required_argument, NULL, BLOCKOP}, - {"debug", no_argument, NULL, DEBUG}, - {"equal", required_argument, NULL, EQFILTER}, - {"eq", required_argument, NULL, EQFILTER}, - {"greater-than", required_argument, NULL, GTFILTER}, - {"gt", required_argument, NULL, GTFILTER}, - {"greater-than-equal", required_argument, NULL, GEFILTER}, - {"ge", required_argument, NULL, GEFILTER}, - {"less-than", required_argument, NULL, LTFILTER}, - {"lt", required_argument, NULL, LTFILTER}, - {"less-than-equal", required_argument, NULL, LEFILTER}, - {"le", required_argument, NULL, LEFILTER}, - {"not-equal", required_argument, NULL, NEFILTER}, - {"ne", required_argument, NULL, NEFILTER}, - {"bop", optional_argument, NULL, BOP}, - {"list", no_argument, NULL, LISTOP}, - {"loopback", optional_argument, NULL, LOOPBACKOP}, - {"concurrent", no_argument, NULL, CONCURRENT}, - {"lbid-trace", no_argument, NULL, LBIDTRACE}, - {"lb", no_argument, NULL, LBIDTRACE}, - {"pm-prof", no_argument, NULL, PMPROFILE}, - {"pm-profile", no_argument, NULL, PMPROFILE}, - {"pmp", no_argument, NULL, PMPROFILE}, - {"batchscan", required_argument, NULL, BATCHSCANOP}, - {"batchstep", required_argument, NULL, BATCHSTEPOP}, - {"batchfilt", required_argument, NULL, BATCHFILTOP}, - {"queryop", no_argument, NULL, QUERYOP}, - {0, 0, 0, 0}}; - - OidOperation* currOp = NULL; - OperationList OpList; - - if (argc <= 1) - { - usage(); - } - - // process command line arguments - while ((ch = getopt_long_only(argc, argv, "B:Z:F:ds:t:lcqp:", long_options, NULL)) != -1) - { - pid_t pidId = getpid(); - - switch (ch) - { - case SCANOP: - case 's': - if (optarg) - scanOid = getInt(optarg); - - // cout << "OPT=" << ch << " ARG " << scanOid << endl; - currOp = NULL; - - if (scanOid > 0) - currOp = new OidOperation(scanOid, OidOperation::SCAN, pidId); - else - { - cout << "PingProc: scan missing or invalid OID parameter value" << endl; - break; - } - - if (currOp && currOp->isIntegralDataType()) - OpList.push_back(currOp); - else - { - cout << "PingProc cannot process this ColumnType-oid: " << scanOid << endl; - delete currOp; - currOp = NULL; - } - - break; - - case BATCHSCANOP: - case 'B': - if (optarg) - scanOid = getInt(optarg); - else - cout << "no optarg\n"; - - cout << "OPT=" << ch << " ARG " << scanOid << endl; - currOp = NULL; - - if (scanOid > 0) - currOp = new OidOperation(scanOid, OidOperation::BATCHSCAN, pidId); - else - { - cout << "PingProc: batch scan missing or invalid OID parameter value" << endl; - break; - } - - if (currOp && currOp->isIntegralDataType()) - OpList.push_back(currOp); - else - { - cout << "PingProc cannot process this ColumnType-oid: " << scanOid << endl; - delete currOp; - currOp = NULL; - } - - break; - - case BATCHSTEPOP: - case 'Z': - if (optarg) - scanOid = getInt(optarg); - else - cout << "no optarg\n"; - - cout << "OPT=" << ch << " ARG " << scanOid << endl; - currOp = NULL; - - if (scanOid > 0) - currOp = new OidOperation(scanOid, OidOperation::BATCHSTEP, pidId); - else - { - cout << "PingProc: batch step missing or invalid OID parameter value" << endl; - break; - } - - if (currOp && currOp->isIntegralDataType()) - OpList.push_back(currOp); - else - { - cout << "PingProc cannot process this ColumnType-oid: " << scanOid << endl; - delete currOp; - currOp = NULL; - } - - break; - - case BATCHFILTOP: - case 'F': - if (optarg) - scanOid = getInt(optarg); - else - cout << "no optarg\n"; - - cout << "OPT=" << ch << " ARG " << scanOid << endl; - currOp = NULL; - - if (scanOid > 0) - currOp = new OidOperation(scanOid, OidOperation::BATCHFILT, pidId); - else - { - cout << "PingProc: batch filter missing or invalid OID parameter value" << endl; - break; - } - - if (currOp && currOp->isIntegralDataType()) - OpList.push_back(currOp); - else - { - cout << "PingProc cannot process this ColumnType-oid: " << scanOid << endl; - delete currOp; - currOp = NULL; - } - - break; - - case BLOCKOP: - case 't': - if (optarg) - stepOid = getInt(optarg); - - // cout << "OPT=" << ch << " ARG " << stepOid << endl; - currOp = NULL; - - if (stepOid > 0) - currOp = new OidOperation(stepOid, OidOperation::BLOCK, pidId); - else - { - cout << "PingProc: step missing or invalid OID parameter value" << endl; - break; - } - - if (currOp && currOp->isIntegralDataType()) - OpList.push_back(currOp); - else - { - cout << "PingProc cannot process this ColumnType-oid: " << stepOid << endl; - delete currOp; - currOp = NULL; - } - - break; - - case DEBUG: - case 'd': - // cout << "OPT=" << ch << endl; - debug = true; - break; - - case EQFILTER: - if (optarg) - eq_val = getInt(optarg); - else - eq_val = 0; - - cout << "OPT=" << ch << " ARG=" << eq_val << endl; - - if (currOp) - currOp->addFilter(COMPARE_EQ, eq_val); - else - ; // TODO: error Processing - - break; - - case GTFILTER: - if (optarg) - gt_val = getInt(optarg); - else - gt_val = 0; - - // cout << "OPT=" << ch << " ARG=" << gt_val << endl; - if (currOp) - currOp->addFilter(COMPARE_GT, gt_val); - - break; - - case GEFILTER: - if (optarg) - ge_val = getInt(optarg); - else - ge_val = 0; - - // cout << "OPT=" << ch << " ARG=" << ge_val << endl; - if (currOp) - currOp->addFilter(COMPARE_GE, ge_val); - - break; - - case LTFILTER: - if (optarg) - lt_val = getInt(optarg); - else - lt_val = 0; - - // cout << "OPT=" << ch << " ARG=" << lt_val << endl; - if (currOp) - currOp->addFilter(COMPARE_LT, lt_val); - - break; - - case LEFILTER: - if (optarg) - le_val = getInt(optarg); - else - le_val = 0; - - // cout << "OPT=" << ch << " ARG=" << le_val << endl; - if (currOp) - currOp->addFilter(COMPARE_LE, le_val); - - break; - - case NEFILTER: - case 'n': - if (optarg) - ne_val = getInt(optarg); - else - ne_val = 0; - - // cout << "OPT=" << ch << " ARG=" << ne_val << endl; - if (currOp) - currOp->addFilter(COMPARE_NE, ne_val); - - break; - - case BOP: - case 'b': - if (optarg) - bop_val = getInt(optarg); - else - bop_val = 1; // assume AND - - // cout << "OPT=" << ch << " ARG=" << bop_val << endl; - if (currOp) - currOp->BOP(bop_val ? BOP_AND : BOP_OR); - - break; - - case LISTOP: - case 'l': - /** - if (optarg) - listOid=getInt(optarg); - else - listOid=0; - cout << "OPT=" << ch << " LISTOP " << listOid << endl; - **/ - list = true; - break; - - case LOOPBACKOP: - case 'p': - if (optarg) - loopback_count = getInt(optarg); - - // cout << "OPT=" << ch << " LOOPBACKOP " << loopback_count << endl; - currOp = NULL; - currOp = new OidOperation(0, OidOperation::LOOPBACK, pidId); - OpList.push_back(currOp); - break; - - case 'c': - case CONCURRENT: - concurrent_flag = true; - // cout << "OPT=" << ch << " CONCURRENT " << concurrent_flag << endl; - break; - - case LBIDTRACE: - lbidtrace_flag = true; - - // cout << "OPT=" << ch << " CONCURRENT " << concurrent_flag << endl; - if (currOp) - currOp->setLbidTraceOn(); - - break; - - case PMPROFILE: - pmprofile_flag = true; - - // cout << "OPT=" << ch << " CONCURRENT " << concurrent_flag << endl; - if (currOp) - currOp->setPMProfileOn(); - - break; - - case 'q': - case QUERYOP: - query_flag = true; - // cout << "OPT=" << ch << " QUERY FLAG " << query_flag << endl; - break; - - case '?': - default: cout << "optarg " << optarg << endl; usage(); - } - - if (list == true) - break; - - } // while - - // if list is requested, print the listing and exit - // - vector BatchScanThreads; - vector BatchStepThreads; - vector BatchFiltThreads; - vector ColScanThreads; - vector ColStepThreads; - vector DictScanThreads; - vector DictSigThreads; - vector thrArray; - - if (query_flag) - { - cout << "starting batch query thread\n"; - struct BatchQueryThr* qt = new struct BatchQueryThr(OpList); - thread* t1 = new thread(*qt); - - if (concurrent_flag) - thrArray.push_back(t1); - else - t1->join(); - } - else if (list) - { - doListOp(); - } - else - { - for (uint32_t i = 0; i < OpList.size(); i++) - { - if (OpList[i]->OpType() == OidOperation::LOOPBACK) - { - doLoopBack(loopback_count); - } - - else if (OpList[i]->OpType() == OidOperation::SCAN) - { - struct ColScanThr* cst = new struct ColScanThr(*OpList[i]); - ColScanThreads.push_back(cst); - thread* t1 = new thread(*cst); - - if (concurrent_flag) - thrArray.push_back(t1); - else - { - t1->join(); - delete t1; - } - } - - else if (OpList[i]->OpType() == OidOperation::BLOCK) - { - struct ColStepThr* cst = new struct ColStepThr(*OpList[i]); - ColStepThreads.push_back(cst); - thread* t1 = new thread(*cst); - - if (concurrent_flag) - thrArray.push_back(t1); - else - { - t1->join(); - delete t1; - } - } - - else if (OpList[i]->OpType() == OidOperation::BATCHSCAN) - { - cout << "starting batch scan thread\n"; - struct BatchScanThr* cst = new struct BatchScanThr(*OpList[i]); - BatchScanThreads.push_back(cst); - thread* t1 = new thread(*cst); - - if (concurrent_flag) - thrArray.push_back(t1); - else - { - t1->join(); - delete t1; - } - } - - else if (OpList[i]->OpType() == OidOperation::BATCHSTEP) - { - cout << "starting batch step thread\n"; - struct BatchStepThr* cst = new struct BatchStepThr(*OpList[i]); - BatchStepThreads.push_back(cst); - thread* t1 = new thread(*cst); - - if (concurrent_flag) - thrArray.push_back(t1); - else - { - t1->join(); - delete t1; - } - } - - else if (OpList[i]->OpType() == OidOperation::BATCHFILT) - { - cout << "starting batch filt thread\n"; - struct BatchFiltThr* cst = new struct BatchFiltThr(*OpList[i]); - BatchFiltThreads.push_back(cst); - thread* t1 = new thread(*cst); - - if (concurrent_flag) - thrArray.push_back(t1); - else - { - t1->join(); - delete t1; - } - - i += 2; - } - } // for - - } // else - - // join threads to main - for (uint32_t i = 0; i < thrArray.size(); i++) - thrArray[i]->join(); - - // clean up - for (uint32_t i = 0; i < OpList.size(); i++) - delete OpList[i]; - - OpList.clear(); - - for (uint32_t i = 0; i < BatchScanThreads.size(); i++) - delete BatchScanThreads[i]; - - BatchScanThreads.clear(); - - for (uint32_t i = 0; i < BatchStepThreads.size(); i++) - delete BatchStepThreads[i]; - - BatchStepThreads.clear(); - - for (uint32_t i = 0; i < BatchFiltThreads.size(); i++) - delete BatchFiltThreads[i]; - - BatchFiltThreads.clear(); - - for (uint32_t i = 0; i < ColScanThreads.size(); i++) - delete ColScanThreads[i]; - - ColScanThreads.clear(); - - for (uint32_t i = 0; i < ColStepThreads.size(); i++) - delete ColStepThreads[i]; - - ColStepThreads.clear(); - - for (uint32_t i = 0; i < DictScanThreads.size(); i++) - delete DictScanThreads[i]; - - DictScanThreads.clear(); - - for (uint32_t i = 0; i < DictSigThreads.size(); i++) - delete DictSigThreads[i]; - - DictSigThreads.clear(); - - for (uint32_t i = 0; i < thrArray.size(); i++) - delete thrArray[i]; - - thrArray.clear(); - -} // main() diff --git a/tools/qfe/CMakeLists.txt b/tools/qfe/CMakeLists.txt deleted file mode 100644 index 060ce3425..000000000 --- a/tools/qfe/CMakeLists.txt +++ /dev/null @@ -1,55 +0,0 @@ -include_directories(${KDE4_INCLUDES} ${KDE4_INCLUDE_DIR} ${QT_INCLUDES}) - -# ########## next target ############### - -set(QFE_SRCS - cseputils.cpp - ddlstmts.cpp - parsequery.cpp - returnedrows.cpp - sendcsep.cpp - server.cpp - socketio.cpp - qfelexer.lpp - qfeparser.ypp -) - -kde4_add_executable(QFE ${QFE_SRCS}) - -target_link_libraries(QFE ${KDE4_KDECORE_LIBS}) - -install(TARGETS QFE ${INSTALL_TARGETS_DEFAULT_ARGS}) - -# ########## install files ############### - -# original Makefile.am contents follow: - -# Copyright (C) 2014 InfiniDB, Inc. -# -# This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public -# License as published by the Free Software Foundation; version 2 of the License. -# -# This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied -# warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License along with this program; if not, write to the Free -# Software Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. -# -# $Id$ Process this file with automake to produce Makefile.in -# -# AM_CPPFLAGS = $(idb_cppflags) AM_CFLAGS = $(idb_cflags) AM_CXXFLAGS = $(idb_cxxflags) AM_LDFLAGS = $(idb_ldflags) -# AM_YFLAGS = -d -p qfe AM_LFLAGS = -i -Pqfe -olex.yy.c bin_PROGRAMS = QFE QFE_SOURCES = \ cseputils.cpp \ ddlstmts.cpp -# \ parsequery.cpp \ returnedrows.cpp \ sendcsep.cpp \ server.cpp \ socketio.cpp \ qfelexer.lpp \ qfeparser.ypp -# QFE_CPPFLAGS = @idb_common_includes@ $(AM_CPPFLAGS) QFE_LDFLAGS = @idb_common_ldflags@ @idb_write_libs@ -# @idb_exec_libs@ $(AM_LDFLAGS) BUILT_SOURCES = qfeparser.cpp qfelexer.cpp qfeparser.h -# -# test: -# -# coverage: -# -# leakcheck: -# -# docs: -# -# bootstrap: install-data-am -# diff --git a/tools/qfe/cli.cpp b/tools/qfe/cli.cpp deleted file mode 100644 index 032a44b93..000000000 --- a/tools/qfe/cli.cpp +++ /dev/null @@ -1,148 +0,0 @@ -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -using namespace std; - -#include "socktype.h" -#include "socketio.h" -using namespace qfe; - -namespace -{ -void usage() -{ - cout << "usage: cli [-h] [-s serverip] [-p port] [-c schema] " << endl; - cout << '\t' << "-s serverip connect to serverip instead of 127.0.0.1" << endl; - cout << '\t' << "-p port connect on port instead of 9198" << endl; - cout << '\t' << "-c schema use schema as default instead of tpch1" << endl; - cout << '\t' << "-h display this help" << endl; -} -} // namespace - -int main(int argc, char** argv) -{ - opterr = 0; - int c; - string serverip("127.0.0.1"); - string schema("tpch1"); - short PortNo = 9198; - - while ((c = getopt(argc, argv, "s:c:p:h")) != -1) - switch (c) - { - case 's': serverip = optarg; break; - - case 'c': schema = optarg; break; - - case 'p': - PortNo = atoi(optarg); - - if (PortNo == 0) - { - usage(); - return 1; - } - - break; - - case 'h': - case '?': - default: - usage(); - return 0; - break; - } - - if (argc - optind < 1) - { - usage(); - return 1; - } - - string query(argv[optind + 0]); - - SockType fd = -1; - - fd = ::socket(PF_INET, SOCK_STREAM, IPPROTO_TCP); - - if (fd < 0) - { - cerr << "socket create error: " << strerror(errno) << endl; - return 1; - } - - int rc = 0; - struct sockaddr_in serv_addr; - struct addrinfo hints; - struct addrinfo* res = 0; - memset(&hints, 0, sizeof(struct addrinfo)); - hints.ai_family = PF_INET; - hints.ai_socktype = SOCK_STREAM; - hints.ai_protocol = IPPROTO_TCP; - rc = getaddrinfo(serverip.c_str(), 0, &hints, &res); - - if (rc != 0) - { - cerr << "Error resolving '" << serverip << "': " << gai_strerror(rc) << endl; - return 1; - } - - sockaddr_in* sain = 0; - sain = reinterpret_cast(res->ai_addr); - memset(&serv_addr, 0, sizeof(serv_addr)); - serv_addr.sin_family = PF_INET; - serv_addr.sin_addr = sain->sin_addr; - serv_addr.sin_port = htons(PortNo); - freeaddrinfo(res); - - rc = ::connect(fd, (sockaddr*)&serv_addr, sizeof(serv_addr)); - - if (rc < 0) - { - cerr << "socket connect error: " << strerror(errno) << endl; - return 1; - } - - socketio::writeString(fd, schema); - socketio::writeString(fd, query); - - uint32_t flag = 0; - string row; - row = socketio::readString(fd); - - if (row != "OK") - { - cerr << "query failed: " << row << endl; - goto bailout; - } - - row = socketio::readString(fd); - - while (!row.empty()) - { - cout << row << endl; - row = socketio::readString(fd); - } - - flag = 0; - SockWriteFcn(fd, &flag, 4); -bailout: - ::shutdown(fd, SHUT_RDWR); - ::close(fd); - fd = -1; - - return 0; -} diff --git a/tools/qfe/cseputils.cpp b/tools/qfe/cseputils.cpp deleted file mode 100644 index 3584c8a03..000000000 --- a/tools/qfe/cseputils.cpp +++ /dev/null @@ -1,141 +0,0 @@ -#include -#include -using namespace std; - -#include "constantcolumn.h" -#include "simplefilter.h" -#include "calpontsystemcatalog.h" -#include "parsetree.h" -#include "simplecolumn.h" -#include "calpontselectexecutionplan.h" -using namespace execplan; - -#include "cseputils.h" - -namespace qfe -{ -extern string DefaultSchema; - -namespace utils -{ -ConstantColumn* createConstCol(const string& valstr) -{ - ConstantColumn* cc = new ConstantColumn(valstr); - cc->alias(valstr); - return cc; -} - -template -ConstantColumn* createConstCol(const string& valstr, T val) -{ - ConstantColumn* cc = new ConstantColumn(valstr, val); - cc->alias(valstr); - return cc; -} - -SimpleFilter* createSimpleFilter(boost::shared_ptr& csc, - const CalpontSystemCatalog::TableColName& tcn, const string& opstr, - ConstantColumn* cc) -{ - SimpleFilter* lsf = new SimpleFilter(); - - Operator* op = new Operator(); - op->data(opstr); - CalpontSystemCatalog::ColType ccct; - ccct = op->resultType(); - ccct.colDataType = cc->resultType().colDataType; - op->operationType(ccct); - - SOP sop(op); - lsf->op(sop); - - CalpontSystemCatalog::OID oid = csc->lookupOID(tcn); - CalpontSystemCatalog::ColType ct = csc->colType(oid); - - SimpleColumn* sc = new SimpleColumn(); - sc->schemaName(tcn.schema); - sc->tableName(tcn.table); - sc->tableAlias(tcn.table); - sc->columnName(tcn.column); - sc->oid(oid); - sc->resultType(ct); - sc->alias(tcn.toString()); - - lsf->lhs(sc); - lsf->rhs(cc); - - return lsf; -} - -void appendSimpleFilter(ParseTree*& ptree, SimpleFilter* filter) -{ - if (ptree->data() == 0) - { - // degenerate case, this filter goes at this node - ptree->data(filter); - } - else if (ptree->right() == 0 && ptree->left() == 0) - { - // this will be the case when there is a single node in the tree - // that contains a filter. Here we want to make the root node an - // 'and' operator, push the existing down to the lhs and make a - // new node for the new filter - ParseTree* newLhs = new ParseTree(ptree->data()); - ParseTree* newRhs = new ParseTree(filter); - - Operator* op = new Operator(); - op->data("and"); - - ptree->data(op); - ptree->left(newLhs); - ptree->right(newRhs); - } - else - { - // this will be the case once we have a tree with an 'and' at the - // root node, a filter in the lhs, and an arbitrary height tree - // with the same properties on the rhs. Because all operators - // are guaranteed to be and for now we simply insert a new rhs - // node and "push down" the existing tree - Operator* op = new Operator(); - op->data("and"); - - ParseTree* newRhs = new ParseTree(op); - newRhs->left(new ParseTree(filter)); - newRhs->right(ptree->right()); - ptree->right(newRhs); - } -} - -void updateParseTree(boost::shared_ptr& csc, - execplan::CalpontSelectExecutionPlan*& csep, execplan::SimpleColumn* sc, - const std::string& relop, pair cval) -{ - execplan::ConstantColumn* cc = 0; - - if (cval.first == 0) - cc = createConstCol(cval.second, static_cast(atoll(cval.second.c_str()))); - else - cc = createConstCol(cval.second); - - if (sc->schemaName() == "infinidb_unknown" && !DefaultSchema.empty()) - sc->schemaName(DefaultSchema); - - execplan::SimpleFilter* sf = 0; - sf = createSimpleFilter(csc, execplan::make_tcn(sc->schemaName(), sc->tableName(), sc->columnName()), relop, - cc); - execplan::ParseTree* ptp = 0; - ptp = csep->filters(); - - if (ptp == 0) - ptp = new execplan::ParseTree(); - - appendSimpleFilter(ptp, sf); - csep->filters(ptp); -} - -// template instantiations -template ConstantColumn* createConstCol(const string& valstr, int64_t val); - -} // namespace utils -} // namespace qfe diff --git a/tools/qfe/cseputils.h b/tools/qfe/cseputils.h deleted file mode 100644 index be289554c..000000000 --- a/tools/qfe/cseputils.h +++ /dev/null @@ -1,34 +0,0 @@ -#pragma once - -#include - -namespace execplan -{ -class ConstantColumn; -class SimpleFilter; -class ParseTree; -class SimpleColumn; -} // namespace execplan -#include "calpontsystemcatalog.h" - -namespace qfe -{ -namespace utils -{ -execplan::ConstantColumn* createConstCol(const std::string& valstr); - -template -execplan::ConstantColumn* createConstCol(const std::string& valstr, T val); - -execplan::SimpleFilter* createSimpleFilter(boost::shared_ptr& csc, - const execplan::CalpontSystemCatalog::TableColName& tcn, - const std::string& opstr, execplan::ConstantColumn* cc); - -void appendSimpleFilter(execplan::ParseTree*& ptree, execplan::SimpleFilter* filter); - -void updateParseTree(boost::shared_ptr&, - execplan::CalpontSelectExecutionPlan*&, execplan::SimpleColumn*, const std::string&, - pair); - -} // namespace utils -} // namespace qfe diff --git a/tools/qfe/ddlstmts.cpp b/tools/qfe/ddlstmts.cpp deleted file mode 100644 index d25ffd893..000000000 --- a/tools/qfe/ddlstmts.cpp +++ /dev/null @@ -1,80 +0,0 @@ -#include -#include -#include -using namespace std; - -#include -using namespace boost; - -#include "ddlpkg.h" -#include "sqlparser.h" -using namespace ddlpackage; - -#include "bytestream.h" -#include "messagequeue.h" -using namespace messageqcpp; - -#include "ddlstmts.h" - -namespace qfe -{ -extern string DefaultSchema; -} - -namespace -{ -using namespace qfe; - -int processDDLStmt(const string& stmt, uint32_t sid) -{ - MessageQueueClient* mq = 0; - SqlParser parser; - - parser.setDefaultSchema(DefaultSchema); - parser.Parse(stmt.c_str()); - - if (parser.Good()) - { - const ddlpackage::ParseTree& ptree = parser.GetParseTree(); - SqlStatement& ddlstmt = *ptree.fList[0]; - ddlstmt.fSessionID = sid; - ddlstmt.fSql = stmt; - ddlstmt.fOwner = DefaultSchema; - ByteStream bytestream; - bytestream << ddlstmt.fSessionID; - ddlstmt.serialize(bytestream); - mq = new MessageQueueClient("DDLProc"); - scoped_ptr smq(mq); - ByteStream::byte b = 0; - mq->write(bytestream); - bytestream = mq->read(); - bytestream >> b; - string emsg; - bytestream >> emsg; - - if (b != 0) - throw runtime_error(emsg); - } - else - { - throw runtime_error("syntax error"); - } - - return 0; -} - -} // namespace - -namespace qfe -{ -void processCreateStmt(const string& stmt, uint32_t sid) -{ - processDDLStmt(stmt, sid); -} - -void processDropStmt(const string& stmt, uint32_t sid) -{ - processDDLStmt(stmt, sid); -} - -} // namespace qfe diff --git a/tools/qfe/ddlstmts.h b/tools/qfe/ddlstmts.h deleted file mode 100644 index 3d928f647..000000000 --- a/tools/qfe/ddlstmts.h +++ /dev/null @@ -1,10 +0,0 @@ -#pragma once - -#include -#include - -namespace qfe -{ -void processCreateStmt(const std::string&, uint32_t); -void processDropStmt(const std::string&, uint32_t); -} // namespace qfe diff --git a/tools/qfe/parsequery.cpp b/tools/qfe/parsequery.cpp deleted file mode 100644 index bda94df70..000000000 --- a/tools/qfe/parsequery.cpp +++ /dev/null @@ -1,97 +0,0 @@ -#include -#include -#include -using namespace std; - -#include -#include -using namespace boost; - -#include "cseputils.h" -#include "parsequery.h" - -#include "calpontselectexecutionplan.h" -#include "simplecolumn.h" -#include "simplefilter.h" -#include "constantcolumn.h" -#include "calpontsystemcatalog.h" -#include "sessionmanager.h" -using namespace execplan; - -#include "brmtypes.h" -using namespace BRM; - -extern int qfeparse(); -extern int qfedebug; - -struct yy_buffer_state; -extern yy_buffer_state* qfe_scan_string(const char*); -extern void qfe_delete_buffer(yy_buffer_state*); -extern CalpontSelectExecutionPlan* ParserCSEP; -extern boost::shared_ptr ParserCSC; - -namespace -{ -mutex ParserMutex; -} // namespace - -namespace qfe -{ -extern string DefaultSchema; - -CalpontSelectExecutionPlan* parseQuery(const string& query, const uint32_t sid) -{ - // We're going to make parsing the query single-threaded for now. This makes it a lot - // easier to interface with the parser and doesn;t materially affect overall query - // performance (I think) - mutex::scoped_lock lk(ParserMutex); - - boost::shared_ptr csc = CalpontSystemCatalog::makeCalpontSystemCatalog(sid); - CalpontSelectExecutionPlan* csep = 0; - csep = new CalpontSelectExecutionPlan(); - // we use an auto_ptr here with some trepidation. We only want auto delete on an execption. - // If the parseing and plan build succeed, we want the ptr to stay around. boost::scoped_ptr<> - // doesn't have an API to release ownership, so we use auto_ptr... - auto_ptr scsep(csep); - - yy_buffer_state* ybs = 0; - ybs = qfe_scan_string(query.c_str()); - - if (ybs != 0) - { - ParserCSEP = csep; - ParserCSC = csc; - - if (qfeparse() != 0) - throw runtime_error("syntax error"); - - qfe_delete_buffer(ybs); - } - else - throw runtime_error("Internal parser memory error"); - - csep->data(query); - - SessionManager sm; - TxnID txnID; - txnID = sm.getTxnID(sid); - - if (!txnID.valid) - { - txnID.id = 0; - txnID.valid = true; - } - - QueryContext verID; - verID = sm.verID(); - - csep->txnID(txnID.id); - csep->verID(verID); - csep->sessionID(sid); - - // cout << *csep << endl; - scsep.release(); - return csep; -} - -} // namespace qfe diff --git a/tools/qfe/parsequery.h b/tools/qfe/parsequery.h deleted file mode 100644 index f5b18b9cd..000000000 --- a/tools/qfe/parsequery.h +++ /dev/null @@ -1,12 +0,0 @@ -#pragma once - -#include -#include - -#include "calpontselectexecutionplan.h" - -namespace qfe -{ -execplan::CalpontSelectExecutionPlan* parseQuery(const std::string&, uint32_t); - -} diff --git a/tools/qfe/qfe-cli.py b/tools/qfe/qfe-cli.py deleted file mode 100755 index 70a5ea191..000000000 --- a/tools/qfe/qfe-cli.py +++ /dev/null @@ -1,51 +0,0 @@ -#!/usr/bin/python -''' -Created on May 1, 2013 - -@author: rtw -''' - -import struct -import binascii -import socket -import sys - -QFE_HOST = 'localhost' -#QFE_HOST = 'winsvr2008r2.calpont.com' -QFE_SOCK = 9198 - -def do_query(query): - client_socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM) - client_socket.connect((QFE_HOST, QFE_SOCK)) - - defschem = "tpch1"; - lenstruct = struct.pack("I", len(defschem)) - client_socket.send(lenstruct) - client_socket.send(defschem) - - lenstruct = struct.pack("I", len(query)) - client_socket.send(lenstruct) - client_socket.send(query) - - rowlen = 1 - while rowlen: - rowlen = client_socket.recv(4) - if len(rowlen) < 4: - break - rowval = struct.unpack("I",rowlen)[0] - if rowval == 0: - lenstruct = struct.pack("I", 0); - client_socket.send(lenstruct) - break - row = client_socket.recv(rowval) - print row - - client_socket.close() - -if __name__ == '__main__': - if len(sys.argv) < 2: - query = raw_input ( "QFE-cli > " ) - else: - query = sys.argv[1] - - sys.exit(do_query(query)) diff --git a/tools/qfe/qfelexer.lpp b/tools/qfe/qfelexer.lpp deleted file mode 100644 index 853cec75e..000000000 --- a/tools/qfe/qfelexer.lpp +++ /dev/null @@ -1,65 +0,0 @@ -%{ -#include -#include -#include "returnedcolumn.h" -#ifndef _MSC_VER -#include "qfeparser.h" -#else -#include "bison-win.h" -#endif - -extern int qfeerror(const char *s); - -extern std::string* newstr(const char* cp); - -%} - -%option noyywrap -%option yylineno -%option nounput - -%x CCONST - -digit [0-9] -int_const [+-]?{digit}+ -letter [a-zA-Z_] -objname {letter}({letter}|{digit})* - -%% - -"SELECT" { return QFEP_SELECT; } -"FROM" { return QFEP_FROM; } -"WHERE" { return QFEP_WHERE; } -"GROUP BY" { return GROUPBY; } -"AND" { qfelval.cp = newstr(qfetext); return LOGICOP; } -"OR" { qfelval.cp = newstr(qfetext); return LOGICOP; } -"LIMIT" { return LIMIT; } -"ORDER BY" { return ORDERBY; } -"ASC" { return ASC; } -"DESC" { return DESC; } -"AS" { return AS; } -"SUM" { qfelval.cp = newstr(qfetext); return FUNC; } -"MIN" { qfelval.cp = newstr(qfetext); return FUNC; } -"MAX" { qfelval.cp = newstr(qfetext); return FUNC; } -"AVG" { qfelval.cp = newstr(qfetext); return FUNC; } - -{objname} { qfelval.cp = newstr(qfetext); return OBJNAME; } - -"'" { BEGIN(CCONST); } -[^']* { qfelval.cp = newstr(qfetext); return CHAR_CONST; } -"'" { BEGIN(INITIAL); } - -{int_const} { qfelval.cp = newstr(qfetext); return INT_CONST; } - -">" { qfelval.cp = newstr(qfetext); return RELOP; } -">=" { qfelval.cp = newstr(qfetext); return RELOP; } -"<" { qfelval.cp = newstr(qfetext); return RELOP; } -"<=" { qfelval.cp = newstr(qfetext); return RELOP; } -"<>" { qfelval.cp = newstr(qfetext); return RELOP; } -"=" { qfelval.cp = newstr(qfetext); return RELOP; } - -[ \t]* {} -[\n] { qfelineno++; } - -. { return qfetext[0]; } - diff --git a/tools/qfe/qfeparser.ypp b/tools/qfe/qfeparser.ypp deleted file mode 100644 index 4bf75d59f..000000000 --- a/tools/qfe/qfeparser.ypp +++ /dev/null @@ -1,306 +0,0 @@ -%{ -#include -#include -#include -using namespace std; - -namespace qfe -{ -extern string DefaultSchema; -} - -int qfeerror(const char *s); -int qfelex(void); - -#include "calpontselectexecutionplan.h" -#include "simplecolumn.h" -#include "calpontsystemcatalog.h" -#include "aggregatecolumn.h" -execplan::CalpontSelectExecutionPlan* ParserCSEP; -execplan::CalpontSystemCatalog* ParserCSC; -namespace execplan -{ -class ReturnedColumn; -} - -#include "cseputils.h" - -%} - -%union -{ - execplan::ReturnedColumn* rcp; - std::string* cp; - std::pair* cvp; -} - -//Some of these defines cause conflicts in calpontselectexecutionplan.h, so we prepend QFEP_ -%token CHAR_CONST RELOP LOGICOP QFEP_SELECT QFEP_FROM QFEP_WHERE GROUPBY OBJNAME INT_CONST LIMIT ORDERBY ASC -%token DESC AS FUNC - -%type select_column_list table_name filter_predicate_list obj_name_aux opt_alias opt_as -%type opt_limit_clause -%type OBJNAME CHAR_CONST INT_CONST RELOP LOGICOP FUNC AS - -%type aliased_column_name select_column_name obj_name func_col groupby_item - -%type constant - -%destructor { delete $$; } OBJNAME CHAR_CONST INT_CONST RELOP LOGICOP FUNC AS -%destructor { delete $$; } aliased_column_name select_column_name obj_name func_col groupby_item -%destructor { delete $$; } constant - -%% - -statement: QFEP_SELECT select_column_list - QFEP_FROM table_spec - opt_where_clause - opt_groupby_clause - opt_orderby_clause - opt_limit_clause ';' - ; - -select_column_list: aliased_column_name - { - execplan::SRCP rcp($1); - ParserCSEP->returnedCols().push_back(rcp); - $$ = 0; - } - | select_column_list ',' aliased_column_name - { - execplan::SRCP rcp($3); - ParserCSEP->returnedCols().push_back(rcp); - $$ = 0; - } - ; - -select_column_name: obj_name - { - execplan::SimpleColumn* sc = dynamic_cast($1); - if (sc->schemaName() == "infinidb_unknown" && !qfe::DefaultSchema.empty()) - sc->schemaName(qfe::DefaultSchema); - sc->setOID(); - sc->tableAlias(sc->tableName()); - sc->alias(execplan::make_tcn(sc->schemaName(), sc->tableName(), sc->columnName()).toString()); - $$ = $1; - } - | func_col - ; - -func_col: FUNC '(' select_column_name ')' - { - execplan::AggregateColumn* ag = new execplan::AggregateColumn(execplan::AggregateColumn::agname2num(*($1)), $3); - delete $1; - $$ = ag; - } - ; - -aliased_column_name: select_column_name opt_alias - ; - -opt_alias: /* empty */ - { $$ = 0; } - | opt_as OBJNAME - { - delete $2; - $$ = $1; - } - | opt_as CHAR_CONST - { - delete $2; - $$ = $1; - } - ; - -opt_as: /* empty */ - { $$ = 0; } - | AS - ; - -table_spec: aliased_table_name - ; - -/* this is too loose for a table name, but it's easier to parse */ -table_name: obj_name - { - execplan::SimpleColumn* sc=0; - sc = dynamic_cast($1); - execplan::CalpontSystemCatalog::TableAliasName tan; - tan.schema = sc->tableName(); - if (tan.schema == "infinidb_unknown" && !qfe::DefaultSchema.empty()) - tan.schema = qfe::DefaultSchema; - if (qfe::DefaultSchema.empty()) - qfe::DefaultSchema = tan.schema; - tan.table = sc->columnName(); - tan.alias = tan.table; - execplan::CalpontSelectExecutionPlan::TableList tl; - tl.push_back(tan); - ParserCSEP->tableList(tl); - delete sc; - $$ = 0; - } - ; - -aliased_table_name: table_name opt_alias - ; - -opt_where_clause: /* empty */ - | QFEP_WHERE filter_predicate_list - ; - -filter_predicate_list: obj_name RELOP constant - { - pair cval = *($3); - delete $3; - execplan::SimpleColumn* sc=0; - sc = dynamic_cast($1); - qfe::utils::updateParseTree(ParserCSC, ParserCSEP, - sc, *($2), cval); - delete sc; - delete $2; - $$ = 0; - } - | filter_predicate_list LOGICOP obj_name RELOP constant - { - //string logicop = *($2); - delete $2; - pair cval = *($5); - delete $5; - execplan::SimpleColumn* sc=0; - sc = dynamic_cast($3); - qfe::utils::updateParseTree(ParserCSC, ParserCSEP, - sc, *($4), cval); - delete sc; - delete $4; - $$ = 0; - } - ; - -obj_name: obj_name_aux - { - //This is possibly a table name, but we shove it into a SimpleColumn. We'll - // fix this in the table_spec production - execplan::SimpleColumn* sc = new execplan::SimpleColumn("infinidb_unknown", "infinidb_unknown", *($1)); - sc->tableAlias("infinidb_unknown"); - sc->alias(execplan::make_tcn("infinidb_unknown", "infinidb_unknown", *($1)).toString()); - //cerr << "inside parser: " << *sc << endl; - delete $1; - $$ = sc; - } - | obj_name_aux '.' obj_name_aux - { - //This is possibly a table name, but we shove it into a SimpleColumn. We'll - // fix this in the table_spec production - execplan::SimpleColumn* sc = new execplan::SimpleColumn("infinidb_unknown", *($1), *($3)); - sc->tableAlias(*($1)); - sc->alias(execplan::make_tcn("infinidb_unknown", *($1), *($3)).toString()); - //cerr << "inside parser: " << *sc << endl; - delete $1; - delete $3; - $$ = sc; - } - | obj_name_aux '.' obj_name_aux '.' obj_name_aux - { - if (qfe::DefaultSchema.empty()) - qfe::DefaultSchema = *($1); - execplan::SimpleColumn* sc = new execplan::SimpleColumn(*($1), *($3), *($5)); - sc->tableAlias(*($3)); - sc->alias(execplan::make_tcn(*($1), *($3), *($5)).toString()); - //cerr << "inside parser: " << *sc << endl; - delete $1; - delete $3; - delete $5; - $$ = sc; - } - ; - -obj_name_aux: OBJNAME - | '`' OBJNAME '`' - { - $$ = $2; - } - ; - -constant: CHAR_CONST - { - pair* p = new pair(1, *($1)); - delete $1; - $$ = p; - } - | INT_CONST - { - pair* p = new pair(0, *($1)); - delete $1; - $$ = p; - } - ; - -opt_groupby_clause: /* empty */ - | GROUPBY groupby_items_list - { YYERROR; } - ; - -groupby_items_list: groupby_item - | groupby_items_list ',' groupby_item - { delete $3; } - ; - -groupby_item: obj_name - | INT_CONST - { - delete $1; - $$ = 0; - } - ; - -opt_orderby_clause: /* empty */ - | ORDERBY groupby_items_list opt_direction - { YYERROR; } - ; - -opt_direction: /* empty */ - | ASC - | DESC - ; - -opt_limit_clause: /* empty */ - { $$ = 0; } - | LIMIT INT_CONST - { - delete $2; - YYERROR; - $$ = 0; - } - | LIMIT INT_CONST ',' INT_CONST - { - delete $2; - delete $4; - YYERROR; - $$ = 0; - } - ; - -%% - -int qfeerror(const string& s) -{ - extern int qfelineno; // defined and maintained in lex.c - extern char *qfetext; // defined and maintained in lex.c - - cerr << "ERROR: " << s << " at symbol \"" << qfetext; - cerr << "\" on line " << qfelineno << endl; - return -1; -} - -int qfeerror(const char *s) -{ - return qfeerror(string(s)); -} - -string* newstr(const char* cp) -{ - string* strp; - strp = new string(cp); - return strp; -} - diff --git a/tools/qfe/returnedrows.cpp b/tools/qfe/returnedrows.cpp deleted file mode 100644 index f84b0a143..000000000 --- a/tools/qfe/returnedrows.cpp +++ /dev/null @@ -1,95 +0,0 @@ -#include -#include -#include -#include -using namespace std; - -#include -#include -using namespace boost; - -#include "mcsconfig.h" - -#include "socktype.h" - -#include "exceptclasses.h" - -#include "socketio.h" - -#include "bytestream.h" -#include "messagequeue.h" -using namespace messageqcpp; - -#include "rowgroup.h" -using namespace rowgroup; - -namespace qfe -{ -void processReturnedRows(MessageQueueClient* mqc, SockType fd) -{ - scoped_ptr cleaner(mqc); - SBS sbs; - sbs = mqc->read(); - // cerr << "got a bs of " << sbs->length() << " bytes" << endl; - - RowGroup rg; - rg.deserialize(*sbs); - - // cerr << "got a base rowgroup with rows of " << rg.getRowSize() << " bytes" << endl; - // cerr << rg.toString() << endl; - - ByteStream bs; - ByteStream::quadbyte tableOID = 100; - bs.reset(); - bs << tableOID; - mqc->write(bs); - - sbs = mqc->read(); - // cerr << "got a bs of " << sbs->length() << " bytes" << endl; - RGData rgd; - rgd.deserialize(*sbs, true); - rg.setData(&rgd); - // cerr << "got a rowgroup with: " << rg.getRowCount() << " rows" << endl; - - socketio::writeString(fd, "OK"); - Row r; - - while (rg.getRowCount() > 0) - { - rg.initRow(&r); - rg.getRow(0, &r); - string csv; - bs.reset(); - - for (unsigned i = 0; i < rg.getRowCount(); i++) - { - csv = r.toCSV(); - bs << csv; - r.nextRow(); - } - - // cerr << "writing " << bs.length() << " bytes back to client" << endl; - SockWriteFcn(fd, bs.buf(), bs.length()); - - bs.reset(); - bs << tableOID; - mqc->write(bs); - - sbs = mqc->read(); - // cerr << "got a bs of " << sbs->length() << " bytes" << endl; - rgd.deserialize(*sbs, true); - rg.setData(&rgd); - // cerr << "got a rowgroup with: " << rg.getRowCount() << " rows" << endl; - } - - tableOID = 0; - bs.reset(); - bs << tableOID; - mqc->write(bs); - - // sync with the client on end-of-results - SockWriteFcn(fd, &tableOID, 4); - SockReadFcn(fd, &tableOID, 4); -} - -} // namespace qfe diff --git a/tools/qfe/returnedrows.h b/tools/qfe/returnedrows.h deleted file mode 100644 index c9451b16b..000000000 --- a/tools/qfe/returnedrows.h +++ /dev/null @@ -1,10 +0,0 @@ -#pragma once - -#include "socktype.h" -#include "messagequeue.h" - -namespace qfe -{ -void processReturnedRows(messageqcpp::MessageQueueClient*, SockType); - -} diff --git a/tools/qfe/sendcsep.cpp b/tools/qfe/sendcsep.cpp deleted file mode 100644 index b720f16b4..000000000 --- a/tools/qfe/sendcsep.cpp +++ /dev/null @@ -1,53 +0,0 @@ -#include -#include -#include -using namespace std; - -#include -using namespace boost; - -#include "calpontselectexecutionplan.h" -using namespace execplan; -#include "bytestream.h" -#include "messagequeue.h" -using namespace messageqcpp; - -namespace qfe -{ -MessageQueueClient* sendCSEP(CalpontSelectExecutionPlan* csep) -{ - scoped_ptr cleaner(csep); - - ByteStream bs; - - MessageQueueClient* mqc = 0; - - mqc = new MessageQueueClient("ExeMgr1"); - auto_ptr smqc(mqc); - - bs.reset(); - ByteStream::quadbyte wantTuples = 4; - bs << wantTuples; - mqc->write(bs); - - bs.reset(); - csep->serialize(bs); - mqc->write(bs); - - SBS sbs; - sbs = mqc->read(); - *sbs >> wantTuples; - // cerr << "got flag: " << wantTuples << endl; - string msg; - sbs = mqc->read(); - *sbs >> msg; - // cerr << "got msg: " << msg << endl; - - if (wantTuples != 0) - throw runtime_error(msg); - - smqc.release(); - return mqc; -} - -} // namespace qfe diff --git a/tools/qfe/sendcsep.h b/tools/qfe/sendcsep.h deleted file mode 100644 index 8c655febb..000000000 --- a/tools/qfe/sendcsep.h +++ /dev/null @@ -1,12 +0,0 @@ -#pragma once - -#include "messagequeue.h" -#include "calpontselectexecutionplan.h" - -namespace qfe -{ -// Takes ownership of the alloc'd ptr -// Returns an alloc'd mqc -messageqcpp::MessageQueueClient* sendCSEP(execplan::CalpontSelectExecutionPlan*); - -} // namespace qfe diff --git a/tools/qfe/server.cpp b/tools/qfe/server.cpp deleted file mode 100644 index db96e19c6..000000000 --- a/tools/qfe/server.cpp +++ /dev/null @@ -1,429 +0,0 @@ -/* Copyright (C) 2014 InfiniDB, Inc. - - This program is free software; you can redistribute it and/or - modify it under the terms of the GNU General Public License - as published by the Free Software Foundation; version 2 of - the License. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU General Public License for more details. - - You should have received a copy of the GNU General Public License - along with this program; if not, write to the Free Software - Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, - MA 02110-1301, USA. */ - -/* - Protocol definition: -*/ - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -using namespace std; - -#include -#include -#include -#include -#include -using namespace boost; - -#include "mcsconfig.h" - -#include "socktype.h" -#include "parsequery.h" -#include "sendcsep.h" -#include "returnedrows.h" -#include "socketio.h" -#include "ddlstmts.h" - -#include "exceptclasses.h" //brings in idbassert_s macro - -#include "messagequeue.h" -using namespace messageqcpp; - -#include "atomicops.h" - -namespace execplan -{ -class CalpontSelectExecutionPlan; -} - -#define SINGLE_THREADED - -namespace qfe -{ -string DefaultSchema; -} - -namespace -{ -using namespace qfe; - -enum StmtType -{ - UNKNOWN, - QUERY, - CREATE, - DROP, - SHOW, -}; - -volatile uint32_t SystemSID; - -void log(const string& s) -{ - cerr << s << endl; -} - -struct QueryMessage -{ - QueryMessage() : isValid(false) - { - } - ~QueryMessage() - { - } - - string toString() const; - - bool isValid; - string queryText; - string defaultSchema; -}; - -string QueryMessage::toString() const -{ - ostringstream oss; - oss << "valid: " << boolalpha << isValid << ", " - << "queryText: " << queryText << ", " - << "defaultSchema: " << defaultSchema; - return oss.str(); -} - -ostream& operator<<(ostream& os, const QueryMessage& rhs) -{ - os << rhs.toString(); - return os; -} - -class ThreadFunc -{ - public: - ThreadFunc(SockType fd) : fFd(fd) - { - } - ~ThreadFunc() - { - } - - void run(); - void operator()() - { - run(); - } - - private: - ThreadFunc(const ThreadFunc& rhs); - ThreadFunc& operator=(const ThreadFunc& rhs); - - SockType fFd; -}; - -bool serverInit() -{ - - setsid(); - - // Handle certain signals (we want these to return EINTR so we can throw) - // SIGPIPE - // I don't think we'll get any of these from init (except possibly HUP, but that's an indication - // of bad things anyway) - // SIGHUP? - // SIGUSR1? - // SIGUSR2? - // SIGPOLL? - struct sigaction sa; - memset(&sa, 0, sizeof(struct sigaction)); - sa.sa_handler = SIG_IGN; - sigaction(SIGPIPE, &sa, 0); - sigaction(SIGHUP, &sa, 0); - sigaction(SIGUSR1, &sa, 0); - sigaction(SIGUSR2, &sa, 0); -#ifndef __FreeBSD__ - sigaction(SIGPOLL, &sa, 0); -#endif -#if 0 - int fd; - close(2); - fd = open("/tmp/qfe.err", O_CREAT | O_TRUNC | O_WRONLY, 0644); - - if (fd >= 0 && fd != 2) - { - dup2(fd, 2); - close(fd); - } - -#endif - return true; -} - -SockType initListenSock(short portNo) -{ - SockType listenSock = -1; - listenSock = socket(PF_INET, SOCK_STREAM, IPPROTO_TCP); - idbassert_s(listenSock >= 0, string("socket create error: ") + strerror(errno)); - // if (listenSock < 0) throw runtime_error(string("socket create error: ") + strerror(errno)); - int optval = 1; - setsockopt(listenSock, SOL_SOCKET, SO_REUSEADDR, reinterpret_cast(&optval), sizeof(optval)); - int rc = 0; - struct sockaddr_in serv_addr; - memset(&serv_addr, 0, sizeof(serv_addr)); - serv_addr.sin_family = AF_INET; - serv_addr.sin_addr.s_addr = INADDR_ANY; - serv_addr.sin_port = htons(portNo); - const int MaxTries = 5 * 60 / 10; - int tries = 0; -again: - rc = ::bind(listenSock, (sockaddr*)&serv_addr, sizeof(serv_addr)); - - if (rc < 0) - { - if (errno == EADDRINUSE) - { - // cerr << "Addr in use..." << endl; - if (++tries >= MaxTries) - { - log("Waited too long for socket to bind...giving up"); - // cerr << "Waited too long for socket to bind...giving up" << endl; - exit(1); - } - - sleep(10); - goto again; - } - - idbassert_s(0, string("socket bind error: ") + strerror(errno)); - // throw runtime_error(string("socket bind error: ") + strerror(errno)); - } - - rc = listen(listenSock, 16); - idbassert_s(rc >= 0, string("socket listen error") + strerror(errno)); - // if (rc < 0) throw runtime_error(string("socket listen error") + strerror(errno)); - - return listenSock; -} - -QueryMessage getNextMsg(SockType fd) -{ - QueryMessage msg; - - try - { - msg.defaultSchema = socketio::readString(fd); - msg.queryText = socketio::readString(fd); - msg.isValid = true; - } - catch (runtime_error& rex) - { - cerr << "re reading ctl msg: " << rex.what() << endl; - msg.queryText = ""; - } - catch (...) - { - cerr << "ex reading ctl msg" << endl; - msg.queryText = ""; - } - - return msg; -} - -StmtType guessStatementType(const string& stmt) -{ - typedef boost::tokenizer > tokenizer; - char_separator sep; - tokenizer tokens(stmt, sep); - tokenizer::iterator tok_iter = tokens.begin(); - string first_word; - first_word = *tok_iter; - algorithm::to_lower(first_word); - - if (first_word == "select") - return QUERY; - - if (first_word == "create") - return CREATE; - - if (first_word == "drop") - return DROP; - - if (first_word == "show") - return SHOW; - - return UNKNOWN; -} - -struct ScopedCleaner -{ - ScopedCleaner(SockType fd = -1) : fFd(fd) - { - } - ~ScopedCleaner() - { - if (fFd >= 0) - shutdown(fFd, SHUT_RDWR); - - close(fFd); - } - - SockType fFd; -}; - -void ThreadFunc::run() -{ - QueryMessage m; - execplan::CalpontSelectExecutionPlan* csep = 0; - MessageQueueClient* msgqcl; - - ScopedCleaner cleaner(fFd); - - uint32_t sid = 1; - sid = atomicops::atomicInc(&SystemSID); - - try - { - m = getNextMsg(fFd); - - if (m.isValid) - { - DefaultSchema = m.defaultSchema; - StmtType st = guessStatementType(m.queryText); - - switch (st) - { - case QUERY: - csep = parseQuery(m.queryText, sid); - // sendCSEP takes ownership of the ptr from parseQuery - msgqcl = sendCSEP(csep); - // processReturnedRows takes ownership of the ptr from sendCSEP - processReturnedRows(msgqcl, fFd); - break; - - case CREATE: processCreateStmt(m.queryText, sid); break; - - case DROP: processDropStmt(m.queryText, sid); break; - - case SHOW: - { - ostringstream oss; - oss << "select calpontsys.systable.tablename from calpontsys.systable where " - "calpontsys.systable.schema='" - << m.defaultSchema << "';"; - csep = parseQuery(oss.str(), sid); - msgqcl = sendCSEP(csep); - processReturnedRows(msgqcl, fFd); - break; - } - - default: throw runtime_error("couldn't guess the statement type"); break; - } - } - } - catch (std::exception& ex) - { - socketio::writeString(fFd, ex.what()); - throw; // in a multi-threaded server this will simply cause this thread to exit - } - catch (...) - { - socketio::writeString(fFd, "internal query processing error"); - throw; - } -} - -} // namespace - -int main(int argc, char** argv) -{ - int c; - SockType listenSock; - short portNo; - - portNo = 0; - char* p = getenv("IDB_QFE_PORT"); - - if (p && *p) - portNo = atoi(p); - - if (portNo <= 0) - portNo = 9198; - - listenSock = -1; - opterr = 0; - - while ((c = getopt(argc, argv, "p:")) != -1) - switch (c) - { - case 'p': portNo = atoi(optarg); break; - - case '?': - default: break; - } - - if (!serverInit()) - { - log("Could not initialize the QFE Server!"); - cerr << "Could not initialize the QFE Server!" << endl; - return 1; - } - - listenSock = initListenSock(portNo); - - SystemSID = 0; - - for (;;) - { - int querySock = -1; - querySock = accept(listenSock, 0, 0); - idbassert_s(querySock >= 0, string("socket accept error: ") + strerror(errno)); - - // ThreadFunc now owns querySock and is responsible for cleaning it up - ThreadFunc tf(querySock); - -#ifdef SINGLE_THREADED - - try - { - tf.run(); - } - catch (std::exception& ex) - { - cerr << "ThreadFunc run threw an exception: " << ex.what() << endl; - } - catch (...) - { - cerr << "ThreadFunc run threw an exception" << endl; - } - -#else - thread t(tf); -#endif - } - - return 0; -} diff --git a/tools/qfe/socketio.cpp b/tools/qfe/socketio.cpp deleted file mode 100644 index 15cc2d228..000000000 --- a/tools/qfe/socketio.cpp +++ /dev/null @@ -1,133 +0,0 @@ -#include -#include -#include -#include -#include -#include -#include -using namespace std; - -#include -using namespace boost; - -#include "mcsconfig.h" - -#include "exceptclasses.h" - -#include "socketio.h" - -namespace qfe -{ -namespace socketio -{ -void readn(int fd, void* buf, const size_t wanted) -{ - size_t needed = wanted; - size_t sofar = 0; - char* p = static_cast(buf); - ssize_t rrc = -1; - pollfd fds[1]; - int en = 0; - - fds[0].fd = fd; - fds[0].events = POLLIN; - - while (wanted > sofar) - { - fds[0].revents = 0; - poll(fds, 1, -1); - errno = 0; - rrc = read(fd, (p + sofar), needed); - en = errno; - - if (rrc < 0) - { - if (en == EAGAIN || en == EINTR || en == 512) - continue; - - ostringstream oss; - oss << "qfe: readn: read() returned " << rrc << " (" << strerror(en) << ")"; - idbassert_s(0, oss.str()); - } - - needed -= rrc; - sofar += rrc; - } -} - -size_t writen(int fd, const void* data, const size_t nbytes) -{ - size_t nleft; - ssize_t nwritten; - const char* bufp = static_cast(data); - nleft = nbytes; - - while (nleft > 0) - { - // the O_NONBLOCK flag is not set, this is a blocking I/O. - if ((nwritten = ::write(fd, bufp, nleft)) < 0) - { - if (errno == EINTR) - nwritten = 0; - else - { - // save the error no first - int e = errno; - string errorMsg = "qfe: writen: write() error: "; - scoped_array buf(new char[80]); -#if STRERROR_R_CHAR_P - const char* p; - - if ((p = strerror_r(e, buf.get(), 80)) != 0) - errorMsg += p; - -#else - int p; - - if ((p = strerror_r(e, buf.get(), 80)) == 0) - errorMsg += buf.get(); - -#endif - idbassert_s(0, errorMsg); - } - } - - nleft -= nwritten; - bufp += nwritten; - } - - return nbytes; -} - -uint32_t readNumber32(SockType fd) -{ - uint32_t np; - SockReadFcn(fd, &np, 4); - return np; -} - -string readString(SockType fd) -{ - string s; - uint32_t len = readNumber32(fd); - - if (len > 0) - { - scoped_array buf(new char[len + 1]); - SockReadFcn(fd, buf.get(), len); - buf[len] = 0; - s = buf.get(); - } - - return s; -} - -void writeString(SockType fd, const string& data) -{ - uint32_t len = data.length(); - SockWriteFcn(fd, &len, 4); - SockWriteFcn(fd, data.c_str(), len); -} - -} // namespace socketio -} // namespace qfe diff --git a/tools/qfe/socketio.h b/tools/qfe/socketio.h deleted file mode 100644 index 81beb3e3f..000000000 --- a/tools/qfe/socketio.h +++ /dev/null @@ -1,20 +0,0 @@ -#pragma once - -#include -#include -#include - -#include "socktype.h" - -namespace qfe -{ -namespace socketio -{ -void readn(int fd, void* buf, const size_t wanted); -size_t writen(int fd, const void* data, const size_t nbytes); -uint32_t readNumber32(SockType); -std::string readString(SockType); -void writeString(SockType, const std::string&); - -} // namespace socketio -} // namespace qfe diff --git a/tools/qfe/socktype.h b/tools/qfe/socktype.h deleted file mode 100644 index 0e48ad719..000000000 --- a/tools/qfe/socktype.h +++ /dev/null @@ -1,6 +0,0 @@ -#pragma once - - -typedef int SockType; -#define SockReadFcn qfe::socketio::readn -#define SockWriteFcn qfe::socketio::writen diff --git a/tools/rebuildEM/CMakeLists.txt b/tools/rebuildEM/CMakeLists.txt index d7d907566..581f8e4c4 100644 --- a/tools/rebuildEM/CMakeLists.txt +++ b/tools/rebuildEM/CMakeLists.txt @@ -1,10 +1,5 @@ include_directories(${ENGINE_COMMON_INCLUDES}) set(rebuildEM_SRCS main.cpp rebuildEM.cpp) -add_executable(mcsRebuildEM ${rebuildEM_SRCS}) -target_link_libraries(mcsRebuildEM ${ENGINE_LDFLAGS} ${ENGINE_WRITE_LIBS} boost_system boost_filesystem) -install( - TARGETS mcsRebuildEM - DESTINATION ${ENGINE_BINDIR} - COMPONENT columnstore-engine -) +columnstore_executable(mcsRebuildEM ${rebuildEM_SRCS}) +columnstore_link(mcsRebuildEM ${ENGINE_LDFLAGS} ${ENGINE_WRITE_LIBS} boost_system boost_filesystem) diff --git a/tools/rgprint/CMakeLists.txt b/tools/rgprint/CMakeLists.txt index 68b0f7bd9..788079df9 100644 --- a/tools/rgprint/CMakeLists.txt +++ b/tools/rgprint/CMakeLists.txt @@ -4,12 +4,6 @@ include_directories(${ENGINE_COMMON_INCLUDES}) set(rgprint_SRCS rgprint.cpp) -add_executable(rgprint ${rgprint_SRCS}) +columnstore_executable(rgprint ${rgprint_SRCS}) -target_link_libraries(rgprint ${ENGINE_LDFLAGS} ${NETSNMP_LIBRARIES} ${ENGINE_WRITE_LIBS}) - -install( - TARGETS rgprint - DESTINATION ${ENGINE_BINDIR} - COMPONENT columnstore-engine -) +columnstore_link(rgprint ${ENGINE_LDFLAGS} ${NETSNMP_LIBRARIES} ${ENGINE_WRITE_LIBS}) diff --git a/tools/sendPlan/CMakeLists.txt b/tools/sendPlan/CMakeLists.txt deleted file mode 100644 index 58adc132a..000000000 --- a/tools/sendPlan/CMakeLists.txt +++ /dev/null @@ -1,32 +0,0 @@ -include_directories(${KDE4_INCLUDES} ${KDE4_INCLUDE_DIR} ${QT_INCLUDES}) - -# ########## next target ############### - -set(sendPlan_SRCS sendplan.cpp) - -kde4_add_executable(sendPlan ${sendPlan_SRCS}) - -target_link_libraries(sendPlan ${KDE4_KDECORE_LIBS}) - -install(TARGETS sendPlan ${INSTALL_TARGETS_DEFAULT_ARGS}) - -# ########## install files ############### - -# original Makefile.am contents follow: - -# $Id: Makefile.am 333 2009-04-03 20:35:04Z rdempsey $ Process this file with automake to produce Makefile.in -# -# AM_CPPFLAGS = $(idb_cppflags) AM_CFLAGS = $(idb_cflags) AM_CXXFLAGS = $(idb_cxxflags) AM_LDFLAGS = $(idb_ldflags) -# bin_PROGRAMS = sendPlan sendPlan_SOURCES = sendplan.cpp sendPlan_CPPFLAGS = @idb_common_includes@ $(AM_CPPFLAGS) -# sendPlan_LDFLAGS = @idb_common_ldflags@ @idb_exec_libs@ $(AM_LDFLAGS) -# -# test: -# -# coverage: -# -# leakcheck: -# -# docs: -# -# bootstrap: install-data-am -# diff --git a/tools/sendPlan/sendplan.cpp b/tools/sendPlan/sendplan.cpp deleted file mode 100644 index bd8f4bb4e..000000000 --- a/tools/sendPlan/sendplan.cpp +++ /dev/null @@ -1,392 +0,0 @@ -// $Id: sendplan.cpp 1739 2012-03-22 12:57:59Z pleblanc $ -#include -#include -#include -#include -#include -#include -using namespace std; - -#include "bytestream.h" -using namespace messageqcpp; - -#include "calpontselectexecutionplan.h" -#include "sessionmanager.h" -using namespace execplan; - -#include "calpontsystemcatalog.h" -using namespace execplan; - -#include "tableband.h" -#include "joblist.h" -#include "joblistfactory.h" -using namespace joblist; - -#include "configcpp.h" -#include "errorcodes.h" - -#include "rowgroup.h" -using namespace rowgroup; - -typedef CalpontSystemCatalog::OID OID; - -namespace -{ -bool vflg; - -void usage() -{ - cout << "usage: sendPlan [-v|h|d|B] [-t lvl] [-s sid] plan_file ..." << endl; - cout << "-v verbose output" << endl; - cout << "-t lvl set trace level to lvl" << endl; - cout << "-s sid set session id to sid" << endl; - cout << "-d display the query, but don't run it" << endl; - cout << "-B Bob's personal preferred output format" << endl; - cout << "-h display this help" << endl; -} - -double tm_diff(const struct timeval* st, const struct timeval* et) -{ - double sd = (double)st->tv_sec + (double)st->tv_usec / 1000000.0; - double ed = (double)et->tv_sec + (double)et->tv_usec / 1000000.0; - - return (ed - sd); -} - -//...Extract the stats string to be printed and the runtime start time from -//...the full stats string we receive from ExeMgr. -void parseStatsString(const string& fullStatsString, string& printStatsString, - struct timeval& queryRunStartTime) -{ - string::size_type delimPos = fullStatsString.find('|'); - printStatsString = fullStatsString.substr(0, delimPos); - - istringstream startTimeString(fullStatsString.substr(delimPos + 1)); - startTimeString >> queryRunStartTime.tv_sec >> queryRunStartTime.tv_usec; -} - -} // namespace - -int main(int argc, char** argv) -{ - vflg = false; - uint32_t tlvl = 0; - bool dflg = false; - int c; - int32_t sid = -1; - bool Bflg = false; - - opterr = 0; - - while ((c = getopt(argc, argv, "vt:ds:Bh")) != EOF) - switch (c) - { - case 't': tlvl = static_cast(strtoul(optarg, 0, 0)); break; - - case 'v': vflg = true; break; - - case 'd': dflg = true; break; - - case 's': sid = static_cast(strtol(optarg, 0, 0)); break; - - case 'B': Bflg = true; break; - - case 'h': - case '?': - default: - usage(); - return (c == 'h' ? 0 : 1); - break; - } - - if (dflg) - vflg = true; - - if ((argc - optind) < 1) - { - usage(); - return 1; - } - - ifstream inputf; - ByteStream bs; - ByteStream dbs; - ByteStream eoq; - ByteStream tbs; - ByteStream statsStream; - ByteStream::quadbyte q = 0; - eoq << q; - uint32_t sessionid; - time_t t; - SJLP jl; - DeliveredTableMap tm; - DeliveredTableMap::iterator iter; - DeliveredTableMap::iterator end; - CalpontSelectExecutionPlan csep; - struct timeval start_time; - struct timeval end_time; - - MessageQueueClient* mqc = 0; - - if (!dflg) - mqc = new MessageQueueClient("ExeMgr1"); - - if (sid == -1) - { - time(&t); - sessionid = static_cast(t); - } - else - { - sessionid = static_cast(sid); - } - - sessionid &= 0x7fffffff; - logging::ErrorCodes errorCodes; - - for (; optind < argc; optind++) - { - inputf.open(argv[optind]); - - if (!inputf.good()) - { - cerr << "error opening plan stream " << argv[optind] << endl; - return 1; - } - - bs.reset(); - inputf >> bs; - - inputf.close(); - - csep.unserialize(bs); - - csep.sessionID(sessionid); - SessionManager sm; - csep.verID(sm.verID()); - - csep.traceFlags(0); - ResourceManager* rm = ResourceManager::instance(); - jl = JobListFactory::makeJobList(&csep, rm); - csep.traceFlags(tlvl); - - if (vflg) - { - if (dflg) - cout << endl << "Query:" << endl; - else - { - cout << endl << "Session: " << sessionid << ", Sending Query"; - - if (Bflg) - cout << " (" << argv[optind] << ')'; - - cout << ':' << endl; - } - - if (!Bflg) - cout << csep.data() << endl << endl; - } - - if (dflg) - continue; - - try - { - dbs.reset(); - csep.serialize(dbs); - - gettimeofday(&start_time, 0); - - // try tuples first, but expect the worst... - bool expectTuples = false; - ByteStream tbs; - ByteStream::quadbyte tqb = 4; - tbs << tqb; - mqc->write(tbs); - - // send the CSEP - mqc->write(dbs); - - // read the response to the tuple request - tbs = mqc->read(); - idbassert(tbs.length() == 4); - tbs >> tqb; - - if (tqb == 4) - expectTuples = true; - - if (!expectTuples) - cout << "Using TableBand I/F" << endl; - else - cout << "Using tuple I/F" << endl; - - tm = jl->deliveredTables(); - - iter = tm.begin(); - end = tm.end(); - - OID toid; - uint64_t rowTot; - bool reported = false; - bool needRGCtor = true; - - while (iter != end) - { - toid = iter->first; - q = static_cast(toid); - tbs.reset(); - tbs << q; - mqc->write(tbs); - - ByteStream tbbs; - TableBand tb; - RowGroup rg; - rowTot = 0; - uint16_t status = 0; - TableBand::VBA::size_type rc; - ofstream out; - - for (;;) - { - tbbs = mqc->read(); -#if 0 - cout << tbbs.length() << endl; - out.open("bs1.dat"); - idbassert(out.good()); - out << tbbs; - out.close(); - tbbs = mqc->read(); - cout << tbbs.length() << endl; - out.open("bs2.dat"); - idbassert(out.good()); - out << tbbs; - out.close(); - tbbs = mqc->read(); - cout << tbbs.length() << endl; - out.open("bs3.dat"); - idbassert(out.good()); - out << tbbs; - out.close(); -#endif - - if (tbbs.length()) - { - if (!expectTuples) - tb.unserialize(tbbs); - else - { - if (needRGCtor) - { - rg.deserialize(tbbs); - needRGCtor = false; - tbbs = mqc->read(); - } - - rg.setData((uint8_t*)tbbs.buf()); - } - } - else - { - //@bug 1346 - if (!status) - status = logging::makeJobListErr; - - break; - } - - if (!expectTuples) - { - rc = tb.getRowCount(); - status = tb.getStatus(); - } - else - { - rc = rg.getRowCount(); - status = rg.getStatus(); - - if (rc == 0) - status = 0; - } - - if (rc == 0) - break; - - rowTot += rc; - } - - BatchPrimitive* step = dynamic_cast(iter->second.get()); - - if (vflg && step) - { - cout << "For table " << step->tableName(); - - if (!Bflg) - cout << " " << toid; - - cout << ": read " << rowTot << " rows" << endl; - } - - if (status && !reported) - { - cout << "### Query failed: " << errorCodes.errorString(status) << " Check crit.log\n"; - reported = true; - } - - if (!step && !reported) - { - cout << "### Query failed: Did not return project BatchPrimitive. Check crit.log\n"; - reported = true; - } - - ++iter; - } - - if (vflg) - { - gettimeofday(&end_time, 0); - cout << "Query time: " << fixed << setprecision(1) << tm_diff(&start_time, &end_time) << " secs" - << endl; - - //...Ask for query stats through special table id of 3 - const OID TABLE_ID_TO_GET_QUERY_STATS = 3; - - if (!Bflg) - cout << "Retrieving stats..." << endl; - - toid = TABLE_ID_TO_GET_QUERY_STATS; - q = static_cast(toid); - statsStream.reset(); - statsStream << q; - mqc->write(statsStream); - - ByteStream bs_statsString; - bs_statsString = mqc->read(); - string statsString; - bs_statsString >> statsString; - - string printStatsString; - struct timeval startRunTime; - parseStatsString(statsString, printStatsString, startRunTime); - cout << printStatsString << "; QuerySetupTime-" << tm_diff(&start_time, &startRunTime) << "secs" - << endl; - } - - //...Close this query/session - mqc->write(eoq); - jl.reset(); - } - catch (const exception& ex) - { - cout << "### SendPlan caught an exception: " << ex.what() << endl; - } - } - - // jl.reset(); - CalpontSystemCatalog::removeCalpontSystemCatalog(sessionid); - config::Config::deleteInstanceMap(); - - delete mqc; - - return 0; -} diff --git a/tools/setConfig/CMakeLists.txt b/tools/setConfig/CMakeLists.txt index a7c2429b9..b43566721 100644 --- a/tools/setConfig/CMakeLists.txt +++ b/tools/setConfig/CMakeLists.txt @@ -4,12 +4,5 @@ include_directories(${ENGINE_COMMON_INCLUDES}) set(setConfig_SRCS main.cpp) -add_executable(mcsSetConfig ${setConfig_SRCS}) - -target_link_libraries(mcsSetConfig ${ENGINE_LDFLAGS} ${NETSNMP_LIBRARIES} ${ENGINE_EXEC_LIBS}) - -install( - TARGETS mcsSetConfig - DESTINATION ${ENGINE_BINDIR} - COMPONENT columnstore-engine -) +columnstore_executable(mcsSetConfig ${setConfig_SRCS}) +columnstore_link(mcsSetConfig ${ENGINE_LDFLAGS} ${NETSNMP_LIBRARIES} ${ENGINE_EXEC_LIBS}) diff --git a/tools/snconvert/snconvert.pl b/tools/snconvert/snconvert.pl deleted file mode 100755 index f68aa2f65..000000000 --- a/tools/snconvert/snconvert.pl +++ /dev/null @@ -1,132 +0,0 @@ -#!/usr/bin/perl -w -s - - -################################### -# Control parameters -################################### - -# location of the data files to process -$srcDir=""; -$srcFileName=""; - -if ( $#ARGV>-1 ) -{ - $srcDir = $ARGV[0]; - $srcFileName= $ARGV[1]; - print "\n\nCLA $srcDir $srcFileName\n\n"; -} -else -{ - $srcDir = "."; - $srcFileName = "FILE000.cdf"; -} - -# full path of the data file -$srcFile = $srcDir . $srcFileName; - -# name of the new file/extent file -$newFileName = $srcFileName . '_'; - -# get current directory -$currDir = `pwd`; -chomp $currDir; - -# number of file system files -$fsFileCnt=8; - -# once file data has been reorganized they will be moved -# to these locations. -@pmDest = ( - "root\@srvswdev2\:\/tmp", - "root\@srvswdev2\:\/tmp", - "root\@srvswdev2\:\/tmp", - "root\@srvswdev2\:\/tmp" -); - -# split column file into seperate extent files -@sysCmd = ("split", "-d", "-b", "8m", "$srcFile", "$newFileName"); -print "@sysCmd\n"; -print "Starting split . . . .\n"; -system(@sysCmd); -print "Split complete.\n"; - -# get names of extent files -@files = <$newFileName*>; - -$extentCnt = @files; -$i=0; - -# list of extent files to place into a file system file -@extList = (""); - -#list of file system files -@fsFiles = (""); - -#init file lists -while ( $i<$fsFileCnt ) -{ - $fsFiles[$i]="fsFile" . $i; - $extList[$i]=""; - $i++; -} - -# organize extents into files per file systems -#in this case there are 8 file systems -$i=0; -$j=0; -while ( $i<$extentCnt ) -{ - $j=0; - while ( $j<$fsFileCnt && $i<$extentCnt ) - { - $extList[$j] = $extList[$j] . " " . $files[$i]; - $i++; - $j++; - } -} - -$i=0; -while ( $i < $fsFileCnt ) -{ - print "List $i: $extList[$i]\n"; - $Cmd="cat $extList[$i] >> $fsFiles[$i]"; - print "concat $i: $Cmd\n"; - `$Cmd`; - $i++; -} - -# copy the file system files to the appropriate PM -# in this case there are 4 PMs -$pmCnt=4; -$i=0; -$j=0; -@pmFiles =(""); - -while ( $i<$fsFileCnt ) -{ - $j=0; - while ( $j < $pmCnt && $i < $fsFileCnt) - { -# $pmFiles[$j] = $pmFiles[$j] . " " . $fsFiles[$i++]; - $Cmd = "./remote_scp_put.sh $fsFiles[$i++] $pmDest[$j++] Calpont1"; - print "scp command: $Cmd\n"; - `$Cmd`; - } -} - -#$j=0; -#while ( $j < $pmCnt ) -#{ -# print "pmFileList $j : $pmFiles[$j]\n"; -# $j++; -#} -# -#$j=0; -#while ( $j < $pmCnt ) -#{ -# $Cmd = "scp $pmFiles[$j] $pmDest[$j]"; -# print "scp command: $Cmd\n"; -# system($Cmd); -# $j++; -#} - diff --git a/tools/updateGenii/listRepos.pl b/tools/updateGenii/listRepos.pl deleted file mode 100755 index 62954280e..000000000 --- a/tools/updateGenii/listRepos.pl +++ /dev/null @@ -1,58 +0,0 @@ -#!/usr/bin/perl -w -# -# $Id: listRepos.pl 192 2007-05-14 14:42:49Z rdempsey $ -# - -use File::Basename; - -$reporoot = '/Calpont/repos/genii/*'; - -$opt_a = 0; -if (defined($ENV{'QUERY_STRING'})) -{ - $opt_a = 1 if ($ENV{'QUERY_STRING'} eq "All"); -} - -sub findRepos -{ - my ($root) = @_; - - my @repolist = (); - - my @repodirs = glob($root); - - my $dir; - - for $dir (@repodirs) - { - $dir = basename($dir); - - if ($opt_a == 0) - { - next if ($dir eq "net-snmp"); - # deprecated projects - next if ($dir eq "altera"); - next if ($dir eq "diskmgr"); - next if ($dir eq "emulator"); - next if ($dir eq "message"); - next if ($dir eq "sqlengine"); - } - next if ($dir eq "demo"); - next if ($dir eq "doc"); - - push(@repolist, $dir); - } - - return @repolist; -} - -@repos = findRepos($reporoot); - -print 'content-type: text/plain', "\r\n"; -print "\r\n"; - -foreach $repo (@repos) -{ - print $repo, "\r\n"; -} - diff --git a/tools/updateGenii/test.dat b/tools/updateGenii/test.dat deleted file mode 100644 index 2690a2a07..000000000 --- a/tools/updateGenii/test.dat +++ /dev/null @@ -1,14 +0,0 @@ -A OID3046.dat -? brmtest/brmtest -? dumpcol/dumpcol -? fixdate/fixdate.cdf -? fixdate/f1.cdf -? fixdate/fixdate -? evalcol/evalcol -? editem/editem -? dbbuilder/dbbuilder -? oid2file/oid2file -M Makefile -G updateGenii -C updateGenii/updateGenii.pl -G updateGenii/listRepos.pl diff --git a/tools/updateGenii/updateGenii.pl b/tools/updateGenii/updateGenii.pl deleted file mode 100755 index 7766a162b..000000000 --- a/tools/updateGenii/updateGenii.pl +++ /dev/null @@ -1,186 +0,0 @@ -#!/usr/bin/perl -w -# -# $Id: updateGenii.pl 1265 2010-09-17 12:25:42Z rdempsey $ -# - -use Getopt::Std; - -sub usage -{ - my $msg = <<'EOD' -usage: updateGenii.pl [-h] [-b|-t branch/tag] [-r root] [-a] - -h Display this help - -b branch Update a branch wc - -t tag Update a tag wc - -r root Use root as the starting point rather than $HOME - -a Update wc for all svn projects, even deprecated ones -EOD -; - print "$msg\n"; -} - -sub findRepos -{ - my @repolist = (); - my $all = ''; - - $all = '?All' if ($opt_a); - - open(FH, 'curl -s http://srvengcm1.calpont.com/cgi-bin/listRepos.pl' . $all . ' |'); - while () - { - $_ =~ s/[\r\n]*$//; - push(@repolist, $_); - } - close(FH); - - return @repolist; -} - -if (!getopts('hab:r:t:')) -{ - usage(); - exit 1; -} - -if (defined($opt_h)) -{ - $opt_h = 1; - usage(); - exit 0; -} - -$opt_a = 1 if (defined($opt_a)); - -$prjroot = 'genii'; -$prjroot = $opt_r if (defined($opt_r)); -$prjroot = $opt_b if (defined($opt_b)); -$prjroot = $opt_t if (defined($opt_t)); - -if (!defined($opt_b) && !defined($opt_t) && $#ARGV >= 0) -{ - usage(); - exit 1; -} - -if (defined($opt_b) || defined($opt_t)) -{ - $cmd = 'svn list http://srvengcm1.calpont.com/svn/genii/dbcon/'; - if (defined($opt_b)) - { - $cmd = $cmd . 'branches'; - } - else - { - $cmd = $cmd . 'tags'; - } - $listcmd = $cmd; - $tmp = $prjroot; - $tmp =~ s/\./\\./g; - $cmd = $cmd . " 2>/dev/null | egrep '^" . $tmp . "/' | wc -l"; - $brcnt = `$cmd`; - if ($brcnt != 1) - { - print "branch/tag $prjroot not found. Choices are:\n"; - $listcmd = $listcmd . ' | sed s,/\$,,'; - system($listcmd); - exit 1; - } -} - -chdir $ENV{'HOME'} or die; - -$destDir = $prjroot; -if ((defined($opt_b) || defined($opt_t)) && defined($opt_r)) -{ - $destDir = $opt_r; -} -mkdir $destDir if (! -d $destDir); - -chdir $destDir or die; - -@repos = findRepos(); -@mergelines = (); -@conflictlines = (); - -foreach $repo (@repos) -{ - #print $repo, "\n"; next; - if (-d $repo . '/.svn') - { - print $repo, ": \n"; - chdir $repo or die; - open FH, 'svn update |'; - @svnlines = ; - close FH; - foreach $line (@svnlines) - { - print "$line"; - if ($line =~ /^C/) - { - push(@conflictlines, $repo . ": " . $line); - } - elsif ($line =~ /^G/) - { - push(@mergelines, $repo . ": " . $line); - } - } - - chdir '..' or die; - } - else - { - print $repo, ": \n"; - if (defined($opt_b)) - { - $svncmd = 'svn checkout http://srvengcm1.calpont.com/svn/genii/' . $repo . '/branches/' . - $prjroot . ' ' . $repo; - } - elsif (defined($opt_t)) - { - $svncmd = 'svn checkout http://srvengcm1.calpont.com/svn/genii/' . $repo . '/tags/' . - $prjroot . ' ' . $repo; - } - else - { - $svncmd = 'svn checkout http://srvengcm1.calpont.com/svn/genii/' . $repo . '/trunk/' . ' ' . - $repo; - } - system $svncmd; - } -} - -unlink 'merges.txt'; -unlink 'conflicts.txt'; - -if ($#mergelines >= 0) -{ - print "\n** Warning! merges:\n"; - foreach $merge (@mergelines) - { - print "$merge"; - } - open FH, ">merges.txt"; - print FH @mergelines; - close FH; - print "This information has been saved in merges.txt\n"; -} - -if ($#conflictlines >= 0) -{ - print "\n** Warning! conflicts:\n"; - foreach $conflict (@conflictlines) - { - print "$conflict"; - } - open FH, ">conflicts.txt"; - print FH @conflictlines; - close FH; - print "This information has been saved in conflicts.txt\n"; -} - -if (! -e 'Makefile') -{ - system './build/bootstrap'; -} - diff --git a/tools/valprint/valprint.cpp b/tools/valprint/valprint.cpp deleted file mode 100644 index 2fcfd9079..000000000 --- a/tools/valprint/valprint.cpp +++ /dev/null @@ -1,48 +0,0 @@ -/* Copyright (C) 2014 InfiniDB, Inc. - - This program is free software; you can redistribute it and/or - modify it under the terms of the GNU General Public License - as published by the Free Software Foundation; version 2 of - the License. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU General Public License for more details. - - You should have received a copy of the GNU General Public License - along with this program; if not, write to the Free Software - Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, - MA 02110-1301, USA. */ - -// $Id: valprint.cpp 2101 2013-01-21 14:12:52Z rdempsey $ - -#include -#include -using namespace std; - -#include "configcpp.h" -using namespace config; - -int main(int argc, char** argv) -{ - int c; - opterr = 0; - - while ((c = getopt(argc, argv, "")) != EOF) - switch (c) - { - case '?': - default: break; - } - - string s; - - for (int i = optind; i < argc; i++) - { - s = argv[i]; - cout << s << " = " << Config::fromText(s) << endl; - } - - return 0; -} diff --git a/tools/viewtablelock/CMakeLists.txt b/tools/viewtablelock/CMakeLists.txt index 820c1fec4..785057a4f 100644 --- a/tools/viewtablelock/CMakeLists.txt +++ b/tools/viewtablelock/CMakeLists.txt @@ -4,12 +4,5 @@ include_directories(${ENGINE_COMMON_INCLUDES}) set(viewtablelock_SRCS viewtablelock.cpp) -add_executable(viewtablelock ${viewtablelock_SRCS}) - -target_link_libraries(viewtablelock ${ENGINE_LDFLAGS} ${ENGINE_EXEC_LIBS}) - -install( - TARGETS viewtablelock - DESTINATION ${ENGINE_BINDIR} - COMPONENT columnstore-engine -) +columnstore_executable(viewtablelock ${viewtablelock_SRCS}) +columnstore_link(viewtablelock ${ENGINE_LDFLAGS} ${ENGINE_EXEC_LIBS}) diff --git a/utils/basic/string_utils.h b/utils/basic/string_utils.h new file mode 100644 index 000000000..94f17636a --- /dev/null +++ b/utils/basic/string_utils.h @@ -0,0 +1,15 @@ +#pragma once + +#include +#include + +namespace datatypes +{ +inline bool ASCIIStringCaseInsensetiveEquals(const std::string& left, const std::string& right) +{ + auto asciiHigher = [](char c) { return (c >= 'a' && c <= 'z') ? c - 'a' + 'A' : c; }; + return left.size() == right.size() && + std::equal(left.begin(), left.end(), right.begin(), + [&asciiHigher](char l, char r) { return asciiHigher(l) == asciiHigher(r); }); +} +} // namespace datatypes \ No newline at end of file diff --git a/utils/batchloader/CMakeLists.txt b/utils/batchloader/CMakeLists.txt index d724d20ad..25f58197e 100644 --- a/utils/batchloader/CMakeLists.txt +++ b/utils/batchloader/CMakeLists.txt @@ -6,12 +6,4 @@ set(batchloader_LIB_SRCS batchloader.cpp) columnstore_library(batchloader ${batchloader_LIB_SRCS}) -add_dependencies(batchloader loggingcpp) - -target_link_libraries(batchloader ${NETSNMP_LIBRARIES}) - -install( - TARGETS batchloader - DESTINATION ${ENGINE_LIBDIR} - COMPONENT columnstore-engine -) +columnstore_link(batchloader ${NETSNMP_LIBRARIES} loggingcpp) diff --git a/utils/cacheutils/CMakeLists.txt b/utils/cacheutils/CMakeLists.txt index ef7d80c0d..0c9c5d63c 100644 --- a/utils/cacheutils/CMakeLists.txt +++ b/utils/cacheutils/CMakeLists.txt @@ -5,11 +5,4 @@ include_directories(${ENGINE_COMMON_INCLUDES}) set(cacheutils_LIB_SRCS cacheutils.cpp) columnstore_library(cacheutils ${cacheutils_LIB_SRCS}) - -add_dependencies(cacheutils loggingcpp) - -install( - TARGETS cacheutils - DESTINATION ${ENGINE_LIBDIR} - COMPONENT columnstore-engine -) +columnstore_link(cacheutils messageqcpp) diff --git a/utils/cloudio/CMakeLists.txt b/utils/cloudio/CMakeLists.txt index b52a76fc4..c5bb954d0 100755 --- a/utils/cloudio/CMakeLists.txt +++ b/utils/cloudio/CMakeLists.txt @@ -14,22 +14,15 @@ columnstore_library(cloudio ${cloudio_LIB_SRCS}) # IDBDataFile currently depends on cloudio, which is backward. Once cloudio has been turned into a proper plugin for # idbdatafile, we should be able to reverse the dependency like so: -target_link_libraries(cloudio idbdatafile messageqcpp loggingcpp) - -install( - TARGETS cloudio - DESTINATION ${ENGINE_LIBDIR} - COMPONENT columnstore-engine -) +columnstore_link(cloudio idbdatafile messageqcpp) add_executable(cloudio_component_test component_test.cpp) - -add_executable(end_to_end_test end_to_end_test.cpp) +add_executable(cloudio_end_to_end_test end_to_end_test.cpp) # see the comment above and change this dependency to cloudio. Hm our lib dependencies seem not to be declared. Punting # on that, maybe in the future we can have some poor unfortunate intern untangle all of that and declare lib # dependencies properly. For now I'm going to do like the other executables, which means nearly everything AFAICT. -target_link_libraries(cloudio_component_test ${ENGINE_LDFLAGS} ${ENGINE_EXEC_LIBS} cloudio) -target_link_libraries(end_to_end_test ${ENGINE_LDFLAGS} ${ENGINE_EXEC_LIBS} cloudio) -# target_link_libraries(cloudio_component_test cloudio) Copy end_to_end_test dataFile to binary directory +columnstore_link(cloudio_component_test ${ENGINE_LDFLAGS} ${ENGINE_EXEC_LIBS} cloudio) +columnstore_link(cloudio_end_to_end_test ${ENGINE_LDFLAGS} ${ENGINE_EXEC_LIBS} cloudio) +# columnstore_link(cloudio_component_test cloudio) Copy end_to_end_test dataFile to binary directory file(COPY testData DESTINATION ${CMAKE_CURRENT_BINARY_DIR}/) diff --git a/utils/common/CMakeLists.txt b/utils/common/CMakeLists.txt index 678d70f17..2f7aa2bc9 100644 --- a/utils/common/CMakeLists.txt +++ b/utils/common/CMakeLists.txt @@ -15,13 +15,4 @@ set(common_LIB_SRCS ) columnstore_library(common ${common_LIB_SRCS}) - -target_link_libraries(common boost_filesystem) - -add_dependencies(common loggingcpp) - -install( - TARGETS common - DESTINATION ${ENGINE_LIBDIR} - COMPONENT columnstore-engine -) +columnstore_link(common boost_filesystem configcpp loggingcpp messageqcpp ${MARIADB_STRING_LIBS}) diff --git a/utils/common/collation.h b/utils/common/collation.h index 5aa3fafc1..e5781b8d9 100644 --- a/utils/common/collation.h +++ b/utils/common/collation.h @@ -275,12 +275,4 @@ class CollationAwareComparator : public Charset } }; -inline bool ASCIIStringCaseInsensetiveEquals(const std::string& left, const std::string& right) -{ - auto asciiHigher = [](char c) { return (c >= 'a' && c <= 'z') ? c - 'a' + 'A' : c; }; - return left.size() == right.size() && - std::equal(left.begin(), left.end(), right.begin(), - [&asciiHigher](char l, char r) { return asciiHigher(l) == asciiHigher(r); }); -} - } // end of namespace datatypes diff --git a/utils/compress/CMakeLists.txt b/utils/compress/CMakeLists.txt index ca81e2204..dbb61a2b4 100644 --- a/utils/compress/CMakeLists.txt +++ b/utils/compress/CMakeLists.txt @@ -11,19 +11,10 @@ set_property(DIRECTORY PROPERTY INCLUDE_DIRECTORIES "${dirs}") set(compress_LIB_SRCS idbcompress.cpp) -add_definitions(-DNDEBUG) - columnstore_library(compress ${compress_LIB_SRCS}) -add_dependencies(compress loggingcpp external_boost) +columnstore_link(compress PRIVATE loggingcpp boost_filesystem ${SNAPPY_LIBRARIES}) -target_link_libraries(compress ${SNAPPY_LIBRARIES}) if(HAVE_LZ4) message_once(STATUS "LINK WITH LZ4") - target_link_libraries(compress ${LZ4_LIBRARIES}) + columnstore_link(compress PRIVATE ${LZ4_LIBRARIES}) endif() - -install( - TARGETS compress - DESTINATION ${ENGINE_LIBDIR} - COMPONENT columnstore-engine -) diff --git a/utils/configcpp/CMakeLists.txt b/utils/configcpp/CMakeLists.txt index c3a8f7bc1..2dc60715e 100644 --- a/utils/configcpp/CMakeLists.txt +++ b/utils/configcpp/CMakeLists.txt @@ -1,16 +1,4 @@ +set(configcpp_LIB_SRCS configcpp.cpp xmlparser.cpp) include_directories(${ENGINE_COMMON_INCLUDES}) - -# ########## next target ############### - -set(configcpp_LIB_SRCS configcpp.cpp xmlparser.cpp configstream.cpp) - columnstore_library(configcpp ${configcpp_LIB_SRCS}) -add_dependencies(configcpp loggingcpp) - -target_compile_definitions(configcpp PUBLIC BOOST_NO_CXX11_SCOPED_ENUMS) - -install( - TARGETS configcpp - DESTINATION ${ENGINE_LIBDIR} - COMPONENT columnstore-engine -) +columnstore_link(configcpp PRIVATE LibXml2::LibXml2 boost_filesystem) diff --git a/utils/configcpp/Calpont.xml b/utils/configcpp/Calpont.xml deleted file mode 100644 index 52df4dfd6..000000000 --- a/utils/configcpp/Calpont.xml +++ /dev/null @@ -1,12 +0,0 @@ - - - - Message - . - 875 - V1.0 - - - V1.0 - - diff --git a/utils/configcpp/XCalpont.xml b/utils/configcpp/XCalpont.xml deleted file mode 100644 index 25564dea7..000000000 --- a/utils/configcpp/XCalpont.xml +++ /dev/null @@ -1,12 +0,0 @@ - - - - Message - . - 875 - V1.0 - - - V1.0 - - diff --git a/utils/configcpp/configcpp.cpp b/utils/configcpp/configcpp.cpp index 11a8b0a0c..3f76689e9 100644 --- a/utils/configcpp/configcpp.cpp +++ b/utils/configcpp/configcpp.cpp @@ -25,8 +25,6 @@ #include #include #include -#include -#include #include using namespace std; @@ -36,31 +34,20 @@ using namespace std; using namespace boost; namespace fs = boost::filesystem; -#include -#include -#include -#include -#include -#include -#include -#include -#include -#ifdef HAVE_ALLOCA_H -#include -#endif -#include -//#define NDEBUG #include -#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include #include "configcpp.h" -#include "exceptclasses.h" -#include "installdir.h" -#include - -#include "bytestream.h" - namespace config { @@ -460,44 +447,6 @@ void Config::write(const string& configFile) const } } -void Config::writeConfigFile(messageqcpp::ByteStream msg) const -{ - struct flock fl; - int fd; - - // get config file name being udated - string fileName; - msg >> fileName; - - fl.l_type = F_WRLCK; // write lock - fl.l_whence = SEEK_SET; - fl.l_start = 0; - fl.l_len = 0; - fl.l_pid = getpid(); - - // lock file if it exists - if ((fd = open(fileName.c_str(), O_WRONLY)) >= 0) - { - if (fcntl(fd, F_SETLKW, &fl) == -1) - throw runtime_error("Config::write: file lock error " + fileName); - - ofstream out(fileName.c_str()); - out << msg; - - fl.l_type = F_UNLCK; // unlock - - if (fcntl(fd, F_SETLK, &fl) == -1) - throw runtime_error("Config::write: file unlock error " + fileName); - - close(fd); - } - else - { - ofstream out(fileName.c_str()); - out << msg; - } -} - /* static */ void Config::deleteInstanceMap() { diff --git a/utils/configcpp/configcpp.h b/utils/configcpp/configcpp.h index 9dd033620..86e2db67b 100644 --- a/utils/configcpp/configcpp.h +++ b/utils/configcpp/configcpp.h @@ -42,7 +42,6 @@ class ByteStream; } - namespace config { /** @brief a config file I/F class @@ -55,7 +54,6 @@ namespace config class Config { public: - /** @brief Config factory method * * Creates a singleton Config object @@ -96,8 +94,7 @@ class Config * @param name the param name whose value is to be returned * @param values the values in the section are returned in this vector */ - void getConfig(const std::string& section, const std::string& name, - std::vector& values); + void getConfig(const std::string& section, const std::string& name, std::vector& values); /** @brief set name's value in section * @@ -132,13 +129,6 @@ class Config */ void write(const std::string& fileName) const; - /** @brief write a stream copy of config file to disk - * - * write a stream copy of config file to disk. used to distributed mass updates to system nodes - * - */ - void writeConfigFile(messageqcpp::ByteStream msg) const; - /** @brief return the name of this config file * * return the name of this config file. @@ -256,12 +246,10 @@ class Config static Config& globConfigInstance(); - Config(const Config& rhs); Config& operator=(const Config& rhs); Config(const std::string& configFile); - xmlDocPtr fDoc; const std::string fConfigFile; time_t fMtime; @@ -272,11 +260,8 @@ class Config * */ void checkAndReloadConfig(); - }; - } // namespace config #undef EXPORT - diff --git a/utils/configcpp/configstream.cpp b/utils/configcpp/configstream.cpp deleted file mode 100644 index cfb1a4acf..000000000 --- a/utils/configcpp/configstream.cpp +++ /dev/null @@ -1,66 +0,0 @@ -/* Copyright (C) 2014 InfiniDB, Inc. - - This program is free software; you can redistribute it and/or - modify it under the terms of the GNU General Public License - as published by the Free Software Foundation; version 2 of - the License. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU General Public License for more details. - - You should have received a copy of the GNU General Public License - along with this program; if not, write to the Free Software - Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, - MA 02110-1301, USA. */ - -/****************************************************************************************** - * $Id$ - * - ******************************************************************************************/ -#include "mcsconfig.h" - -#include -#include -#include -#include -using namespace std; - -#include "bytestream.h" -using namespace messageqcpp; - -#include "configstream.h" - -namespace config -{ -ConfigStream::ConfigStream(const ByteStream& bs) : fParser() -{ - init(reinterpret_cast(bs.buf())); -} - -ConfigStream::ConfigStream(const string& str) : fParser() -{ - init(reinterpret_cast(str.c_str())); -} - -ConfigStream::ConfigStream(const char* cptr) : fParser() -{ - init(reinterpret_cast(cptr)); -} - -ConfigStream::~ConfigStream() -{ - if (fDoc != NULL) - xmlFreeDoc(fDoc); -} - -void ConfigStream::init(const xmlChar* xp) -{ - fDoc = xmlParseDoc(xp); - - if (fDoc == NULL) - throw runtime_error("ConfigStream::ConfigStream: bad XML stream"); -} - -} // namespace config diff --git a/utils/configcpp/configstream.h b/utils/configcpp/configstream.h deleted file mode 100644 index deefb02f7..000000000 --- a/utils/configcpp/configstream.h +++ /dev/null @@ -1,63 +0,0 @@ -/* Copyright (C) 2014 InfiniDB, Inc. - - This program is free software; you can redistribute it and/or - modify it under the terms of the GNU General Public License - as published by the Free Software Foundation; version 2 of - the License. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU General Public License for more details. - - You should have received a copy of the GNU General Public License - along with this program; if not, write to the Free Software - Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, - MA 02110-1301, USA. */ - -/****************************************************************************************** - * $Id$ - * - ******************************************************************************************/ -/** - * @file - */ -#pragma once - -#include -#include - -#include "bytestream.h" - -#include "xmlparser.h" - -namespace config -{ -/** @brief a config ByteStream I/F class - * - */ -class ConfigStream -{ - public: - ConfigStream(const messageqcpp::ByteStream& bs); - ConfigStream(const std::string& str); - ConfigStream(const char* cptr); - ~ConfigStream(); - - const std::string getConfig(const std::string& section, const std::string& name) const - { - return fParser.getConfig(fDoc, section, name); - } - - private: - ConfigStream(const ConfigStream& rhs); - ConfigStream& operator=(const ConfigStream& rhs); - - void init(const xmlChar* xp); - - XMLParser fParser; - xmlDocPtr fDoc; -}; - -} // namespace config - diff --git a/utils/configcpp/resource.h b/utils/configcpp/resource.h deleted file mode 100644 index 148f1ec8e..000000000 --- a/utils/configcpp/resource.h +++ /dev/null @@ -1,14 +0,0 @@ -//{{NO_DEPENDENCIES}} -// Microsoft Visual C++ generated include file. -// Used by libconfigcpp.rc - -// Next default values for new objects -// -#ifdef APSTUDIO_INVOKED -#ifndef APSTUDIO_READONLY_SYMBOLS -#define _APS_NEXT_RESOURCE_VALUE 101 -#define _APS_NEXT_COMMAND_VALUE 40001 -#define _APS_NEXT_CONTROL_VALUE 1001 -#define _APS_NEXT_SYMED_VALUE 101 -#endif -#endif diff --git a/utils/configcpp/tdriver.cpp b/utils/configcpp/tdriver.cpp deleted file mode 100644 index 62d5e5948..000000000 --- a/utils/configcpp/tdriver.cpp +++ /dev/null @@ -1,371 +0,0 @@ -/* Copyright (C) 2014 InfiniDB, Inc. - - This program is free software; you can redistribute it and/or - modify it under the terms of the GNU General Public License - as published by the Free Software Foundation; version 2 of - the License. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU General Public License for more details. - - You should have received a copy of the GNU General Public License - along with this program; if not, write to the Free Software - Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, - MA 02110-1301, USA. */ - -#include -#include -#include -#include -#include -#include -#include -#include -using namespace std; - -#include -using namespace boost; - -#include - -#include "bytestream.h" -using namespace messageqcpp; - -#include "configcpp.h" -#include "configstream.h" -using namespace config; - -class ConfigFileTest : public CppUnit::TestFixture -{ - CPPUNIT_TEST_SUITE(ConfigFileTest); - - CPPUNIT_TEST(test1); - - CPPUNIT_TEST_EXCEPTION(test2, std::runtime_error); - CPPUNIT_TEST(test3); - CPPUNIT_TEST(test4); - CPPUNIT_TEST_EXCEPTION(test5, std::runtime_error); - CPPUNIT_TEST_EXCEPTION(test6, std::runtime_error); - CPPUNIT_TEST_EXCEPTION(test7, std::invalid_argument); - CPPUNIT_TEST_EXCEPTION(test8, std::invalid_argument); - CPPUNIT_TEST(test9); - CPPUNIT_TEST(test10); - CPPUNIT_TEST(test11); - CPPUNIT_TEST(test12); - CPPUNIT_TEST_EXCEPTION(test13_1, std::runtime_error); - CPPUNIT_TEST_EXCEPTION(test13_2, std::runtime_error); - CPPUNIT_TEST(test14); - CPPUNIT_TEST_SUITE_END(); - - private: - public: - void setUp() - { - } - - void tearDown() - { - } - - void test1() - { - Config* c1 = Config::makeConfig("./Columnstore.xml"); - string value; - value = c1->getConfig("Message", "Name"); - CPPUNIT_ASSERT(value == "Message"); - - value = c1->getConfig("Message", "xName"); - CPPUNIT_ASSERT(value.size() == 0); - Config::deleteInstanceMap(); - } - - void test2() - { - Config* c1 = Config::makeConfig("./yadayada.xml"); - string value; - - value = c1->getConfig("Message", "Name"); - CPPUNIT_ASSERT(value.size() == 0); - - value = c1->getConfig("Message", "xName"); - CPPUNIT_ASSERT(value.size() == 0); - - Config::deleteInstanceMap(); - } - - void test3() - { - Config* c1; - string value; - - for (int i = 0; i < 1000; i++) - { - c1 = Config::makeConfig("./Columnstore.xml"); - value = c1->getConfig("Message", "Name"); - assert(value == "Message"); - } - - Config::deleteInstanceMap(); - } - - void test4() - { - Config* c1 = Config::makeConfig("./Columnstore.xml"); - string value; - - value = c1->getConfig("SystemConfig", "SystemVersion"); - c1->setConfig("SystemConfig", "SystemVersion", "2.2.versionversionversion"); - value = c1->getConfig("SystemConfig", "SystemVersion"); - CPPUNIT_ASSERT(value == "2.2.versionversionversion"); - - ::unlink("./Columnstore.xml.new"); - c1->write("./Columnstore.xml.new"); - - value = c1->getConfig("SystemConfig", "SystemVersion"); - CPPUNIT_ASSERT(value == "2.2.versionversionversion"); - - c1->setConfig("SystemConfig", "SystemVersion1", "V1.x"); - value = c1->getConfig("SystemConfig", "SystemVersion1"); - CPPUNIT_ASSERT(value == "V1.x"); - - c1->setConfig("SystemConfig1", "SystemVersion1", "Vx.x"); - value = c1->getConfig("SystemConfig1", "SystemVersion1"); - CPPUNIT_ASSERT(value == "Vx.x"); - - c1->write("./Columnstore.xml.new"); - Config* c2 = Config::makeConfig("./Columnstore.xml.new"); - value = c2->getConfig("SystemConfig1", "SystemVersion1"); - CPPUNIT_ASSERT(value == "Vx.x"); - c2->setConfig("SystemConfig", "SystemVersion1", "V1.1"); - value = c2->getConfig("SystemConfig", "SystemVersion1"); - CPPUNIT_ASSERT(value == "V1.1"); - c2->write(); - Config::deleteInstanceMap(); - } - - void test5() - { - Config* c1 = Config::makeConfig("./Columnstore.xml"); - c1->write("/cantwritethis"); - Config::deleteInstanceMap(); - } - - void test6() - { - Config* c1 = Config::makeConfig("./XColumnstore.xml"); - // compiler warning...we won't actually get here - c1 = 0; - } - - void test7() - { - Config* c1 = Config::makeConfig("./Columnstore.xml"); - string s; - string n; - string v; - c1->setConfig(s, n, v); - Config::deleteInstanceMap(); - } - - void test8() - { - Config* c1 = Config::makeConfig("./Columnstore.xml"); - string s; - string n; - string v; - v = c1->getConfig(s, n); - Config::deleteInstanceMap(); - } - - void test9() - { - string value; - - Config* c1 = Config::makeConfig("./Columnstore.xml"); - Config* c2 = Config::makeConfig("./Columnstore.xml.new"); - - value = c1->getConfig("Message", "Name"); - CPPUNIT_ASSERT(value == "Message"); - - value = c2->getConfig("SystemConfig", "SystemVersion1"); - CPPUNIT_ASSERT(value == "V1.1"); - Config::deleteInstanceMap(); - } - - void test10() - { - string value; - - setenv("CALPONT_CONFIG_FILE", "./Columnstore.xml", 1); - Config* c1 = Config::makeConfig(); - - value = c1->getConfig("Message", "Name"); - CPPUNIT_ASSERT(value == "Message"); - Config::deleteInstanceMap(); - } - - void test11() - { - string value; - struct stat stat_buf; - struct utimbuf utime_buf; - - CPPUNIT_ASSERT(stat("./Columnstore.xml.new", &stat_buf) == 0); - - Config* c1 = Config::makeConfig("./Columnstore.xml.new"); - - value = c1->getConfig("Message", "Name"); - CPPUNIT_ASSERT(value == "Message"); - - utime_buf.actime = utime_buf.modtime = stat_buf.st_mtime + 1; - CPPUNIT_ASSERT(utime("./Columnstore.xml.new", &utime_buf) == 0); - - value = c1->getConfig("Message", "Name"); - CPPUNIT_ASSERT(value == "Message"); - Config::deleteInstanceMap(); - } - - void test12() - { - string value; - int64_t ival; - uint64_t uval; - - value = "10"; - ival = Config::fromText(value); - CPPUNIT_ASSERT(ival == 10); - uval = Config::uFromText(value); - CPPUNIT_ASSERT(uval == 10); - - value = "0x10"; - ival = Config::fromText(value); - CPPUNIT_ASSERT(ival == 0x10); - uval = Config::uFromText(value); - CPPUNIT_ASSERT(uval == 0x10); - - value = "010"; - ival = Config::fromText(value); - CPPUNIT_ASSERT(ival == 010); - uval = Config::uFromText(value); - CPPUNIT_ASSERT(uval == 010); - - value = "-10"; - ival = Config::fromText(value); - CPPUNIT_ASSERT(ival == -10); - - value = "10K"; - ival = Config::fromText(value); - CPPUNIT_ASSERT(ival == (10 * 1024)); - uval = Config::uFromText(value); - CPPUNIT_ASSERT(uval == (10 * 1024)); - - value = "10k"; - ival = Config::fromText(value); - CPPUNIT_ASSERT(ival == (10 * 1024)); - uval = Config::uFromText(value); - CPPUNIT_ASSERT(uval == (10 * 1024)); - - value = "10M"; - ival = Config::fromText(value); - CPPUNIT_ASSERT(ival == (10 * 1024 * 1024)); - uval = Config::uFromText(value); - CPPUNIT_ASSERT(uval == (10 * 1024 * 1024)); - - value = "10m"; - ival = Config::fromText(value); - CPPUNIT_ASSERT(ival == (10 * 1024 * 1024)); - uval = Config::uFromText(value); - CPPUNIT_ASSERT(uval == (10 * 1024 * 1024)); - - value = "10G"; - ival = Config::fromText(value); - CPPUNIT_ASSERT(ival == (10 * 1024 * 1024 * 1024LL)); - uval = Config::uFromText(value); - CPPUNIT_ASSERT(uval == (10 * 1024 * 1024 * 1024ULL)); - - value = "10g"; - ival = Config::fromText(value); - CPPUNIT_ASSERT(ival == (10 * 1024 * 1024 * 1024LL)); - uval = Config::uFromText(value); - CPPUNIT_ASSERT(uval == (10 * 1024 * 1024 * 1024ULL)); - - value = "10MB"; - ival = Config::fromText(value); - CPPUNIT_ASSERT(ival == (10 * 1024 * 1024)); - uval = Config::uFromText(value); - CPPUNIT_ASSERT(uval == (10 * 1024 * 1024)); - - value = "0x7afafafafafafafa"; - ival = Config::fromText(value); - CPPUNIT_ASSERT(ival == 0x7afafafafafafafaLL); - uval = Config::uFromText(value); - CPPUNIT_ASSERT(uval == 0x7afafafafafafafaULL); - - value = "-0x7afafafafafafafa"; - uval = Config::uFromText(value); - CPPUNIT_ASSERT(uval == 0x8505050505050506ULL); - - value = "-1"; - uval = Config::uFromText(value); - CPPUNIT_ASSERT(uval == 0xffffffffffffffffULL); - } - - void test13_1() - { - string value; - int64_t ival; - - value = "2.2MB"; // invalid char causes throw - ival = Config::fromText(value); - } - - void test13_2() - { - string value; - int64_t ival; - - value = "10,000"; // invalid char causes throw - ival = Config::fromText(value); - } - - void test14() - { - ByteStream bs; - ifstream ifs("./Columnstore.xml"); - ifs >> bs; - string id("."); - string value; - { - ConfigStream cs(bs, id); - value = cs.getConfig("Message", "Name"); - CPPUNIT_ASSERT(value == "Message"); - } - string bss(reinterpret_cast(bs.buf()), bs.length()); - { - ConfigStream cs(bss, id); - value = cs.getConfig("Message", "Name"); - CPPUNIT_ASSERT(value == "Message"); - } - { - ConfigStream cs(bss.c_str(), id); - value = cs.getConfig("Message", "Name"); - CPPUNIT_ASSERT(value == "Message"); - } - } -}; - -CPPUNIT_TEST_SUITE_REGISTRATION(ConfigFileTest); - -#include -#include - -int main(int argc, char** argv) -{ - CppUnit::TextUi::TestRunner runner; - CppUnit::TestFactoryRegistry& registry = CppUnit::TestFactoryRegistry::getRegistry(); - runner.addTest(registry.makeTest()); - bool wasSuccessful = runner.run("", false); - return (wasSuccessful ? 0 : 1); -} diff --git a/utils/configcpp/tdriver2.cpp b/utils/configcpp/tdriver2.cpp deleted file mode 100644 index f6d00cc49..000000000 --- a/utils/configcpp/tdriver2.cpp +++ /dev/null @@ -1,123 +0,0 @@ -/* Copyright (C) 2014 InfiniDB, Inc. - - This program is free software; you can redistribute it and/or - modify it under the terms of the GNU General Public License - as published by the Free Software Foundation; version 2 of - the License. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU General Public License for more details. - - You should have received a copy of the GNU General Public License - along with this program; if not, write to the Free Software - Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, - MA 02110-1301, USA. */ - -#include -#include -#include -#include -#include -#include -#include -using namespace std; - -#include -using namespace boost; - -#include - -#include "configcpp.h" -#include "writeonce.h" -using namespace config; - -namespace -{ -const string cf("./woparms.dat"); -} - -class WOConfigFileTest : public CppUnit::TestFixture -{ - CPPUNIT_TEST_SUITE(WOConfigFileTest); - - CPPUNIT_TEST(test1); - CPPUNIT_TEST_EXCEPTION(test2, std::runtime_error); - CPPUNIT_TEST_EXCEPTION(test3, std::runtime_error); - CPPUNIT_TEST_EXCEPTION(test4, std::runtime_error); - CPPUNIT_TEST(test5); - - CPPUNIT_TEST_SUITE_END(); - - private: - public: - void setUp() - { - unlink(cf.c_str()); - } - - void tearDown() - { - unlink(cf.c_str()); - } - - void test1() - { - WriteOnceConfig woc(cf); - CPPUNIT_ASSERT(woc.owns("SystemConfig", "DBRootCount")); - CPPUNIT_ASSERT(woc.owns("SystemConfig", "DBRMRoot")); - - CPPUNIT_ASSERT(!woc.owns("dummy", "dummy")); - - int vali; - - woc.setConfig("SystemConfig", "DBRootCount", "10"); - vali = Config::fromText(woc.getConfig("SystemConfig", "DBRootCount")); - CPPUNIT_ASSERT(vali == 10); - - WriteOnceConfig woc2(cf.c_str()); - vali = Config::fromText(woc2.getConfig("SystemConfig", "DBRootCount")); - CPPUNIT_ASSERT(vali == 10); - } - - void test2() - { - WriteOnceConfig woc(cf); - woc.getConfig("dummy", "dummy"); - } - - void test3() - { - WriteOnceConfig woc(cf); - woc.setConfig("dummy", "dummy", "100"); - } - - void test4() - { - WriteOnceConfig woc(cf); - woc.setConfig("SystemConfig", "DBRootCount", "10"); - woc.setConfig("SystemConfig", "DBRootCount", "11"); - } - - void test5() - { - WriteOnceConfig woc(cf); - woc.setConfig("SystemConfig", "DBRootCount", "10"); - woc.setConfig("SystemConfig", "DBRootCount", "11", true); - } -}; - -CPPUNIT_TEST_SUITE_REGISTRATION(WOConfigFileTest); - -#include -#include - -int main(int argc, char** argv) -{ - CppUnit::TextUi::TestRunner runner; - CppUnit::TestFactoryRegistry& registry = CppUnit::TestFactoryRegistry::getRegistry(); - runner.addTest(registry.makeTest()); - bool wasSuccessful = runner.run("", false); - return (wasSuccessful ? 0 : 1); -} diff --git a/utils/configcpp/xmlparser.cpp b/utils/configcpp/xmlparser.cpp index a12e5944f..29753165e 100644 --- a/utils/configcpp/xmlparser.cpp +++ b/utils/configcpp/xmlparser.cpp @@ -19,7 +19,9 @@ * $Id$ * ******************************************************************************************/ -#include "mcsconfig.h" +// #include "mcsconfig.h" + +#include "basic/string_utils.h" #include #include @@ -27,7 +29,6 @@ #include #include -#include "collation.h" using namespace std; #include "xmlparser.h" diff --git a/utils/dataconvert/CMakeLists.txt b/utils/dataconvert/CMakeLists.txt index f41813740..5bb417b9c 100644 --- a/utils/dataconvert/CMakeLists.txt +++ b/utils/dataconvert/CMakeLists.txt @@ -5,11 +5,4 @@ include_directories(${ENGINE_COMMON_INCLUDES}) set(dataconvert_LIB_SRCS dataconvert.cpp) columnstore_library(dataconvert ${dataconvert_LIB_SRCS}) - -add_dependencies(dataconvert loggingcpp) - -install( - TARGETS dataconvert - DESTINATION ${ENGINE_LIBDIR} - COMPONENT columnstore-engine -) +columnstore_link(dataconvert PUBLIC loggingcpp datatypes) diff --git a/utils/dataconvert/dataconvert.cpp b/utils/dataconvert/dataconvert.cpp index 5deffaa8f..44f671447 100644 --- a/utils/dataconvert/dataconvert.cpp +++ b/utils/dataconvert/dataconvert.cpp @@ -38,6 +38,8 @@ using namespace boost::algorithm; #include "calpontselectexecutionplan.h" #include "columnresult.h" +#include "basic/string_utils.h" + #include "joblisttypes.h" #define DATACONVERT_DLLEXPORT diff --git a/utils/dataconvert/tdriver.cpp b/utils/dataconvert/tdriver.cpp deleted file mode 100644 index 6d011d676..000000000 --- a/utils/dataconvert/tdriver.cpp +++ /dev/null @@ -1,1375 +0,0 @@ -/* Copyright (C) 2014 InfiniDB, Inc. - - This program is free software; you can redistribute it and/or - modify it under the terms of the GNU General Public License - as published by the Free Software Foundation; version 2 of - the License. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU General Public License for more details. - - You should have received a copy of the GNU General Public License - along with this program; if not, write to the Free Software - Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, - MA 02110-1301, USA. */ - -/** - * $Id: tdriver.cpp 3495 2013-01-21 14:09:51Z rdempsey $ - */ - -#include -#include -using namespace std; - -#include -#include -using namespace boost; - -#include - -#include "calpontsystemcatalog.h" -using namespace execplan; - -#include "dataconvert.h" -using namespace dataconvert; - -class DataConvertTest : public CppUnit::TestFixture -{ - CPPUNIT_TEST_SUITE(DataConvertTest); - - CPPUNIT_TEST(dc1); - CPPUNIT_TEST(dc2); - CPPUNIT_TEST(dc3); - CPPUNIT_TEST(dc4); - CPPUNIT_TEST(dc5); - CPPUNIT_TEST(dc6); - CPPUNIT_TEST(dc7); - CPPUNIT_TEST(dc8); - CPPUNIT_TEST(dc_datetest); - CPPUNIT_TEST(dc_datetostrtest); - CPPUNIT_TEST(dc_datetostr1test); - CPPUNIT_TEST(dc_datetomysqlinttest); - CPPUNIT_TEST(dc_datetest_perf); - CPPUNIT_TEST(dc_datetimetest); - CPPUNIT_TEST(dc_datetimetostrtest); - CPPUNIT_TEST(dc_datetimetostr1test); - CPPUNIT_TEST(dc_datetimetest_perf); - CPPUNIT_TEST(dc_datetimetomysqlinttest); - CPPUNIT_TEST(dc_datevalidtest); - CPPUNIT_TEST(dc9); - CPPUNIT_TEST(dc10); - CPPUNIT_TEST_SUITE_END(); - - private: - DataConvert converter; - boost::any anyval; - CalpontSystemCatalog::ColType ct; - string data; - - public: - void setUp() - { - } - - void tearDown() - { - } - - /* from calpontsystemcatalog.h - struct ColType - { - int colWidth; - ConstraintType constraintType; - ColDataType colDataType; - DictOID ddn; - boost::any defaultValue; - int colPosition; // temporally put here. may need to have ColInfo struct later - int scale; //number after decimal points - int precision; - }; - colWidth - constraintType - colDataType - defaultValue - scale - precision - */ - - // mid-range positive integers - void dc1() - { - ct.colWidth = 1; - ct.constraintType = CalpontSystemCatalog::NO_CONSTRAINT; - ct.colDataType = CalpontSystemCatalog::TINYINT; - ct.scale = -1; - ct.precision = 2; - data = "12"; - bool pushWarning; - - anyval = converter.convertColumnData(ct, data, pushWarning, false); - - CPPUNIT_ASSERT(anyval.type() == typeid(char)); - char tinyintval = any_cast(anyval); - CPPUNIT_ASSERT(tinyintval == 12); - - ct.colWidth = 2; - ct.constraintType = CalpontSystemCatalog::NO_CONSTRAINT; - ct.colDataType = CalpontSystemCatalog::SMALLINT; - ct.scale = -1; - ct.precision = 4; - data = "1234"; - - anyval = converter.convertColumnData(ct, data, pushWarning, false); - - CPPUNIT_ASSERT(anyval.type() == typeid(int16_t)); - int16_t smallintval = any_cast(anyval); - CPPUNIT_ASSERT(smallintval == 1234); - - ct.colWidth = 4; - ct.constraintType = CalpontSystemCatalog::NO_CONSTRAINT; - ct.colDataType = CalpontSystemCatalog::INT; - ct.scale = -1; - ct.precision = 9; - data = "12345678"; - - anyval = converter.convertColumnData(ct, data, pushWarning, false); - - CPPUNIT_ASSERT(anyval.type() == typeid(int32_t)); - int32_t intval = any_cast(anyval); - CPPUNIT_ASSERT(intval == 12345678); - - ct.colWidth = 4; - ct.constraintType = CalpontSystemCatalog::NO_CONSTRAINT; - ct.colDataType = CalpontSystemCatalog::MEDINT; - ct.scale = -1; - ct.precision = 9; - data = "12345678"; - - anyval = converter.convertColumnData(ct, data, pushWarning, false); - - CPPUNIT_ASSERT(anyval.type() == typeid(int32_t)); - intval = any_cast(anyval); - CPPUNIT_ASSERT(intval == 12345678); - - ct.colWidth = 8; - ct.constraintType = CalpontSystemCatalog::NO_CONSTRAINT; - ct.colDataType = CalpontSystemCatalog::BIGINT; - ct.scale = -1; - ct.precision = 18; - data = "1234567890123456"; - - anyval = converter.convertColumnData(ct, data, pushWarning, false); - - CPPUNIT_ASSERT(anyval.type() == typeid(long long)); - int64_t bigintval = static_cast(any_cast(anyval)); - CPPUNIT_ASSERT(bigintval == 1234567890123456LL); - } - - // mid-range negative integers - void dc2() - { - ct.colWidth = 1; - ct.constraintType = CalpontSystemCatalog::NO_CONSTRAINT; - ct.colDataType = CalpontSystemCatalog::TINYINT; - ct.scale = -1; - ct.precision = 2; - data = "-12"; - bool pushWarning; - - anyval = converter.convertColumnData(ct, data, pushWarning, false); - - CPPUNIT_ASSERT(anyval.type() == typeid(char)); - char tinyintval = any_cast(anyval); - CPPUNIT_ASSERT(tinyintval == -12); - - ct.colWidth = 2; - ct.constraintType = CalpontSystemCatalog::NO_CONSTRAINT; - ct.colDataType = CalpontSystemCatalog::SMALLINT; - ct.scale = -1; - ct.precision = 4; - data = "-1234"; - - anyval = converter.convertColumnData(ct, data, pushWarning, false); - - CPPUNIT_ASSERT(anyval.type() == typeid(int16_t)); - int16_t smallintval = any_cast(anyval); - CPPUNIT_ASSERT(smallintval == -1234); - - ct.colWidth = 4; - ct.constraintType = CalpontSystemCatalog::NO_CONSTRAINT; - ct.colDataType = CalpontSystemCatalog::INT; - ct.scale = -1; - ct.precision = 9; - data = "-12345678"; - - anyval = converter.convertColumnData(ct, data, pushWarning, false); - - CPPUNIT_ASSERT(anyval.type() == typeid(int32_t)); - int32_t intval = any_cast(anyval); - CPPUNIT_ASSERT(intval == -12345678); - - ct.colWidth = 4; - ct.constraintType = CalpontSystemCatalog::NO_CONSTRAINT; - ct.colDataType = CalpontSystemCatalog::MEDINT; - ct.scale = -1; - ct.precision = 9; - data = "-12345678"; - - anyval = converter.convertColumnData(ct, data, pushWarning, false); - - CPPUNIT_ASSERT(anyval.type() == typeid(int32_t)); - intval = any_cast(anyval); - CPPUNIT_ASSERT(intval == -12345678); - - ct.colWidth = 8; - ct.constraintType = CalpontSystemCatalog::NO_CONSTRAINT; - ct.colDataType = CalpontSystemCatalog::BIGINT; - ct.scale = -1; - ct.precision = 18; - data = "-1234567890123456"; - - anyval = converter.convertColumnData(ct, data, pushWarning, false); - - CPPUNIT_ASSERT(anyval.type() == typeid(long long)); - int64_t bigintval = static_cast(any_cast(anyval)); - CPPUNIT_ASSERT(bigintval == -1234567890123456LL); - } - - // limit positive integers - void dc3() - { - ct.colWidth = 1; - ct.constraintType = CalpontSystemCatalog::NO_CONSTRAINT; - ct.colDataType = CalpontSystemCatalog::TINYINT; - ct.scale = -1; - ct.precision = 3; - data = "127"; - bool pushWarning; - - anyval = converter.convertColumnData(ct, data, pushWarning, false); - - CPPUNIT_ASSERT(anyval.type() == typeid(char)); - char tinyintval = any_cast(anyval); - CPPUNIT_ASSERT(tinyintval == 127); - - ct.colWidth = 2; - ct.constraintType = CalpontSystemCatalog::NO_CONSTRAINT; - ct.colDataType = CalpontSystemCatalog::SMALLINT; - ct.scale = -1; - ct.precision = 5; - data = "32767"; - - anyval = converter.convertColumnData(ct, data, pushWarning, false); - - CPPUNIT_ASSERT(anyval.type() == typeid(int16_t)); - int16_t smallintval = any_cast(anyval); - CPPUNIT_ASSERT(smallintval == 32767); - - ct.colWidth = 4; - ct.constraintType = CalpontSystemCatalog::NO_CONSTRAINT; - ct.colDataType = CalpontSystemCatalog::INT; - ct.scale = -1; - ct.precision = 10; - data = "2147483647"; - - anyval = converter.convertColumnData(ct, data, pushWarning, false); - - CPPUNIT_ASSERT(anyval.type() == typeid(int32_t)); - int32_t intval = any_cast(anyval); - CPPUNIT_ASSERT(intval == 2147483647); - - ct.colWidth = 4; - ct.constraintType = CalpontSystemCatalog::NO_CONSTRAINT; - ct.colDataType = CalpontSystemCatalog::MEDINT; - ct.scale = -1; - ct.precision = 10; - data = "2147483647"; - - anyval = converter.convertColumnData(ct, data, pushWarning, false); - - CPPUNIT_ASSERT(anyval.type() == typeid(int32_t)); - intval = any_cast(anyval); - CPPUNIT_ASSERT(intval == 2147483647); - - ct.colWidth = 8; - ct.constraintType = CalpontSystemCatalog::NO_CONSTRAINT; - ct.colDataType = CalpontSystemCatalog::BIGINT; - ct.scale = 0; - ct.precision = 18; - data = "9223372036854775807"; - - anyval = converter.convertColumnData(ct, data, pushWarning, false); - - CPPUNIT_ASSERT(anyval.type() == typeid(long long)); - int64_t bigintval = static_cast(any_cast(anyval)); - CPPUNIT_ASSERT(bigintval == 9223372036854775807LL); - } - - // limit negative integers - void dc4() - { - ct.colWidth = 1; - ct.constraintType = CalpontSystemCatalog::NO_CONSTRAINT; - ct.colDataType = CalpontSystemCatalog::TINYINT; - ct.scale = -1; - ct.precision = 3; - data = "-126"; - bool pushWarning; - - anyval = converter.convertColumnData(ct, data, pushWarning, false); - - CPPUNIT_ASSERT(anyval.type() == typeid(char)); - char tinyintval = any_cast(anyval); - CPPUNIT_ASSERT(tinyintval == -126); - - ct.colWidth = 2; - ct.constraintType = CalpontSystemCatalog::NO_CONSTRAINT; - ct.colDataType = CalpontSystemCatalog::SMALLINT; - ct.scale = -1; - ct.precision = 4; - data = "-32766"; - - anyval = converter.convertColumnData(ct, data, pushWarning, false); - - CPPUNIT_ASSERT(anyval.type() == typeid(int16_t)); - int16_t smallintval = any_cast(anyval); - CPPUNIT_ASSERT(smallintval == -32766); - - ct.colWidth = 4; - ct.constraintType = CalpontSystemCatalog::NO_CONSTRAINT; - ct.colDataType = CalpontSystemCatalog::INT; - ct.scale = -1; - ct.precision = 9; - data = "-2147483646"; - - anyval = converter.convertColumnData(ct, data, pushWarning, false); - - CPPUNIT_ASSERT(anyval.type() == typeid(int32_t)); - int32_t intval = any_cast(anyval); - CPPUNIT_ASSERT(intval == -2147483646); - - ct.colWidth = 4; - ct.constraintType = CalpontSystemCatalog::NO_CONSTRAINT; - ct.colDataType = CalpontSystemCatalog::MEDINT; - ct.scale = -1; - ct.precision = 9; - data = "-2147483646"; - - anyval = converter.convertColumnData(ct, data, pushWarning, false); - - CPPUNIT_ASSERT(anyval.type() == typeid(int32_t)); - intval = any_cast(anyval); - CPPUNIT_ASSERT(intval == -2147483646); - - ct.colWidth = 8; - ct.constraintType = CalpontSystemCatalog::NO_CONSTRAINT; - ct.colDataType = CalpontSystemCatalog::BIGINT; - ct.scale = -1; - ct.precision = 18; - data = "-9223372036854775806"; - - anyval = converter.convertColumnData(ct, data, pushWarning, false); - - CPPUNIT_ASSERT(anyval.type() == typeid(long long)); - int64_t bigintval = static_cast(any_cast(anyval)); - CPPUNIT_ASSERT(bigintval == -9223372036854775806LL); - } - - // mid-range positive decimals (2 fixed points) - void dc5() - { - ct.colWidth = 2; - ct.constraintType = CalpontSystemCatalog::NO_CONSTRAINT; - ct.colDataType = CalpontSystemCatalog::SMALLINT; - ct.scale = 2; - ct.precision = 4; - bool pushWarning; - - data = "12.34"; - - anyval = converter.convertColumnData(ct, data, pushWarning, false); - - CPPUNIT_ASSERT(anyval.type() == typeid(int16_t)); - int16_t smallintval = any_cast(anyval); - CPPUNIT_ASSERT(smallintval == 1234); - - data = "2.345"; - - anyval = converter.convertColumnData(ct, data, pushWarning, false, true); - - CPPUNIT_ASSERT(anyval.type() == typeid(int16_t)); - smallintval = any_cast(anyval); - CPPUNIT_ASSERT(smallintval == 234); - - data = "(2.34)"; - - anyval = converter.convertColumnData(ct, data, pushWarning, false); - - CPPUNIT_ASSERT(anyval.type() == typeid(int16_t)); - smallintval = any_cast(anyval); - CPPUNIT_ASSERT(smallintval == 234); - - data = "2.3"; - - anyval = converter.convertColumnData(ct, data, pushWarning, false); - - CPPUNIT_ASSERT(anyval.type() == typeid(int16_t)); - smallintval = any_cast(anyval); - CPPUNIT_ASSERT(smallintval == 230); - - data = "2."; - - anyval = converter.convertColumnData(ct, data, pushWarning, false); - - CPPUNIT_ASSERT(anyval.type() == typeid(int16_t)); - smallintval = any_cast(anyval); - CPPUNIT_ASSERT(smallintval == 200); - - data = "2"; - - anyval = converter.convertColumnData(ct, data, pushWarning, false); - - CPPUNIT_ASSERT(anyval.type() == typeid(int16_t)); - smallintval = any_cast(anyval); - CPPUNIT_ASSERT(smallintval == 200); - - ct.colWidth = 4; - ct.constraintType = CalpontSystemCatalog::NO_CONSTRAINT; - ct.colDataType = CalpontSystemCatalog::INT; - ct.scale = 6; - ct.precision = 9; - data = "0.000481"; - - anyval = converter.convertColumnData(ct, data, pushWarning, false); - - CPPUNIT_ASSERT(anyval.type() == typeid(int32_t)); - int32_t intval = any_cast(anyval); - CPPUNIT_ASSERT(intval == 481); - - ct.colWidth = 4; - ct.constraintType = CalpontSystemCatalog::NO_CONSTRAINT; - ct.colDataType = CalpontSystemCatalog::INT; - ct.scale = 2; - ct.precision = 9; - data = "1234567.89"; - - anyval = converter.convertColumnData(ct, data, pushWarning, false); - - CPPUNIT_ASSERT(anyval.type() == typeid(int32_t)); - intval = any_cast(anyval); - CPPUNIT_ASSERT(intval == 123456789); - - data = "34567.8999"; - - anyval = converter.convertColumnData(ct, data, pushWarning, false, true); - - CPPUNIT_ASSERT(anyval.type() == typeid(int32_t)); - intval = any_cast(anyval); - CPPUNIT_ASSERT(intval == 3456789); - - data = "(34567.89)"; - - anyval = converter.convertColumnData(ct, data, pushWarning, false); - - CPPUNIT_ASSERT(anyval.type() == typeid(int32_t)); - intval = any_cast(anyval); - CPPUNIT_ASSERT(intval == 3456789); - - data = "34567.8"; - - anyval = converter.convertColumnData(ct, data, pushWarning, false); - - CPPUNIT_ASSERT(anyval.type() == typeid(int32_t)); - intval = any_cast(anyval); - CPPUNIT_ASSERT(intval == 3456780); - - data = "34567."; - - anyval = converter.convertColumnData(ct, data, pushWarning, false); - - CPPUNIT_ASSERT(anyval.type() == typeid(int32_t)); - intval = any_cast(anyval); - CPPUNIT_ASSERT(intval == 3456700); - - data = "34567"; - - anyval = converter.convertColumnData(ct, data, pushWarning, false); - - CPPUNIT_ASSERT(anyval.type() == typeid(int32_t)); - intval = any_cast(anyval); - CPPUNIT_ASSERT(intval == 3456700); - - ct.colWidth = 8; - ct.constraintType = CalpontSystemCatalog::NO_CONSTRAINT; - ct.colDataType = CalpontSystemCatalog::BIGINT; - ct.scale = 2; - ct.precision = 18; - - data = "12345678901234.56"; - - anyval = converter.convertColumnData(ct, data, pushWarning, false); - - CPPUNIT_ASSERT(anyval.type() == typeid(long long)); - int64_t bigintval = static_cast(any_cast(anyval)); - CPPUNIT_ASSERT(bigintval == 1234567890123456LL); - - data = "45678901234.56093"; - - anyval = converter.convertColumnData(ct, data, pushWarning, false); - - CPPUNIT_ASSERT(anyval.type() == typeid(long long)); - bigintval = static_cast(any_cast(anyval)); - CPPUNIT_ASSERT(bigintval == 4567890123456LL); - - data = "(45678901234.56)"; - - anyval = converter.convertColumnData(ct, data, pushWarning, false); - - CPPUNIT_ASSERT(anyval.type() == typeid(long long)); - bigintval = static_cast(any_cast(anyval)); - CPPUNIT_ASSERT(bigintval == 4567890123456LL); - - data = "12345678901234.5"; - - anyval = converter.convertColumnData(ct, data, pushWarning, false); - - CPPUNIT_ASSERT(anyval.type() == typeid(long long)); - bigintval = static_cast(any_cast(anyval)); - CPPUNIT_ASSERT(bigintval == 1234567890123450LL); - - data = "12345678901234."; - - anyval = converter.convertColumnData(ct, data, pushWarning, false); - - CPPUNIT_ASSERT(anyval.type() == typeid(long long)); - bigintval = static_cast(any_cast(anyval)); - CPPUNIT_ASSERT(bigintval == 1234567890123400LL); - - data = "12345678901234"; - - anyval = converter.convertColumnData(ct, data, pushWarning, false); - - CPPUNIT_ASSERT(anyval.type() == typeid(long long)); - bigintval = static_cast(any_cast(anyval)); - CPPUNIT_ASSERT(bigintval == 1234567890123400LL); - } - - // mid-range positive decimals (fixed-point) - void dc6() - { - ct.colWidth = 8; - ct.constraintType = CalpontSystemCatalog::NO_CONSTRAINT; - ct.colDataType = CalpontSystemCatalog::BIGINT; - ct.scale = 1; - ct.precision = 18; - bool pushWarning; - - data = "123456789012345.6"; - - anyval = converter.convertColumnData(ct, data, pushWarning, false); - - CPPUNIT_ASSERT(anyval.type() == typeid(long long)); - int64_t bigintval = static_cast(any_cast(anyval)); - CPPUNIT_ASSERT(bigintval == 1234567890123456LL); - - ct.scale = 3; - - data = "1234567890123.456"; - - anyval = converter.convertColumnData(ct, data, pushWarning, false); - - CPPUNIT_ASSERT(anyval.type() == typeid(long long)); - bigintval = static_cast(any_cast(anyval)); - CPPUNIT_ASSERT(bigintval == 1234567890123456LL); - - ct.scale = 4; - - data = "123456789012.3456"; - - anyval = converter.convertColumnData(ct, data, pushWarning, false); - - CPPUNIT_ASSERT(anyval.type() == typeid(long long)); - bigintval = static_cast(any_cast(anyval)); - CPPUNIT_ASSERT(bigintval == 1234567890123456LL); - } - - // mid-range negative decimals (2 fixed points) - void dc7() - { - ct.colWidth = 2; - ct.constraintType = CalpontSystemCatalog::NO_CONSTRAINT; - ct.colDataType = CalpontSystemCatalog::SMALLINT; - ct.scale = 2; - ct.precision = 4; - bool pushWarning; - - data = "-12.34"; - - anyval = converter.convertColumnData(ct, data, pushWarning, false); - - CPPUNIT_ASSERT(anyval.type() == typeid(int16_t)); - int16_t smallintval = any_cast(anyval); - CPPUNIT_ASSERT(smallintval == -1234); - - data = "-2.345"; - - anyval = converter.convertColumnData(ct, data, pushWarning, false, true); - - CPPUNIT_ASSERT(anyval.type() == typeid(int16_t)); - smallintval = any_cast(anyval); - CPPUNIT_ASSERT(smallintval == -234); - - data = "(-2.34)"; - - anyval = converter.convertColumnData(ct, data, pushWarning, false); - - CPPUNIT_ASSERT(anyval.type() == typeid(int16_t)); - smallintval = any_cast(anyval); - CPPUNIT_ASSERT(smallintval == -234); - - data = "-2.3"; - - anyval = converter.convertColumnData(ct, data, pushWarning, false); - - CPPUNIT_ASSERT(anyval.type() == typeid(int16_t)); - smallintval = any_cast(anyval); - CPPUNIT_ASSERT(smallintval == -230); - - data = "-2."; - - anyval = converter.convertColumnData(ct, data, pushWarning, false); - - CPPUNIT_ASSERT(anyval.type() == typeid(int16_t)); - smallintval = any_cast(anyval); - CPPUNIT_ASSERT(smallintval == -200); - - ct.colWidth = 4; - ct.constraintType = CalpontSystemCatalog::NO_CONSTRAINT; - ct.colDataType = CalpontSystemCatalog::INT; - ct.scale = 2; - ct.precision = 9; - data = "-1234567.89"; - - anyval = converter.convertColumnData(ct, data, pushWarning, false); - - CPPUNIT_ASSERT(anyval.type() == typeid(int32_t)); - int32_t intval = any_cast(anyval); - CPPUNIT_ASSERT(intval == -123456789); - - data = "-34567.8999"; - - anyval = converter.convertColumnData(ct, data, pushWarning, false, true); - - CPPUNIT_ASSERT(anyval.type() == typeid(int32_t)); - intval = any_cast(anyval); - CPPUNIT_ASSERT(intval == -3456789); - - data = "(-34567.89)"; - - anyval = converter.convertColumnData(ct, data, pushWarning, false); - - CPPUNIT_ASSERT(anyval.type() == typeid(int32_t)); - intval = any_cast(anyval); - CPPUNIT_ASSERT(intval == -3456789); - - data = "-34567.8"; - - anyval = converter.convertColumnData(ct, data, pushWarning, false); - - CPPUNIT_ASSERT(anyval.type() == typeid(int32_t)); - intval = any_cast(anyval); - CPPUNIT_ASSERT(intval == -3456780); - - data = "-34567."; - - anyval = converter.convertColumnData(ct, data, pushWarning, false); - - CPPUNIT_ASSERT(anyval.type() == typeid(int32_t)); - intval = any_cast(anyval); - CPPUNIT_ASSERT(intval == -3456700); - - ct.colWidth = 8; - ct.constraintType = CalpontSystemCatalog::NO_CONSTRAINT; - ct.colDataType = CalpontSystemCatalog::BIGINT; - ct.scale = 2; - ct.precision = 18; - - data = "-12345678901234.56"; - - anyval = converter.convertColumnData(ct, data, pushWarning, false); - - CPPUNIT_ASSERT(anyval.type() == typeid(long long)); - int64_t bigintval = static_cast(any_cast(anyval)); - CPPUNIT_ASSERT(bigintval == -1234567890123456LL); - - data = "-45678901234.56093"; - - anyval = converter.convertColumnData(ct, data, pushWarning, false); - - CPPUNIT_ASSERT(anyval.type() == typeid(long long)); - bigintval = static_cast(any_cast(anyval)); - CPPUNIT_ASSERT(bigintval == -4567890123456LL); - - data = "(-45678901234.56)"; - - anyval = converter.convertColumnData(ct, data, pushWarning, false); - - CPPUNIT_ASSERT(anyval.type() == typeid(long long)); - bigintval = static_cast(any_cast(anyval)); - CPPUNIT_ASSERT(bigintval == -4567890123456LL); - - data = "-12345678901234.5"; - - anyval = converter.convertColumnData(ct, data, pushWarning, false); - - CPPUNIT_ASSERT(anyval.type() == typeid(long long)); - bigintval = static_cast(any_cast(anyval)); - CPPUNIT_ASSERT(bigintval == -1234567890123450LL); - - data = "-12345678901234."; - - anyval = converter.convertColumnData(ct, data, pushWarning, false); - - CPPUNIT_ASSERT(anyval.type() == typeid(long long)); - bigintval = static_cast(any_cast(anyval)); - CPPUNIT_ASSERT(bigintval == -1234567890123400LL); - } - - void dc8() - { - data = "2007-05-11"; - int status = 0; - uint32_t anyval; - anyval = converter.convertColumnDate(data.c_str(), CALPONTDATE_ENUM, status, data.length()); - CPPUNIT_ASSERT(anyval == 131551998); - std::string backToString = converter.dateToString(anyval); - CPPUNIT_ASSERT(backToString == data); - } - - void show_date_debugs(const Date& d1) - { - cout << "d1.spare = " << d1.spare << endl; - cout << "d1.day = " << d1.day << endl; - cout << "d1.month = " << d1.month << endl; - cout << "d1.year = " << d1.year << endl; - } - -#include -#include - - /* Subtract the `struct timeval' values X and Y, - storing the result in RESULT. - Return 1 if the difference is negative, otherwise 0. */ - - int timeval_subtract(struct timeval* result, struct timeval* x, struct timeval* y) - { - /* Perform the carry for the later subtraction by updating y. */ - if (x->tv_usec < y->tv_usec) - { - int nsec = (y->tv_usec - x->tv_usec) / 1000000 + 1; - y->tv_usec -= 1000000 * nsec; - y->tv_sec += nsec; - } - - if (x->tv_usec - y->tv_usec > 1000000) - { - int nsec = (x->tv_usec - y->tv_usec) / 1000000; - y->tv_usec += 1000000 * nsec; - y->tv_sec -= nsec; - } - - /* Compute the time remaining to wait. - tv_usec is certainly positive. */ - result->tv_sec = x->tv_sec - y->tv_sec; - result->tv_usec = x->tv_usec - y->tv_usec; - - /* Return 1 if result is negative. */ - return x->tv_sec < y->tv_sec; - } - - void dc_datetest_perf() - { - data = "2007-05-11"; - - struct timeval tv1, tv2, delta; - const int iters = 10000000; - - // test the performance of the default date conversion with full format support - gettimeofday(&tv1, 0); - - for (int i = 0; i < iters; i++) - { - converter.stringToDate(data); - } - - gettimeofday(&tv2, 0); - - timeval_subtract(&delta, &tv2, &tv1); - cout << iters << " operations performed in " << delta.tv_sec << "." << delta.tv_usec << " seconds" - << " using the default stringToDate method" << endl; - unsigned long total_usec = delta.tv_sec * 1000000 + delta.tv_usec; - CPPUNIT_ASSERT(total_usec < 3000000); - - // now run the performance of the "fast path" date conversions to check relative performance - int status = 0; - gettimeofday(&tv1, 0); - - for (int i = 0; i < iters; i++) - { - converter.convertColumnDate(data.c_str(), CALPONTDATE_ENUM, status, data.length()); - } - - gettimeofday(&tv2, 0); - - timeval_subtract(&delta, &tv2, &tv1); - cout << iters << " operations performed in " << delta.tv_sec << "." << delta.tv_usec << " seconds" - << " using the \"fastpath\" convertColumnDate method" << endl; - } - - void dc_datetimetest_perf() - { - data = "2007-05-11 10:30:45"; - - struct timeval tv1, tv2, delta; - const int iters = 10000000; - - // test the performance of the default datetime conversion with full format support - gettimeofday(&tv1, 0); - - for (int i = 0; i < iters; i++) - { - converter.stringToDatetime(data); - } - - gettimeofday(&tv2, 0); - - timeval_subtract(&delta, &tv2, &tv1); - cout << iters << " operations performed in " << delta.tv_sec << "." << delta.tv_usec << " seconds" - << " using the default stringToDatetime method" << endl; - unsigned long total_usec = delta.tv_sec * 1000000 + delta.tv_usec; - CPPUNIT_ASSERT(total_usec < 3000000); - - // now run the performance of the "fast path" datetime conversions to check relative performance - int status = 0; - gettimeofday(&tv1, 0); - - for (int i = 0; i < iters; i++) - { - converter.convertColumnDatetime(data.c_str(), CALPONTDATETIME_ENUM, status, data.length()); - } - - gettimeofday(&tv2, 0); - - timeval_subtract(&delta, &tv2, &tv1); - cout << iters << " operations performed in " << delta.tv_sec << "." << delta.tv_usec << " seconds" - << " using the \"fastpath\" convertColumnDatetime method" << endl; - } - - void dc_datetest() - { - struct DateCheck - { - const char* str; - Date date; - }; - - DateCheck date_tests[] = { - {"0000-00-00", Date()}, // bogus date - {"2000-01-01", Date(2000, 1, 1)}, - {"2000-00-01", Date()}, // 0 not a valid monht - {"2000-Jan-01", Date()}, // no conversion from month names - {"2000-January-01", Date()}, // no conversion from month names - {"2000-Jac-01", Date()}, // no conversion from month names - {"2000-jan-01", Date()}, // no conversion from month names - {"20000101", Date(2000, 1, 1)}, - {"2000", Date()}, // no month or day - {"200001", Date()}, // no day - {"000101", Date(2000, 1, 1)}, - {"01-01-2000", Date()}, // can not swap field order - //{ "100-10-1", Date(100,10,1) }, // boost min date is 1400-1-1 - {"10-100-1", Date()}, // invalid month=100 - //{ "5-02-28", Date(5,2,28) }, // boost min date is 1400-1-1 - {"2000-02-28", Date(2000, 2, 28)}, - {"2000-02-29", Date(2000, 2, 29)}, - {"2004-02-29", Date(2004, 2, 29)}, - {"2003-12-31", Date(2003, 12, 31)}, - {"2003-12-32", Date()}, // no Dec. 32nd - {"03-11-30", Date(2003, 11, 30)}, - {"07-07-31", Date(2007, 7, 31)}, - {"2005-06-23 ", Date(2005, 6, 23)}, - {"2005-06-23 tomorrow", Date()}, // no trailing text - {"2005-06-23tomorrow ", Date()}, // no trailing text - {"2006-011-29 ", Date(2006, 11, 29)}, - {"2006-11-029 ", Date(2006, 11, 29)}, - {"10000-01-02 ", Date()}, - {"1999-0102 ", Date()}, - }; - - for (unsigned i = 0; i < sizeof(date_tests) / sizeof(DateCheck); i++) - { - uint32_t val = converter.stringToDate(date_tests[i].str); - bool check = (((uint32_t)(val & 0xFFFFFFC0)) == - ((*(reinterpret_cast(&date_tests[i].date))) & 0xFFFFFFC0)); - - if (!check) - { - printf("For input \"%s\", check 0x%08x vs 0x%08x\n", date_tests[i].str, (val & 0xFFFFFFC0), - (*(reinterpret_cast(&date_tests[i].date))) & 0xFFFFFFC0); - } - - CPPUNIT_ASSERT(check); - } - } - - void dc_datetostrtest() - { - struct DateCheck - { - const char* str; - Date date; - }; - - DateCheck date_tests[] = {{"2000-01-01", Date(2000, 1, 1)}, {"0100-10-01", Date(100, 10, 1)}, - {"0005-02-28", Date(5, 2, 28)}, {"2000-02-28", Date(2000, 2, 28)}, - {"2000-02-29", Date(2000, 2, 29)}, {"2004-02-29", Date(2004, 2, 29)}, - {"2003-12-31", Date(2003, 12, 31)}, {"2003-11-30", Date(2003, 11, 30)}, - {"2007-07-31", Date(2007, 7, 31)}, {"2005-06-23", Date(2005, 6, 23)}, - {"2006-11-29", Date(2006, 11, 29)}}; - - for (unsigned i = 0; i < sizeof(date_tests) / sizeof(DateCheck); i++) - { - string val = converter.dateToString(*(reinterpret_cast(&date_tests[i].date))); - - bool check = (val == date_tests[i].str); - - if (!check) - { - cout << "\"" << date_tests[i].str << "\" != \"" << val << endl; - } - - CPPUNIT_ASSERT(check); - } - } - - void dc_datetostr1test() - { - struct DateCheck - { - const char* str; - Date date; - }; - - DateCheck date_tests[] = { - {"20000101", Date(2000, 1, 1)}, {"01001001", Date(100, 10, 1)}, {"00050228", Date(5, 2, 28)}, - {"20000228", Date(2000, 2, 28)}, {"20000229", Date(2000, 2, 29)}, {"20040229", Date(2004, 2, 29)}, - {"20031231", Date(2003, 12, 31)}, {"20031130", Date(2003, 11, 30)}, {"20070731", Date(2007, 7, 31)}, - {"20050623", Date(2005, 6, 23)}, {"20061129", Date(2006, 11, 29)}}; - - for (unsigned i = 0; i < sizeof(date_tests) / sizeof(DateCheck); i++) - { - string val = converter.dateToString1(*(reinterpret_cast(&date_tests[i].date))); - - bool check = (val == date_tests[i].str); - - if (!check) - { - cout << "\"" << date_tests[i].str << "\" != \"" << val << endl; - } - - CPPUNIT_ASSERT(check); - } - } - - void dc_datetomysqlinttest() - { - struct DateCheck - { - int32_t intval; - Date date; - }; - - DateCheck date_tests[] = { - {20000101, Date(2000, 1, 1)}, {1001001, Date(100, 10, 1)}, {50228, Date(5, 2, 28)}, - {20000228, Date(2000, 2, 28)}, {20000229, Date(2000, 2, 29)}, {20040229, Date(2004, 2, 29)}, - {20031231, Date(2003, 12, 31)}, {20031130, Date(2003, 11, 30)}, {20070731, Date(2007, 7, 31)}, - {20050623, Date(2005, 6, 23)}, {20061129, Date(2006, 11, 29)}}; - - for (unsigned i = 0; i < sizeof(date_tests) / sizeof(DateCheck); i++) - { - int64_t val = date_tests[i].date.convertToMySQLint(); - - bool check = (val == date_tests[i].intval); - - if (!check) - { - cout << val << " != " << date_tests[i].intval << endl; - } - - CPPUNIT_ASSERT(check); - } - } - - void show_datetime_debugs(const DateTime& d1) - { - cout << "d1.day = " << setw(10) << d1.day << endl; - cout << "d1.month = " << setw(10) << d1.month << endl; - cout << "d1.year = " << setw(10) << d1.year << endl; - cout << "d1.hour = " << setw(10) << d1.hour << endl; - cout << "d1.minute = " << setw(10) << d1.minute << endl; - cout << "d1.second = " << setw(10) << d1.second << endl; - cout << "d1.msecond = " << setw(10) << d1.msecond << endl; - } - - void dc_datetimetest() - { - struct DateTimeCheck - { - const char* str; - DateTime dtime; - }; - - DateTimeCheck dtime_tests[] = { - {"2010-03-31 10:30:45.000PM", DateTime()}, // AM/PM not supported - {"2010-03-31 10:30:45.000 PM", DateTime()}, // AM/PM not supported - {"2010-03-31 10:30:45PM", DateTime()}, // AM/PM not allowed without microsecs - {"2010-03-31 10:30:45AM", DateTime()}, // AM/PM not allowed without microsecs - {"2010-03-31 10:30:45 PM", DateTime()}, // AM/PM not allowed without microsecs - {"2008-01-01 10:00", DateTime(2008, 1, 1, 10, 0, 0, 0)}, - {"2008-01-01 10:30:02", DateTime(2008, 1, 1, 10, 30, 2, 0)}, - {"2008-01-01 10:45:01.1111", DateTime(2008, 1, 1, 10, 45, 1, 111100)}, - {"2008-01-01 10:45:02.222222", DateTime(2008, 1, 1, 10, 45, 2, 222222)}, - {"2008-01-01 10:45:02.2222223", DateTime()}, // microsecs field too long - {"2008-01-01 10", DateTime(2008, 1, 1, 10, 0, 0, 0)}, - {"200903311030", DateTime()}, // Interprets as YYMMDDHHMMSS so hour(31) is invalid - {"20090331103017", DateTime(2009, 03, 31, 10, 30, 17, 0)}, - // { "2009-04-28 11.34.55", DateTime(2009,4,28,11,34,55,0) }, // no more support for arbitrary - // separators - {"20100228T134501", DateTime(2010, 02, 28, 13, 45, 1, 0)}, - {"10:45:00 2010-01-31", DateTime()}, // can't switch time/date order - {"0000-00-00 00:00:00", DateTime()}, // invalid date/time - {"2009-12-31 23:59:56", DateTime(2009, 12, 31, 23, 59, 56, 0)} // invalid date/time - }; - - for (unsigned i = 0; i < sizeof(dtime_tests) / sizeof(DateTimeCheck); i++) - { - u_int64_t val = converter.stringToDatetime(dtime_tests[i].str); - - // this is a little screwy, but the way things are working is that stringToDateTime - // returns a -1 when the stringToDatetime operation fails. The default value for - // datetime, though is -2, so if we get back the error value then we need to mask - // off the LSB - if (val == 0xFFFFFFFFFFFFFFFF) - val = (val & 0xFFFFFFFFFFFFFFFE); - - bool check = (val == *(reinterpret_cast(&dtime_tests[i].dtime))); - - if (!check) - { - printf("For input \"%s\", check 0x%016lx vs 0x%016lx\n", dtime_tests[i].str, val, - *(reinterpret_cast(&dtime_tests[i].dtime))); - } - - if (!check) - show_datetime_debugs(DateTime(val)); - - CPPUNIT_ASSERT(check); - } - } - - void dc_datetimetostrtest() - { - struct DateTimeCheck - { - const char* str; - DateTime dtime; - }; - - DateTimeCheck dtime_tests[] = { - {"2008-01-01 10:00:00", DateTime(2008, 1, 1, 10, 0, 0, 0)}, - {"2008-01-01 10:30:02", DateTime(2008, 1, 1, 10, 30, 2, 0)}, - {"2008-01-01 10:45:01", DateTime(2008, 1, 1, 10, 45, 1, 111100)}, - {"2008-01-01 10:45:02", DateTime(2008, 1, 1, 10, 45, 2, 222222)}, - {"2008-01-01 10:00:00", DateTime(2008, 1, 1, 10, 0, 0, 0)}, - {"2009-03-31 10:30:17", DateTime(2009, 03, 31, 10, 30, 17, 0)}, - {"2009-04-28 11:34:55", DateTime(2009, 4, 28, 11, 34, 55, 0)}, - {"2010-02-28 13:45:01", DateTime(2010, 02, 28, 13, 45, 1, 0)}, - {"2009-12-31 23:59:56", DateTime(2009, 12, 31, 23, 59, 56, 0)} // invalid date/time - }; - - for (unsigned i = 0; i < sizeof(dtime_tests) / sizeof(DateTimeCheck); i++) - { - string val = converter.datetimeToString(*(reinterpret_cast(&dtime_tests[i].dtime))); - - bool check = (val == dtime_tests[i].str); - - if (!check) - { - cout << "\"" << dtime_tests[i].str << "\" != \"" << val << endl; - } - - CPPUNIT_ASSERT(check); - } - } - - void dc_datetimetostr1test() - { - struct DateTimeCheck - { - const char* str; - DateTime dtime; - }; - - DateTimeCheck dtime_tests[] = {{"20080101100000000000", DateTime(2008, 1, 1, 10, 0, 0, 0)}, - {"20080101103002000000", DateTime(2008, 1, 1, 10, 30, 2, 0)}, - {"20080101104501111100", DateTime(2008, 1, 1, 10, 45, 1, 111100)}, - {"20080101104502222222", DateTime(2008, 1, 1, 10, 45, 2, 222222)}, - {"20080101100000000000", DateTime(2008, 1, 1, 10, 0, 0, 0)}, - {"20090331103017000000", DateTime(2009, 03, 31, 10, 30, 17, 0)}, - {"20090428113455000000", DateTime(2009, 4, 28, 11, 34, 55, 0)}, - {"20100228134501000000", DateTime(2010, 02, 28, 13, 45, 1, 0)}, - {"20091231235956000000", DateTime(2009, 12, 31, 23, 59, 56, 0)}}; - - for (unsigned i = 0; i < sizeof(dtime_tests) / sizeof(DateTimeCheck); i++) - { - string val = converter.datetimeToString1(*(reinterpret_cast(&dtime_tests[i].dtime))); - - bool check = (val == dtime_tests[i].str); - - if (!check) - { - cout << "\"" << dtime_tests[i].str << "\" != \"" << val << endl; - } - - CPPUNIT_ASSERT(check); - } - } - - void dc_datetimetomysqlinttest() - { - struct DateTimeCheck - { - int64_t intval; - DateTime dtime; - }; - - DateTimeCheck dtime_tests[] = {{20080101100000, DateTime(2008, 1, 1, 10, 0, 0, 0)}, - {20080101103002, DateTime(2008, 1, 1, 10, 30, 2, 0)}, - {20080101104501, DateTime(2008, 1, 1, 10, 45, 1, 111100)}, - {20080101104502, DateTime(2008, 1, 1, 10, 45, 2, 222222)}, - {20080101100000, DateTime(2008, 1, 1, 10, 0, 0, 0)}, - {20090331103017, DateTime(2009, 03, 31, 10, 30, 17, 0)}, - {20090428113455, DateTime(2009, 4, 28, 11, 34, 55, 0)}, - {20100228134501, DateTime(2010, 02, 28, 13, 45, 1, 0)}, - {20091231235956, DateTime(2009, 12, 31, 23, 59, 56, 0)}}; - - for (unsigned i = 0; i < sizeof(dtime_tests) / sizeof(DateTimeCheck); i++) - { - int64_t val = dtime_tests[i].dtime.convertToMySQLint(); - - bool check = (val == dtime_tests[i].intval); - - if (!check) - { - cout << val << " != " << dtime_tests[i].intval << endl; - } - - CPPUNIT_ASSERT(check); - } - } - - void check_date(int year, int month, int day, bool valid, bool boost_only = false) - { - bool idbval = isDateValid(day, month, year); - - bool boostval; - - try - { - boost::gregorian::date d(year, month, day); - boostval = true; - } - catch (...) - { - boostval = false; - } - - bool check = boost_only ? (idbval == boostval) : (idbval == valid && ((boostval == idbval) || boostval)); - - if (!check) - { - cout << "(" << year << "," << month << "," << day << "), expected:" << valid << ", idb=" << idbval - << ", boost=" << boostval << endl; - } - - CPPUNIT_ASSERT(check); - } - - void dc_datevalidtest() - { - struct Check - { - int year; - int month; - int day; - bool valid; - }; - - Check tests[] = { - {2000, 2, 29, true}, {2001, 2, 29, false}, {1900, 2, 29, false}, {1900, 2, 28, true}, - {1399, 12, 31, false}, {10000, 1, 1, false}, {2000, 0, 29, false}, {2000, 13, 29, false}, - {2000, 1, 0, false}, {2000, 1, 32, false}, - }; - - // predefined tests shown above - for (unsigned i = 0; i < sizeof(tests) / sizeof(Check); i++) - { - check_date(tests[i].year, tests[i].month, tests[i].day, tests[i].valid); - } - - // randomly generated date tests - for (unsigned i = 0; i < 1000000; i++) - { - int year = random() % 9000 + 1200; // some spillage on either side - - if (year == 10000) - --year; // year 10000 is not good because boost calls it good but we don't want to - - int month = random() % 14; // some spillage on either side - int day = random() % 33; // some spillage on either side - check_date(year, month, day, true, true); - } - } - - // Testing equality of floating point numbers is not good practice, rather make sure - // the conversion is within a tolerance (usually +/- 1 LSD). - inline bool inTolerance(double d, double a, double t) - { - return ((d > (a - t)) && (d < (a + t))); - } - - // Float tests 6 digits of accuracy - void dc9() - { - ct.colWidth = 4; - ct.constraintType = CalpontSystemCatalog::NO_CONSTRAINT; - ct.colDataType = CalpontSystemCatalog::FLOAT; - ct.scale = 0; - ct.precision = 4; - bool pushWarning; - - data = "0.123456"; - - anyval = converter.convertColumnData(ct, data, pushWarning, false); - - CPPUNIT_ASSERT(anyval.type() == typeid(float)); - float floatval = any_cast(anyval); - CPPUNIT_ASSERT(inTolerance(floatval, 0.123456, 0.000001)); - - data = "3456.01"; - - anyval = converter.convertColumnData(ct, data, pushWarning, false); - - CPPUNIT_ASSERT(anyval.type() == typeid(float)); - floatval = any_cast(anyval); - CPPUNIT_ASSERT(inTolerance(floatval, 3456.01, 0.01)); - - data = "(3456.01)"; - - anyval = converter.convertColumnData(ct, data, pushWarning, false); - - CPPUNIT_ASSERT(anyval.type() == typeid(float)); - floatval = any_cast(anyval); - CPPUNIT_ASSERT(inTolerance(floatval, 3456.01, 0.01)); - - data = "6.02214E+23"; - - anyval = converter.convertColumnData(ct, data, pushWarning, false); - - CPPUNIT_ASSERT(anyval.type() == typeid(float)); - floatval = any_cast(anyval); - CPPUNIT_ASSERT(inTolerance(floatval, 6.02214E+23, 0.00001E+23)); - - data = "1.60217E-19"; - - anyval = converter.convertColumnData(ct, data, pushWarning, false); - - CPPUNIT_ASSERT(anyval.type() == typeid(float)); - floatval = any_cast(anyval); - CPPUNIT_ASSERT(inTolerance(floatval, 1.60217E-19, 0.00001E-19)); - } - - // Double tests 15 digits of accuracy - void dc10() - { - ct.colWidth = 8; - ct.constraintType = CalpontSystemCatalog::NO_CONSTRAINT; - ct.colDataType = CalpontSystemCatalog::DOUBLE; - ct.scale = 0; - ct.precision = 4; - bool pushWarning; - - data = "0.123456789012345"; - - anyval = converter.convertColumnData(ct, data, pushWarning, false); - - CPPUNIT_ASSERT(anyval.type() == typeid(double)); - double doubleval = any_cast(anyval); - CPPUNIT_ASSERT(inTolerance(doubleval, 0.123456789012345, 0.000000000000001)); - - data = "123456.000000001"; - - anyval = converter.convertColumnData(ct, data, pushWarning, false); - - CPPUNIT_ASSERT(anyval.type() == typeid(double)); - doubleval = any_cast(anyval); - CPPUNIT_ASSERT(inTolerance(doubleval, 123456.000000001, 0.000000001)); - - data = "(123456.000000001)"; - - anyval = converter.convertColumnData(ct, data, pushWarning, false); - - CPPUNIT_ASSERT(anyval.type() == typeid(double)); - doubleval = any_cast(anyval); - CPPUNIT_ASSERT(inTolerance(doubleval, 123456.000000001, 0.000000001)); - - data = "6.02214179000000E+23"; - - anyval = converter.convertColumnData(ct, data, pushWarning, false); - - CPPUNIT_ASSERT(anyval.type() == typeid(double)); - doubleval = any_cast(anyval); - CPPUNIT_ASSERT(inTolerance(doubleval, 6.02214179000000E+23, 0.00000000000001E+23)); - - data = "1.60217653140000E-19"; - - anyval = converter.convertColumnData(ct, data, pushWarning, false); - - CPPUNIT_ASSERT(anyval.type() == typeid(double)); - doubleval = any_cast(anyval); - CPPUNIT_ASSERT(inTolerance(doubleval, 1.60217653140000E-19, 0.00000000000001E-19)); - - data = "3.14159265358979"; - - anyval = converter.convertColumnData(ct, data, pushWarning, false); - - CPPUNIT_ASSERT(anyval.type() == typeid(double)); - doubleval = any_cast(anyval); - CPPUNIT_ASSERT(inTolerance(doubleval, 3.14159265358979, 0.00000000000001)); - } -}; - -CPPUNIT_TEST_SUITE_REGISTRATION(DataConvertTest); - -#include -#include - -int main(int argc, char** argv) -{ - CppUnit::TextUi::TestRunner runner; - CppUnit::TestFactoryRegistry& registry = CppUnit::TestFactoryRegistry::getRegistry(); - runner.addTest(registry.makeTest()); - bool wasSuccessful = runner.run("", false); - return (wasSuccessful ? 0 : 1); -} diff --git a/utils/ddlcleanup/CMakeLists.txt b/utils/ddlcleanup/CMakeLists.txt index 78756a2e7..c813fe428 100644 --- a/utils/ddlcleanup/CMakeLists.txt +++ b/utils/ddlcleanup/CMakeLists.txt @@ -1,17 +1,5 @@ include_directories(${ENGINE_COMMON_INCLUDES}) -# ########## next target ############### - set(ddlcleanuputil_LIB_SRCS ddlcleanuputil.cpp) - columnstore_library(ddlcleanuputil ${ddlcleanuputil_LIB_SRCS}) - -add_dependencies(ddlcleanuputil loggingcpp) - -target_link_libraries(ddlcleanuputil ${NETSNMP_LIBRARIES}) - -install( - TARGETS ddlcleanuputil - DESTINATION ${ENGINE_LIBDIR} - COMPONENT columnstore-engine -) +columnstore_link(ddlcleanuputil PRIVATE loggingcpp ${NETSNMP_LIBRARIES}) diff --git a/utils/funcexp/CMakeLists.txt b/utils/funcexp/CMakeLists.txt index d8e54ab88..6f1c758a7 100644 --- a/utils/funcexp/CMakeLists.txt +++ b/utils/funcexp/CMakeLists.txt @@ -143,13 +143,13 @@ set(funcexp_LIB_SRCS ) columnstore_library(funcexp ${funcexp_LIB_SRCS}) - -add_dependencies(funcexp loggingcpp) - -target_link_libraries(funcexp ${NETSNMP_LIBRARIES} pron pcre2-8 ${MARIADB_STRING_LIBS}) - -install( - TARGETS funcexp - DESTINATION ${ENGINE_LIBDIR} - COMPONENT columnstore-engine +columnstore_link( + funcexp + PRIVATE + pcre2-8 + pron + loggingcpp + dataconvert + ${MARIADB_STRING_LIBS} + ${NETSNMP_LIBRARIES} ) diff --git a/utils/idbdatafile/CMakeLists.txt b/utils/idbdatafile/CMakeLists.txt index b22023242..06a1ebf44 100644 --- a/utils/idbdatafile/CMakeLists.txt +++ b/utils/idbdatafile/CMakeLists.txt @@ -14,13 +14,4 @@ set(idbdatafile_LIB_SRCS ) columnstore_library(idbdatafile ${idbdatafile_LIB_SRCS}) - -target_link_libraries(idbdatafile ${NETSNMP_LIBRARIES} ${ENGINE_OAM_LIBS} boost_filesystem boost_system) - -target_compile_definitions(idbdatafile PUBLIC BOOST_NO_CXX11_SCOPED_ENUMS) - -install( - TARGETS idbdatafile - DESTINATION ${ENGINE_LIBDIR} - COMPONENT columnstore-engine -) +columnstore_link(idbdatafile PRIVATE ${NETSNMP_LIBRARIES} ${ENGINE_OAM_LIBS} boost_filesystem boost_system) diff --git a/utils/joiner/CMakeLists.txt b/utils/joiner/CMakeLists.txt index 35f241a46..0df5c60f7 100644 --- a/utils/joiner/CMakeLists.txt +++ b/utils/joiner/CMakeLists.txt @@ -5,11 +5,4 @@ include_directories(${ENGINE_COMMON_INCLUDES}) set(joiner_LIB_SRCS tuplejoiner.cpp joinpartition.cpp) columnstore_library(joiner ${joiner_LIB_SRCS}) - -add_dependencies(joiner loggingcpp) - -install( - TARGETS joiner - DESTINATION ${ENGINE_LIBDIR} - COMPONENT columnstore-engine -) +columnstore_link(joiner PRIVATE loggingcpp) diff --git a/utils/libmarias3/CMakeLists.txt b/utils/libmarias3/CMakeLists.txt index 21b8fc7a9..bc62e13a6 100644 --- a/utils/libmarias3/CMakeLists.txt +++ b/utils/libmarias3/CMakeLists.txt @@ -15,10 +15,10 @@ set(S3_SOURCES ${S3API_DIR}/src/assume_role.c ) -add_library(marias3 SHARED ${S3_SOURCES}) - -target_link_libraries(marias3 curl m) +columnstore_shared_library(marias3 ${S3_SOURCES}) +columnstore_link(marias3 curl m) include_directories(${CMAKE_CURRENT_SOURCE_DIR} ${S3API_DIR}) + add_definitions(-D_GNU_SOURCE) set(S3API_DEPS diff --git a/utils/libmysql_client/CMakeLists.txt b/utils/libmysql_client/CMakeLists.txt index 86254147a..aef36f51c 100644 --- a/utils/libmysql_client/CMakeLists.txt +++ b/utils/libmysql_client/CMakeLists.txt @@ -3,17 +3,6 @@ add_definitions(-DMYSQL_SERVICE_THD_TIMEZONE_INCLUDED) include_directories(${ENGINE_COMMON_INCLUDES}) -# ########## next target ############### - set(libmysql_client_LIB_SRCS libmysql_client.cpp) - columnstore_library(libmysql_client ${libmysql_client_LIB_SRCS}) -target_link_libraries(libmysql_client ${MARIADB_CLIENT_LIBS}) - -add_dependencies(libmysql_client loggingcpp) - -install( - TARGETS libmysql_client - DESTINATION ${ENGINE_LIBDIR} - COMPONENT columnstore-engine -) +columnstore_link(libmysql_client PRIVATE loggingcpp ${MARIADB_CLIENT_LIBS}) diff --git a/utils/loggingcpp/CMakeLists.txt b/utils/loggingcpp/CMakeLists.txt index 3faab8775..3617c4c96 100644 --- a/utils/loggingcpp/CMakeLists.txt +++ b/utils/loggingcpp/CMakeLists.txt @@ -31,16 +31,8 @@ columnstore_library( ${CMAKE_CURRENT_BINARY_DIR}/messageids.h ${CMAKE_CURRENT_BINARY_DIR}/errorids.h ) -add_dependencies(loggingcpp external_boost) -install( - TARGETS loggingcpp - DESTINATION ${ENGINE_LIBDIR} - COMPONENT columnstore-engine -) +columnstore_link(loggingcpp configcpp boost_filesystem) -install( - FILES MessageFile.txt ErrorMessage.txt - DESTINATION ${ENGINE_SYSCONFDIR}/columnstore - COMPONENT columnstore-engine -) +columnstore_install_file(MessageFile.txt ${ENGINE_SYSCONFDIR}/columnstore) +columnstore_install_file(ErrorMessage.txt ${ENGINE_SYSCONFDIR}/columnstore) diff --git a/utils/loggingcpp/message.cpp b/utils/loggingcpp/message.cpp index 765c1bd47..33e41b7f0 100644 --- a/utils/loggingcpp/message.cpp +++ b/utils/loggingcpp/message.cpp @@ -29,7 +29,6 @@ #include using namespace std; - #include #include using namespace boost; @@ -42,7 +41,6 @@ using namespace config; #include "installdir.h" #include "format.h" -#include "mcs_int128.h" namespace { @@ -129,11 +127,6 @@ void Message::Args::add(uint64_t u64) fArgs.push_back(u64); } -void Message::Args::add(int128_t i128) -{ - fArgs.push_back(datatypes::TSInt128(i128).toString()); -} - void Message::Args::add(const string& s) { fArgs.push_back(s); diff --git a/utils/messageqcpp/CMakeLists.txt b/utils/messageqcpp/CMakeLists.txt index 598824c5a..527e9395a 100644 --- a/utils/messageqcpp/CMakeLists.txt +++ b/utils/messageqcpp/CMakeLists.txt @@ -1,21 +1,16 @@ include_directories(${ENGINE_COMMON_INCLUDES}) -# ########## next target ############### - set(messageqcpp_LIB_SRCS + bytestream.cpp + bytestreampool.cpp + compressed_iss.cpp + inetstreamsocket.cpp + iosocket.cpp messagequeue.cpp messagequeuepool.cpp - bytestream.cpp - socketparms.cpp - inetstreamsocket.cpp samenodepseudosocket.cpp - iosocket.cpp - compressed_iss.cpp - bytestreampool.cpp + socketparms.cpp ) -add_library(messageqcpp STATIC ${messageqcpp_LIB_SRCS}) - -add_dependencies(messageqcpp loggingcpp) - -# We don't isntall static library install(TARGETS messageqcpp DESTINATION ${ENGINE_LIBDIR} COMPONENT columnstore-engine) +columnstore_static_library(messageqcpp ${messageqcpp_LIB_SRCS}) +columnstore_link(messageqcpp PUBLIC loggingcpp compress) diff --git a/utils/multicast/CMakeLists.txt b/utils/multicast/CMakeLists.txt deleted file mode 100644 index 7d5016336..000000000 --- a/utils/multicast/CMakeLists.txt +++ /dev/null @@ -1,33 +0,0 @@ -# -# Not used -# - -# original Makefile.am contents follow: - -# Copyright (C) 2014 InfiniDB, Inc. -# -# This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public -# License as published by the Free Software Foundation; version 2 of the License. -# -# This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied -# warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License along with this program; if not, write to the Free -# Software Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. -# -# $Id$ Process this file with automake to produce Makefile.in -# -# AM_CPPFLAGS = $(idb_common_includes) $(idb_cppflags) AM_CFLAGS = $(idb_cflags) AM_CXXFLAGS = $(idb_cxxflags) -# AM_LDFLAGS = -version-info 1:0:0 $(idb_ldflags) lib_LTLIBRARIES = libmulticast.la libmulticast_la_SOURCES = -# multicast.cpp include_HEADERS = multicast.h -# -# test: -# -# coverage: -# -# leakcheck: -# -# docs: -# -# bootstrap: install-data-am -# diff --git a/utils/multicast/impl.cpp b/utils/multicast/impl.cpp deleted file mode 100644 index 09ca6f854..000000000 --- a/utils/multicast/impl.cpp +++ /dev/null @@ -1,3820 +0,0 @@ -/* Copyright (C) 2014 InfiniDB, Inc. - - This program is free software; you can redistribute it and/or - modify it under the terms of the GNU General Public License - as published by the Free Software Foundation; version 2 of - the License. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU General Public License for more details. - - You should have received a copy of the GNU General Public License - along with this program; if not, write to the Free Software - Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, - MA 02110-1301, USA. */ - -// $Id: impl.cpp 3495 2013-01-21 14:09:51Z rdempsey $ - -/* This code is based on udpcast-20090830. Most of the source code in that release - contains no copyright or licensing notices at all. The exception is fec.c, which - is not used here. The udpcast website, http://udpcast.linux.lu/, implies that - the source is covered under GPL. */ - -#include -#include -#include -#include -#include -#include -#include -using namespace std; - -#include "bytestream.h" -using namespace messageqcpp; - -#include "udp-sender.h" -#include "udpc-protoc.h" -#include "util.h" -#include "mc_fifo.h" - -#include "impl.h" - -struct participantsDb -{ - int nrParticipants; - - struct clientDesc - { - struct sockaddr_in addr; - int used; - int capabilities; - unsigned int rcvbuf; - } clientTable[MAX_CLIENTS]; -}; -struct produconsum -{ - unsigned int size; - volatile unsigned int produced; - unsigned int consumed; - volatile int atEnd; - pthread_mutex_t mutex; - volatile int consumerIsWaiting; - pthread_cond_t cond; - const char* name; -}; - -struct stats -{ - int fd; - struct timeval lastPrinted; - long statPeriod; - int printUncompressedPos; -}; - -struct sender_stats -{ - FILE* log; - unsigned long long totalBytes; - unsigned long long retransmissions; - int clNo; - unsigned long periodBytes; - struct timeval periodStart; - long bwPeriod; - struct stats s; -}; - -struct receiver_stats -{ - struct timeval tv_start; - int bytesOrig; - long long totalBytes; - int timerStarted; - struct stats s; -}; - -#define SLICEMAGIC 0x41424344 - -typedef struct slice -{ - int base; /* base address of slice in buffer */ - int sliceNo; - int bytes; /* bytes in slice */ - int nextBlock; /* index of next buffer to be transmitted */ - enum slice_state - { - SLICE_FREE, /* free slice, and in the queue of free slices */ - SLICE_NEW, /* newly allocated. FEC calculation and first - * transmission */ - SLICE_XMITTED, /* transmitted */ - SLICE_ACKED, /* acknowledged (if applicable) */ - SLICE_PRE_FREE, /* no longer used, but not returned to queue */ - SLICE_RECEIVING, - SLICE_DONE, - }; - volatile enum slice_state state; - char rxmitMap[MAX_SLICE_SIZE / BITS_PER_CHAR]; - /* blocks to be retransmitted */ - - char isXmittedMap[MAX_SLICE_SIZE / BITS_PER_CHAR]; - /* blocks which have already been retransmitted during this round*/ - - int rxmitId; /* used to distinguish among several retransmission - * requests, so that we can easily discard answers to "old" - * requests */ - - /* This structure is used to keep track of clients who answered, and - * to make the reqack message - */ - struct reqackBm - { - struct reqack ra; - char readySet[MAX_CLIENTS / BITS_PER_CHAR]; /* who is already ok? */ - } sl_reqack; - - char answeredSet[MAX_CLIENTS / BITS_PER_CHAR]; /* who answered at all? */ - - int nrReady; /* number of participants who are ready */ - int nrAnswered; /* number of participants who answered; */ - int needRxmit; /* does this need retransmission? */ - int lastGoodBlock; /* last good block of slice (i.e. last block having not - * needed retransmission */ - - int lastReqack; /* last req ack sent (debug) */ -#ifdef BB_FEATURE_UDPCAST_FEC - unsigned char* fec_data; -#endif - int magic; - int blocksTransferred; /* blocks transferred during this slice */ - int dataBlocksTransferred; /* data blocks transferred during this slice */ - struct retransmit retransmit; - int freePos; /* where the next data part will be stored to */ - int bytesKnown; /* is number of bytes known yet? */ - short missing_data_blocks[MAX_FEC_INTERLEAVE]; -} * slice_t; - -#define QUEUE_SIZE 256 - -struct returnChannel -{ - pthread_t thread; /* message receiving thread */ - int rcvSock; /* socket on which we receive the messages */ - produconsum_t incoming; /* where to enqueue incoming messages */ - produconsum_t freeSpace; /* free space */ - struct - { - int clNo; /* client number */ - union message msg; /* its message */ - } q[QUEUE_SIZE]; - struct net_config* config; - participantsDb_t participantsDb; -}; -#define NR_SLICES 2 - -typedef struct senderState -{ - struct returnChannel rc; - struct fifo* fifo; - - struct net_config* config; - sender_stats_t stats; - int socket; - - struct slice slices[NR_SLICES]; - - produconsum_t free_slices_pc; - - unsigned char* fec_data; - pthread_t fec_thread; - produconsum_t fec_data_pc; -} * sender_state_t; - -struct clientState -{ - struct fifo* fifo; - struct client_config* client_config; - struct net_config* net_config; - union serverDataMsg Msg; - - struct msghdr data_hdr; - - /* pre-prepared messages */ - struct iovec data_iov[2]; - - struct slice* currentSlice; - int currentSliceNo; - receiver_stats_t stats; - - produconsum_t free_slices_pc; - struct slice slices[NR_SLICES]; - - /* Completely received slices */ - int receivedPtr; - int receivedSliceNo; - -#ifdef BB_FEATURE_UDPCAST_FEC - int use_fec; /* do we use forward error correction ? */ -#endif - produconsum_t fec_data_pc; - struct slice* fec_slices[NR_SLICES]; - pthread_t fec_thread; - - /* A reservoir of free blocks for FEC */ - produconsum_t freeBlocks_pc; - unsigned char** blockAddresses; /* adresses of blocks in local queue */ - - unsigned char** localBlockAddresses; - /* local blocks: freed FEC blocks after we - * have received the corresponding data */ - int localPos; - - unsigned char* blockData; - unsigned char* nextBlock; - - int endReached; /* end of transmission reached: - 0: transmission in progress - 2: network transmission _and_ FEC - processing finished - */ - - int netEndReached; /* In case of a FEC transmission; network - * transmission finished. This is needed to avoid - * a race condition, where the receiver thread would - * already prepare to wait for more data, at the same - * time that the FEC would set endReached. To avoid - * this, we do a select without timeout before - * receiving the last few packets, so that if the - * race condition strikes, we have a way to protect - * against - */ - - int selectedFd; - - int promptPrinted; /* Has "Press any key..." prompt already been printed */ - -#ifdef BB_FEATURE_UDPCAST_FEC - fec_code_t fec_code; -#endif -}; - -#define S_UCAST socks[0] -#define S_BCAST socks[1] -#define S_MCAST_CTRL socks[2] -#define S_MCAST_DATA socks[3] - -#define SSEND(x) SEND(client_config->S_UCAST, x, client_config->serverAddr) - -/** - * Receiver will passively listen to sender. Works best if sender runs - * in async mode - */ -#define FLAG_PASSIVE 0x0010 - -/** - * Do not write file synchronously - */ -#define FLAG_NOSYNC 0x0040 - -/* - * Don't ask for keyboard input on receiver end. - */ -#define FLAG_NOKBD 0x0080 - -/** - * Do write file synchronously - */ -#define FLAG_SYNC 0x0100 - -namespace -{ -int udpc_isFullDuplex(int s, const char* ifname) -{ -#ifdef ETHTOOL_GLINK - struct ifreq ifr; - struct ethtool_cmd ecmd; - - ecmd.cmd = ETHTOOL_GSET; - - strncpy(ifr.ifr_name, ifname, sizeof(ifr.ifr_name) - 1); - ifr.ifr_data = (char*)&ecmd; - - if (ioctl(s, SIOCETHTOOL, &ifr) == -1) - { - /* Operation not supported */ - return -1; - } - else - { - return ecmd.duplex; - } - -#else - return -1; -#endif -} - -#define getSinAddr(addr) (((struct sockaddr_in*)addr)->sin_addr) - -int udpc_ipIsZero(struct sockaddr_in* ip) -{ - return getSinAddr(ip).s_addr == 0; -} - -int hasLink(int s, const char* ifname) -{ -#ifdef ETHTOOL_GLINK - struct ifreq ifr; - struct ethtool_value edata; - - edata.cmd = ETHTOOL_GLINK; - - strncpy(ifr.ifr_name, ifname, sizeof(ifr.ifr_name) - 1); - ifr.ifr_data = (char*)&edata; - - if (ioctl(s, SIOCETHTOOL, &ifr) == -1) - { - /* Operation not supported */ - return -1; - } - else - { - return edata.data; - } - -#else - return -1; -#endif -} - -#define INET_ATON(a, i) inet_aton(a, i) - -int udpc_doSend(int s, void* message, size_t len, struct sockaddr_in* to) -{ - /* flprintf("sent: %08x %d\n", *(int*) message, len);*/ -#ifdef LOSSTEST - loseSendPacket(); -#endif - return sendto(s, message, len, 0, (struct sockaddr*)to, sizeof(*to)); -} - -void udpc_copyToMessage(unsigned char* dst, struct sockaddr_in* src) -{ - memcpy(dst, (char*)&((struct sockaddr_in*)src)->sin_addr, sizeof(struct in_addr)); -} - -void udpc_sendHello(struct net_config* net_config, int sock, int streaming) -{ - // cerr << "sending hello..." << endl; - struct hello hello; - - /* send hello message */ - if (streaming) - hello.opCode = htons(CMD_HELLO_STREAMING); - else - hello.opCode = htons(CMD_HELLO); - - hello.reserved = 0; - hello.capabilities = htonl(net_config->capabilities); - udpc_copyToMessage(hello.mcastAddr, &net_config->dataMcastAddr); - hello.blockSize = htons(net_config->blockSize); - // TODO: FIXME - // rgWaitAll(net_config, sock, net_config->controlMcastAddr.sin_addr.s_addr, sizeof(hello)); - BCAST_CONTROL(sock, hello); -} - -char* udpc_getIpString(struct sockaddr_in* addr, char* buffer) -{ - long iaddr = htonl(getSinAddr(addr).s_addr); - sprintf(buffer, "%ld.%ld.%ld.%ld", (iaddr >> 24) & 0xff, (iaddr >> 16) & 0xff, (iaddr >> 8) & 0xff, - iaddr & 0xff); - return buffer; -} - -net_if_t* udpc_getNetIf(const char* wanted) -{ -#ifndef __MINGW32__ - struct ifreq ibuf[100]; - struct ifreq *ifrp, *ifend, *chosen; - struct ifconf ifc; - int s; -#else /* __MINGW32__ */ - int i; - - int etherNo = -1; - int wantedEtherNo = -2; /* Wanted ethernet interface */ - - MIB_IPADDRTABLE* iptab = NULL; - MIB_IFTABLE* iftab = NULL; - - MIB_IPADDRROW *iprow, *chosen = NULL; - MIB_IFROW* chosenIf = NULL; - WORD wVersionRequested; /* Version of Winsock to load */ - WSADATA wsaData; /* Winsock implementation details */ - ULONG a; - - int r; -#endif /* __MINGW32__ */ - - int lastGoodness = 0; - struct in_addr wantedAddress; - int isAddress = 0; - int wantedLen = 0; - net_if_t* net_if; - - if (wanted == NULL) - { - wanted = getenv("IFNAME"); - } - - if (wanted && INET_ATON(wanted, &wantedAddress)) - isAddress = 1; - else - wantedAddress.s_addr = 0; - - if (wanted) - wantedLen = strlen(wanted); - - net_if = MALLOC(net_if_t); - // TODO: FIXME - // if(net_if == NULL) - // udpc_fatal(1, "Out of memory error"); - -#ifndef __MINGW32__ - - s = socket(PF_INET, SOCK_DGRAM, 0); - - if (s < 0) - { - perror("make socket"); - exit(1); - } - - ifc.ifc_len = sizeof(ibuf); - ifc.ifc_buf = (caddr_t)ibuf; - - if (ioctl(s, SIOCGIFCONF, (char*)&ifc) < 0 || ifc.ifc_len < (signed int)sizeof(struct ifreq)) - { - perror("udpcast: SIOCGIFCONF: "); - exit(1); - } - - ifend = (struct ifreq*)((char*)ibuf + ifc.ifc_len); - chosen = NULL; - - for (ifrp = ibuf; ifrp < ifend; -#ifdef IFREQ_SIZE - ifrp = IFREQ_SIZE(*ifrp) + (char*)ifrp -#else - ifrp++ -#endif - ) - { - unsigned long iaddr = getSinAddr(&ifrp->ifr_addr).s_addr; - int goodness; - - if (ifrp->ifr_addr.sa_family != PF_INET) - continue; - - if (wanted) - { - if (isAddress && iaddr == wantedAddress.s_addr) - { - goodness = 8; - } - else if (strcmp(wanted, ifrp->ifr_name) == 0) - { - /* perfect match on interface name */ - goodness = 12; - } - else if (wanted != NULL && strncmp(wanted, ifrp->ifr_name, wantedLen) == 0) - { - /* prefix match on interface name */ - goodness = 7; - } - else - { - /* no match, try next */ - continue; - } - } - else - { - if (iaddr == 0) - { - /* disregard interfaces whose address is zero */ - goodness = 1; - } - else if (iaddr == htonl(0x7f000001)) - { - /* disregard localhost type devices */ - goodness = 2; - } - else if (strcmp("eth0", ifrp->ifr_name) == 0 || strcmp("en0", ifrp->ifr_name) == 0) - { - /* prefer first ethernet interface */ - goodness = 6; - } - else if (strncmp("eth0:", ifrp->ifr_name, 5) == 0) - { - /* second choice: any secondary addresses of first ethernet */ - goodness = 5; - } - else if (strncmp("eth", ifrp->ifr_name, 3) == 0 || strncmp("en", ifrp->ifr_name, 2) == 0) - { - /* and, if not available, any other ethernet device */ - goodness = 4; - } - else - { - goodness = 3; - } - } - - if (hasLink(s, ifrp->ifr_name)) - /* Good or unknown link status privileged over known - * disconnected */ - goodness += 3; - - /* If all else is the same, prefer interfaces that - * have broadcast */ - goodness = goodness * 2; - - if (goodness >= lastGoodness) - { - /* Privilege broadcast-enabled interfaces */ - if (ioctl(s, SIOCGIFBRDADDR, ifrp) < 0) - { - // TODO: FIXME - // udpc_fatal(-1, "Error getting broadcast address for %s: %s", ifrp->ifr_name, strerror(errno)); - } - - if (getSinAddr(&ifrp->ifr_ifru.ifru_broadaddr).s_addr) - goodness++; - } - - if (goodness > lastGoodness) - { - chosen = ifrp; - lastGoodness = goodness; - net_if->addr.s_addr = iaddr; - } - } - - if (!chosen) - { - fprintf(stderr, "No suitable network interface found\n"); - fprintf(stderr, "The following interfaces are available:\n"); - - for (ifrp = ibuf; ifrp < ifend; -#ifdef IFREQ_SIZE - ifrp = IFREQ_SIZE(*ifrp) + (char*)ifrp -#else - ifrp++ -#endif - ) - { - char buffer[16]; - - if (ifrp->ifr_addr.sa_family != PF_INET) - continue; - - fprintf(stderr, "\t%s\t%s\n", ifrp->ifr_name, - udpc_getIpString((struct sockaddr_in*)&ifrp->ifr_addr, buffer)); - } - - exit(1); - } - - net_if->name = strdup(chosen->ifr_name); - -#ifdef HAVE_STRUCT_IP_MREQN_IMR_IFINDEX - - /* Index for multicast subscriptions */ - if (ioctl(s, SIOCGIFINDEX, chosen) < 0) - { - // TODO: FIXME - // udpc_fatal(-1, "Error getting index for %s: %s", net_if->name, strerror(errno)); - } - - net_if->index = chosen->ifr_ifindex; -#endif - - /* Broadcast */ - if (ioctl(s, SIOCGIFBRDADDR, chosen) < 0) - { - // TODO: FIXME - // udpc_fatal(-1, "Error getting broadcast address for %s: %s", net_if->name, strerror(errno)); - } - - net_if->bcast = getSinAddr(&chosen->ifr_ifru.ifru_broadaddr); - - close(s); - -#else /* __MINGW32__ */ - - /* WINSOCK initialization */ - wVersionRequested = MAKEWORD(2, 0); /* Request Winsock v2.0 */ - - if (WSAStartup(wVersionRequested, &wsaData) != 0) /* Load Winsock DLL */ - { - fprintf(stderr, "WSAStartup() failed"); - exit(1); - } - - /* End WINSOCK initialization */ - - a = 0; - r = GetIpAddrTable(iptab, &a, TRUE); - iptab = malloc(a); - r = GetIpAddrTable(iptab, &a, TRUE); - - a = 0; - r = GetIfTable(iftab, &a, TRUE); - iftab = malloc(a); - r = GetIfTable(iftab, &a, TRUE); - - if (wanted && !strncmp(wanted, "eth", 3) && wanted[3]) - { - char* ptr; - int n = strtoul(wanted + 3, &ptr, 10); - - if (!*ptr) - wantedEtherNo = n; - } - - for (i = 0; i < iptab->dwNumEntries; i++) - { - int goodness = -1; - unsigned long iaddr; - int isEther = 0; - MIB_IFROW* ifrow; - - iprow = &iptab->table[i]; - iaddr = iprow->dwAddr; - - ifrow = getIfRow(iftab, iprow->dwIndex); - - if (ifrow && ifrow->dwPhysAddrLen == 6 && iprow->dwBCastAddr) - { - isEther = 1; - etherNo++; - } - - if (wanted) - { - if (isAddress && iaddr == wantedAddress.s_addr) - { - goodness = 8; - } - else if (isEther && wantedEtherNo == etherNo) - { - goodness = 9; - } - else if (ifrow->dwPhysAddrLen) - { - int j; - const char* ptr = wanted; - - for (j = 0; *ptr && j < ifrow->dwPhysAddrLen; j++) - { - int digit = strtoul(ptr, (char**)&ptr, 16); - - if (digit != ifrow->bPhysAddr[j]) - break; /* Digit mismatch */ - - if (*ptr == '-' || *ptr == ':') - { - ptr++; - } - } - - if (!*ptr && j == ifrow->dwPhysAddrLen) - { - goodness = 9; - } - } - } - else - { - if (iaddr == 0) - { - /* disregard interfaces whose address is zero */ - goodness = 1; - } - else if (iaddr == htonl(0x7f000001)) - { - /* disregard localhost type devices */ - goodness = 2; - } - else if (isEther) - { - /* prefer ethernet */ - goodness = 6; - } - else if (ifrow->dwPhysAddrLen) - { - /* then prefer interfaces which have a physical address */ - goodness = 4; - } - else - { - goodness = 3; - } - } - - goodness = goodness * 2; - - /* If all else is the same, prefer interfaces that - * have broadcast */ - if (goodness >= lastGoodness) - { - /* Privilege broadcast-enabled interfaces */ - if (iprow->dwBCastAddr) - goodness++; - } - - if (goodness > lastGoodness) - { - chosen = iprow; - chosenIf = ifrow; - lastGoodness = goodness; - } - } - - if (!chosen) - { - fprintf(stderr, "No suitable network interface found%s%s\n", wanted ? " for " : "", wanted ? wanted : ""); - fprintf(stderr, "The following interfaces are available:\n"); - - for (i = 0; i < iptab->dwNumEntries; i++) - { - char buffer[16]; - struct sockaddr_in addr; - MIB_IFROW* ifrow; - char* name = NULL; - iprow = &iptab->table[i]; - addr.sin_addr.s_addr = iprow->dwAddr; - ifrow = getIfRow(iftab, iprow->dwIndex); - name = fmtName(ifrow); - fprintf(stderr, " %15s %s\n", udpc_getIpString(&addr, buffer), name ? name : ""); - - if (name) - free(name); - } - - exit(1); - } - - net_if->bcast.s_addr = net_if->addr.s_addr = chosen->dwAddr; - - if (chosen->dwBCastAddr) - net_if->bcast.s_addr |= ~chosen->dwMask; - - if (chosenIf) - { - net_if->name = fmtName(chosenIf); - } - else - { - net_if->name = "*"; - } - - free(iftab); - free(iptab); -#endif /* __MINGW32__ */ - - return net_if; -} - -#define IP_MREQN ip_mreqn - -int fillMreq(net_if_t* net_if, struct in_addr addr, struct IP_MREQN* mreq) -{ -#ifdef HAVE_STRUCT_IP_MREQN_IMR_IFINDEX - mreq->imr_ifindex = net_if->index; - mreq->imr_address.s_addr = 0; -#else - mreq->imr_interface = net_if->addr; -#endif - mreq->imr_multiaddr = addr; - - return 0; -} - -int mcastOp(int sock, net_if_t* net_if, struct in_addr addr, int code, const char* message) -{ - struct IP_MREQN mreq; - int r; - - fillMreq(net_if, addr, &mreq); - r = setsockopt(sock, SOL_IP, code, (char*)&mreq, sizeof(mreq)); - - if (r < 0) - { - perror(message); - exit(1); - } - - return 0; -} - -int udpc_setMcastDestination(int sock, net_if_t* net_if, struct sockaddr_in* addr) -{ -#ifdef WINDOWS - int r; - struct sockaddr_in interface_addr; - struct in_addr if_addr; - getMyAddress(net_if, &interface_addr); - if_addr = getSinAddr(&interface_addr); - r = setsockopt(sock, IPPROTO_IP, IP_MULTICAST_IF, (char*)&if_addr, sizeof(if_addr)); - - if (r < 0) - fatal(1, "Set multicast send interface"); - - return 0; -#else - /* IP_MULTICAST_IF not correctly supported on Cygwin */ - return mcastOp(sock, net_if, getSinAddr(addr), IP_MULTICAST_IF, "Set multicast send interface"); -#endif -} - -int initSockAddress(addr_type_t addr_type, net_if_t* net_if, in_addr_t ip, unsigned short port, - struct sockaddr_in* addr) -{ - memset((char*)addr, 0, sizeof(struct sockaddr_in)); - addr->sin_family = AF_INET; - addr->sin_port = htons(port); - - // TODO: FIXME - // if(!net_if && addr_type != ADDR_TYPE_MCAST) - // udpc_fatal(1, "initSockAddr without ifname\n"); - - switch (addr_type) - { - case ADDR_TYPE_UCAST: addr->sin_addr = net_if->addr; break; - - case ADDR_TYPE_BCAST: addr->sin_addr = net_if->bcast; break; - - case ADDR_TYPE_MCAST: addr->sin_addr.s_addr = ip; break; - } - - return 0; -} - -int mcastListen(int sock, net_if_t* net_if, struct sockaddr_in* addr) -{ - return mcastOp(sock, net_if, getSinAddr(addr), IP_ADD_MEMBERSHIP, "Subscribe to multicast group"); -} - -int udpc_makeSocket(addr_type_t addr_type, net_if_t* net_if, struct sockaddr_in* tmpl, int port) -{ - int ret, s; - struct sockaddr_in myaddr; - in_addr_t ip = 0; - -#ifdef WINDOWS - static int lastSocket = -1; - /* Very ugly hack, but hey!, this is for Windows */ - - if (addr_type == ADDR_TYPE_MCAST) - { - mcastListen(lastSocket, net_if, tmpl); - return -1; - } - else if (addr_type != ADDR_TYPE_UCAST) - return -1; - -#endif - - s = socket(PF_INET, SOCK_DGRAM, 0); - - if (s < 0) - { - perror("make socket"); - exit(1); - } - - if (addr_type == ADDR_TYPE_MCAST && tmpl != NULL) - { - ip = tmpl->sin_addr.s_addr; - } - - ret = initSockAddress(addr_type, net_if, ip, port, &myaddr); - - // TODO: FIXME - // if(ret < 0) - // udpc_fatal(1, "Could not get socket address fot %d/%s", - // addr_type, net_if->name); - if (addr_type == ADDR_TYPE_BCAST) - { - // cerr << "for addr_type == ADDR_TYPE_BCAST, myaddr.sin_addr.s_addr = 0x" << hex << - // myaddr.sin_addr.s_addr << dec << endl; - } - - if (addr_type == ADDR_TYPE_BCAST && myaddr.sin_addr.s_addr == 0) - { - /* Attempting to bind to broadcast address on not-broadcast media ... */ - closesocket(s); - return -1; - } - - ret = bind(s, (struct sockaddr*)&myaddr, sizeof(myaddr)); - // TODO: FIXME - // if (ret < 0) { - // char buffer[16]; - // udpc_fatal(1, "bind socket to %s:%d (%s)\n", - // udpc_getIpString(&myaddr, buffer), - // udpc_getPort(&myaddr), - // strerror(errno)); - // } - - if (addr_type == ADDR_TYPE_MCAST) - mcastListen(s, net_if, &myaddr); - -#ifdef WINDOWS - lastSocket = s; -#endif - return s; -} - -int udpc_setSocketToBroadcast(int sock) -{ - /* set the socket to broadcast */ - int p = 1; - return setsockopt(sock, SOL_SOCKET, SO_BROADCAST, (char*)&p, sizeof(int)); -} - -int udpc_getBroadCastAddress(net_if_t* net_if, struct sockaddr_in* addr, short port) -{ - int r = initSockAddress(ADDR_TYPE_BCAST, net_if, INADDR_ANY, port, addr); - - if (addr->sin_addr.s_addr == 0) - { - /* Quick hack to make it work for loopback */ - struct sockaddr_in ucast; - initSockAddress(ADDR_TYPE_UCAST, net_if, INADDR_ANY, port, &ucast); - - if ((ntohl(ucast.sin_addr.s_addr) & 0xff000000) == 0x7f000000) - addr->sin_addr.s_addr = ucast.sin_addr.s_addr; - } - - return r; -} - -int safe_inet_aton(const char* address, struct in_addr* ip) -{ - if (!INET_ATON(address, ip)) - { - // TODO: FIXME - // udpc_fatal(-1, "Bad address %s", address); - } - - return 0; -} - -int udpc_getMcastAllAddress(struct sockaddr_in* addr, const char* address, short port) -{ - struct in_addr ip; - int ret; - - if (address == NULL || address[0] == '\0') - safe_inet_aton("224.0.0.1", &ip); - else - { - if ((ret = safe_inet_aton(address, &ip)) < 0) - return ret; - } - - return initSockAddress(ADDR_TYPE_MCAST, NULL, ip.s_addr, port, addr); -} - -void setPort(struct sockaddr_in* addr, unsigned short port) -{ - ((struct sockaddr_in*)addr)->sin_port = htons(port); -} - -int isMcastAddress(struct sockaddr_in* addr) -{ - int ip = ntohl(addr->sin_addr.s_addr) >> 24; - return ip >= 0xe0 && ip < 0xf0; -} - -void udpc_clearIp(struct sockaddr_in* addr) -{ - addr->sin_addr.s_addr = 0; - addr->sin_family = AF_INET; -} - -void udpc_setSendBuf(int sock, unsigned int bufsize) -{ - if (setsockopt(sock, SOL_SOCKET, SO_SNDBUF, (char*)&bufsize, sizeof(bufsize)) < 0) - perror("Set send buffer"); -} - -int udpc_setTtl(int sock, int ttl) -{ - /* set the socket to broadcast */ - return setsockopt(sock, SOL_IP, IP_MULTICAST_TTL, (char*)&ttl, sizeof(int)); -} - -int udpc_getMyAddress(net_if_t* net_if, struct sockaddr_in* addr) -{ - return initSockAddress(ADDR_TYPE_UCAST, net_if, INADDR_ANY, 0, addr); -} - -void udpc_getDefaultMcastAddress(net_if_t* net_if, struct sockaddr_in* mcast) -{ - udpc_getMyAddress(net_if, mcast); - mcast->sin_addr.s_addr &= htonl(0x07ffffff); - mcast->sin_addr.s_addr |= htonl(0xe8000000); -} - -void udpc_copyIpFrom(struct sockaddr_in* dst, struct sockaddr_in* src) -{ - dst->sin_addr = src->sin_addr; - dst->sin_family = src->sin_family; -} - -int udpc_getSelectedSock(int* socks, int nr, fd_set* read_set) -{ - int i; - int maxFd; - maxFd = -1; - - for (i = 0; i < nr; i++) - { - if (socks[i] == -1) - continue; - - if (FD_ISSET(socks[i], read_set)) - return socks[i]; - } - - return -1; -} - -int udpc_ipIsEqual(struct sockaddr_in* left, struct sockaddr_in* right) -{ - return getSinAddr(left).s_addr == getSinAddr(right).s_addr; -} - -void udpc_closeSock(int* socks, int nr, int target) -{ - int i; - int sock = socks[target]; - - socks[target] = -1; - - for (i = 0; i < nr; i++) - if (socks[i] == sock) - return; - - closesocket(sock); -} -int prepareForSelect(int* socks, int nr, fd_set* read_set) -{ - int i; - int maxFd; - FD_ZERO(read_set); - maxFd = -1; - - for (i = 0; i < nr; i++) - { - if (socks[i] == -1) - continue; - - FD_SET(socks[i], read_set); - - if (socks[i] > maxFd) - maxFd = socks[i]; - } - - return maxFd; -} - -int doReceive(int s, void* message, size_t len, struct sockaddr_in* from, int portBase) -{ - socklen_t slen; - int r; - unsigned short port; - - slen = sizeof(*from); -#ifdef LOSSTEST - loseRecvPacket(s); -#endif - r = recvfrom(s, message, len, 0, (struct sockaddr*)from, &slen); - - if (r < 0) - return r; - - port = ntohs(from->sin_port); - - if (port != RECEIVER_PORT(portBase) && port != SENDER_PORT(portBase)) - { - return -1; - } - - /* flprintf("recv: %08x %d\n", *(int*) message, r);*/ - return r; -} - -participantsDb_t udpc_makeParticipantsDb(void) -{ - return MALLOC(struct participantsDb); -} - -int udpc_removeParticipant(struct participantsDb* db, int i) -{ - if (db->clientTable[i].used) - { - db->clientTable[i].used = 0; - db->nrParticipants--; - } - - return 0; -} - -int udpc_lookupParticipant(struct participantsDb* db, struct sockaddr_in* addr) -{ - int i; - - for (i = 0; i < MAX_CLIENTS; i++) - { - if (db->clientTable[i].used && udpc_ipIsEqual(&db->clientTable[i].addr, addr)) - { - return i; - } - } - - return -1; -} - -int udpc_nrParticipants(participantsDb_t db) -{ - return db->nrParticipants; -} - -int udpc_addParticipant(participantsDb_t db, struct sockaddr_in* addr, int capabilities, unsigned int rcvbuf, - int pointopoint) -{ - // cerr << "adding a participant..." << endl; - int i; - - if ((i = udpc_lookupParticipant(db, addr)) >= 0) - return i; - - for (i = 0; i < MAX_CLIENTS; i++) - { - if (!db->clientTable[i].used) - { - db->clientTable[i].addr = *addr; - db->clientTable[i].used = 1; - db->clientTable[i].capabilities = capabilities; - db->clientTable[i].rcvbuf = rcvbuf; - db->nrParticipants++; - - return i; - } - else if (pointopoint) - return -1; - } - - return -1; /* no space left in participant's table */ -} - -int selectWithoutConsole(int maxFd, fd_set* read_set, struct timeval* tv) -{ - int ret; - - ret = select(maxFd, read_set, NULL, NULL, tv); - - if (ret < 0) - return -1; - - return ret; -} - -#if 0 -void sendHello(struct net_config* net_config, int sock, - int streaming) -{ - struct hello hello; - - /* send hello message */ - if (streaming) - hello.opCode = htons(CMD_HELLO_STREAMING); - else - hello.opCode = htons(CMD_HELLO); - - hello.reserved = 0; - hello.capabilities = htonl(net_config->capabilities); - udpc_copyToMessage(hello.mcastAddr, &net_config->dataMcastAddr); - hello.blockSize = htons(net_config->blockSize); - //rgWaitAll(net_config, sock, net_config->controlMcastAddr.sin_addr.s_addr, sizeof(hello)); - BCAST_CONTROL(sock, hello); -} -#endif - -int checkClientWait(participantsDb_t db, struct net_config* net_config, time_t* firstConnected) -{ - time_t now; - - if (!udpc_nrParticipants(db) || !firstConnected || !*firstConnected) - return 0; /* do not start: no receivers */ - - now = time(0); - - /* - * If we have a max_client_wait, start the transfer after first client - * connected + maxSendWait - */ - if (net_config->max_receivers_wait && (now >= *firstConnected + net_config->max_receivers_wait)) - { -#ifdef USE_SYSLOG - syslog(LOG_INFO, "max wait[%d] passed: starting", net_config->max_receivers_wait); -#endif - return 1; /* send-wait passed: start */ - } - - /* - * Otherwise check to see if the minimum of clients - * have checked in. - */ - else if (udpc_nrParticipants(db) >= net_config->min_receivers && - /* - * If there are enough clients and there's a min wait time, we'll - * wait around anyway until then. - * Otherwise, we always transfer - */ - (!net_config->min_receivers_wait || now >= *firstConnected + net_config->min_receivers_wait)) - { -#ifdef USE_SYSLOG - syslog(LOG_INFO, "min receivers[%d] reached: starting", net_config->min_receivers); -#endif - return 1; - } - else - return 0; -} - -int sendConnectionReply(participantsDb_t db, int sock, struct net_config* config, struct sockaddr_in* client, - int capabilities, unsigned int rcvbuf) -{ - struct connectReply reply; - - if (rcvbuf == 0) - rcvbuf = 65536; - - if (capabilities & CAP_BIG_ENDIAN) - { - reply.opCode = htons(CMD_CONNECT_REPLY); - reply.clNr = - htonl(udpc_addParticipant(db, client, capabilities, rcvbuf, config->flags & FLAG_POINTOPOINT)); - reply.blockSize = htonl(config->blockSize); - } - else - { - // TODO: FIXME - // udpc_fatal(1, "Little endian protocol no longer supported"); - } - - reply.reserved = 0; - - if (config->flags & FLAG_POINTOPOINT) - { - udpc_copyIpFrom(&config->dataMcastAddr, client); - } - - /* new parameters: always big endian */ - reply.capabilities = ntohl(config->capabilities); - udpc_copyToMessage(reply.mcastAddr, &config->dataMcastAddr); - - /*reply.mcastAddress = mcastAddress;*/ - // rgWaitAll(config, sock, client->sin_addr.s_addr, sizeof(reply)); - if (SEND(sock, reply, *client) < 0) - { - perror("reply add new client"); - return -1; - } - - return 0; -} - -int mainDispatcher(int* fd, int nr, participantsDb_t db, struct net_config* net_config, int* tries, - time_t* firstConnected) -{ - struct sockaddr_in client; - union message fromClient; - fd_set read_set; - int ret; - int msgLength; - int startNow = 0; - int selected; - - if (firstConnected && !*firstConnected && udpc_nrParticipants(db)) - { - *firstConnected = time(0); -#ifdef USE_SYSLOG - syslog(LOG_INFO, "first connection: min wait[%d] secs - max wait[%d] - min clients[%d]", - net_config->min_receivers_wait, net_config->max_receivers_wait, net_config->min_receivers); -#endif - } - - while (!startNow) - { - struct timeval tv; - struct timeval* tvp; - int nr_desc; - - int maxFd = prepareForSelect(fd, nr, &read_set); - - if (net_config->rexmit_hello_interval) - { - tv.tv_usec = (net_config->rexmit_hello_interval % 1000) * 1000; - tv.tv_sec = net_config->rexmit_hello_interval / 1000; - tvp = &tv; - } - else if (firstConnected && udpc_nrParticipants(db)) - { - tv.tv_usec = 0; - tv.tv_sec = 2; - tvp = &tv; - } - else - tvp = 0; - - nr_desc = selectWithoutConsole(maxFd + 1, &read_set, tvp); - - if (nr_desc < 0) - { - perror("select"); - return -1; - } - - if (nr_desc > 0) - /* key pressed, or receiver activity */ - break; - - if (net_config->rexmit_hello_interval) - { - /* retransmit hello message */ - udpc_sendHello(net_config, fd[0], 0); - (*tries)++; - - if (net_config->autostart != 0 && *tries > net_config->autostart) - startNow = 1; - } - - if (firstConnected) - startNow = startNow || checkClientWait(db, net_config, firstConnected); - } - - selected = udpc_getSelectedSock(fd, nr, &read_set); - - if (selected == -1) - return startNow; - - BZERO(fromClient); /* Zero it out in order to cope with short messages - * from older versions */ - - msgLength = RECV(selected, fromClient, client, net_config->portBase); - - if (msgLength < 0) - { - perror("problem getting data from client"); - return 0; /* don't panic if we get weird messages */ - } - - if (net_config->flags & FLAG_ASYNC) - return 0; - - switch (ntohs(fromClient.opCode)) - { - case CMD_CONNECT_REQ: - sendConnectionReply(db, fd[0], net_config, &client, - CAP_BIG_ENDIAN | ntohl(fromClient.connectReq.capabilities), - ntohl(fromClient.connectReq.rcvbuf)); - return startNow; - - case CMD_GO: return 1; - - case CMD_DISCONNECT: - ret = udpc_lookupParticipant(db, &client); - - if (ret >= 0) - udpc_removeParticipant(db, ret); - - return startNow; - - default: break; - } - - return startNow; -} - -static int isPointToPoint(participantsDb_t db, int flags) -{ - if (flags & FLAG_POINTOPOINT) - return 1; - - if (flags & (FLAG_NOPOINTOPOINT | FLAG_ASYNC)) - return 0; - - return udpc_nrParticipants(db) == 1; -} - -int getProducedAmount(produconsum_t pc) -{ - unsigned int produced = pc->produced; - unsigned int consumed = pc->consumed; - - if (produced < consumed) - return produced + 2 * pc->size - consumed; - else - return produced - consumed; -} - -int _consumeAny(produconsum_t pc, unsigned int minAmount, struct timespec* ts) -{ - unsigned int amount; -#if DEBUG - flprintf("%s: Waiting for %d bytes (%d:%d)\n", pc->name, minAmount, pc->consumed, pc->produced); -#endif - pc->consumerIsWaiting = 1; - amount = getProducedAmount(pc); - - if (amount >= minAmount || pc->atEnd) - { - pc->consumerIsWaiting = 0; -#if DEBUG - flprintf("%s: got %d bytes\n", pc->name, amount); -#endif - return amount; - } - - pthread_mutex_lock(&pc->mutex); - - while ((amount = getProducedAmount(pc)) < minAmount && !pc->atEnd) - { -#if DEBUG - flprintf("%s: ..Waiting for %d bytes (%d:%d)\n", pc->name, minAmount, pc->consumed, pc->produced); -#endif - - if (ts == 0) - pthread_cond_wait(&pc->cond, &pc->mutex); - else - { - int r; -#if DEBUG - flprintf("Before timed wait\n"); -#endif - r = pthread_cond_timedwait(&pc->cond, &pc->mutex, ts); -#if DEBUG - flprintf("After timed wait %d\n", r); -#endif - - if (r == ETIMEDOUT) - { - amount = getProducedAmount(pc); - break; - } - } - } - - pthread_mutex_unlock(&pc->mutex); -#if DEBUG - flprintf("%s: Got them %d (for %d) %d\n", pc->name, amount, minAmount, pc->atEnd); -#endif - pc->consumerIsWaiting = 0; - return amount; -} - -produconsum_t pc_makeProduconsum(int size, const char* name) -{ - produconsum_t pc = MALLOC(struct produconsum); - pc->size = size; - pc->produced = 0; - pc->consumed = 0; - pc->atEnd = 0; - pthread_mutex_init(&pc->mutex, NULL); - pc->consumerIsWaiting = 0; - pthread_cond_init(&pc->cond, NULL); - pc->name = name; - return pc; -} - -int pc_consumeAnyWithTimeout(produconsum_t pc, struct timespec* ts) -{ - return _consumeAny(pc, 1, ts); -} - -unsigned int pc_getProducerPosition(produconsum_t pc) -{ - return pc->produced % pc->size; -} - -unsigned int pc_getWaiting(produconsum_t pc) -{ - return getProducedAmount(pc); -} - -int pc_consumeAny(produconsum_t pc) -{ - return _consumeAny(pc, 1, 0); -} - -int pc_consume(produconsum_t pc, int amount) -{ - return _consumeAny(pc, amount, 0); -} - -void wakeConsumer(produconsum_t pc) -{ - if (pc->consumerIsWaiting) - { - pthread_mutex_lock(&pc->mutex); - pthread_cond_signal(&pc->cond); - pthread_mutex_unlock(&pc->mutex); - } -} - -void pc_produceEnd(produconsum_t pc) -{ - pc->atEnd = 1; - wakeConsumer(pc); -} - -int pc_consumed(produconsum_t pc, int amount) -{ - unsigned int consumed = pc->consumed; - - if (consumed >= 2 * pc->size - amount) - { - consumed += amount - 2 * pc->size; - } - else - { - consumed += amount; - } - - pc->consumed = consumed; - return amount; -} - -unsigned int pc_getConsumerPosition(produconsum_t pc) -{ - return pc->consumed % pc->size; -} - -void pc_produce(produconsum_t pc, unsigned int amount) -{ - unsigned int produced = pc->produced; - unsigned int consumed = pc->consumed; - - /* sanity checks: - * 1. should not produce more than size - * 2. do not pass consumed+size - */ - if (amount > pc->size) - { - // TODO: FIXME - // udpc_fatal(1, "Buffer overflow in produce %s: %d > %d \n", pc->name, amount, pc->size); - } - - produced += amount; - - if (produced >= 2 * pc->size) - produced -= 2 * pc->size; - - if (produced > consumed + pc->size || (produced < consumed && produced > consumed - pc->size)) - { - // TODO: FIXME - // udpc_fatal(1, "Buffer overflow in produce %s: %d > %d [%d] \n", pc->name, produced, consumed, - // pc->size); - } - - pc->produced = produced; - wakeConsumer(pc); -} - -void udpc_initFifo(struct fifo* fifo, int blockSize) -{ - fifo->dataBufSize = blockSize * 4096; - fifo->dataBuffer = (unsigned char*)malloc(fifo->dataBufSize + 4096); - fifo->dataBuffer += 4096 - (((unsigned long)fifo->dataBuffer) % 4096); - - /* Free memory queue is initially full */ - fifo->freeMemQueue = pc_makeProduconsum(fifo->dataBufSize, "free mem"); - pc_produce(fifo->freeMemQueue, fifo->dataBufSize); - - fifo->data = pc_makeProduconsum(fifo->dataBufSize, "receive"); -} - -THREAD_RETURN returnChannelMain(void* args) -{ - struct returnChannel* returnChannel = (struct returnChannel*)args; - - while (1) - { - struct sockaddr_in from; - int clNo; - int pos = pc_getConsumerPosition(returnChannel->freeSpace); - pc_consumeAny(returnChannel->freeSpace); - - RECV(returnChannel->rcvSock, returnChannel->q[pos].msg, from, returnChannel->config->portBase); - clNo = udpc_lookupParticipant(returnChannel->participantsDb, &from); - - if (clNo < 0) - { - /* packet from unknown provenance */ - continue; - } - - returnChannel->q[pos].clNo = clNo; - pc_consumed(returnChannel->freeSpace, 1); - pc_produce(returnChannel->incoming, 1); - } - - return 0; -} - -void initReturnChannel(struct returnChannel* returnChannel, struct net_config* config, int sock) -{ - returnChannel->config = config; - returnChannel->rcvSock = sock; - returnChannel->freeSpace = pc_makeProduconsum(QUEUE_SIZE, "msg:free-queue"); - pc_produce(returnChannel->freeSpace, QUEUE_SIZE); - returnChannel->incoming = pc_makeProduconsum(QUEUE_SIZE, "msg:incoming"); - - pthread_create(&returnChannel->thread, NULL, returnChannelMain, returnChannel); -} - -void senderStatsAddBytes(sender_stats_t ss, long bytes) -{ - if (ss != NULL) - { - ss->totalBytes += bytes; - - if (ss->bwPeriod) - { - double tdiff, bw; - struct timeval tv; - gettimeofday(&tv, 0); - ss->periodBytes += bytes; - - if (tv.tv_sec - ss->periodStart.tv_sec < ss->bwPeriod - 1) - return; - - tdiff = (tv.tv_sec - ss->periodStart.tv_sec) * 1000000.0 + tv.tv_usec - ss->periodStart.tv_usec; - - if (tdiff < ss->bwPeriod * 1000000.0) - return; - - bw = ss->periodBytes * 8.0 / tdiff; - ss->periodBytes = 0; - ss->periodStart = tv; - } - } -} -int ackSlice(struct slice* slice, struct net_config* net_config, struct fifo* fifo, sender_stats_t stats) -{ - if (slice->state == slice::SLICE_ACKED) - /* already acked */ - return 0; - - if (!(net_config->flags & FLAG_SN)) - { - if (net_config->discovery == net_config::DSC_DOUBLING) - { - net_config->sliceSize += net_config->sliceSize / 4; - - if (net_config->sliceSize >= net_config->max_slice_size) - { - net_config->sliceSize = net_config->max_slice_size; - net_config->discovery = net_config::DSC_REDUCING; - } - } - } - - slice->state = slice::SLICE_ACKED; - pc_produce(fifo->freeMemQueue, slice->bytes); - - /* Statistics */ - senderStatsAddBytes(stats, slice->bytes); - - /* End Statistics */ - - return 0; -} - -int isSliceAcked(struct slice* slice) -{ - if (slice->state == slice::SLICE_ACKED) - { - return 1; - } - else - { - return 0; - } -} - -int freeSlice(sender_state_t sendst, struct slice* slice) -{ - int i; - i = slice - sendst->slices; - slice->state = slice::SLICE_PRE_FREE; - - while (1) - { - int pos = pc_getProducerPosition(sendst->free_slices_pc); - - if (sendst->slices[pos].state == slice::SLICE_PRE_FREE) - sendst->slices[pos].state = slice::SLICE_FREE; - else - break; - - pc_produce(sendst->free_slices_pc, 1); - } - - return 0; -} - -int isSliceXmitted(struct slice* slice) -{ - if (slice->state == slice::SLICE_XMITTED) - { - return 1; - } - else - { - return 0; - } -} - -int getSliceBlocks(struct slice* slice, struct net_config* net_config) -{ - return (slice->bytes + net_config->blockSize - 1) / net_config->blockSize; -} - -int sendReqack(struct slice* slice, struct net_config* net_config, struct fifo* fifo, sender_stats_t stats, - int sock) -{ - /* in async mode, just confirm slice... */ - if ((net_config->flags & FLAG_ASYNC) && slice->bytes != 0) - { - ackSlice(slice, net_config, fifo, stats); - return 0; - } - - if ((net_config->flags & FLAG_ASYNC) -#ifdef BB_FEATURE_UDPCAST_FEC - && (net_config->flags & FLAG_FEC) -#endif - ) - { - return 0; - } - - if (!(net_config->flags & FLAG_SN) && slice->rxmitId != 0) - { - int nrBlocks; - nrBlocks = getSliceBlocks(slice, net_config); - - if (slice->lastGoodBlock != 0 && slice->lastGoodBlock < nrBlocks) - { - net_config->discovery = net_config::DSC_REDUCING; - - if (slice->lastGoodBlock < net_config->sliceSize / 2) - { - net_config->sliceSize = net_config->sliceSize / 2; - } - else - { - net_config->sliceSize = slice->lastGoodBlock; - } - - if (net_config->sliceSize < 32) - { - /* a minimum of 32 */ - net_config->sliceSize = 32; - } - } - } - - slice->lastGoodBlock = 0; - slice->sl_reqack.ra.opCode = htons(CMD_REQACK); - slice->sl_reqack.ra.sliceNo = htonl(slice->sliceNo); - slice->sl_reqack.ra.bytes = htonl(slice->bytes); - - slice->sl_reqack.ra.reserved = 0; - memcpy((void*)&slice->answeredSet, (void*)&slice->sl_reqack.readySet, sizeof(slice->answeredSet)); - slice->nrAnswered = slice->nrReady; - - /* not everybody is ready yet */ - slice->needRxmit = 0; - memset(slice->rxmitMap, 0, sizeof(slice->rxmitMap)); - memset(slice->isXmittedMap, 0, sizeof(slice->isXmittedMap)); - slice->sl_reqack.ra.rxmit = htonl(slice->rxmitId); - - // rgWaitAll(net_config, sock, net_config->dataMcastAddr.sin_addr.s_addr, sizeof(slice->sl_reqack)); - BCAST_DATA(sock, slice->sl_reqack); - return 0; -} - -struct slice* findSlice(struct slice* slice1, struct slice* slice2, int sliceNo) -{ - if (slice1 != NULL && slice1->sliceNo == sliceNo) - return slice1; - - if (slice2 != NULL && slice2->sliceNo == sliceNo) - return slice2; - - return NULL; -} - -void markParticipantAnswered(slice_t slice, int clNo) -{ - if (BIT_ISSET(clNo, slice->answeredSet)) - /* client already has answered */ - return; - - slice->nrAnswered++; - SET_BIT(clNo, slice->answeredSet); -} - -int udpc_isParticipantValid(struct participantsDb* db, int i) -{ - return db->clientTable[i].used; -} - -void senderSetAnswered(sender_stats_t ss, int clNo) -{ - if (ss != NULL) - ss->clNo = clNo; -} - -int handleOk(sender_state_t sendst, struct slice* slice, int clNo) -{ - if (slice == NULL) - return 0; - - if (!udpc_isParticipantValid(sendst->rc.participantsDb, clNo)) - { - // udpc_flprintf("Invalid participant %d\n", clNo); - return 0; - } - - if (BIT_ISSET(clNo, slice->sl_reqack.readySet)) - { - /* client is already marked ready */ - } - else - { - SET_BIT(clNo, slice->sl_reqack.readySet); - slice->nrReady++; - senderSetAnswered(sendst->stats, clNo); - markParticipantAnswered(slice, clNo); - } - - return 0; -} - -int handleDisconnect1(struct slice* slice, int clNo) -{ - if (slice != NULL) - { - if (BIT_ISSET(clNo, slice->sl_reqack.readySet)) - { - /* avoid counting client both as left and ready */ - CLR_BIT(clNo, slice->sl_reqack.readySet); - slice->nrReady--; - } - - if (BIT_ISSET(clNo, slice->answeredSet)) - { - slice->nrAnswered--; - CLR_BIT(clNo, slice->answeredSet); - } - } - - return 0; -} - -int handleDisconnect(participantsDb_t db, struct slice* slice1, struct slice* slice2, int clNo) -{ - handleDisconnect1(slice1, clNo); - handleDisconnect1(slice2, clNo); - udpc_removeParticipant(db, clNo); - return 0; -} - -int handleRetransmit(sender_state_t sendst, struct slice* slice, int clNo, unsigned char* map, int rxmit) -{ - unsigned int i; - - if (!udpc_isParticipantValid(sendst->rc.participantsDb, clNo)) - { - // udpc_flprintf("Invalid participant %d\n", clNo); - return 0; - } - - if (slice == NULL) - return 0; - - if (rxmit < slice->rxmitId) - { - /* late answer to previous Req Ack */ - return 0; - } - - for (i = 0; i < sizeof(slice->rxmitMap) / sizeof(char); i++) - { - slice->rxmitMap[i] |= ~map[i]; - } - - slice->needRxmit = 1; - markParticipantAnswered(slice, clNo); - return 0; -} -int handleNextMessage(sender_state_t sendst, struct slice* xmitSlice, struct slice* rexmitSlice) -{ - int pos = pc_getConsumerPosition(sendst->rc.incoming); - union message* msg = &sendst->rc.q[pos].msg; - int clNo = sendst->rc.q[pos].clNo; - - pc_consumeAny(sendst->rc.incoming); - - switch (ntohs(msg->opCode)) - { - case CMD_OK: handleOk(sendst, findSlice(xmitSlice, rexmitSlice, ntohl(msg->ok.sliceNo)), clNo); break; - - case CMD_DISCONNECT: handleDisconnect(sendst->rc.participantsDb, xmitSlice, rexmitSlice, clNo); break; - - case CMD_RETRANSMIT: - handleRetransmit(sendst, findSlice(xmitSlice, rexmitSlice, ntohl(msg->retransmit.sliceNo)), clNo, - msg->retransmit.map, msg->retransmit.rxmit); - break; - - default: - // TODO: FIXME - // udpc_flprintf("Bad command %04x\n", (unsigned short) msg->opCode); - break; - } - - pc_consumed(sendst->rc.incoming, 1); - pc_produce(sendst->rc.freeSpace, 1); - return 0; -} - -int sendRawData(int sock, struct net_config* config, char* header, int headerSize, unsigned char* data, - int dataSize) -{ - struct iovec iov[2]; - struct msghdr hdr; - int packetSize; - int ret; - - iov[0].iov_base = header; - iov[0].iov_len = headerSize; - - iov[1].iov_base = data; - iov[1].iov_len = dataSize; - - hdr.msg_name = &config->dataMcastAddr; - hdr.msg_namelen = sizeof(struct sockaddr_in); - hdr.msg_iov = iov; - hdr.msg_iovlen = 2; - initMsgHdr(&hdr); - - packetSize = dataSize + headerSize; - // rgWaitAll(config, sock, config->dataMcastAddr.sin_addr.s_addr, packetSize); - ret = sendmsg(sock, &hdr, 0); - - if (ret < 0) - { - // TODO: FIXME - } - - return 0; -} -int transmitDataBlock(sender_state_t sendst, struct slice* slice, int i) -{ - struct fifo* fifo = sendst->fifo; - struct net_config* config = sendst->config; - struct dataBlock msg; - - idbassert(i < MAX_SLICE_SIZE); - - msg.opCode = htons(CMD_DATA); - msg.sliceNo = htonl(slice->sliceNo); - msg.blockNo = htons(i); - - msg.reserved = 0; - msg.reserved2 = 0; - msg.bytes = htonl(slice->bytes); - - sendRawData(sendst->socket, config, (char*)&msg, sizeof(msg), - fifo->dataBuffer + (slice->base + i * config->blockSize) % fifo->dataBufSize, - config->blockSize); - return 0; -} - -void senderStatsAddRetransmissions(sender_stats_t ss, int retransmissions) -{ - if (ss != NULL) - { - ss->retransmissions += retransmissions; - } -} - -int sendSlice(sender_state_t sendst, struct slice* slice, int retransmitting) -{ - struct net_config* config = sendst->config; - - int nrBlocks, i, rehello; -#ifdef BB_FEATURE_UDPCAST_FEC - int fecBlocks; -#endif - int retransmissions = 0; - - if (retransmitting) - { - slice->nextBlock = 0; - - if (slice->state != slice::SLICE_XMITTED) - return 0; - } - else - { - if (slice->state != slice::SLICE_NEW) - return 0; - } - - nrBlocks = getSliceBlocks(slice, config); -#ifdef BB_FEATURE_UDPCAST_FEC - - if ((config->flags & FLAG_FEC) && !retransmitting) - { - fecBlocks = config->fec_redundancy * config->fec_stripes; - } - else - { - fecBlocks = 0; - } - -#endif - - if ((sendst->config->flags & FLAG_STREAMING)) - { - rehello = nrBlocks - sendst->config->rehelloOffset; - - if (rehello < 0) - rehello = 0; - } - else - { - rehello = -1; - } - - /* transmit the data */ - for (i = slice->nextBlock; i < nrBlocks -#ifdef BB_FEATURE_UDPCAST_FEC - + fecBlocks -#endif - ; - i++) - { - if (retransmitting) - { - if (!BIT_ISSET(i, slice->rxmitMap) || BIT_ISSET(i, slice->isXmittedMap)) - { - /* if slice is not in retransmit list, or has _already_ - * been retransmitted, skip it */ - if (i > slice->lastGoodBlock) - slice->lastGoodBlock = i; - - continue; - } - - SET_BIT(i, slice->isXmittedMap); - retransmissions++; - } - - if (i == rehello) - { - udpc_sendHello(sendst->config, sendst->socket, 1); - } - - if (i < nrBlocks) - transmitDataBlock(sendst, slice, i); - -#ifdef BB_FEATURE_UDPCAST_FEC - else - transmitFecBlock(sendst, slice, i - nrBlocks); - -#endif - - if (!retransmitting && pc_getWaiting(sendst->rc.incoming)) - { - i++; - break; - } - } - - if (retransmissions) - senderStatsAddRetransmissions(sendst->stats, retransmissions); - - slice->nextBlock = i; - - if (i == nrBlocks -#ifdef BB_FEATURE_UDPCAST_FEC - + fecBlocks -#endif - ) - { - slice->needRxmit = 0; - - if (!retransmitting) - slice->state = slice::SLICE_XMITTED; - - return 2; - } - - return 1; -} - -int doRetransmissions(sender_state_t sendst, struct slice* slice) -{ - if (slice->state == slice::SLICE_ACKED) - return 0; /* nothing to do */ - - /* FIXME: reduce slice size if needed */ - if (slice->needRxmit) - { - /* do some retransmissions */ - sendSlice(sendst, slice, 1); - } - - return 0; -} - -struct slice* makeSlice(sender_state_t sendst, int sliceNo) -{ - struct net_config* config = sendst->config; - struct fifo* fifo = sendst->fifo; - int i; - struct slice* slice = NULL; - - pc_consume(sendst->free_slices_pc, 1); - i = pc_getConsumerPosition(sendst->free_slices_pc); - slice = &sendst->slices[i]; - idbassert(slice->state == slice::SLICE_FREE); - BZERO(*slice); - pc_consumed(sendst->free_slices_pc, 1); - - slice->base = pc_getConsumerPosition(sendst->fifo->data); - slice->sliceNo = sliceNo; - slice->bytes = pc_consume(fifo->data, 10 * config->blockSize); - - /* fixme: use current slice size here */ - if (slice->bytes > config->blockSize * config->sliceSize) - slice->bytes = config->blockSize * config->sliceSize; - - pc_consumed(fifo->data, slice->bytes); - slice->nextBlock = 0; - slice->state = slice::SLICE_NEW; - BZERO(slice->sl_reqack.readySet); - slice->nrReady = 0; -#ifdef BB_FEATURE_UDPCAST_FEC - slice->fec_data = sendst->fec_data + (i * config->fec_stripes * config->fec_redundancy * config->blockSize); -#endif - return slice; -} - -void cancelReturnChannel(struct returnChannel* returnChannel) -{ - /* No need to worry about the pthread_cond_wait in produconsum, because - * at the point where we enter here (to cancel the thread), we are sure - * that nobody else uses that produconsum any more - */ - pthread_cancel(returnChannel->thread); - pthread_join(returnChannel->thread, NULL); -} - -THREAD_RETURN netSenderMain(void* args0) -{ - sender_state_t sendst = (sender_state_t)args0; - struct net_config* config = sendst->config; - struct timeval tv; - struct timespec ts; - int atEnd = 0; - int nrWaited = 0; - unsigned long waitAverage = 10000; /* Exponential average of last wait times */ - - struct slice* xmitSlice = NULL; /* slice being transmitted a first time */ - struct slice* rexmitSlice = NULL; /* slice being re-transmitted */ - int sliceNo = 0; - - /* transmit the data */ - if (config->default_slice_size == 0) - { -#ifdef BB_FEATURE_UDPCAST_FEC - - if (config->flags & FLAG_FEC) - { - config->sliceSize = config->fec_stripesize * config->fec_stripes; - } - else -#endif - if (config->flags & FLAG_ASYNC) - config->sliceSize = 1024; - else if (sendst->config->flags & FLAG_SN) - { - sendst->config->sliceSize = 112; - } - else - sendst->config->sliceSize = 130; - - sendst->config->discovery = net_config::DSC_DOUBLING; - } - else - { - config->sliceSize = config->default_slice_size; -#ifdef BB_FEATURE_UDPCAST_FEC - - if ((config->flags & FLAG_FEC) && (config->sliceSize > 128 * config->fec_stripes)) - config->sliceSize = 128 * config->fec_stripes; - -#endif - } - -#ifdef BB_FEATURE_UDPCAST_FEC - - if ((sendst->config->flags & FLAG_FEC) && config->max_slice_size > config->fec_stripes * 128) - config->max_slice_size = config->fec_stripes * 128; - -#endif - - if (config->sliceSize > config->max_slice_size) - config->sliceSize = config->max_slice_size; - - idbassert(config->sliceSize <= MAX_SLICE_SIZE); - - do - { - /* first, cleanup rexmit Slice if needed */ - - if (rexmitSlice != NULL) - { - if (rexmitSlice->nrReady == udpc_nrParticipants(sendst->rc.participantsDb)) - { -#if DEBUG - flprintf("slice is ready\n"); -#endif - ackSlice(rexmitSlice, sendst->config, sendst->fifo, sendst->stats); - } - - if (isSliceAcked(rexmitSlice)) - { - freeSlice(sendst, rexmitSlice); - rexmitSlice = NULL; - } - } - - /* then shift xmit slice to rexmit slot, if possible */ - if (rexmitSlice == NULL && xmitSlice != NULL && isSliceXmitted(xmitSlice)) - { - rexmitSlice = xmitSlice; - xmitSlice = NULL; - sendReqack(rexmitSlice, sendst->config, sendst->fifo, sendst->stats, sendst->socket); - } - - /* handle any messages */ - if (pc_getWaiting(sendst->rc.incoming)) - { -#if DEBUG - flprintf("Before message %d\n", pc_getWaiting(sendst->rc.incoming)); -#endif - handleNextMessage(sendst, xmitSlice, rexmitSlice); - - /* restart at beginning of loop: we may have acked the rxmit - * slice, makeing it possible to shift the pipe */ - continue; - } - - /* do any needed retransmissions */ - if (rexmitSlice != NULL && rexmitSlice->needRxmit) - { - doRetransmissions(sendst, rexmitSlice); - /* restart at beginning: new messages may have arrived during - * retransmission */ - continue; - } - - /* if all participants answered, send req ack */ - if (rexmitSlice != NULL && rexmitSlice->nrAnswered == udpc_nrParticipants(sendst->rc.participantsDb)) - { - rexmitSlice->rxmitId++; - sendReqack(rexmitSlice, sendst->config, sendst->fifo, sendst->stats, sendst->socket); - } - - if (xmitSlice == NULL && !atEnd) - { -#if DEBUG - flprintf("SN=%d\n", sendst->config->flags & FLAG_SN); -#endif - - if ((sendst->config->flags & FLAG_SN) || rexmitSlice == NULL) - { -#ifdef BB_FEATURE_UDPCAST_FEC - - if (sendst->config->flags & FLAG_FEC) - { - int i; - pc_consume(sendst->fec_data_pc, 1); - i = pc_getConsumerPosition(sendst->fec_data_pc); - xmitSlice = &sendst->slices[i]; - pc_consumed(sendst->fec_data_pc, 1); - } - else -#endif - { - xmitSlice = makeSlice(sendst, sliceNo++); - } - - if (xmitSlice->bytes == 0) - atEnd = 1; - } - } - - if (xmitSlice != NULL && xmitSlice->state == slice::SLICE_NEW) - { - sendSlice(sendst, xmitSlice, 0); -#if DEBUG - flprintf("%d Interrupted at %d/%d\n", xmitSlice->sliceNo, xmitSlice->nextBlock, - getSliceBlocks(xmitSlice, sendst->config)); -#endif - continue; - } - - if (atEnd && rexmitSlice == NULL && xmitSlice == NULL) - break; - - if (sendst->config->flags & FLAG_ASYNC) - break; - -#if DEBUG - flprintf("Waiting for timeout...\n"); -#endif - gettimeofday(&tv, 0); - ts.tv_sec = tv.tv_sec; - ts.tv_nsec = (long int)((tv.tv_usec + 1.1 * waitAverage) * 1000); - -#ifdef WINDOWS - /* Windows has a granularity of 1 millisecond in its timer. Take this - * into account here */ -#define GRANULARITY 1000000 - ts.tv_nsec += 3 * GRANULARITY / 2; - ts.tv_nsec -= ts.tv_nsec % GRANULARITY; -#endif - -#define BILLION 1000000000 - - while (ts.tv_nsec >= BILLION) - { - ts.tv_nsec -= BILLION; - ts.tv_sec++; - } - - if (rexmitSlice->rxmitId > 10) - /* after tenth retransmission, wait minimum one second */ - ts.tv_sec++; - - if (pc_consumeAnyWithTimeout(sendst->rc.incoming, &ts) != 0) - { -#if DEBUG - flprintf("Have data\n"); -#endif - { - struct timeval tv2; - unsigned long timeout; - gettimeofday(&tv2, 0); - timeout = (tv2.tv_sec - tv.tv_sec) * 1000000 + tv2.tv_usec - tv.tv_usec; - - if (nrWaited) - timeout += waitAverage; - - waitAverage += 9; /* compensate against rounding errors */ - waitAverage = (long unsigned int)((0.9 * waitAverage + 0.1 * timeout)); - } - nrWaited = 0; - continue; - } - - if (rexmitSlice == NULL) - { - // TODO: FIXME - // udpc_flprintf("Weird. Timeout and no rxmit slice"); - break; - } - - if (nrWaited > 5) - { -#ifndef WINDOWS - /* on Cygwin, we would get too many of those messages... */ - nrWaited = 0; -#endif - } - - nrWaited++; - - if (rexmitSlice->rxmitId > config->retriesUntilDrop) - { - int i; - - for (i = 0; i < MAX_CLIENTS; i++) - { - if (udpc_isParticipantValid(sendst->rc.participantsDb, i) && - !BIT_ISSET(i, rexmitSlice->sl_reqack.readySet)) - { - udpc_removeParticipant(sendst->rc.participantsDb, i); - - if (udpc_nrParticipants(sendst->rc.participantsDb) == 0) - goto exit_main_loop; - } - } - - continue; - } - - rexmitSlice->rxmitId++; - sendReqack(rexmitSlice, sendst->config, sendst->fifo, sendst->stats, sendst->socket); - } while (udpc_nrParticipants(sendst->rc.participantsDb) || (config->flags & FLAG_ASYNC)); - -exit_main_loop: - cancelReturnChannel(&sendst->rc); - pc_produceEnd(sendst->fifo->freeMemQueue); - return 0; -} - -int spawnNetSender(struct fifo* fifo, int sock, struct net_config* config, participantsDb_t db) -{ - int i; - - sender_state_t sendst = MALLOC(struct senderState); - sendst->fifo = fifo; - sendst->socket = sock; - sendst->config = config; - // sendst->stats = stats; -#ifdef BB_FEATURE_UDPCAST_FEC - - if (sendst->config->flags & FLAG_FEC) - sendst->fec_data = xmalloc(NR_SLICES * config->fec_stripes * config->fec_redundancy * config->blockSize); - -#endif - sendst->rc.participantsDb = db; - initReturnChannel(&sendst->rc, sendst->config, sendst->socket); - - sendst->free_slices_pc = pc_makeProduconsum(NR_SLICES, "free slices"); - pc_produce(sendst->free_slices_pc, NR_SLICES); - - for (i = 0; i < NR_SLICES; i++) - sendst->slices[i].state = slice::SLICE_FREE; - -#ifdef BB_FEATURE_UDPCAST_FEC - - if (sendst->config->flags & FLAG_FEC) - { - /* Free memory queue is initially full */ - fec_init(); - sendst->fec_data_pc = pc_makeProduconsum(NR_SLICES, "fec data"); - - pthread_create(&sendst->fec_thread, NULL, fecMain, sendst); - } - -#endif - - pthread_create(&fifo->thread, NULL, netSenderMain, sendst); - return 0; -} - -struct sockaddr_in* udpc_getParticipantIp(participantsDb_t db, int i) -{ - return &db->clientTable[i].addr; -} - -int udpc_getParticipantCapabilities(participantsDb_t db, int i) -{ - return db->clientTable[i].capabilities; -} - -unsigned int udpc_getParticipantRcvBuf(participantsDb_t db, int i) -{ - return db->clientTable[i].rcvbuf; -} - -int pc_consumeContiguousMinAmount(produconsum_t pc, int amount) -{ - int n = _consumeAny(pc, amount, 0); - int l = pc->size - (pc->consumed % pc->size); - - if (n > l) - n = l; - - return n; -} - -#define BLOCKSIZE 4096 - -void localReader(struct fifo* fifo, const uint8_t* buf, uint32_t len) -{ - // cerr << "starting to send " << len << " bytes" << endl; - uint32_t offset = 0; - - while (1) - { - int pos = pc_getConsumerPosition(fifo->freeMemQueue); - int bytes = pc_consumeContiguousMinAmount(fifo->freeMemQueue, BLOCKSIZE); - - if (bytes > (pos + bytes) % BLOCKSIZE) - bytes -= (pos + bytes) % BLOCKSIZE; - - if (offset + bytes > len) - bytes = len - offset; - - // cerr << "sending " << bytes << " bytes from bs..." << endl; - // bytes = read(in, fifo->dataBuffer + pos, bytes); - memcpy(fifo->dataBuffer + pos, buf + offset, bytes); - offset += bytes; - - if (bytes == 0) - { - /* the end */ - pc_produceEnd(fifo->data); - break; - } - else - { - pc_consumed(fifo->freeMemQueue, bytes); - pc_produce(fifo->data, bytes); - } - } - - // cerr << "done sending" << endl; -} - -unsigned int udpc_getRcvBuf(int sock) -{ - unsigned int bufsize; - socklen_t len = sizeof(int); - - if (getsockopt(sock, SOL_SOCKET, SO_RCVBUF, (char*)&bufsize, &len) < 0) - return -1; - - return bufsize; -} - -int sendConnectReq(struct client_config* client_config, struct net_config* net_config, int haveServerAddress) -{ - // cerr << "sending a connect request" << endl; - struct connectReq connectReq; - - if (net_config->flags & FLAG_PASSIVE) - return 0; - - connectReq.opCode = htons(CMD_CONNECT_REQ); - connectReq.reserved = 0; - connectReq.capabilities = htonl(RECEIVER_CAPABILITIES); - connectReq.rcvbuf = htonl(udpc_getRcvBuf(client_config->S_UCAST)); - - if (haveServerAddress) - return SSEND(connectReq); - else - return BCAST_CONTROL(client_config->S_UCAST, connectReq); -} - -void udpc_setRcvBuf(int sock, unsigned int bufsize) -{ - if (setsockopt(sock, SOL_SOCKET, SO_RCVBUF, (char*)&bufsize, sizeof(bufsize)) < 0) - { - // TODO: FIXME - // perror("Set receiver buffer"); - } -} - -void udpc_copyFromMessage(struct sockaddr_in* dst, unsigned char* src) -{ - memcpy((char*)&dst->sin_addr, src, sizeof(struct in_addr)); -} - -unsigned short udpc_getPort(struct sockaddr_in* addr) -{ - return ntohs(((struct sockaddr_in*)addr)->sin_port); -} - -int udpc_selectSock(int* socks, int nr, int startTimeout) -{ - fd_set read_set; - int r; - int maxFd; - struct timeval tv, *tvp; - - if (startTimeout) - { - tv.tv_sec = startTimeout; - tv.tv_usec = 0; - tvp = &tv; - } - else - { - tvp = NULL; - } - - maxFd = prepareForSelect(socks, nr, &read_set); - r = select(maxFd + 1, &read_set, NULL, NULL, tvp); - - if (r < 0) - return r; - - return udpc_getSelectedSock(socks, nr, &read_set); -} - -void udpc_zeroSockArray(int* socks, int nr) -{ - int i; - - for (i = 0; i < nr; i++) - socks[i] = -1; -} - -unsigned char* getBlockSpace(struct clientState* clst) -{ - int pos; - - if (clst->localPos) - { - clst->localPos--; - return clst->localBlockAddresses[clst->localPos]; - } - - pc_consume(clst->freeBlocks_pc, 1); - pos = pc_getConsumerPosition(clst->freeBlocks_pc); - pc_consumed(clst->freeBlocks_pc, 1); - return clst->blockAddresses[pos]; -} - -void setNextBlock(struct clientState* clst) -{ - clst->nextBlock = getBlockSpace(clst); -} - -int setupMessages(struct clientState* clst) -{ - /* the messages received from the server */ - clst->data_iov[0].iov_base = (void*)&clst->Msg; - clst->data_iov[0].iov_len = sizeof(clst->Msg); - - /* namelen set just before reception */ - clst->data_hdr.msg_iov = clst->data_iov; - /* iovlen set just before reception */ - - initMsgHdr(&clst->data_hdr); - return 0; -} - -struct slice* initSlice(struct clientState* clst, struct slice* slice, int sliceNo) -{ - idbassert(slice->state == slice::SLICE_FREE || slice->state == slice::SLICE_RECEIVING); - - slice->magic = SLICEMAGIC; - slice->state = slice::SLICE_RECEIVING; - slice->blocksTransferred = 0; - slice->dataBlocksTransferred = 0; - BZERO(slice->retransmit); - slice->freePos = 0; - slice->bytes = 0; - - if (clst->currentSlice != NULL && !clst->currentSlice->bytesKnown) - { - // udpc_fatal(1, "Previous slice size not known\n"); - } - - if (clst->currentSliceNo != sliceNo - 1) - { - // udpc_fatal(1, "Slice no mismatch %d <-> %d\n", sliceNo, clst->currentSliceNo); - } - - slice->bytesKnown = 0; - slice->sliceNo = sliceNo; - - BZERO(slice->missing_data_blocks); -#ifdef BB_FEATURE_UDPCAST_FEC - BZERO(slice->fec_stripes); - BZERO(slice->fec_blocks); - BZERO(slice->fec_descs); -#endif - clst->currentSlice = slice; - clst->currentSliceNo = sliceNo; - return slice; -} -struct slice* newSlice(struct clientState* clst, int sliceNo) -{ - struct slice* slice = NULL; - int i; - - pc_consume(clst->free_slices_pc, 1); - i = pc_getConsumerPosition(clst->free_slices_pc); - pc_consumed(clst->free_slices_pc, 1); - slice = &clst->slices[i]; - idbassert(slice->state == slice::SLICE_FREE); - - /* wait for free data memory */ - slice->base = pc_getConsumerPosition(clst->fifo->freeMemQueue); - pc_consume(clst->fifo->freeMemQueue, clst->net_config->blockSize * MAX_SLICE_SIZE); - initSlice(clst, slice, sliceNo); - return slice; -} - -#define ADR(x, bs) (fifo->dataBuffer + (slice->base + (x)*bs) % fifo->dataBufSize) - -void closeAllExcept(struct clientState* clst, int fd) -{ - int i; - int* socks = clst->client_config->socks; - - if (clst->selectedFd >= 0) - return; - - clst->selectedFd = fd; - - for (i = 1; i < NR_CLIENT_SOCKS; i++) - if (socks[i] != -1 && socks[i] != fd) - udpc_closeSock(socks, NR_CLIENT_SOCKS, i); -} - -struct slice* rcvFindSlice(struct clientState* clst, int sliceNo) -{ - if (!clst->currentSlice) - { - /* Streaming mode? */ - clst->currentSliceNo = sliceNo - 1; - return newSlice(clst, sliceNo); - } - - if (sliceNo <= clst->currentSliceNo) - { - struct slice* slice = clst->currentSlice; - int pos = slice - clst->slices; - idbassert(slice == NULL || slice->magic == SLICEMAGIC); - - while (slice->sliceNo != sliceNo) - { - if (slice->state == slice::SLICE_FREE) - return NULL; - - idbassert(slice->magic == SLICEMAGIC); - pos--; - - if (pos < 0) - pos += NR_SLICES; - - slice = &clst->slices[pos]; - } - - return slice; - } - - if (sliceNo != clst->currentSliceNo + 1) - { - // TODO: FIXME - // udpc_flprintf("Slice %d skipped\n", sliceNo-1); - // exit(1); - } - - if ((clst->net_config->flags & FLAG_STREAMING) && sliceNo != clst->currentSliceNo) - { - return initSlice(clst, clst->currentSlice, sliceNo); - } - - if (sliceNo > clst->receivedSliceNo + 2) - { - // TODO: FIXME -#if 0 - slice_t slice = rcvFindSlice(clst, clst->receivedSliceNo + 1); - udpc_flprintf("Dropped by server now=%d last=%d\n", sliceNo, clst->receivedSliceNo); - - if (slice != NULL) - printMissedBlockMap(clst, slice); - - exit(1); -#endif - } - - return newSlice(clst, sliceNo); -} - -void setSliceBytes(struct slice* slice, struct clientState* clst, int bytes) -{ - idbassert(slice->magic == SLICEMAGIC); - - if (slice->bytesKnown) - { - if (slice->bytes != bytes) - { - // TODO: FIXME - // udpc_fatal(1, "Byte number mismatch %d <-> %d\n", bytes, slice->bytes); - } - } - else - { - slice->bytesKnown = 1; - slice->bytes = bytes; - - if (bytes == 0) - clst->netEndReached = 1; - } -} - -void advanceReceivedPointer(struct clientState* clst) -{ - int pos = clst->receivedPtr; - - while (1) - { - slice_t slice = &clst->slices[pos]; - - if ( -#ifdef BB_FEATURE_UDPCAST_FEC - slice->state != SLICE_FEC && slice->state != SLICE_FEC_DONE && -#endif - slice->state != slice::SLICE_DONE) - break; - - pos++; - clst->receivedSliceNo = slice->sliceNo; - - if (pos >= NR_SLICES) - pos -= NR_SLICES; - } - - clst->receivedPtr = pos; -} - -void receiverStatsAddBytes(receiver_stats_t rs, long bytes) -{ - if (rs != NULL) - rs->totalBytes += bytes; -} - -void cleanupSlices(struct clientState* clst, unsigned int doneState) -{ - while (1) - { - int pos = pc_getProducerPosition(clst->free_slices_pc); - int bytes; - slice_t slice = &clst->slices[pos]; - - if (slice->state != (signed)doneState) - break; - - receiverStatsAddBytes(clst->stats, slice->bytes); - bytes = slice->bytes; - - /* signal data received */ - if (bytes == 0) - { - pc_produceEnd(clst->fifo->data); - } - else - pc_produce(clst->fifo->data, slice->bytes); - - /* free up slice structure */ - clst->slices[pos].state = slice::SLICE_FREE; - pc_produce(clst->free_slices_pc, 1); - - /* if at end, exit this thread */ - if (!bytes) - { - clst->endReached = 2; - } - } -} - -void checkSliceComplete(struct clientState* clst, struct slice* slice) -{ - int blocksInSlice; - - idbassert(slice->magic == SLICEMAGIC); - - if (slice->state != slice::SLICE_RECEIVING) - /* bad starting state */ - return; - - /* is this slice ready ? */ - idbassert(clst->net_config->blockSize != 0); - blocksInSlice = (slice->bytes + clst->net_config->blockSize - 1) / clst->net_config->blockSize; - - if (blocksInSlice == slice->blocksTransferred) - { - pc_consumed(clst->fifo->freeMemQueue, slice->bytes); - clst->net_config->flags &= ~FLAG_STREAMING; - - if (blocksInSlice == slice->dataBlocksTransferred) - slice->state = slice::SLICE_DONE; - else - { -#ifdef BB_FEATURE_UDPCAST_FEC - idbassert(clst->use_fec == 1); - slice->state = SLICE_FEC; -#else - idbassert(0); -#endif - } - - advanceReceivedPointer(clst); -#ifdef BB_FEATURE_UDPCAST_FEC - - if (clst->use_fec) - { - int n = pc_getProducerPosition(clst->fec_data_pc); - idbassert(slice->state == SLICE_DONE || slice->state == SLICE_FEC); - clst->fec_slices[n] = slice; - pc_produce(clst->fec_data_pc, 1); - } - else -#endif - cleanupSlices(clst, slice::SLICE_DONE); - } -} - -int processDataBlock(struct clientState* clst, int sliceNo, int blockNo, int bytes) -{ - // cerr << "processDataBlock(): " << sliceNo << ", " << blockNo << ", " << bytes << endl; - struct fifo* fifo = clst->fifo; - struct slice* slice = rcvFindSlice(clst, sliceNo); - unsigned char *shouldAddress, *isAddress; - - idbassert(slice == NULL || slice->magic == SLICEMAGIC); - - if (slice == NULL || slice->state == slice::SLICE_FREE || slice->state == slice::SLICE_DONE -#ifdef BB_FEATURE_UDPCAST_FEC - || slice->state == SLICE_FEC || slice->state == SLICE_FEC_DONE -#endif - ) - { - /* an old slice. Ignore */ - // cerr << "ignore" << endl; - return 0; - } - - if (sliceNo > clst->currentSliceNo + 2) - { - // TODO: FIXME - // udpc_fatal(1, "We have been dropped by sender\n"); - } - - if (BIT_ISSET(blockNo, slice->retransmit.map)) - { - /* we already have this packet, ignore */ -#if 0 - flprintf("Packet %d:%d not for us\n", sliceNo, blockNo); -#endif - // cerr << "dup" << endl; - return 0; - } - - if (slice->base % clst->net_config->blockSize) - { - // TODO: FIXME - // udpc_fatal(1, "Bad base %d, not multiple of block size %d\n", slice->base, - // clst->net_config->blockSize); - // cerr << "bad base" << endl; - } - - // cerr << "good slice" << endl; - - shouldAddress = ADR(blockNo, clst->net_config->blockSize); - isAddress = (unsigned char*)clst->data_hdr.msg_iov[1].iov_base; - - if (shouldAddress != isAddress) - { - /* copy message to the correct place */ - memcpy(shouldAddress, isAddress, clst->net_config->blockSize); - } - - if (clst->client_config->sender_is_newgen && bytes != 0) - setSliceBytes(slice, clst, bytes); - - if (clst->client_config->sender_is_newgen && bytes == 0) - clst->netEndReached = 0; - - SET_BIT(blockNo, slice->retransmit.map); -#ifdef BB_FEATURE_UDPCAST_FEC - - if (slice->fec_stripes) - { - int stripe = blockNo % slice->fec_stripes; - slice->missing_data_blocks[stripe]--; - idbassert(slice->missing_data_blocks[stripe] >= 0); - - if (slice->missing_data_blocks[stripe] < slice->fec_blocks[stripe]) - { - int blockIdx; - /* FIXME: FEC block should be enqueued in local queue here...*/ - slice->fec_blocks[stripe]--; - blockIdx = stripe + slice->fec_blocks[stripe] * slice->fec_stripes; - idbassert(slice->fec_descs[blockIdx].adr != 0); - clst->localBlockAddresses[clst->localPos++] = slice->fec_descs[blockIdx].adr; - slice->fec_descs[blockIdx].adr = 0; - slice->blocksTransferred--; - } - } - -#endif - slice->dataBlocksTransferred++; - slice->blocksTransferred++; - - while (slice->freePos < MAX_SLICE_SIZE && BIT_ISSET(slice->freePos, slice->retransmit.map)) - slice->freePos++; - - checkSliceComplete(clst, slice); - return 0; -} - -int sendOk(struct client_config* client_config, unsigned int sliceNo) -{ - struct ok ok; - ok.opCode = htons(CMD_OK); - ok.reserved = 0; - ok.sliceNo = htonl(sliceNo); - return SSEND(ok); -} - -int sendRetransmit(struct clientState* clst, struct slice* slice, int rxmit) -{ - struct client_config* client_config = clst->client_config; - - idbassert(slice->magic == SLICEMAGIC); - slice->retransmit.opCode = htons(CMD_RETRANSMIT); - slice->retransmit.reserved = 0; - slice->retransmit.sliceNo = htonl(slice->sliceNo); - slice->retransmit.rxmit = htonl(rxmit); - return SSEND(slice->retransmit); -} - -int processReqAck(struct clientState* clst, int sliceNo, int bytes, int rxmit) -{ - struct slice* slice = rcvFindSlice(clst, sliceNo); - int blocksInSlice; - char* readySet = (char*)clst->data_hdr.msg_iov[1].iov_base; - - idbassert(slice == NULL || slice->magic == SLICEMAGIC); - - { - struct timeval tv; - gettimeofday(&tv, 0); - /* usleep(1); DEBUG: FIXME */ - } - - if (BIT_ISSET(clst->client_config->clientNumber, readySet)) - { - /* not for us */ - return 0; - } - - if (slice == NULL) - { - /* an old slice => send ok */ - return sendOk(clst->client_config, sliceNo); - } - - setSliceBytes(slice, clst, bytes); - idbassert(clst->net_config->blockSize != 0); - blocksInSlice = (slice->bytes + clst->net_config->blockSize - 1) / clst->net_config->blockSize; - - if (blocksInSlice == slice->blocksTransferred) - { - /* send ok */ - sendOk(clst->client_config, slice->sliceNo); - } - else - { - sendRetransmit(clst, slice, rxmit); - } - - checkSliceComplete(clst, slice); /* needed for the final 0 sized slice */ - advanceReceivedPointer(clst); -#ifdef BB_FEATURE_UDPCAST_FEC - - if (!clst->use_fec) - cleanupSlices(clst, SLICE_DONE); - -#endif - return 0; -} - -int udpc_isAddressEqual(struct sockaddr_in* a, struct sockaddr_in* b) -{ - return !memcmp((char*)a, (char*)b, 8); -} -int dispatchMessage(struct clientState* clst) -{ - int ret; - struct sockaddr_in lserver; - struct fifo* fifo = clst->fifo; - int fd = -1; - struct client_config* client_config = clst->client_config; - - /* set up message header */ - if (clst->currentSlice != NULL && clst->currentSlice->freePos < MAX_SLICE_SIZE) - { - struct slice* slice = clst->currentSlice; - idbassert(slice == NULL || slice->magic == SLICEMAGIC); - clst->data_iov[1].iov_base = ADR(slice->freePos, clst->net_config->blockSize); - } - else - { - clst->data_iov[1].iov_base = clst->nextBlock; - } - - clst->data_iov[1].iov_len = clst->net_config->blockSize; - clst->data_hdr.msg_iovlen = 2; - - clst->data_hdr.msg_name = &lserver; - clst->data_hdr.msg_namelen = sizeof(struct sockaddr_in); - - while (clst->endReached || clst->netEndReached) - { - int oldEndReached = clst->endReached; - int nr_desc; - struct timeval tv; - fd_set read_set; - - int maxFd = prepareForSelect(client_config->socks, NR_CLIENT_SOCKS, &read_set); - - tv.tv_sec = clst->net_config->exitWait / 1000; - tv.tv_usec = (clst->net_config->exitWait % 1000) * 1000; - nr_desc = select(maxFd, &read_set, 0, 0, &tv); - - if (nr_desc < 0) - { - break; - } - - fd = udpc_getSelectedSock(client_config->socks, NR_CLIENT_SOCKS, &read_set); - - if (fd >= 0) - break; - - /* Timeout expired */ - if (oldEndReached >= 2) - { - clst->endReached = 3; - return 0; - } - } - - if (fd < 0) - fd = clst->selectedFd; - - if (fd < 0) - { - struct timeval tv, *tvp; - fd_set read_set; - int maxFd = prepareForSelect(client_config->socks, NR_CLIENT_SOCKS, &read_set); - clst->promptPrinted = 1; - - if (clst->net_config->startTimeout == 0) - { - tvp = NULL; - } - else - { - tv.tv_sec = clst->net_config->startTimeout; - tv.tv_usec = 0; - tvp = &tv; - } - - // cerr << "waiting for data..." << endl; - ret = selectWithoutConsole(maxFd + 1, &read_set, tvp); - - if (ret < 0) - { - perror("Select"); - return 0; - } - - if (ret == 0) - { - clst->endReached = 3; - clst->netEndReached = 3; - pc_produceEnd(clst->fifo->data); - return 1; - } - - fd = udpc_getSelectedSock(clst->client_config->socks, NR_CLIENT_SOCKS, &read_set); - } - -#ifdef LOSSTEST - loseRecvPacket(fd); - ret = RecvMsg(fd, &clst->data_hdr, 0); -#else - ret = recvmsg(fd, &clst->data_hdr, 0); -// cerr << "got some data on fd " << fd << endl; -#endif - - if (ret < 0) - { -#if DEBUG - flprintf("data recvfrom %d: %s\n", fd, strerror(errno)); -#endif - return -1; - } - -#if 0 - fprintf(stderr, "received packet for slice %d, block %d\n", - ntohl(Msg.sliceNo), ntohs(db.blockNo)); -#endif - - if (!udpc_isAddressEqual(&lserver, &clst->client_config->serverAddr)) - { - return -1; - } - - switch (ntohs(clst->Msg.opCode)) - { - case CMD_DATA: - // cerr << "got CMD_DATA" << endl; - closeAllExcept(clst, fd); - // udpc_receiverStatsStartTimer(clst->stats); - clst->client_config->isStarted = 1; - return processDataBlock(clst, ntohl(clst->Msg.dataBlock.sliceNo), ntohs(clst->Msg.dataBlock.blockNo), - ntohl(clst->Msg.dataBlock.bytes)); -#ifdef BB_FEATURE_UDPCAST_FEC - - case CMD_FEC: - // cerr << "got CMD_FEC" << endl; - closeAllExcept(clst, fd); - // receiverStatsStartTimer(clst->stats); - clst->client_config->isStarted = 1; - return processFecBlock(clst, ntohs(clst->Msg.fecBlock.stripes), ntohl(clst->Msg.fecBlock.sliceNo), - ntohs(clst->Msg.fecBlock.blockNo), ntohl(clst->Msg.fecBlock.bytes)); -#endif - - case CMD_REQACK: - // cerr << "got CMD_REQACK" << endl; - closeAllExcept(clst, fd); - // receiverStatsStartTimer(clst->stats); - clst->client_config->isStarted = 1; - return processReqAck(clst, ntohl(clst->Msg.reqack.sliceNo), ntohl(clst->Msg.reqack.bytes), - ntohl(clst->Msg.reqack.rxmit)); - - case CMD_HELLO_STREAMING: - case CMD_HELLO_NEW: - case CMD_HELLO: - // cerr << "got CMD_HELLO" << endl; - /* retransmission of hello to find other participants ==> ignore */ - return 0; - - default: break; - } - - return -1; -} - -THREAD_RETURN netReceiverMain(void* args0) -{ - struct clientState* clst = (struct clientState*)args0; - - clst->currentSliceNo = 0; - setupMessages(clst); - - clst->currentSliceNo = -1; - clst->currentSlice = NULL; - clst->promptPrinted = 0; - - if (!(clst->net_config->flags & FLAG_STREAMING)) - newSlice(clst, 0); - else - { - clst->currentSlice = NULL; - clst->currentSliceNo = 0; - } - - while (clst->endReached < 3) - { - dispatchMessage(clst); - } - -#ifdef BB_FEATURE_UDPCAST_FEC - - if (clst->use_fec) - pthread_join(clst->fec_thread, NULL); - -#endif - return 0; -} - -int spawnNetReceiver(struct fifo* fifo, struct client_config* client_config, struct net_config* net_config) -{ - int i; - struct clientState* clst = MALLOC(struct clientState); - clst->fifo = fifo; - clst->client_config = client_config; - clst->net_config = net_config; - // clst->stats = stats; - clst->endReached = 0; - clst->netEndReached = 0; - clst->selectedFd = -1; - - clst->free_slices_pc = pc_makeProduconsum(NR_SLICES, "free slices"); - pc_produce(clst->free_slices_pc, NR_SLICES); - - for (i = 0; i < NR_SLICES; i++) - clst->slices[i].state = slice::SLICE_FREE; - - clst->receivedPtr = 0; - clst->receivedSliceNo = 0; - -#ifdef BB_FEATURE_UDPCAST_FEC - fec_init(); /* fec new involves memory - * allocation. Better do it here */ - clst->use_fec = 0; - clst->fec_data_pc = pc_makeProduconsum(NR_SLICES, "fec data"); -#endif - -#define NR_BLOCKS 4096 - clst->freeBlocks_pc = pc_makeProduconsum(NR_BLOCKS, "free blocks"); - pc_produce(clst->freeBlocks_pc, NR_BLOCKS); - clst->blockAddresses = (unsigned char**)calloc(NR_BLOCKS, sizeof(char*)); - clst->localBlockAddresses = (unsigned char**)calloc(NR_BLOCKS, sizeof(char*)); - clst->blockData = (unsigned char*)malloc(NR_BLOCKS * net_config->blockSize); - - for (i = 0; i < NR_BLOCKS; i++) - clst->blockAddresses[i] = clst->blockData + i * net_config->blockSize; - - clst->localPos = 0; - - setNextBlock(clst); - return pthread_create(&client_config->thread, NULL, netReceiverMain, clst); -} - -unsigned int pc_getSize(produconsum_t pc) -{ - return pc->size; -} - -int writer(struct fifo* fifo, SBS outbs) -{ - // cerr << "start appending to outbs" << endl; - outbs->restart(); - int fifoSize = pc_getSize(fifo->data); - - while (1) - { - int pos = pc_getConsumerPosition(fifo->data); - int bytes = pc_consumeContiguousMinAmount(fifo->data, BLOCKSIZE); - - if (bytes == 0) - { - // cerr << "done appending to outbs: " << outbs->length() << endl; - return 0; - } - - /* - * If we have more than blocksize, round down to nearest blocksize - * multiple - */ - if (pos + bytes != fifoSize && bytes > (pos + bytes) % BLOCKSIZE) - bytes -= (pos + bytes) % BLOCKSIZE; - - /* make sure we don't write to big a chunk... Better to - * liberate small chunks one by one rather than attempt to - * write out a bigger chunk and block reception for too - * long */ - if (bytes > 128 * 1024) - bytes = 64 * 1024; - - // bytes = write(outFile, fifo->dataBuffer + pos, bytes); - // cerr << "appending " << bytes << " bytes to outbs..." << endl; - outbs->append(fifo->dataBuffer + pos, bytes); - pc_consumed(fifo->data, bytes); - pc_produce(fifo->freeMemQueue, bytes); - } -} -} // namespace - -namespace multicast -{ -MulticastImpl::MulticastImpl(int min_receivers, const string& ifName, int portBase, int bufSize) - : fIfName(ifName), fDb(0) -{ - udpc_clearIp(&fNet_config.dataMcastAddr); - fNet_config.mcastRdv = 0; - fNet_config.blockSize = 1024; // 1456; - fNet_config.sliceSize = 16; - fNet_config.portBase = portBase; - fNet_config.nrGovernors = 0; - fNet_config.flags = FLAG_NOKBD; - fNet_config.capabilities = 0; - fNet_config.min_slice_size = 16; - fNet_config.max_slice_size = 1024; - fNet_config.default_slice_size = 0; - fNet_config.ttl = 1; - fNet_config.rexmit_hello_interval = 2000; - fNet_config.autostart = 0; - fNet_config.requestedBufSize = bufSize; - - fNet_config.min_receivers = min_receivers; - fNet_config.max_receivers_wait = 0; - fNet_config.min_receivers_wait = 0; - - fNet_config.retriesUntilDrop = 200; - - fNet_config.rehelloOffset = 50; - - fStat_config.log = 0; - fStat_config.bwPeriod = 0; - fStat_config.printUncompressedPos = -1; - fStat_config.statPeriod = DEFLT_STAT_PERIOD; - - fNet_config.net_if = 0; - - // full-duplex - fNet_config.flags |= FLAG_SN; - - if (fIfName.empty()) - fIfName = "eth0"; - - fSock[0] = -1; - fSock[1] = -1; - fSock[2] = -1; -} - -MulticastImpl::~MulticastImpl() -{ - delete fDb; - - for (int i = 0; i < 3; i++) - { - if (fSock[i] >= 0) - { - shutdown(fSock[i], SHUT_RDWR); - close(fSock[i]); - } - } -} - -void MulticastImpl::startSender() -{ - int tries; - time_t firstConnected = 0; - time_t* firstConnectedP; - - // participantsDb_t db; - - /* make the socket and print banner */ - // int fSock[3]; - int nr = 0; - int fd; - int r; - int j; - - fNet_config.net_if = udpc_getNetIf(fIfName.c_str()); - - fSock[nr++] = udpc_makeSocket(ADDR_TYPE_UCAST, fNet_config.net_if, NULL, SENDER_PORT(fNet_config.portBase)); - // cerr << "sock[" << (nr-1) << "] = " << fSock[(nr-1)] << endl; - - if (!(fNet_config.flags & (FLAG_SN | FLAG_NOTSN))) - { - if (udpc_isFullDuplex(fSock[0], fNet_config.net_if->name) == 1) - { - fNet_config.flags |= FLAG_SN; - } - } - - fd = udpc_makeSocket(ADDR_TYPE_BCAST, fNet_config.net_if, NULL, SENDER_PORT(fNet_config.portBase)); - - if (fd >= 0) - fSock[nr++] = fd; - - // cerr << "sock[" << (nr-1) << "] = " << fSock[(nr-1)] << endl; - - if (fNet_config.requestedBufSize) - udpc_setSendBuf(fSock[0], fNet_config.requestedBufSize); - - fNet_config.controlMcastAddr.sin_addr.s_addr = 0; - - if (fNet_config.ttl == 1 && fNet_config.mcastRdv == NULL) - { - udpc_getBroadCastAddress(fNet_config.net_if, &fNet_config.controlMcastAddr, - RECEIVER_PORT(fNet_config.portBase)); - udpc_setSocketToBroadcast(fSock[0]); - } - - if (fNet_config.controlMcastAddr.sin_addr.s_addr == 0) - { - udpc_getMcastAllAddress(&fNet_config.controlMcastAddr, fNet_config.mcastRdv, - RECEIVER_PORT(fNet_config.portBase)); - - /* Only do the following if controlMcastAddr is indeed an - mcast address ... */ - if (isMcastAddress(&fNet_config.controlMcastAddr)) - { - udpc_setMcastDestination(fSock[0], fNet_config.net_if, &fNet_config.controlMcastAddr); - udpc_setTtl(fSock[0], fNet_config.ttl); - fSock[nr++] = udpc_makeSocket(ADDR_TYPE_MCAST, fNet_config.net_if, &fNet_config.controlMcastAddr, - SENDER_PORT(fNet_config.portBase)); - // cerr << "sock[" << (nr-1) << "] = " << fSock[(nr-1)] << endl; - } - } - - if (!(fNet_config.flags & FLAG_POINTOPOINT) && udpc_ipIsZero(&fNet_config.dataMcastAddr)) - { - udpc_getDefaultMcastAddress(fNet_config.net_if, &fNet_config.dataMcastAddr); - } - - if (fNet_config.flags & FLAG_POINTOPOINT) - { - udpc_clearIp(&fNet_config.dataMcastAddr); - } - - setPort(&fNet_config.dataMcastAddr, RECEIVER_PORT(fNet_config.portBase)); - - fNet_config.capabilities = SENDER_CAPABILITIES; - - if (fNet_config.flags & FLAG_ASYNC) - fNet_config.capabilities |= CAP_ASYNC; - - udpc_sendHello(&fNet_config, fSock[0], 0); - - fDb = udpc_makeParticipantsDb(); - tries = 0; - - if (fNet_config.min_receivers || fNet_config.min_receivers_wait || fNet_config.max_receivers_wait) - firstConnectedP = &firstConnected; - else - firstConnectedP = NULL; - - while (!(r = mainDispatcher(fSock, nr, fDb, &fNet_config, &tries, firstConnectedP))) - ; - - for (j = 1; j < nr; j++) - if (fSock[j] != fSock[0]) - closesocket(fSock[j]); - - if (r == 1) - { - int i; - - for (i = 1; i < nr; i++) - udpc_closeSock(fSock, nr, i); - } - - // doTransfer(fSock[0], fDb, &fNet_config, &fStat_config); -} - -void MulticastImpl::doTransfer(const uint8_t* buf, uint32_t len) -{ - int i; - int ret; - struct fifo fifo; - int isPtP = isPointToPoint(fDb, fNet_config.flags); - - fNet_config.rcvbuf = 0; - - for (i = 0; i < MAX_CLIENTS; i++) - if (udpc_isParticipantValid(fDb, i)) - { - unsigned int pRcvBuf = udpc_getParticipantRcvBuf(fDb, i); - - if (isPtP) - udpc_copyIpFrom(&fNet_config.dataMcastAddr, udpc_getParticipantIp(fDb, i)); - - fNet_config.capabilities &= udpc_getParticipantCapabilities(fDb, i); - - if (pRcvBuf != 0 && (fNet_config.rcvbuf == 0 || fNet_config.rcvbuf > pRcvBuf)) - fNet_config.rcvbuf = pRcvBuf; - } - - if (isMcastAddress(&fNet_config.dataMcastAddr)) - udpc_setMcastDestination(fSock[0], fNet_config.net_if, &fNet_config.dataMcastAddr); - - if (!(fNet_config.capabilities & CAP_BIG_ENDIAN)) - { - // TODO: FIXME - // udpc_fatal(1, "Peer with incompatible endianness"); - } - - if (!(fNet_config.capabilities & CAP_NEW_GEN)) - { - fNet_config.dataMcastAddr = fNet_config.controlMcastAddr; - fNet_config.flags &= ~(FLAG_SN | FLAG_ASYNC); - } - - if (fNet_config.flags & FLAG_BCAST) - fNet_config.dataMcastAddr = fNet_config.controlMcastAddr; - - udpc_initFifo(&fifo, fNet_config.blockSize); - ret = spawnNetSender(&fifo, fSock[0], &fNet_config, fDb); - localReader(&fifo, buf, len); - - pthread_join(fifo.thread, NULL); -} - -void MulticastImpl::startReceiver() -{ - union serverControlMsg Msg; - int connectReqSent = 0; - struct sockaddr_in myIp; - int haveServerAddress; - - fClient_config.sender_is_newgen = 0; - - fNet_config.net_if = udpc_getNetIf(fIfName.c_str()); - { - fprintf(stderr, "net_if:\n\taddr = 0x%x\n\tbcast = 0x%x\n\tname = %s\n\tindex = %d\n", - fNet_config.net_if->addr.s_addr, fNet_config.net_if->bcast.s_addr, fNet_config.net_if->name, - fNet_config.net_if->index); - } - - udpc_zeroSockArray(fClient_config.socks, NR_CLIENT_SOCKS); - - fClient_config.S_UCAST = - udpc_makeSocket(ADDR_TYPE_UCAST, fNet_config.net_if, 0, RECEIVER_PORT(fNet_config.portBase)); - // cerr << "S_UCAST = " << fClient_config.S_UCAST << endl; - fClient_config.S_BCAST = - udpc_makeSocket(ADDR_TYPE_BCAST, fNet_config.net_if, 0, RECEIVER_PORT(fNet_config.portBase)); - // cerr << "S_BCAST = " << fClient_config.S_BCAST << endl; - - if (fNet_config.ttl == 1 && fNet_config.mcastRdv == NULL) - { - udpc_getBroadCastAddress(fNet_config.net_if, &fNet_config.controlMcastAddr, - SENDER_PORT(fNet_config.portBase)); - udpc_setSocketToBroadcast(fClient_config.S_UCAST); - } - else - { - udpc_getMcastAllAddress(&fNet_config.controlMcastAddr, fNet_config.mcastRdv, - SENDER_PORT(fNet_config.portBase)); - - if (isMcastAddress(&fNet_config.controlMcastAddr)) - { - udpc_setMcastDestination(fClient_config.S_UCAST, fNet_config.net_if, &fNet_config.controlMcastAddr); - udpc_setTtl(fClient_config.S_UCAST, fNet_config.ttl); - - fClient_config.S_MCAST_CTRL = - udpc_makeSocket(ADDR_TYPE_MCAST, fNet_config.net_if, &fNet_config.controlMcastAddr, - RECEIVER_PORT(fNet_config.portBase)); - // cerr << "S_MCAST_CTRL = " << fClient_config.S_MCAST_CTRL << endl; - // TODO: subscribe address as receiver to! - } - } - - udpc_clearIp(&fNet_config.dataMcastAddr); - - connectReqSent = 0; - haveServerAddress = 0; - - fClient_config.clientNumber = 0; /*default number for asynchronous transfer*/ - - while (1) - { - // int len; - int msglen; - int sock; - - if (!connectReqSent) - { - if (sendConnectReq(&fClient_config, &fNet_config, haveServerAddress) < 0) - { - // TODO: FIXME - // perror("sendto to locate server"); - } - - connectReqSent = 1; - } - - haveServerAddress = 0; - - // cerr << "waiting for msg..." << flush << endl; - sock = udpc_selectSock(fClient_config.socks, NR_CLIENT_SOCKS, fNet_config.startTimeout); - - // cerr << "got something" << endl; - if (sock < 0) - { - // TODO: FIXME - } - - // len = sizeof(server); - msglen = RECV(sock, Msg, fClient_config.serverAddr, fNet_config.portBase); - - if (msglen < 0) - { - // TODO: FIXME - // perror("recvfrom to locate server"); - // exit(1); - } - - if (udpc_getPort(&fClient_config.serverAddr) != SENDER_PORT(fNet_config.portBase)) - /* not from the right port */ - continue; - - switch (ntohs(Msg.opCode)) - { - case CMD_CONNECT_REPLY: - // cerr << "got conrep" << endl; - fClient_config.clientNumber = ntohl(Msg.connectReply.clNr); - fNet_config.blockSize = ntohl(Msg.connectReply.blockSize); - - if (ntohl(Msg.connectReply.capabilities) & CAP_NEW_GEN) - { - fClient_config.sender_is_newgen = 1; - udpc_copyFromMessage(&fNet_config.dataMcastAddr, Msg.connectReply.mcastAddr); - } - - if (fClient_config.clientNumber == -1) - { - // TODO: FIXME - // udpc_fatal(1, "Too many clients already connected\n"); - } - - goto break_loop; - - case CMD_HELLO_STREAMING: - case CMD_HELLO_NEW: - case CMD_HELLO: - // cerr << "got hello" << endl; - connectReqSent = 0; - - if (ntohs(Msg.opCode) == CMD_HELLO_STREAMING) - fNet_config.flags |= FLAG_STREAMING; - - if (ntohl(Msg.hello.capabilities) & CAP_NEW_GEN) - { - fClient_config.sender_is_newgen = 1; - udpc_copyFromMessage(&fNet_config.dataMcastAddr, Msg.hello.mcastAddr); - fNet_config.blockSize = ntohs(Msg.hello.blockSize); - - if (ntohl(Msg.hello.capabilities) & CAP_ASYNC) - fNet_config.flags |= FLAG_PASSIVE; - - if (fNet_config.flags & FLAG_PASSIVE) - goto break_loop; - } - - haveServerAddress = 1; - continue; - - case CMD_CONNECT_REQ: - case CMD_DATA: - case CMD_FEC: continue; - - default: break; - } - - // TODO: FIXME - // udpc_fatal(1, "Bad server reply %04x. Other transfer in progress?\n", (unsigned short) - // ntohs(Msg.opCode)); - } - -break_loop: - - udpc_getMyAddress(fNet_config.net_if, &myIp); - - if (!udpc_ipIsZero(&fNet_config.dataMcastAddr) && !udpc_ipIsEqual(&fNet_config.dataMcastAddr, &myIp) && - (udpc_ipIsZero(&fNet_config.controlMcastAddr) || - !udpc_ipIsEqual(&fNet_config.dataMcastAddr, &fNet_config.controlMcastAddr))) - { - fClient_config.S_MCAST_DATA = udpc_makeSocket( - ADDR_TYPE_MCAST, fNet_config.net_if, &fNet_config.dataMcastAddr, RECEIVER_PORT(fNet_config.portBase)); - // cerr << "S_MCAST_DATA = " << fClient_config.S_MCAST_DATA << endl; - } - - if (fNet_config.requestedBufSize) - { - int i; - - for (i = 0; i < NR_CLIENT_SOCKS; i++) - if (fClient_config.socks[i] != -1) - udpc_setRcvBuf(fClient_config.socks[i], fNet_config.requestedBufSize); - } -} - -void MulticastImpl::receive(SBS outbs) -{ - struct fifo fifo; - - udpc_initFifo(&fifo, fNet_config.blockSize); - fifo.data = pc_makeProduconsum(fifo.dataBufSize, "receive"); - fClient_config.isStarted = 0; - - spawnNetReceiver(&fifo, &fClient_config, &fNet_config); - writer(&fifo, outbs); - - pthread_join(fClient_config.thread, NULL); -} - -} // namespace multicast diff --git a/utils/multicast/impl.h b/utils/multicast/impl.h deleted file mode 100644 index ce3f41b0b..000000000 --- a/utils/multicast/impl.h +++ /dev/null @@ -1,63 +0,0 @@ -/* Copyright (C) 2014 InfiniDB, Inc. - - This program is free software; you can redistribute it and/or - modify it under the terms of the GNU General Public License - as published by the Free Software Foundation; version 2 of - the License. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU General Public License for more details. - - You should have received a copy of the GNU General Public License - along with this program; if not, write to the Free Software - Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, - MA 02110-1301, USA. */ - -/****************************************************************************** - * $Id: impl.h 3495 2013-01-21 14:09:51Z rdempsey $ - * - *****************************************************************************/ - -/* This code is based on udpcast-20090830. Most of the source code in that release - contains no copyright or licensing notices at all. The exception is fec.c, which - is not used here. The udpcast website, http://udpcast.linux.lu/, implies that - the source is covered under GPL. */ - -/** @file */ - -#pragma once - -#include -#include - -#include "bytestream.h" - -#include "udpcast.h" -#include "participants.h" - -namespace multicast -{ -class MulticastImpl -{ - public: - MulticastImpl(int min_receivers, const std::string& ifName, int portBase = 9000, - int bufSize = 8 * 1024 * 1024); - ~MulticastImpl(); - - void startSender(); - void doTransfer(const uint8_t* buf, uint32_t len); - - void startReceiver(); - void receive(messageqcpp::SBS obs); - - struct net_config fNet_config; - struct stat_config fStat_config; - struct client_config fClient_config; - std::string fIfName; - int fSock[3]; - participantsDb_t fDb; -}; - -} // namespace multicast diff --git a/utils/multicast/mc_fifo.h b/utils/multicast/mc_fifo.h deleted file mode 100644 index cf02f1018..000000000 --- a/utils/multicast/mc_fifo.h +++ /dev/null @@ -1,33 +0,0 @@ -/* Copyright (C) 2014 InfiniDB, Inc. - - This program is free software; you can redistribute it and/or - modify it under the terms of the GNU General Public License - as published by the Free Software Foundation; version 2 of - the License. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU General Public License for more details. - - You should have received a copy of the GNU General Public License - along with this program; if not, write to the Free Software - Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, - MA 02110-1301, USA. */ - -#pragma once - -#include "threads.h" -#include "produconsum.h" - -typedef struct fifo -{ - unsigned char* dataBuffer; - unsigned int dataBufSize; - - produconsum_t freeMemQueue; /* queue for free memory */ - produconsum_t data; /* queue for received data or data received - * from disk */ - - pthread_t thread; -} * fifo_t; diff --git a/utils/multicast/multicast.cpp b/utils/multicast/multicast.cpp deleted file mode 100644 index b471c170f..000000000 --- a/utils/multicast/multicast.cpp +++ /dev/null @@ -1,95 +0,0 @@ -/* Copyright (C) 2014 InfiniDB, Inc. - - This program is free software; you can redistribute it and/or - modify it under the terms of the GNU General Public License - as published by the Free Software Foundation; version 2 of - the License. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU General Public License for more details. - - You should have received a copy of the GNU General Public License - along with this program; if not, write to the Free Software - Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, - MA 02110-1301, USA. */ - -/****************************************************************************** - * $Id: multicast.cpp 3495 2013-01-21 14:09:51Z rdempsey $ - * - *****************************************************************************/ - -#include -#include -#include -#include -using namespace std; - -#include "bytestream.h" -using namespace messageqcpp; - -#include "configcpp.h" -using namespace config; - -#include "multicast.h" - -namespace multicast -{ -Multicast::Multicast() : fPMCount(1), fIFName("eth0"), fPortBase(9000), fBufSize(8 * 1024 * 1024) -{ - int tmp; - string stmp; - - Config* cf = Config::makeConfig(); - - tmp = Config::fromText(cf->getConfig("PrimitiveServers", "Count")); - - if (tmp > 0) - fPMCount = tmp; - - stmp = cf->getConfig("Multicast", "Interface"); - - if (!stmp.empty()) - fIFName = stmp; - - tmp = Config::fromText(cf->getConfig("Multicast", "PortBase")); - - if (tmp > 0) - fPortBase = tmp; - - tmp = Config::fromText(cf->getConfig("Multicast", "BufSize")); - - if (tmp > 0) - fBufSize = tmp; -} - -MulticastReceiver::MulticastReceiver() : fPimpl(0) -{ -} - -MulticastReceiver::~MulticastReceiver() -{ -} - -SBS MulticastReceiver::receive() -{ - throw runtime_error("Multicast is not available"); - return fByteStream; -} - -MulticastSender::MulticastSender() : fPimpl(0) -{ -} - -MulticastSender::~MulticastSender() -{ -} - -void MulticastSender::send(const ByteStream& msg) -{ - throw runtime_error("Multicast is not available"); -} - -} // namespace multicast - diff --git a/utils/multicast/multicast.h b/utils/multicast/multicast.h deleted file mode 100644 index 82c5702cc..000000000 --- a/utils/multicast/multicast.h +++ /dev/null @@ -1,128 +0,0 @@ -/* Copyright (C) 2014 InfiniDB, Inc. - - This program is free software; you can redistribute it and/or - modify it under the terms of the GNU General Public License - as published by the Free Software Foundation; version 2 of - the License. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU General Public License for more details. - - You should have received a copy of the GNU General Public License - along with this program; if not, write to the Free Software - Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, - MA 02110-1301, USA. */ - -/****************************************************************************** - * $Id: multicast.h 3495 2013-01-21 14:09:51Z rdempsey $ - * - *****************************************************************************/ - -/** @file - * class Multicast interface - */ - -#pragma once - -#include "messagequeue.h" - -namespace multicast -{ -/** @brief MulticastReceive - * Wrapper for multicast proto - */ - -class MulticastImpl; - -class Multicast -{ - public: - /** @brief ctor - * Base class - */ - Multicast(); - - /** @brief dtor - */ - virtual ~Multicast() - { - destroy(); - } - - virtual void destroy() - { - } - - int PMCount() const - { - return fPMCount; - } - std::string iFName() const - { - return fIFName; - } - int portBase() const - { - return fPortBase; - } - int bufSize() const - { - return fBufSize; - } - - private: - int fPMCount; - std::string fIFName; - int fPortBase; - int fBufSize; -}; - -class MulticastReceiver : public Multicast -{ - public: - /** @brief ctor - * - */ - MulticastReceiver(); - - ~MulticastReceiver(); - - messageqcpp::SBS receive(); - - private: - // not copyable - MulticastReceiver(const MulticastReceiver& rhs); - MulticastReceiver& operator=(const MulticastReceiver& rhs); - - messageqcpp::SBS fByteStream; - - MulticastImpl* fPimpl; -}; - -class MulticastSender : public Multicast -{ - public: - /** @brief ctor - * - */ - MulticastSender(); - - ~MulticastSender(); - - /** @brief receive - * - * @param bytestream to send - */ - void send(const messageqcpp::ByteStream& bs); - - private: - // Not copyable - MulticastSender(const MulticastSender& rhs); - MulticastSender& operator=(const MulticastSender& rhs); - - MulticastImpl* fPimpl; -}; - -} // namespace multicast diff --git a/utils/multicast/participants.h b/utils/multicast/participants.h deleted file mode 100644 index dd0f37325..000000000 --- a/utils/multicast/participants.h +++ /dev/null @@ -1,28 +0,0 @@ -/* Copyright (C) 2014 InfiniDB, Inc. - - This program is free software; you can redistribute it and/or - modify it under the terms of the GNU General Public License - as published by the Free Software Foundation; version 2 of - the License. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU General Public License for more details. - - You should have received a copy of the GNU General Public License - along with this program; if not, write to the Free Software - Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, - MA 02110-1301, USA. */ - -#pragma once - -#define MAX_CLIENTS 1024 - -typedef struct participantsDb* participantsDb_t; - -#define printNotSet udpc_printNotSet -#define printSet udpc_printSet - -void udpc_printNotSet(participantsDb_t db, char* d); -void udpc_printSet(participantsDb_t db, char* d); diff --git a/utils/multicast/produconsum.h b/utils/multicast/produconsum.h deleted file mode 100644 index 098f2abc5..000000000 --- a/utils/multicast/produconsum.h +++ /dev/null @@ -1,25 +0,0 @@ -/* Copyright (C) 2014 InfiniDB, Inc. - - This program is free software; you can redistribute it and/or - modify it under the terms of the GNU General Public License - as published by the Free Software Foundation; version 2 of - the License. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU General Public License for more details. - - You should have received a copy of the GNU General Public License - along with this program; if not, write to the Free Software - Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, - MA 02110-1301, USA. */ - -#pragma once - -typedef struct produconsum* produconsum_t; - -/** - * Get contiguous chunk of data - */ -int pc_consumeAnyContiguous(produconsum_t pc); diff --git a/utils/multicast/socklib.h b/utils/multicast/socklib.h deleted file mode 100644 index 74936d958..000000000 --- a/utils/multicast/socklib.h +++ /dev/null @@ -1,169 +0,0 @@ -/* Copyright (C) 2014 InfiniDB, Inc. - - This program is free software; you can redistribute it and/or - modify it under the terms of the GNU General Public License - as published by the Free Software Foundation; version 2 of - the License. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU General Public License for more details. - - You should have received a copy of the GNU General Public License - along with this program; if not, write to the Free Software - Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, - MA 02110-1301, USA. */ - -#pragma once - -#ifndef UDPCAST_CONFIG_H -#define UDPCAST_CONFIG_H -#include "mcsconfig.h" -#endif - -#include -#include - -#ifdef HAVE_SYS_SOCKET_H -#include -#endif - -#ifdef HAVE_SYS_SELECT_H -#include -#endif - -#ifdef HAVE_SYS_SOCKIO_H -#include -#endif - -#ifdef HAVE_NETINET_IN_H -#include -#endif - -#ifdef HAVE_NETDB_H -#include -#endif - -#ifdef HAVE_WINSOCK2_H -#include -#endif - -#ifdef HAVE_SYS_UIO_H -#include -#endif - -#ifdef __MINGW32__ -#define WINDOWS -#undef USE_SYSLOG -#endif /* __MINGW32__ */ - -#ifdef __CYGWIN__ -/* Untested so far ... */ -#define WINDOWS -#endif - -#define RECEIVER_PORT(x) (x) -#define SENDER_PORT(x) ((x) + 1) - -#define loseSendPacket udpc_loseSendPacket -#define loseRecvPacket udpc_loseRecvPacket -#define setWriteLoss udpc_setWriteLoss -#define setReadLoss udpc_setReadLoss -#define setReadSwap udpc_setReadSwap -#define srandomTime udpc_srandomTime -#define RecvMsg udpc_RecvMsg -#define doAutoRateLimit udpc_doAutoRateLimit -#define makeSockAddr udpc_makeSockAddr -#define printMyIp udpc_printMyIp -#define getSendBuf udpc_getSendBuf -#define setIpFromString udpc_setIpFromString -#define parseSize udpc_parseSize - -#ifdef LOSSTEST -int loseSendPacket(void); -void loseRecvPacket(int s); -void setWriteLoss(char* l); -void setReadLoss(char* l); -void setReadSwap(char* l); -void srandomTime(int printSeed); -int RecvMsg(int s, struct msghdr* msg, int flags); -#endif - -struct net_if -{ - struct in_addr addr; - struct in_addr bcast; - const char* name; -#ifdef SIOCGIFINDEX - int index; -#endif -}; -typedef struct net_if net_if_t; - -typedef enum addr_type_t -{ - ADDR_TYPE_UCAST, - ADDR_TYPE_MCAST, - ADDR_TYPE_BCAST -} addr_type_t; - -void doAutoRateLimit(int sock, int dir, int qsize, int size); - -int makeSockAddr(char* hostname, short port, struct sockaddr_in* addr); - -void printMyIp(net_if_t* net_if); - -int getSendBuf(int sock); - -#define SEND(s, msg, to) doSend(s, &msg, sizeof(msg), &to) - -#define RECV(s, msg, from, portBase) doReceive((s), &msg, sizeof(msg), &from, (portBase)) - -#define BCAST_CONTROL(s, msg) doSend(s, &msg, sizeof(msg), &net_config->controlMcastAddr) - -void setIpFromString(struct sockaddr_in* addr, char* ip); - -unsigned long parseSize(char* sizeString); - -int udpc_socklibFatal(int code); - -#ifdef __MINGW32__ /* __MINGW32__ */ - -struct iovec -{ - void* iov_base; - int iov_len; -}; -struct msghdr -{ - void* msg_name; - int msg_namelen; - struct iovec* msg_iov; - int msg_iovlen; -}; - -ssize_t sendmsg(int s, const struct msghdr* msg, int flags); -ssize_t recvmsg(int fd, struct msghdr* msg, int flags); - -#define usleep(x) Sleep((x) / 1000) -#define sleep(x) Sleep(1000L * (x)) -#endif /* __MINGW32__ */ - -static inline void initMsgHdr(struct msghdr* hdr) -{ -#ifndef WINDOWS - hdr->msg_control = 0; - hdr->msg_controllen = 0; - hdr->msg_flags = 0; -#endif -} - -#ifndef __MINGW32__ -#undef closesocket -#define closesocket(x) close(x) -#endif - -#ifndef HAVE_IN_ADDR_T -typedef unsigned long in_addr_t; -#endif diff --git a/utils/multicast/statistics.h b/utils/multicast/statistics.h deleted file mode 100644 index 91e992fda..000000000 --- a/utils/multicast/statistics.h +++ /dev/null @@ -1,36 +0,0 @@ -/* Copyright (C) 2014 InfiniDB, Inc. - - This program is free software; you can redistribute it and/or - modify it under the terms of the GNU General Public License - as published by the Free Software Foundation; version 2 of - the License. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU General Public License for more details. - - You should have received a copy of the GNU General Public License - along with this program; if not, write to the Free Software - Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, - MA 02110-1301, USA. */ - -#pragma once - -typedef struct receiver_stats* receiver_stats_t; -typedef struct sender_stats* sender_stats_t; - -#define allocReadStats udpc_allocReadStats -#define receiverStatsStartTimer udpc_receiverStatsStartTimer -#define displayReceiverStats udpc_displayReceiverStats - -receiver_stats_t udpc_allocReadStats(int fd, long statPeriod, int printUncompressedPos); -void udpc_receiverStatsStartTimer(receiver_stats_t); -void udpc_displayReceiverStats(receiver_stats_t, int isFinal); - -#define allocSenderStats udpc_allocSenderStats -#define displaySenderStats udpc_displaySenderStats - -sender_stats_t udpc_allocSenderStats(int fd, FILE* logfile, long bwPeriod, long statPeriod, - int printUncompressedPos); -void udpc_displaySenderStats(sender_stats_t, int blockSize, int sliceSize, int isFinal); diff --git a/utils/multicast/threads.h b/utils/multicast/threads.h deleted file mode 100644 index 077344185..000000000 --- a/utils/multicast/threads.h +++ /dev/null @@ -1,141 +0,0 @@ -/* Copyright (C) 2014 InfiniDB, Inc. - - This program is free software; you can redistribute it and/or - modify it under the terms of the GNU General Public License - as published by the Free Software Foundation; version 2 of - the License. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU General Public License for more details. - - You should have received a copy of the GNU General Public License - along with this program; if not, write to the Free Software - Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, - MA 02110-1301, USA. */ - -#pragma once - -#ifdef __MINGW32__ - -#include "socklib.h" -#include -#include -#include -typedef HANDLE pthread_t; -typedef CRITICAL_SECTION pthread_mutex_t; -typedef HANDLE pthread_cond_t; - -struct timespec -{ - unsigned long tv_sec; - unsigned long tv_nsec; -}; - -static inline int pthread_create(pthread_t* thread, void* dummy1, LPTHREAD_START_ROUTINE start_routine, - void* arg) -{ - /* Start thread ... - * see http://msdn.microsoft.com/library/default.asp?url=/library/en-us/dllproc/base/createthread.asp - */ - *thread = CreateThread(NULL, /* lpThreadAttributes */ - 0, /* dwStackSize */ - start_routine, arg, /* lpParameter */ - 0, /* dwCreationFlags */ - NULL /* lpThreadId */); - return *thread != NULL ? 0 : -1; -} - -static inline int pthread_join(pthread_t th, void** thread_return) -{ - return WaitForSingleObject(th, INFINITE) == WAIT_OBJECT_0 ? 0 : -1; -} - -static inline int pthread_mutex_init(pthread_mutex_t* mutex, void* dummy) -{ - InitializeCriticalSection(mutex); - return 0; -} - -static inline int pthread_mutex_lock(pthread_mutex_t* mutex) -{ - EnterCriticalSection(mutex); - return 0; -} - -static inline int pthread_mutex_unlock(pthread_mutex_t* mutex) -{ - LeaveCriticalSection(mutex); - return 0; -} - -static inline int pthread_cond_init(pthread_cond_t* cond, void* dummy) -{ - *cond = CreateEvent(NULL, TRUE, TRUE, NULL); - - if (*cond == NULL) - return -1; - else - return 0; -} - -static inline int pthread_cond_signal(pthread_cond_t* cond) -{ - return SetEvent(*cond) ? 0 : -1; -} - -static inline int pthread_cond_wait(pthread_cond_t* cond, pthread_mutex_t* mutex) -{ - int r; - ResetEvent(*cond); - LeaveCriticalSection(mutex); - r = WaitForSingleObject(*cond, INFINITE) == WAIT_OBJECT_0 ? 0 : -1; - EnterCriticalSection(mutex); - return r; -} - -static inline void pthread_cancel(pthread_t* thread) -{ - TerminateThread(thread, 0); -} - -#define ETIMEDOUT -2 -#define MILLION 1000000 -#define BILLION 1000000000 - -static inline int pthread_cond_timedwait(pthread_cond_t* cond, pthread_mutex_t* mutex, struct timespec* ts) -{ - int r; - struct timeval tv; - long delta; - - gettimeofday(&tv, NULL); - - delta = (ts->tv_sec - tv.tv_sec) * 1000 + (ts->tv_nsec / BILLION - tv.tv_usec / MILLION); - - if (delta < 0) - delta = 0; - - ResetEvent(*cond); - LeaveCriticalSection(mutex); - - switch (WaitForSingleObject(*cond, delta)) - { - case WAIT_OBJECT_0: r = 0; break; - - case WAIT_TIMEOUT: r = ETIMEDOUT; break; - - default: r = -1; break; - } - - EnterCriticalSection(mutex); - return r; -} - -#define THREAD_RETURN DWORD WINAPI - -#else /* __MINGW32__ */ -#include -#define THREAD_RETURN void* -#endif /* __MINGW32__ */ diff --git a/utils/multicast/udp-sender.h b/utils/multicast/udp-sender.h deleted file mode 100644 index 60afc3255..000000000 --- a/utils/multicast/udp-sender.h +++ /dev/null @@ -1,100 +0,0 @@ -/* Copyright (C) 2014 InfiniDB, Inc. - - This program is free software; you can redistribute it and/or - modify it under the terms of the GNU General Public License - as published by the Free Software Foundation; version 2 of - the License. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU General Public License for more details. - - You should have received a copy of the GNU General Public License - along with this program; if not, write to the Free Software - Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, - MA 02110-1301, USA. */ - -#pragma once - -#include "udp-sender.h" -#include "udpcast.h" -#include "participants.h" -#include "statistics.h" -#include "socklib.h" - -extern FILE* udpc_log; - -struct fifo; - -#define openFile udpc_openFile -#define openPipe udpcs_openPipe -#define localReader udpc_localReader -#define doSend udpc_doSend - -int openFile(struct disk_config* config); -int openPipe(struct disk_config* config, int in, int* pid); -int localReader(struct fifo* fifo, int in); - -#define BCAST_DATA(s, msg) doSend(s, &msg, sizeof(msg), &net_config->dataMcastAddr) - -/** - * "switched network" mode: server already starts sending next slice before - * first one is acknowledged. Do not use on old coax networks - */ -#define FLAG_SN 0x0001 - -/** - * "not switched network" mode: network is known not to be switched - */ -#define FLAG_NOTSN 0x0002 - -/** - * Asynchronous mode: do not any confirmation at all from clients. - * Useful in situations where no return channel is available - */ -#define FLAG_ASYNC 0x0004 - -/** - * Point-to-point transmission mode: use unicast in the (frequent) - * special case where there is only one receiver. - */ -#define FLAG_POINTOPOINT 0x0008 - -/** - * Do automatic rate limitation by monitoring socket's send buffer - * size. Not very useful, as this still doesn't protect against the - * switch dropping packets because its queue (which might be slightly slower) - * overruns - */ -#ifndef WINDOWS -#define FLAG_AUTORATE 0x0008 -#endif - -#ifdef BB_FEATURE_UDPCAST_FEC -/** - * Forward error correction - */ -#define FLAG_FEC 0x0010 -#endif - -/** - * Use broadcast rather than multicast (useful for cards that don't support - * multicast correctly - */ -#define FLAG_BCAST 0x0020 - -/** - * Never use point-to-point, even if only one receiver - */ -#define FLAG_NOPOINTOPOINT 0x0040 - -/* - * Don't ask for keyboard input on sender end. - */ -#define FLAG_NOKBD 0x0080 - -/** - * Streaming mode: allow receiver to join a running transmission - */ -#define FLAG_STREAMING 0x0100 diff --git a/utils/multicast/udpc-protoc.h b/utils/multicast/udpc-protoc.h deleted file mode 100644 index a38ff6cd9..000000000 --- a/utils/multicast/udpc-protoc.h +++ /dev/null @@ -1,204 +0,0 @@ -/* Copyright (C) 2014 InfiniDB, Inc. - - This program is free software; you can redistribute it and/or - modify it under the terms of the GNU General Public License - as published by the Free Software Foundation; version 2 of - the License. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU General Public License for more details. - - You should have received a copy of the GNU General Public License - along with this program; if not, write to the Free Software - Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, - MA 02110-1301, USA. */ - -#pragma once - -#include "udpcast.h" - -#define MAX_BLOCK_SIZE 1456 -#define MAX_FEC_INTERLEAVE 256 - -/** - * This file describes the UDPCast protocol - */ -enum opCode -{ - /* Receiver to sender */ - - CMD_OK, /* all is ok, no need to retransmit anything */ - CMD_RETRANSMIT, /* receiver asks for some data to be retransmitted */ - CMD_GO, /* receiver tells server to start */ - CMD_CONNECT_REQ, /* receiver tries to find out server's address */ - CMD_DISCONNECT, /* receiver wants to disconnect itself */ - - CMD_UNUSED, /* obsolete version of CMD_HELLO, dating back to the - * time when we had little endianness (PC). This - * opcode contained a long unnoticed bug with parsing of - * blocksize */ - - /* Sender to receiver */ - CMD_REQACK, /* server request acknowledgments from receiver */ - CMD_CONNECT_REPLY, /* receiver tries to find out server's address */ - - CMD_DATA, /* a block of data */ - CMD_FEC, /* a forward-error-correction block */ - - CMD_HELLO_NEW, /* sender says he's up */ - CMD_HELLO_STREAMING, /* retransmitted hello during streaming mode */ -}; - -/* Sender says he's up. This is not in the enum with the others, - * because of some endianness Snafu in early versions. However,since - * 2005-12-23, new receivers now understand a CMD_HELLO_NEW which is - * in sequence. Once enough of those are out in the field, we'll send - * CMD_HELLO_NEW by default, and then phase out the old variant. */ -/* Tried to remove this on 2009-08-30, but noticed that receiver was printing - * "unexpected opcode" on retransmitted hello */ -#define CMD_HELLO 0x0500 - -struct connectReq -{ - unsigned short opCode; - short reserved; - int capabilities; - unsigned int rcvbuf; -}; -struct retransmit -{ - unsigned short opCode; - short reserved; - int sliceNo; - int rxmit; - unsigned char map[MAX_SLICE_SIZE / BITS_PER_CHAR]; -}; -struct ok -{ - unsigned short opCode; - short reserved; - int sliceNo; -} ok; - -union message -{ - unsigned short opCode; - struct ok ok; - - struct retransmit retransmit; - - struct connectReq connectReq; - - struct go - { - unsigned short opCode; - short reserved; - } go; - - struct disconnect - { - unsigned short opCode; - short reserved; - } disconnect; -}; - -struct connectReply -{ - unsigned short opCode; - short reserved; - int clNr; - int blockSize; - int capabilities; - unsigned char mcastAddr[16]; /* provide enough place for IPV6 */ -}; - -struct hello -{ - unsigned short opCode; - short reserved; - int capabilities; - unsigned char mcastAddr[16]; /* provide enough place for IPV6 */ - short blockSize; -}; - -union serverControlMsg -{ - unsigned short opCode; - short reserved; - struct hello hello; - struct connectReply connectReply; -}; - -struct dataBlock -{ - unsigned short opCode; - short reserved; - int sliceNo; - unsigned short blockNo; - unsigned short reserved2; - int bytes; -}; - -struct fecBlock -{ - unsigned short opCode; - short stripes; - int sliceNo; - unsigned short blockNo; - unsigned short reserved2; - int bytes; -}; - -struct reqack -{ - unsigned short opCode; - short reserved; - int sliceNo; - int bytes; - int rxmit; -}; - -union serverDataMsg -{ - unsigned short opCode; - struct reqack reqack; - struct dataBlock dataBlock; - struct fecBlock fecBlock; -}; - -/* ============================================ - * Capabilities - */ - -/* Does the receiver support the new CMD_DATA command, which carries - * capabilities mask? - * "new generation" receiver: - * - capabilities word included in hello/connectReq commands - * - receiver multicast capable - * - receiver can receive ASYNC and SN - */ -#define CAP_NEW_GEN 0x0001 - -/* Use multicast instead of Broadcast for data */ -/*#define CAP_MULTICAST 0x0002*/ - -#ifdef BB_FEATURE_UDPCAST_FEC -/* Forward error correction */ -#define CAP_FEC 0x0004 -#endif - -/* Supports big endians (a.k.a. network) */ -#define CAP_BIG_ENDIAN 0x0008 - -/* Support little endians (a.k.a. PC) ==> obsolete! */ -#define CAP_LITTLE_ENDIAN 0x0010 - -/* This transmission is asynchronous (no receiver reply) */ -#define CAP_ASYNC 0x0020 - -/* Sender currently supports CAPABILITIES and MULTICAST */ -#define SENDER_CAPABILITIES (CAP_NEW_GEN | CAP_BIG_ENDIAN) - -#define RECEIVER_CAPABILITIES (CAP_NEW_GEN | CAP_BIG_ENDIAN) diff --git a/utils/multicast/udpcast.h b/utils/multicast/udpcast.h deleted file mode 100644 index 144d191b4..000000000 --- a/utils/multicast/udpcast.h +++ /dev/null @@ -1,157 +0,0 @@ -/* Copyright (C) 2014 InfiniDB, Inc. - - This program is free software; you can redistribute it and/or - modify it under the terms of the GNU General Public License - as published by the Free Software Foundation; version 2 of - the License. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU General Public License for more details. - - You should have received a copy of the GNU General Public License - along with this program; if not, write to the Free Software - Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, - MA 02110-1301, USA. */ - -#pragma once - -#include "socklib.h" -#include -#include -#include - -#define BITS_PER_INT (sizeof(int) * 8) -#define BITS_PER_CHAR 8 - -#define MAP_ZERO(l, map) (memset(map, 0, ((l) + BITS_PER_INT - 1) / BIT_PER_INT)) -#define BZERO(data) (memset((void*)&data, 0, sizeof(data))) - -#define RDATABUFSIZE (2 * (MAX_SLICE_SIZE + 1) * MAX_BLOCK_SIZE) - -#define DATABUFSIZE (RDATABUFSIZE + 4096 - RDATABUFSIZE % 4096) - -int udpc_writeSize(void); -int udpc_largeReadSize(void); -int udpc_smallReadSize(void); -int udpc_makeDataBuffer(int blocksize); -int udpc_parseCommand(char* pipeName, char** arg); - -int udpc_printLongNum(unsigned long long x); -int udpc_waitForProcess(int pid, const char* message); - -struct disk_config -{ - int origOutFile; - const char* fileName; - char* pipeName; - int flags; - - struct timeval stats_last_printed; -}; - -#define MAX_GOVERNORS 10 - -struct net_config -{ - net_if_t* net_if; /* Network interface (eth0, isdn0, etc.) on which to - * multicast */ - int portBase; /* Port base */ - int blockSize; - int sliceSize; - struct sockaddr_in controlMcastAddr; - struct sockaddr_in dataMcastAddr; - const char* mcastRdv; - int ttl; - int nrGovernors; - struct rateGovernor_t* rateGovernor[MAX_GOVERNORS]; - void* rateGovernorData[MAX_GOVERNORS]; - /*int async;*/ - /*int pointopoint;*/ - struct timeval ref_tv; - - enum discovery - { - DSC_DOUBLING, - DSC_REDUCING - } discovery; - - /* int autoRate; do queue watching using TIOCOUTQ, to avoid overruns */ - - int flags; /* non-capability command line flags */ - int capabilities; - - int min_slice_size; - int default_slice_size; - int max_slice_size; - unsigned int rcvbuf; - - int rexmit_hello_interval; /* retransmission interval between hello's. - * If 0, hello message won't be retransmitted - */ - int autostart; /* autostart after that many retransmits */ - - int requestedBufSize; /* requested receiver buffer */ - - /* sender-specific parameters */ - int min_receivers; - int max_receivers_wait; - int min_receivers_wait; - - int retriesUntilDrop; - - /* receiver-specif parameters */ - int exitWait; /* How many milliseconds to wait on program exit */ - - int startTimeout; /* Timeout at start */ - - /* FEC config */ -#ifdef BB_FEATURE_UDPCAST_FEC - int fec_redundancy; /* how much fec blocks are added per group */ - int fec_stripesize; /* size of FEC group */ - int fec_stripes; /* number of FEC stripes per slice */ -#endif - - int rehelloOffset; /* how far before end will rehello packet will - be retransmitted */ -}; - -struct stat_config -{ - FILE* log; /* Log file for statistics */ - long bwPeriod; /* How often are bandwidth estimations logged? */ - - int statPeriod; - int printUncompressedPos; -}; - -#define NR_CLIENT_SOCKS 4 - -struct client_config -{ - int socks[NR_CLIENT_SOCKS]; - struct sockaddr_in serverAddr; - int clientNumber; - int isStarted; - pthread_t thread; - int sender_is_newgen; -}; - -void* rgInitGovernor(struct net_config* cfg, struct rateGovernor_t* gov); -void rgParseRateGovernor(struct net_config* net_config, char* rg); -void rgWaitAll(struct net_config* cfg, int sock, in_addr_t ip, int size); -void rgShutdownAll(struct net_config* cfg); - -/** - * Answers whether given fd is seekable - */ -int udpc_shouldPrintUncompressedPos(int deflt, int fd, int pipe); - -#define MAX_SLICE_SIZE 1024 - -#define DEFLT_STAT_PERIOD 500000 - -#ifndef DEBUG -#define DEBUG 0 -#endif diff --git a/utils/multicast/util.h b/utils/multicast/util.h deleted file mode 100644 index 545eed187..000000000 --- a/utils/multicast/util.h +++ /dev/null @@ -1,30 +0,0 @@ -/* Copyright (C) 2014 InfiniDB, Inc. - - This program is free software; you can redistribute it and/or - modify it under the terms of the GNU General Public License - as published by the Free Software Foundation; version 2 of - the License. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU General Public License for more details. - - You should have received a copy of the GNU General Public License - along with this program; if not, write to the Free Software - Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, - MA 02110-1301, USA. */ - -#pragma once - -#include - -#define MALLOC(type) ((type*)calloc(1, sizeof(type))) - -/* bitmap manipulation */ -#define BITS_PER_ITEM(map) (sizeof(map[0]) * 8) -#define MASK(pos, map) (1 << ((pos) % (BITS_PER_ITEM(map)))) -#define POS(pos, map) ((pos) / BITS_PER_ITEM(map)) -#define SET_BIT(x, map) (map[POS(x, map)] |= MASK(x, map)) -#define CLR_BIT(x, map) (map[POS(x, map)] &= ~MASK(x, map)) -#define BIT_ISSET(x, map) (map[POS(x, map)] & MASK(x, map)) diff --git a/utils/pron/CMakeLists.txt b/utils/pron/CMakeLists.txt index e02c6d916..645084df7 100644 --- a/utils/pron/CMakeLists.txt +++ b/utils/pron/CMakeLists.txt @@ -1,6 +1,5 @@ include_directories(${ENGINE_COMMON_INCLUDES}) -set(pron_LIB_SRCS pron.cpp) -add_library(pron STATIC ${pron_LIB_SRCS}) -target_link_libraries(pron messageqcpp loggingcpp) -# We don't isntall static library install(TARGETS pron DESTINATION ${ENGINE_LIBDIR} COMPONENT columnstore-engine) +set(pron_LIB_SRCS pron.cpp) +columnstore_static_library(pron ${pron_LIB_SRCS}) +columnstore_link(pron PRIVATE messageqcpp loggingcpp) diff --git a/utils/querystats/CMakeLists.txt b/utils/querystats/CMakeLists.txt index 609bd8473..0d91ead9e 100644 --- a/utils/querystats/CMakeLists.txt +++ b/utils/querystats/CMakeLists.txt @@ -3,16 +3,6 @@ add_definitions(-DMYSQL_SERVICE_THD_TIMEZONE_INCLUDED) include_directories(${ENGINE_COMMON_INCLUDES}) -# ########## next target ############### - set(querystats_LIB_SRCS querystats.cpp) - columnstore_library(querystats ${querystats_LIB_SRCS}) - -add_dependencies(querystats loggingcpp) - -install( - TARGETS querystats - DESTINATION ${ENGINE_LIBDIR} - COMPONENT columnstore-engine -) +columnstore_link(querystats PRIVATE loggingcpp) diff --git a/utils/querytele/CMakeLists.txt b/utils/querytele/CMakeLists.txt index f9d3a3926..f5e58ae60 100644 --- a/utils/querytele/CMakeLists.txt +++ b/utils/querytele/CMakeLists.txt @@ -1,18 +1,11 @@ include_directories(${ENGINE_COMMON_INCLUDES}) -# ########## next target ############### - set(querytele_LIB_SRCS querytele.cpp queryteleclient.cpp querytele_constants.cpp querytele_types.cpp QueryTeleService.cpp queryteleprotoimpl.cpp ) columnstore_library(querytele ${querytele_LIB_SRCS}) -add_dependencies(querytele external_boost external_thrift) - +columnstore_link(querytele ${THRIFT_LIBRARY}) target_include_directories(querytele PRIVATE ${THRIFT_INCLUDE_DIRS}) -target_link_libraries(querytele ${THRIFT_LIBRARY}) -install( - TARGETS querytele - DESTINATION ${ENGINE_LIBDIR} - COMPONENT columnstore-engine -) + +add_dependencies(querytele external_boost external_thrift) diff --git a/utils/regr/CMakeLists.txt b/utils/regr/CMakeLists.txt index 3532cc255..f2f378f34 100755 --- a/utils/regr/CMakeLists.txt +++ b/utils/regr/CMakeLists.txt @@ -21,24 +21,11 @@ set(regr_LIB_SRCS add_definitions(-DMYSQL_DYNAMIC_PLUGIN) columnstore_library(regr ${regr_LIB_SRCS}) - -add_dependencies(regr loggingcpp) - -install( - TARGETS regr - DESTINATION ${ENGINE_LIBDIR} - COMPONENT columnstore-engine -) +columnstore_link(regr PRIVATE loggingcpp) set(regr_mysql_LIB_SRCS regrmysql.cpp modamysql.cpp) -columnstore_library(regr_mysql ${regr_mysql_LIB_SRCS}) +columnstore_mysql_plugin_library(regr_mysql SHARED ${regr_mysql_LIB_SRCS}) add_dependencies(regr_mysql external_boost) -install( - TARGETS regr_mysql - DESTINATION ${MARIADB_PLUGINDIR} - COMPONENT columnstore-engine -) - set_target_properties(regr_mysql PROPERTIES LIBRARY_OUTPUT_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/../../../) diff --git a/utils/rowgroup/CMakeLists.txt b/utils/rowgroup/CMakeLists.txt index 75f406bc8..2ba4c2d46 100644 --- a/utils/rowgroup/CMakeLists.txt +++ b/utils/rowgroup/CMakeLists.txt @@ -4,16 +4,6 @@ include_directories(${ENGINE_COMMON_INCLUDES}) set(rowgroup_LIB_SRCS rowaggregation.cpp rowgroup.cpp rowstorage.cpp) -# librowgroup_la_CXXFLAGS = $(march_flags) $(AM_CXXFLAGS) - columnstore_library(rowgroup ${rowgroup_LIB_SRCS}) - -add_dependencies(rowgroup loggingcpp external_boost) - -target_link_libraries(rowgroup ${NETSNMP_LIBRARIES} funcexp) - -install( - TARGETS rowgroup - DESTINATION ${ENGINE_LIBDIR} - COMPONENT columnstore-engine -) +columnstore_link(rowgroup PRIVATE ${NETSNMP_LIBRARIES} funcexp loggingcpp) +add_dependencies(rowgroup external_boost) diff --git a/utils/rwlock/CMakeLists.txt b/utils/rwlock/CMakeLists.txt index 03dda2f2c..04fd8d182 100644 --- a/utils/rwlock/CMakeLists.txt +++ b/utils/rwlock/CMakeLists.txt @@ -1,12 +1,5 @@ include_directories(${ENGINE_COMMON_INCLUDES}) set(rwlock_LIB_SRCS rwlock.cpp rwlock_local.cpp) - columnstore_library(rwlock ${rwlock_LIB_SRCS}) -add_dependencies(rwlock external_boost) - -install( - TARGETS rwlock - DESTINATION ${ENGINE_LIBDIR} - COMPONENT columnstore-engine -) +columnstore_link(rwlock boost_thread) diff --git a/utils/startup/CMakeLists.txt b/utils/startup/CMakeLists.txt index 802576f87..8ba61277b 100644 --- a/utils/startup/CMakeLists.txt +++ b/utils/startup/CMakeLists.txt @@ -1,8 +1,5 @@ include_directories(${ENGINE_COMMON_INCLUDES}) -add_definitions(-fPIC -DPIC) - -add_library(idbboot STATIC installdir.cpp) +set(idbboot_LIB_SRCS installdir.cpp) +columnstore_static_library(idbboot ${idbboot_LIB_SRCS}) add_dependencies(idbboot external_boost) - -install(TARGETS idbboot DESTINATION ${ENGINE_LIBDIR}) diff --git a/utils/testbc/blockcacheclient.cpp b/utils/testbc/blockcacheclient.cpp deleted file mode 100644 index 019aad4c1..000000000 --- a/utils/testbc/blockcacheclient.cpp +++ /dev/null @@ -1,97 +0,0 @@ -/* Copyright (C) 2014 InfiniDB, Inc. - - This program is free software; you can redistribute it and/or - modify it under the terms of the GNU General Public License - as published by the Free Software Foundation; version 2 of - the License. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU General Public License for more details. - - You should have received a copy of the GNU General Public License - along with this program; if not, write to the Free Software - Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, - MA 02110-1301, USA. */ - -/*************************************************************************** - * - * $Id: blockcacheclient.cpp 684 2008-08-19 22:22:59Z pleblanc $ - * - * jrodriguez@calpont.com * - ***************************************************************************/ - -#include -#include -#include "blockcacheclient.h" - -namespace dbbc -{ -blockCacheClient::blockCacheClient() -{ - fBCCBrp = NULL; -} - -blockCacheClient::blockCacheClient(BlockRequestProcessor& brp) -{ - fBCCBrp = &brp; -} - -blockCacheClient::~blockCacheClient() -{ -} - -void blockCacheClient::check(BRM::LBID_t lbid, BRM::VER_t ver, bool flg, bool& wasBlockInCache) -{ - fBCCBrp->check(lbid, ver, flg, wasBlockInCache); -} - -void blockCacheClient::check(const BRM::InlineLBIDRange& range, const BRM::VER_t ver, uint32_t& rCount) -{ - fBCCBrp->check(range, ver, rCount); -} - -FileBuffer* blockCacheClient::getBlockPtr(const BRM::LBID_t& lbid, const BRM::VER_t& ver) -{ - FileBuffer* fb = fBCCBrp->getBlockPtr(lbid, ver); - return fb; -} - -const int blockCacheClient::read(const BRM::LBID_t& lbid, const BRM::VER_t& ver, FileBuffer& fb) -{ - int ret = fBCCBrp->read(lbid, ver, fb); - return ret; -} - -const int blockCacheClient::read(const BRM::LBID_t& lbid, const BRM::VER_t& ver, void* bufferPtr) -{ - int ret = fBCCBrp->read(lbid, ver, bufferPtr); - return ret; -} - -const int blockCacheClient::read(const BRM::InlineLBIDRange& range, FileBufferList_t& fbList, - const BRM::VER_t ver) -{ - int ret = fBCCBrp->read(range, fbList, ver); - return ret; -} - -const int blockCacheClient::getBlock(const BRM::LBID_t& lbid, const BRM::VER_t& ver, void* bufferPtr, - bool flg, bool& wasCached) -{ - int ret = fBCCBrp->getBlock(lbid, ver, bufferPtr, flg, wasCached); - return ret; -} - -bool blockCacheClient::exists(BRM::LBID_t lbid, BRM::VER_t ver) -{ - return fBCCBrp->exists(lbid, ver); -} - -void blockCacheClient::flushCache() -{ - fBCCBrp->flushCache(); -} - -} // namespace dbbc diff --git a/utils/testbc/blockcacheclient.h b/utils/testbc/blockcacheclient.h deleted file mode 100644 index 57cf38190..000000000 --- a/utils/testbc/blockcacheclient.h +++ /dev/null @@ -1,104 +0,0 @@ -/* Copyright (C) 2014 InfiniDB, Inc. - - This program is free software; you can redistribute it and/or - modify it under the terms of the GNU General Public License - as published by the Free Software Foundation; version 2 of - the License. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU General Public License for more details. - - You should have received a copy of the GNU General Public License - along with this program; if not, write to the Free Software - Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, - MA 02110-1301, USA. */ - -/*************************************************************************** - * - * $Id: blockcacheclient.h 684 2008-08-19 22:22:59Z pleblanc $ - * - * * - ***************************************************************************/ - -#pragma once - -#include "blockrequestprocessor.h" -#include "brmtypes.h" - -/** - @author Jason Rodriguez -*/ - -/** - * @brief API for the Disk Block Buffer Cache - * - * - */ - -namespace dbbc -{ -class blockCacheClient -{ - public: - /** - * @brief ctor requires reference to BlockRequestProcessor object - **/ - blockCacheClient(BlockRequestProcessor& brp); - - /** - * @brief dtor - **/ - virtual ~blockCacheClient(); - - /** - * @brief verify that the Disk Block for the LBID lbid, ver are loaded into the Cache. - **/ - void check(BRM::LBID_t lbid, BRM::VER_t ver, bool flg, bool& wasBlockInCache); - - /** - * @brief verify all Disk Blocks for the LBID range are loaded into the Cache - **/ - void check(const BRM::InlineLBIDRange& range, const BRM::VER_t ver, uint32_t& rCount); - - /** - * @brief retrieve the Disk Block at lbid, ver from the Disk Block Buffer Cache - **/ - const int read(const BRM::LBID_t& lbid, const BRM::VER_t& ver, FileBuffer& fb); - - FileBuffer* getBlockPtr(const BRM::LBID_t& lbid, const BRM::VER_t& ver); - - /** - * @brief retrieve the Disk Block at lbid, ver from the Disk Block Buffer Cache - **/ - const int read(const BRM::LBID_t& lbid, const BRM::VER_t& ver, void* bufferPtr); - - /** - * @brief retrieve all disk Blocks in the LBIDRange range and insert them into fbList - **/ - const int read(const BRM::InlineLBIDRange& range, FileBufferList_t& fbList, const BRM::VER_t ver); - - const int getBlock(const BRM::LBID_t& lbid, const BRM::VER_t& ver, void* bufferPtr, bool flg, - bool& wasCached); - - bool exists(BRM::LBID_t lbid, BRM::VER_t ver); - - /** - * @brief flush the cache - **/ - void flushCache(); - - private: - /** - * @brief pointer to the BlockRequestProcessor object on which the API will operate - **/ - BlockRequestProcessor* fBCCBrp; - - // do not implement - blockCacheClient(); - blockCacheClient(const blockCacheClient& bc); - const blockCacheClient& operator=(const blockCacheClient& blk); -}; - -} // namespace dbbc diff --git a/utils/testbc/blockrequestprocessor.cpp b/utils/testbc/blockrequestprocessor.cpp deleted file mode 100644 index 118b63035..000000000 --- a/utils/testbc/blockrequestprocessor.cpp +++ /dev/null @@ -1,227 +0,0 @@ -/* Copyright (C) 2014 InfiniDB, Inc. - - This program is free software; you can redistribute it and/or - modify it under the terms of the GNU General Public License - as published by the Free Software Foundation; version 2 of - the License. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU General Public License for more details. - - You should have received a copy of the GNU General Public License - along with this program; if not, write to the Free Software - Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, - MA 02110-1301, USA. */ - -/*************************************************************************** - * - * $Id: blockrequestprocessor.cpp 725 2008-09-26 16:26:47Z jrodriguez $ - * - * jrodriguez@calpont.com * - * * - ***************************************************************************/ - -#include "blockrequestprocessor.h" -#include "rwlock_local.h" -#include "dbrm.h" -#include -#include -#include -#include -#include -#include -using namespace std; - -namespace dbbc -{ -BlockRequestProcessor::BlockRequestProcessor(uint32_t numBlcks, int thrCount, int blocksPerRead, - uint32_t deleteBlocks, uint32_t blckSz) - : fbMgr(numBlcks, blckSz, deleteBlocks), fIOMgr(fbMgr, fBRPRequestQueue, thrCount, blocksPerRead) -{ -} - -BlockRequestProcessor::~BlockRequestProcessor() -{ -} - -void BlockRequestProcessor::stop() -{ - fBRPRequestQueue.stop(); - fIOMgr.stop(); -} - -int BlockRequestProcessor::check(const BRM::InlineLBIDRange& range, const BRM::VER_t ver, uint32_t& lbidCount) -{ - uint64_t maxLbid = range.start; // highest existent lbid - uint64_t rangeLen = range.size; - uint64_t idx; - uint64_t adjSz; - struct timespec start_tm; - - if (fTrace) - clock_gettime(CLOCK_MONOTONIC, &start_tm); - - for (idx = 0; fbMgr.exists(maxLbid, ver) == true && idx < rangeLen; maxLbid++, idx++) - ; - - if (idx == rangeLen) // range is already loaded - { - uint32_t fbo; - BRM::OID_t oid; - fdbrm.lookup(maxLbid, ver, false, oid, fbo); - fLogFile << oid << " " << maxLbid << " " << fbo << " " << rangeLen << " " << 0 << " " << 0 << " " << 0 - << " " << right << fixed << ((double)(start_tm.tv_sec + (1.e-9 * start_tm.tv_nsec))) << endl; - return 0; - } - - adjSz = rangeLen - idx; - BRM::InlineLBIDRange adjRange; - adjRange.start = maxLbid; - adjRange.size = adjSz; - fileRequest rqstBlk(adjRange, ver); - check(rqstBlk); - lbidCount = rqstBlk.BlocksRead(); - - if (fTrace) - { - uint32_t fbo; - BRM::OID_t oid; - fdbrm.lookup(maxLbid, ver, false, oid, fbo); - fLogFile << oid << " " << maxLbid << " " << fbo << " " << rangeLen << " " << adjSz << " " - << rqstBlk.BlocksRead() << " " << rqstBlk.BlocksLoaded() << " " << right << fixed - << ((double)(start_tm.tv_sec + (1.e-9 * start_tm.tv_nsec))) << endl; - } - - return rqstBlk.BlocksLoaded(); -} // check - -int BlockRequestProcessor::check(fileRequest& rqstBlk) -{ - pthread_mutex_lock(&rqstBlk.frMutex()); - rqstBlk.SetPredicate(fileRequest::SENDING); - sendRequest(rqstBlk); // start file read request - - while (rqstBlk.frPredicate() < fileRequest::COMPLETE) - pthread_cond_wait(&rqstBlk.frCond(), &rqstBlk.frMutex()); - - pthread_mutex_unlock(&rqstBlk.frMutex()); - - return 0; -} - -int BlockRequestProcessor::check(BRM::LBID_t lbid, BRM::VER_t ver, bool flg, bool& wasBlockInCache) -{ - if (fbMgr.exists(lbid, ver) == true) - { - wasBlockInCache = true; - return 0; - } - else - { - wasBlockInCache = false; - fileRequest rqstBlk(lbid, ver, flg); - int ret = check(rqstBlk); - return ret; - } -} - -int BlockRequestProcessor::sendRequest(fileRequest& blk) -{ - int ret = fBRPRequestQueue.push(blk); - return ret; -} - -const int BlockRequestProcessor::read(const BRM::InlineLBIDRange& range, FileBufferList_t& readList, - const BRM::VER_t ver) -{ - int blksLoaded = 0; - HashObject_t fb = {0, 0, 0}; - - for (int idx = 0; (uint64_t)idx < range.size; idx++) - { - fb.lbid = range.start + idx; - fb.ver = ver; - fb.poolIdx = 0; - FileBuffer fbRet(-1, -1); - bool ret = false; // fbMgr.find(fb, fbRet); - - if (ret) - { - blksLoaded++; - readList.push_back(fbRet); - } - } - - return blksLoaded; -} - -FileBuffer* BlockRequestProcessor::getBlockPtr(const BRM::LBID_t lbid, const BRM::VER_t ver) -{ - HashObject_t hashObj = {lbid, ver, 0}; - FileBuffer* fb = fbMgr.findPtr(hashObj); - return fb; -} - -const int BlockRequestProcessor::read(const BRM::LBID_t& lbid, const BRM::VER_t& ver, FileBuffer& fb) -{ - HashObject_t hashObj = {lbid, ver, 0}; - bool ret = fbMgr.find(hashObj, fb); - - if (ret == true) - return 1; - else - return 0; -} - -const int BlockRequestProcessor::read(const BRM::LBID_t& lbid, const BRM::VER_t& ver, void* bufferPtr) -{ - HashObject_t hashObj = {lbid, ver, 0}; - bool ret = fbMgr.find(hashObj, bufferPtr); - - if (ret == true) - return 1; - else - return 0; -} - -const int BlockRequestProcessor::getBlock(const BRM::LBID_t& lbid, const BRM::VER_t& ver, void* bufferPtr, - bool flg, bool& wasCached) -{ - HashObject_t hashObj = {lbid, ver, 0}; - wasCached = fbMgr.find(hashObj, bufferPtr); - - if (wasCached) - return 1; - - wasCached = false; - fileRequest rqstBlk(lbid, ver, flg, (uint8_t*)bufferPtr); - check(rqstBlk); - return 1; -} - -bool BlockRequestProcessor::exists(BRM::LBID_t lbid, BRM::VER_t ver) -{ - HashObject_t ho = {lbid, ver, 0}; - - return fbMgr.exists(ho); -} - -void BlockRequestProcessor::flushCache() -{ - fbMgr.flushCache(); -} -/** -const uint32_t BlockRequestProcessor::resize(const uint32_t s) -{ - int rc = fbMgr.resize(s); - return rc; -} -**/ -ostream& BlockRequestProcessor::formatLRUList(ostream& os) const -{ - return fbMgr.formatLRUList(os); -} - -} // namespace dbbc diff --git a/utils/testbc/blockrequestprocessor.h b/utils/testbc/blockrequestprocessor.h deleted file mode 100644 index abf179c07..000000000 --- a/utils/testbc/blockrequestprocessor.h +++ /dev/null @@ -1,131 +0,0 @@ -/* Copyright (C) 2014 InfiniDB, Inc. - - This program is free software; you can redistribute it and/or - modify it under the terms of the GNU General Public License - as published by the Free Software Foundation; version 2 of - the License. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU General Public License for more details. - - You should have received a copy of the GNU General Public License - along with this program; if not, write to the Free Software - Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, - MA 02110-1301, USA. */ - -#pragma once - -/*************************************************************************** - * - * $Id: blockrequestprocessor.h 725 2008-09-26 16:26:47Z jrodriguez $ - * - * jrodriguez@calpont.com * - * * - ***************************************************************************/ - -#include "blocksize.h" -#include "fileblockrequestqueue.h" -#include "filebuffermgr.h" -#include "iomanager.h" - -/** - @author Jason Rodriguez -*/ - -namespace dbbc -{ -typedef std::list FileBufferList_t; - -/** - * @brief class to control the populating of the Disk Block Buffer Cache and manage Block requests. - **/ - -class BlockRequestProcessor -{ - public: - /** - * @brief default ctor - **/ - BlockRequestProcessor(uint32_t numBlcks, int thrCount, int blocksPerRead, uint32_t deleteBlocks = 0, - uint32_t blckSz = BLOCK_SIZE); - - /** - * @brief default dtor - **/ - virtual ~BlockRequestProcessor(); - - /** - * @brief send a request for disk blocks to the IO manager - **/ - int sendRequest(fileRequest& blk); - - /** - * @brief verify that the lbid@ver disk block is in the block cache. Send request if it is not - **/ - int check(BRM::LBID_t lbid, BRM::VER_t ver, bool flg, bool& wasBlockInCache); - - /** - * @brief verify the LBIDRange of disk blocks is in the block cache. Send request if it is not - **/ - int check(const BRM::InlineLBIDRange& range, const BRM::VER_t ver, uint32_t& lbidCount); - - /** - * @brief retrieve the lbid@ver disk block from the block cache - **/ - FileBuffer* getBlockPtr(const BRM::LBID_t lbid, const BRM::VER_t ver); - - const int read(const BRM::LBID_t& lbid, const BRM::VER_t& ver, FileBuffer& fb); - - /** - * @brief retrieve the lbid@ver disk block from the block cache - **/ - const int read(const BRM::LBID_t& lbid, const BRM::VER_t& ver, void* bufferPtr); - - /** - * @brief retrieve the LBIDRange of disk blocks from the block cache - **/ - const int read(const BRM::InlineLBIDRange& range, FileBufferList_t& fbList, const BRM::VER_t ver); - - const int getBlock(const BRM::LBID_t& lbid, const BRM::VER_t& ver, void* bufferPtr, bool flg, - bool& wasCached); - - bool exists(BRM::LBID_t lbid, BRM::VER_t ver); - - /** - * @brief - **/ - void flushCache(); - - // const uint32_t resize(const uint32_t s); - - std::ostream& formatLRUList(std::ostream& os) const; - - private: - FileBufferMgr fbMgr; - fileBlockRequestQueue fBRPRequestQueue; - ioManager fIOMgr; - pthread_mutex_t check_mutex; - - /** - * helper function for public check functions - **/ - int check(fileRequest& rqstBlk); - - /** - * send stop requests for IOmanager and request Q - **/ - void stop(); - - std::ofstream fLogFile; - bool fTrace; - - BRM::DBRM fdbrm; - - // do not implement - BlockRequestProcessor(const BlockRequestProcessor& brp); - BlockRequestProcessor& operator=(const BlockRequestProcessor& brp); -}; - -} // namespace dbbc diff --git a/utils/testbc/fileblockrequestqueue.cpp b/utils/testbc/fileblockrequestqueue.cpp deleted file mode 100644 index 8fa4c0208..000000000 --- a/utils/testbc/fileblockrequestqueue.cpp +++ /dev/null @@ -1,92 +0,0 @@ -/* Copyright (C) 2014 InfiniDB, Inc. - - This program is free software; you can redistribute it and/or - modify it under the terms of the GNU General Public License - as published by the Free Software Foundation; version 2 of - the License. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU General Public License for more details. - - You should have received a copy of the GNU General Public License - along with this program; if not, write to the Free Software - Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, - MA 02110-1301, USA. */ - -/*************************************************************************** - * - * $Id: fileblockrequestqueue.cpp 666 2008-07-22 14:39:46Z wweeks $ - * - * jrodriguez@calpont.com * - * * - ***************************************************************************/ - -#include "fileblockrequestqueue.h" - -using namespace std; - -namespace dbbc -{ -fileBlockRequestQueue::fileBlockRequestQueue() : queueSize(0), readersWaiting(0) -{ - pthread_mutex_init(&mutex, NULL); - pthread_cond_init(¬Empty, NULL); -} - -fileBlockRequestQueue::~fileBlockRequestQueue() -{ - pthread_cond_destroy(¬Empty); - pthread_mutex_destroy(&mutex); -} - -bool fileBlockRequestQueue::empty() const -{ - return (queueSize == 0); -} - -fileRequest* fileBlockRequestQueue::top() const -{ - return fbQueue.front(); -} - -int fileBlockRequestQueue::push(fileRequest& blk) -{ - pthread_mutex_lock(&mutex); - fbQueue.push_back(&blk); - - // @bug 1007. Changed "== 1" to ">= 1" below. The wake up call was only being fired when the queue size - // was 1 which caused only one i/o thread to be working at a time. - if (++queueSize >= 1 && readersWaiting > 0) - pthread_cond_signal(¬Empty); - - pthread_mutex_unlock(&mutex); - - return 0; -} - -void fileBlockRequestQueue::stop() -{ - pthread_cond_broadcast(¬Empty); -} - -fileRequest* fileBlockRequestQueue::pop(void) -{ - pthread_mutex_lock(&mutex); - - while (queueSize == 0) - { - readersWaiting++; - pthread_cond_wait(¬Empty, &mutex); - readersWaiting--; - } - - fileRequest* blk = fbQueue.front(); - fbQueue.pop_front(); - --queueSize; - pthread_mutex_unlock(&mutex); - return blk; -} - -} // namespace dbbc diff --git a/utils/testbc/fileblockrequestqueue.h b/utils/testbc/fileblockrequestqueue.h deleted file mode 100644 index 060d65279..000000000 --- a/utils/testbc/fileblockrequestqueue.h +++ /dev/null @@ -1,108 +0,0 @@ -/* Copyright (C) 2014 InfiniDB, Inc. - - This program is free software; you can redistribute it and/or - modify it under the terms of the GNU General Public License - as published by the Free Software Foundation; version 2 of - the License. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU General Public License for more details. - - You should have received a copy of the GNU General Public License - along with this program; if not, write to the Free Software - Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, - MA 02110-1301, USA. */ - -#pragma once - -/*************************************************************************** - * - * $Id: fileblockrequestqueue.h 483 2008-02-10 20:23:28Z rdempsey $ - * - * jrodriguez@calpont.com * - * * - ***************************************************************************/ - -#include -#include -#include -#include "filerequest.h" - -/** - @author Jason Rodriguez -*/ - -/** - * @brief definition of the block request queue as stl std::priority_queue - **/ -namespace dbbc -{ -typedef std::deque fileBlockRequestQueue_t; - -/** - * @brief class to hold requests for disk blocks in a queue. sorted by the size of a request - **/ - -class fileBlockRequestQueue -{ - public: - /** - * @brief default ctor - **/ - fileBlockRequestQueue(); - - /** - * @brief dtor - **/ - virtual ~fileBlockRequestQueue(); - - /** - * @brief add a request to the queue - **/ - int push(fileRequest& blk); - - /** - * @brief get the next request from the queue and delete it from the queue - **/ - fileRequest* pop(void); - - /** - * @brief true if no reuquests are in the queue. false if there are requests in the queue - **/ - bool empty() const; - - /** - * @brief number of requests in the queue - **/ - uint32_t size() const - { - return queueSize; - } - - /** - * @brief queue will stop accecpting requests in preparation for the dtor - **/ - void stop(); - - protected: - pthread_mutex_t mutex; - pthread_cond_t notEmpty; - fileBlockRequestQueue_t fbQueue; - uint32_t queueSize; - uint32_t readersWaiting; - - private: - // do not implement - fileBlockRequestQueue(const fileBlockRequestQueue& Q) - { - } - const fileBlockRequestQueue& operator=(const fileBlockRequestQueue& Q); - - /** - * @brief pointer to the next request to be popped from the queue - **/ - fileRequest* top() const; -}; -} // namespace dbbc diff --git a/utils/testbc/filebuffer.cpp b/utils/testbc/filebuffer.cpp deleted file mode 100644 index ba62983a0..000000000 --- a/utils/testbc/filebuffer.cpp +++ /dev/null @@ -1,85 +0,0 @@ -/* Copyright (C) 2014 InfiniDB, Inc. - - This program is free software; you can redistribute it and/or - modify it under the terms of the GNU General Public License - as published by the Free Software Foundation; version 2 of - the License. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU General Public License for more details. - - You should have received a copy of the GNU General Public License - along with this program; if not, write to the Free Software - Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, - MA 02110-1301, USA. */ - -/*************************************************************************** - * - * $Id: filebuffer.cpp 643 2008-06-30 16:39:59Z jrodriguez $ - * - * jrodriguez@calpont.com * - * * - ***************************************************************************/ - -#include "filebuffer.h" -#include -#include -#include - -using namespace std; - -namespace dbbc -{ -FileBuffer::FileBuffer(const FileBuffer& rhs) -{ - if (this == NULL || this == &rhs) - return; - - fLbid = rhs.fLbid; - fVerid = rhs.fVerid; - setData(rhs.fByteData, rhs.fDataLen); - fListLoc = rhs.listLoc(); - fDataLen = rhs.fDataLen; -} - -FileBuffer::FileBuffer(const BRM::LBID_t lbid, const BRM::VER_t ver, const uint8_t* data, const uint32_t len) -{ - fLbid = lbid; - fVerid = ver; - fDataLen = len; - setData(data, fDataLen); -} - -FileBuffer::FileBuffer(const BRM::LBID_t lbid, const BRM::VER_t ver) -{ - fLbid = lbid; - fVerid = ver; - fDataLen = 0; -} - -FileBuffer& FileBuffer::operator=(const FileBuffer& rhs) -{ - fLbid = rhs.fLbid; - fVerid = rhs.fVerid; - fDataLen = rhs.fDataLen; - setData(rhs.fByteData, fDataLen); - fListLoc = rhs.listLoc(); - return *this; -} - -void FileBuffer::setData(const uint8_t* d, const int len) -{ - if (d == NULL || len <= 0) - return; - - fDataLen = len; - memcpy(fByteData, d, len); -} - -FileBuffer::~FileBuffer() -{ -} - -} // namespace dbbc diff --git a/utils/testbc/filebuffer.h b/utils/testbc/filebuffer.h deleted file mode 100644 index 9398ed6e2..000000000 --- a/utils/testbc/filebuffer.h +++ /dev/null @@ -1,182 +0,0 @@ -/* Copyright (C) 2014 InfiniDB, Inc. - - This program is free software; you can redistribute it and/or - modify it under the terms of the GNU General Public License - as published by the Free Software Foundation; version 2 of - the License. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU General Public License for more details. - - You should have received a copy of the GNU General Public License - along with this program; if not, write to the Free Software - Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, - MA 02110-1301, USA. */ - -/*************************************************************************** - * - * $Id: filebuffer.h 643 2008-06-30 16:39:59Z jrodriguez $ - * - * jrodriguez@calpont.com * - * * - ***************************************************************************/ - -#pragma once - -#include -#include -#include -#include -#include -#include -#include -#include "brmtypes.h" -#include -#include - -/** - @author Jason Rodriguez -*/ - -/** - * @brief represents a disk blockrequest - **/ -namespace dbbc -{ -// Set block cache alogorithm to last recently used by defining LRU. -// Otherwise it will be FIFO. -typedef struct FBData -{ - BRM::LBID_t lbid; - BRM::VER_t ver; -} FBData_t; - -//@bug 669 Change to list for last recently used cache -typedef std::list filebuffer_list_t; -typedef std::list::iterator filebuffer_list_iter_t; - -class FileBuffer -{ - public: - /** - * @brief copy ctor - **/ - FileBuffer(const FileBuffer& fb); - - /** - * @brief the disk block from lbid@ver, and a data block len bytes long - **/ - FileBuffer(const BRM::LBID_t lbid, const BRM::VER_t ver, const uint8_t* data, const uint32_t len); - - /** - * @brief disk block lbid@ver and empty data - **/ - FileBuffer(const BRM::LBID_t lbid, const BRM::VER_t ver); - - /** - * @brief class dtor - **/ - ~FileBuffer(); - - /** - * @brief set the data value of this block to d have len bytestream - **/ - void setData(const uint8_t* d, const int len = 8192); - - /** - * @brief retrieve the data in byte* format from this data block - **/ - const uint8_t* getData() const - { - return fByteData; - } - uint8_t* getData() - { - return fByteData; - } - - const uint32_t datLen() const - { - return fDataLen; - } - - /** - * @brief assignment operator - **/ - FileBuffer& operator=(const FileBuffer& rhs); - - /** - * @brief equality operator is based on lbid@ver - **/ - bool operator==(const FileBuffer& rhs) const - { - return (fLbid == rhs.fLbid && fVerid == rhs.fVerid); - } - - /** - * @brief inequality operator - **/ - bool operator!=(const FileBuffer& rhs) const - { - return (!(fLbid == rhs.fLbid && fVerid == rhs.fVerid)); - } - - FileBuffer* thisPtr() - { - return this; - } - /** - * @brief return the lbid value of disk bloc - **/ - const BRM::LBID_t Lbid() const - { - return fLbid; - } - void Lbid(const BRM::LBID_t l) - { - fLbid = l; - } - - /** - * @brief return the version of this disk block. ignored for range retrievals - **/ - const BRM::VER_t Verid() const - { - return fVerid; - } - void Verid(BRM::VER_t v) - { - fVerid = v; - } - - /** - * @brief return the number of bytes in this disk blockrequest - **/ - - void listLoc(const filebuffer_list_iter_t& loc) - { - fListLoc = loc; - } - - const filebuffer_list_iter_t& listLoc() const - { - return fListLoc; - } - - private: - uint8_t fByteData[WriteEngine::BYTE_PER_BLOCK]; - uint32_t fDataLen; - - BRM::LBID_t fLbid; - BRM::VER_t fVerid; - filebuffer_list_iter_t fListLoc; - - // do not implement - FileBuffer(){}; -}; - -typedef std::vector FileBufferPool_t; - -} // namespace dbbc diff --git a/utils/testbc/filebuffermgr.cpp b/utils/testbc/filebuffermgr.cpp deleted file mode 100644 index e9b882347..000000000 --- a/utils/testbc/filebuffermgr.cpp +++ /dev/null @@ -1,299 +0,0 @@ -/* Copyright (C) 2014 InfiniDB, Inc. - - This program is free software; you can redistribute it and/or - modify it under the terms of the GNU General Public License - as published by the Free Software Foundation; version 2 of - the License. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU General Public License for more details. - - You should have received a copy of the GNU General Public License - along with this program; if not, write to the Free Software - Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, - MA 02110-1301, USA. */ - -/*************************************************************************** - * - * $Id: filebuffermgr.cpp 699 2008-09-09 19:44:18Z rdempsey $ - * - * jrodriguez@calpont.com * - * * - ***************************************************************************/ -/** -* InitialDBBCSize - the starting number of elements the unordered set used to store disk blocks. - This does not instantiate InitialDBBCSize disk blocks but only the initial size of the unordered_set - -**/ - -#define NDEBUG // Turn off assert macro -#include -#include -#include -#include "filebuffermgr.h" -using namespace boost; -using namespace std; -#include "stats.h" - -extern dbbc::Stats* gPMStatsPtr; -extern bool gPMProfOn; -extern uint32_t gSession; - -namespace dbbc -{ -FileBufferMgr::FileBufferMgr(const uint32_t numBlcks, const uint32_t blkSz, const uint32_t deleteBlocks) - : fMaxNumBlocks(numBlcks) - , fBlockSz(blkSz) - , fWLock() - , fbSet() - , fbList() - , fCacheSize(0) - , fFBPool() - , aging(false) - , fDeleteBlocks(deleteBlocks) - , fEmptyPoolSlots() -{ - fFBPool.reserve(numBlcks); - fEmptyPoolSlots.reserve(deleteBlocks); -} - -FileBufferMgr::~FileBufferMgr() -{ - flushCache(); -} - -void FileBufferMgr::flushCache() -{ - mutex::scoped_lock lk(fWLock); - fbList.clear(); - fbSet.clear(); - fFBPool.clear(); - fEmptyPoolSlots.clear(); - // TODO:: re-init blocks in pool and HWM -} - -bool FileBufferMgr::exists(const BRM::LBID_t& lbid, const BRM::VER_t& ver) const -{ - const HashObject_t fb = {lbid, ver, 0}; - const bool b = exists(fb); - return b; -} // bool FileBufferMgr::exists(const BRM::LBID_t& lbid, const BRM::VER_t& ver) const - -FileBuffer* FileBufferMgr::findPtr(const HashObject_t& keyFb) -{ - mutex::scoped_lock lk(fWLock); - filebuffer_uset_iter_t it = fbSet.find(keyFb); - - if (fbSet.end() != it) - { - FileBuffer* fb = &(fFBPool[it->poolIdx]); - fbList.splice(fbList.begin(), fbList, (fFBPool[it->poolIdx]).listLoc()); - return fb; - } - - return NULL; - -} // end findPtr(const HashObject_t& keyFB) - -bool FileBufferMgr::find(const HashObject_t& keyFb, FileBuffer& fb) -{ - bool ret = false; - - mutex::scoped_lock lk(fWLock); - filebuffer_uset_iter_t it = fbSet.find(keyFb); - - if (fbSet.end() != it) - { - fbList.splice(fbList.begin(), fbList, (fFBPool[it->poolIdx]).listLoc()); - lk.unlock(); - fb = fFBPool[it->poolIdx]; - ret = true; - } - - return ret; - -} // end find(const HashObject_t& keyFB, HashObject_t& fb) - -bool FileBufferMgr::find(const HashObject_t& keyFb, void* bufferPtr) -{ - bool ret = false; - - if (gPMProfOn && gPMStatsPtr) - gPMStatsPtr->markEvent(keyFb.lbid, pthread_self(), gSession, 'L'); - - mutex::scoped_lock lk(fWLock); - - if (gPMProfOn && gPMStatsPtr) - gPMStatsPtr->markEvent(keyFb.lbid, pthread_self(), gSession, 'M'); - - filebuffer_uset_iter_t it = fbSet.find(keyFb); - - if (fbSet.end() != it) - { - //@bug 669 LRU cache, move block to front of list as last recently used. - fbList.splice(fbList.begin(), fbList, (fFBPool[it->poolIdx]).listLoc()); - lk.unlock(); - memcpy(bufferPtr, (fFBPool[it->poolIdx]).getData(), 8); - - if (gPMProfOn && gPMStatsPtr) - gPMStatsPtr->markEvent(keyFb.lbid, pthread_self(), gSession, 'U'); - - ret = true; - } - - return ret; - -} // end find(const FileBuffer& keyFB, void* bufferPtr) - -bool FileBufferMgr::exists(const HashObject_t& fb) const -{ - bool find_bool = false; - mutex::scoped_lock lk(fWLock); - filebuffer_uset_iter_t it = fbSet.find(fb); - - if (it != fbSet.end()) - { - find_bool = true; - fbList.splice(fbList.begin(), fbList, (fFBPool[it->poolIdx]).listLoc()); - } - - return find_bool; -} - -// default insert operation. -// add a new fb into fbMgr and to fbList -// add to the front and age out from the back -// so add new fbs to the front of the list -//@bug 665: keep filebuffer in a vector. HashObject keeps the index of the filebuffer - -const int FileBufferMgr::insert(const BRM::LBID_t lbid, const BRM::VER_t ver, const uint8_t* data) -{ - int ret = 0; - - if (gPMProfOn && gPMStatsPtr) - gPMStatsPtr->markEvent(lbid, pthread_self(), gSession, 'I'); - - mutex::scoped_lock lk(fWLock); - HashObject_t fbIndex = {lbid, ver, 0}; - filebuffer_pair_t pr = fbSet.insert(fbIndex); - - if (pr.second) - { - // It was inserted (it wasn't there before) - // Right now we have an invalid cache: we have inserted an entry with a -1 index. - // We need to fix this quickly... - fCacheSize++; - FBData_t fbdata = {lbid, ver}; - fbList.push_front(fbdata); - } - else - { - // if it's a duplicate there's nothing to do - if (gPMProfOn && gPMStatsPtr) - gPMStatsPtr->markEvent(lbid, pthread_self(), gSession, 'D'); - - return ret; - } - - uint32_t pi = INT_MAX; - - if (fCacheSize > maxCacheSize()) - { - // If the insert above caused the cache to exceed its max size, find the lru block in - // the cache and use its pool index to store the block data. - FBData_t& fbdata = fbList.back(); // the lru block - HashObject_t lastFB = {fbdata.lbid, fbdata.ver, 0}; - filebuffer_uset_iter_t iter = fbSet.find(lastFB); // should be there - - idbassert(iter != fbSet.end()); - pi = iter->poolIdx; - idbassert(pi < maxCacheSize()); - idbassert(pi < fFBPool.size()); - - /* Why does this iterator return a const HashObject_t? */ - HashObject_t& ref = const_cast(*pr.first); - ref.poolIdx = pi; - - // replace the lru block with this block - FileBuffer fb(lbid, ver, NULL, 0); - fFBPool[pi] = fb; - fFBPool[pi].setData(data, BLOCK_SIZE); - fbSet.erase(iter); - fbList.pop_back(); - - fCacheSize--; - depleteCache(); - ret = 1; - } - else - { - if (!fEmptyPoolSlots.empty()) - { - pi = fEmptyPoolSlots.back(); - fEmptyPoolSlots.pop_back(); - FileBuffer fb(lbid, ver, NULL, 0); - fFBPool[pi] = fb; - fFBPool[pi].setData(data, 8); - } - else - { - pi = fFBPool.size(); - FileBuffer fb(lbid, ver, NULL, 0); - fFBPool.push_back(fb); - fFBPool[pi].setData(data, 8); - } - - /* Why does this iterator return a const? */ - HashObject_t& ref = const_cast(*pr.first); - ref.poolIdx = pi; - ret = 1; - } - - idbassert(pi < fFBPool.size()); - fFBPool[pi].listLoc(fbList.begin()); - - if (gPMProfOn && gPMStatsPtr) - gPMStatsPtr->markEvent(lbid, pthread_self(), gSession, 'J'); - - idbassert(fCacheSize <= maxCacheSize()); - // idbassert(fCacheSize == fbSet.size()); - // idbassert(fCacheSize == fbList.size()); - return ret; -} - -void FileBufferMgr::depleteCache() -{ - for (uint32_t i = 0; i < fDeleteBlocks && !fbList.empty(); ++i) - { - FBData_t fbdata(fbList.back()); // the lru block - HashObject_t lastFB = {fbdata.lbid, fbdata.ver, 0}; - filebuffer_uset_iter_t iter = fbSet.find(lastFB); - - idbassert(iter != fbSet.end()); - uint32_t idx = iter->poolIdx; - idbassert(idx < fFBPool.size()); - // Save position in FileBuffer pool for reuse. - fEmptyPoolSlots.push_back(idx); - fbSet.erase(iter); - fbList.pop_back(); - fCacheSize--; - } -} - -ostream& FileBufferMgr::formatLRUList(ostream& os) const -{ - filebuffer_list_t::const_iterator iter = fbList.begin(); - filebuffer_list_t::const_iterator end = fbList.end(); - - while (iter != end) - { - os << iter->lbid << '\t' << iter->ver << endl; - ++iter; - } - - return os; -} - -} // namespace dbbc diff --git a/utils/testbc/filebuffermgr.h b/utils/testbc/filebuffermgr.h deleted file mode 100644 index c127f98dd..000000000 --- a/utils/testbc/filebuffermgr.h +++ /dev/null @@ -1,197 +0,0 @@ -/* Copyright (C) 2014 InfiniDB, Inc. - - This program is free software; you can redistribute it and/or - modify it under the terms of the GNU General Public License - as published by the Free Software Foundation; version 2 of - the License. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU General Public License for more details. - - You should have received a copy of the GNU General Public License - along with this program; if not, write to the Free Software - Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, - MA 02110-1301, USA. */ - -/*************************************************************************** - * - * $Id: filebuffermgr.h 699 2008-09-09 19:44:18Z rdempsey $ - * - * * - ***************************************************************************/ - -#pragma once -#include -#include "blocksize.h" -#include "filebuffer.h" -#include "rwlock_local.h" -#include -#include - -/** - @author Jason Rodriguez -*/ - -/** - * @brief manages storage of Disk Block Buffers via and LRU cache using the stl classes unordered_set and - *list. - * - **/ - -namespace dbbc -{ -/** - * @brief used as the hasher algorithm for the unordered_set used to store the disk blocks - **/ - -typedef struct -{ - BRM::LBID_t lbid; - BRM::VER_t ver; - uint32_t poolIdx; -} FileBufferIndex_t; - -typedef FileBufferIndex_t HashObject_t; - -class bcHasher -{ - public: - size_t operator()(const HashObject_t& rhs) const - { - return (((rhs.ver & 0xffffULL) << 48) | (rhs.lbid & 0xffffffffffffULL)); - } -}; - -class bcEqual -{ - public: - size_t operator()(const HashObject_t& f1, const HashObject_t& f2) const - { - return ((f1.lbid == f2.lbid) && (f1.ver == f2.ver)); - } -}; - -inline bool operator<(const HashObject_t& f1, const HashObject_t& f2) -{ - // return ((f1.lbid < f2.lbid) || (f1.ver < f2.ver)); -#if 1 - if (f1.lbid < f2.lbid) - return true; - else if (f1.lbid == f2.lbid) - return (f1.ver < f2.ver); - - return false; -#else - bcHasher bh1, bh2; - return (bh1(f1) < bh2(f2)); -#endif -} - -class FileBufferMgr -{ - public: - typedef std::tr1::unordered_set filebuffer_uset_t; - typedef std::tr1::unordered_set::const_iterator filebuffer_uset_iter_t; - typedef std::pair filebuffer_pair_t; // return type for insert - - typedef std::vector intvec_t; - - /** - * @brief ctor. Set max buffer size to numBlcks and block buffer size to blckSz - **/ - - FileBufferMgr(uint32_t numBlcks, uint32_t blckSz = BLOCK_SIZE, uint32_t deleteBlocks = 0); - - /** - * @brief default dtor - **/ - virtual ~FileBufferMgr(); - - /** - * @brief return TRUE if the Disk block lbid@ver is loaded into the Disk Block Buffer cache otherwise return - *FALSE. - **/ - bool exists(const BRM::LBID_t& lbid, const BRM::VER_t& ver) const; - - /** - * @brief return TRUE if the Disk block referenced by fb is loaded into the Disk Block Buffer cache - *otherwise return FALSE. - **/ - bool exists(const HashObject_t& fb) const; - - /** - * @brief add the Disk Block reference by fb into the Disk Block Buffer Cache - **/ - const int insert(const BRM::LBID_t lbid, const BRM::VER_t ver, const uint8_t* data); - - /** - * @brief returns the total number of Disk Blocks in the Cache - **/ - uint32_t size() const - { - return fbSet.size(); - } - - /** - * @brief - **/ - void flushCache(); - - /** - * @brief return the disk Block referenced by fb - **/ - - FileBuffer* findPtr(const HashObject_t& keyFb); - - bool find(const HashObject_t& keyFb, FileBuffer& fb); - - /** - * @brief return the disk Block referenced by bufferPtr - **/ - - bool find(const HashObject_t& keyFb, void* bufferPtr); - - uint32_t maxCacheSize() const - { - return fMaxNumBlocks; - } - - uint32_t listSize() const - { - return fbList.size(); - } - - const filebuffer_uset_iter_t end() const - { - return fbSet.end(); - } - - void displayCounts() const; - - std::ostream& formatLRUList(std::ostream& os) const; - - private: - uint32_t fMaxNumBlocks; // the max number of blockSz blocks to keep in the Cache list - uint32_t fBlockSz; // size in bytes size of a data block - probably 8 - - mutable boost::mutex fWLock; - mutable filebuffer_uset_t fbSet; - - mutable filebuffer_list_t fbList; // rename this - uint32_t fCacheSize; - - FileBufferPool_t fFBPool; // vector - - // do not implement - FileBufferMgr(const FileBufferMgr& fbm); - const FileBufferMgr& operator=(const FileBufferMgr& fbm); - bool aging; - - uint32_t fDeleteBlocks; - intvec_t fEmptyPoolSlots; // keep track of FBPool slots that can be reused - - void depleteCache(); -}; -} // namespace dbbc diff --git a/utils/testbc/filerequest.cpp b/utils/testbc/filerequest.cpp deleted file mode 100644 index 9c9799c7b..000000000 --- a/utils/testbc/filerequest.cpp +++ /dev/null @@ -1,98 +0,0 @@ -/* Copyright (C) 2014 InfiniDB, Inc. - - This program is free software; you can redistribute it and/or - modify it under the terms of the GNU General Public License - as published by the Free Software Foundation; version 2 of - the License. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU General Public License for more details. - - You should have received a copy of the GNU General Public License - along with this program; if not, write to the Free Software - Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, - MA 02110-1301, USA. */ - -/*************************************************************************** - * - * $Id: filerequest.cpp 506 2008-03-14 15:31:55Z jrodriguez $ - * - * jrodriguez@calpont.com * - * * - ***************************************************************************/ - -#include "filerequest.h" -#include - -using namespace std; - -namespace dbbc -{ -fileRequest::fileRequest() -{ - fLBID = -1; - fVer = -1; - data = NULL; - init(); - fRqstType = LBIDREQUEST; -} - -fileRequest::fileRequest(BRM::LBID_t lbid, BRM::VER_t ver, bool flg) - : data(NULL), fLBID(lbid), fVer(ver), fFlg(flg) -{ - init(); - fLength = 1; - fRqstType = LBIDREQUEST; -} - -fileRequest::fileRequest(BRM::LBID_t lbid, BRM::VER_t ver, bool flg, uint8_t* ptr) - : fLBID(lbid), fVer(ver), fFlg(flg) -{ - init(); - fLength = 1; - fRqstType = LBIDREQUEST; - data = ptr; -} - -fileRequest::fileRequest(const BRM::InlineLBIDRange& range, const BRM::VER_t ver) - : data(NULL), fLBID(range.start), fVer(ver), fFlg(false), fLength(range.size), fRqstType(RANGEREQUEST) -{ - init(); - fLength = range.size; -} - -fileRequest::fileRequest(const fileRequest& blk) -{ - fLBID = blk.fLBID; - fVer = blk.fVer; - fLength = blk.fLength; - fblksRead = blk.fblksRead; - fRqstType = blk.fRqstType; - fRqstStatus = blk.fRqstStatus; - data = blk.data; - init(); -} - -void fileRequest::init() -{ - if (pthread_mutex_init(&fFRMutex, NULL) != 0) - throw runtime_error("mutex initialization failure"); - - if (pthread_cond_init(&fFRCond, NULL) != 0) - throw runtime_error("cond var initialization failure"); - - fFRPredicate = INIT; - fLength = 0; - fblksRead = 0; - fRqstStatus = 0; -} - -fileRequest::~fileRequest() -{ - pthread_mutex_destroy(&fFRMutex); - pthread_cond_destroy(&fFRCond); -} - -} // namespace dbbc diff --git a/utils/testbc/filerequest.h b/utils/testbc/filerequest.h deleted file mode 100644 index 227a1592d..000000000 --- a/utils/testbc/filerequest.h +++ /dev/null @@ -1,252 +0,0 @@ -/* Copyright (C) 2014 InfiniDB, Inc. - - This program is free software; you can redistribute it and/or - modify it under the terms of the GNU General Public License - as published by the Free Software Foundation; version 2 of - the License. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU General Public License for more details. - - You should have received a copy of the GNU General Public License - along with this program; if not, write to the Free Software - Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, - MA 02110-1301, USA. */ - -#pragma once - -/*************************************************************************** - * - * $Id: filerequest.h 658 2008-07-10 14:47:24Z jrodriguez $ - * - * jrodriguez@calpont.com * - * * - ***************************************************************************/ - -#include "brmtypes.h" - -/** - @author Jason Rodriguez -*/ - -/** - * @brief request class for the fileblockrequestqueue and the iomanager - **/ -namespace dbbc -{ -class fileRequest -{ - public: - /** - * @brief default ctor - **/ - fileRequest(); - - /** - * @brief copy constructor - **/ - fileRequest(const fileRequest& blk); - - /** - * @brief request for the disk block lbid@ver - **/ - fileRequest(BRM::LBID_t lbid, BRM::VER_t ver, bool flg); - - fileRequest(BRM::LBID_t lbid, BRM::VER_t ver, bool flg, uint8_t* ptr); - /** - * @brief request a range of disk blocks - **/ - fileRequest(const BRM::InlineLBIDRange& range, const BRM::VER_t ver); - - /** - * @brief class dtor - **/ - virtual ~fileRequest(); - - /** - * @brief less-than operator - **/ - bool operator<(const fileRequest& rhs) const - { - return fLength < rhs.fLength; - } - - /** - * @brief greater-than operator - **/ - bool operator>(const fileRequest& rhs) const - { - return fLength > rhs.fLength; - } - - /** - * @brief equality operator - **/ - bool operator==(const fileRequest& rhs) const - { - return fLength == rhs.fLength; - } - - enum request_status_enum - { - SUCCESSFUL, - FAILED - }; - - enum request_type_enum - { - LBIDREQUEST, - RANGEREQUEST - }; - - /** - * @brief used to manage request processing synchronzation - **/ - enum predicate_status_enum - { - INIT, - SENDING, - READING, - COMPLETE, - STOP - }; - - /** - * @brief lbid requested - **/ - const BRM::LBID_t Lbid() const - { - return fLBID; - } - - /** - * @brief version of the lbid requested - **/ - const BRM::VER_t Ver() const - { - return fVer; - } - - /** - * @brief VBBM flag of the LBID/Ver - **/ - const bool Flg() const - { - return fFlg; - } - - /** - * @brief number of blocks requested - **/ - const uint32_t BlocksRequested() const - { - return fLength; - } - - /** - * @brief setter for blocks requested - **/ - void BlocksRequested(const int l) - { - fLength = l; - } - - /** - * @brief number of blocks read from disk - **/ - const uint32_t BlocksRead() const - { - return fblksRead; - } - const uint32_t BlocksLoaded() const - { - return fblksLoaded; - } - - /** - * @brief setter for blocks read from disk - **/ - void BlocksRead(const int l) - { - fblksRead = l; - } - void BlocksLoaded(const int l) - { - fblksLoaded = l; - } - - /** - * @brief did the request succeed for fail - **/ - int RequestStatus() const - { - return fRqstStatus; - } - - /** - * @brief setter for the request status - **/ - void RequestStatus(int s) - { - fRqstStatus = s; - } - - /** - * @brief return BLOCK or RANGE requested - **/ - int RequestType() const - { - return fRqstType; - } - - /** - * @brief mutex to control synchronzation of request processing - **/ - pthread_mutex_t& frMutex() const - { - return fFRMutex; - } - - /** - * @brief condition variable. signal when request is complete - **/ - pthread_cond_t& frCond() const - { - return fFRCond; - } - - /** - * @brief - **/ - const enum predicate_status_enum& frPredicate() const - { - return fFRPredicate; - } - - /** - * @brief setter for the predicate - **/ - void SetPredicate(const enum predicate_status_enum& p) - { - fFRPredicate = p; - } - - uint8_t* data; - - private: - void init(); - BRM::LBID_t fLBID; - BRM::VER_t fVer; - bool fFlg; - mutable pthread_mutex_t fFRMutex; - mutable pthread_cond_t fFRCond; - predicate_status_enum fFRPredicate; - uint32_t fLength; // lbids requested - uint32_t fblksRead; // lbids read - uint32_t fblksLoaded; // lbids loaded into cache - int fRqstStatus; - enum request_type_enum fRqstType; -}; -} // namespace dbbc diff --git a/utils/testbc/iomanager.cpp b/utils/testbc/iomanager.cpp deleted file mode 100644 index b2944e5b7..000000000 --- a/utils/testbc/iomanager.cpp +++ /dev/null @@ -1,327 +0,0 @@ -/* Copyright (C) 2014 InfiniDB, Inc. - - This program is free software; you can redistribute it and/or - modify it under the terms of the GNU General Public License - as published by the Free Software Foundation; version 2 of - the License. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU General Public License for more details. - - You should have received a copy of the GNU General Public License - along with this program; if not, write to the Free Software - Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, - MA 02110-1301, USA. */ - -// $Id: iomanager.cpp 724 2008-09-26 16:16:05Z jrodriguez $ -// -// C++ Implementation: iomanager -// -// Description: -// -// -// Author: Jason Rodriguez -// -// -// - -#define _FILE_OFFSET_BITS 64 -#define _LARGEFILE64_SOURCE -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#define NDEBUG -#include - -using namespace std; - -#include "configcpp.h" -using namespace config; - -#include "iomanager.h" -namespace -{ -using namespace dbbc; -using namespace std; - -void timespec_sub(const struct timespec& tv1, const struct timespec& tv2, double& tm) -{ - tm = (double)(tv2.tv_sec - tv1.tv_sec) + 1.e-9 * (tv2.tv_nsec - tv1.tv_nsec); -} - -struct IOMThreadArg -{ - ioManager* iom; - int32_t thdId; -}; - -typedef IOMThreadArg IOMThreadArg_t; - -void* thr_popper(void* arg) -{ - ioManager* iom = ((IOMThreadArg*)arg)->iom; - int32_t iomThdId = ((IOMThreadArg*)arg)->thdId; - FileBufferMgr* fbm; - int totalRqst = 0; - fileRequest* fr = 0; - BRM::LBID_t lbid = 0; - BRM::OID_t oid = 0; - BRM::VER_t ver = 0; - int blocksLoaded = 0; - int blocksRead = 0; - const unsigned pageSize = 4096; - fbm = &iom->fileBufferManager(); - char fileName[WriteEngine::FILE_NAME_SIZE]; - const uint64_t fileBlockSize = BLOCK_SIZE; - uint32_t offset = 0; - bool flg = false; - char* fileNamePtr = fileName; - uint64_t longSeekOffset = 0; - int err; - uint32_t dlen = 0, acc, readSize, blocksThisRead, j; - uint32_t blocksRequested = 0; - ssize_t i; - uint32_t sz = 0; - char* alignedbuff = 0; - boost::scoped_array realbuff; - pthread_t threadId = 0; - ostringstream iomLogFileName; - ofstream lFile; - - threadId = pthread_self(); - - uint32_t readBufferSz = iom->blocksPerRead * BLOCK_SIZE + pageSize; - - realbuff.reset(new char[readBufferSz]); - - if (realbuff.get() == 0) - { - cerr << "thr_popper: Can't allocate space for a whole extent in memory" << endl; - return 0; - } - -#if __WORDSIZE > 32 - // alignedbuff=(char*)((((ptrdiff_t)&realbuff[0] + pageSize - 1) / pageSize) * pageSize); - // pagesize == (1 << 12) - alignedbuff = (char*)((((ptrdiff_t)realbuff.get() >> 12) << 12) + pageSize); -#else - // alignedbuff=(char*)(((((ptrdiff_t)&realbuff[0] & 0xffffffff) + pageSize - 1) / pageSize) * pageSize); - alignedbuff = (char*)(((((ptrdiff_t)realbuff.get() >> 12) << 12) & 0xffffffff) + pageSize); -#endif - - idbassert(((ptrdiff_t)alignedbuff - (ptrdiff_t)realbuff.get()) < (ptrdiff_t)pageSize); - idbassert(((ptrdiff_t)alignedbuff % pageSize) == 0); - - for (;;) - { - fr = iom->getNextRequest(); - lbid = fr->Lbid(); - ver = fr->Ver(); - flg = fr->Flg(); - blocksLoaded = 0; - blocksRead = 0; - dlen = fr->BlocksRequested(); - blocksRequested = fr->BlocksRequested(); - - err = iom->lbidLookup(lbid, ver, flg, oid, offset); - - if (err < 0) - { - cerr << "lbid=" << lbid << " ver=" << ver << " flg=" << (flg ? 1 : 0) << endl; - throw runtime_error("thr_popper: BRM lookup failure"); - } - - longSeekOffset = (uint64_t)offset * (uint64_t)fileBlockSize; - totalRqst++; - sz = 0; - - uint32_t readCount = 0; - uint32_t bytesRead = 0; - uint32_t jend = blocksRequested / iom->blocksPerRead; - - for (j = 0; j <= jend; j++) - { - blocksThisRead = std::min(dlen, iom->blocksPerRead); - readSize = blocksThisRead * BLOCK_SIZE; - - acc = 0; - - while (acc < readSize) - { - i = readSize; // pread(fd, &alignedbuff[acc], readSize - acc, longSeekOffset); - - /* XXXPAT: Need to decide how to handle errors here */ - if (i < 0 && errno == EINTR) - { - continue; - } - else if (i < 0) - { - perror("thr_popper::read"); - return 0; // shuts down this thread, - // probably not the right thing to do - } - else if (i == 0) - { - try - { - } - catch (exception& exc) - { - cerr << "FileName Err:" << exc.what() << endl; - } - - cerr << "thr_popper: Early EOF in file " << fileNamePtr << endl; - return 0; - } - - acc += i; - longSeekOffset += (uint64_t)i; - readCount++; - bytesRead += i; - } // while(acc... - - blocksRead += blocksThisRead; - - for (i = 0; (unsigned)i < blocksThisRead; ++i) - { - if (fbm->insert((lbid + i) + (j * iom->blocksPerRead), ver, (uint8_t*)&alignedbuff[i * BLOCK_SIZE])) - ++blocksLoaded; - } - - dlen -= blocksThisRead; - } // for (j... - - fr->BlocksRead(blocksRead); - fr->BlocksLoaded(blocksLoaded); - - if (fr->data != 0 && blocksRequested == 1) - memcpy(fr->data, alignedbuff, BLOCK_SIZE); - - pthread_mutex_lock(&fr->frMutex()); - fr->SetPredicate(fileRequest::COMPLETE); - pthread_cond_signal(&fr->frCond()); - pthread_mutex_unlock(&fr->frMutex()); - - } // for(;;) - - lFile.close(); - - return 0; -} // end thr_popper - -} // anonymous namespace - -namespace dbbc -{ -ioManager::ioManager(FileBufferMgr& fbm, fileBlockRequestQueue& fbrq, int thrCount, int bsPerRead) - : blocksPerRead(bsPerRead), fIOMfbMgr(fbm), fIOMRequestQueue(fbrq) -{ - if (thrCount <= 0) - thrCount = 1; - - if (thrCount > 256) - thrCount = 256; - - fConfig = Config::makeConfig(); - string val = fConfig->getConfig("DBBC", "IOMTracing"); - int temp = 0; - - if (val.length() > 0) - temp = static_cast(Config::fromText(val)); - - if (temp > 0) - fIOTrace = true; - else - fIOTrace = false; - - fThreadCount = thrCount; - go(); - -} // ioManager - -void ioManager::buildOidFileName(const BRM::OID_t oid, char* file_name) -{ - if (fFileOp.getFileName(oid, file_name) != WriteEngine::NO_ERROR) - { - file_name[0] = 0; - throw std::runtime_error("fileOp.getFileName failed"); - } -} - -BRM::LBID_t ioManager::lbidLookup(BRM::LBID_t lbid, BRM::VER_t verid, bool vbFlag, BRM::OID_t& oid, - uint32_t& offset) -{ - int rc = fdbrm.lookup(lbid, verid, vbFlag, oid, offset); - return rc; -} - -int ioManager::createReaders() -{ - int realCnt = 0; - IOMThreadArg_t fThdArgArr[256]; - - for (int idx = 0; idx < fThreadCount; idx++) - { - fThdArgArr[realCnt].iom = this; - fThdArgArr[realCnt].thdId = realCnt; - int ret = pthread_create(&fThreadArr[realCnt], 0, thr_popper, &fThdArgArr[realCnt]); - - if (ret != 0) - perror("createReaders::pthread_create"); - else - realCnt++; - } - - fThreadCount = realCnt; - return fThreadCount; -} - -ioManager::~ioManager() -{ - stop(); -} - -void ioManager::go(void) -{ - createReaders(); -} - -void ioManager::stop() -{ - for (int idx = 0; idx < fThreadCount; idx++) - { - pthread_detach(fThreadArr[idx]); - } -} - -fileRequest* ioManager::getNextRequest() -{ - fileRequest* blk = 0; - - try - { - blk = fIOMRequestQueue.pop(); - return blk; - } - catch (exception& e) - { - cerr << "ioManager::getNextRequest() ERROR " << endl; - } - - return blk; -} - -} // namespace dbbc diff --git a/utils/testbc/iomanager.h b/utils/testbc/iomanager.h deleted file mode 100644 index f9dae5b5e..000000000 --- a/utils/testbc/iomanager.h +++ /dev/null @@ -1,100 +0,0 @@ -/* Copyright (C) 2014 InfiniDB, Inc. - - This program is free software; you can redistribute it and/or - modify it under the terms of the GNU General Public License - as published by the Free Software Foundation; version 2 of - the License. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU General Public License for more details. - - You should have received a copy of the GNU General Public License - along with this program; if not, write to the Free Software - Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, - MA 02110-1301, USA. */ - -#pragma once -// $Id: iomanager.h 655 2008-07-08 16:42:54Z jrodriguez $ -// -// C++ Interface: iomanager -// -// Description: -// -// -// Author: Jason Rodriguez -// -// -// - -/** - @author Jason Rodriguez -*/ - -#include -#include - -#include "writeengine.h" -#include "configcpp.h" -#include "brm.h" -#include "fileblockrequestqueue.h" -#include "filebuffermgr.h" - -namespace dbbc -{ -class ioManager -{ - public: - ioManager(FileBufferMgr& fbm, fileBlockRequestQueue& fbrq, int thrCount, int bsPerRead); - // ioManager(FileBufferMgr& fbm, int thrCount); - ~ioManager(); - int readerCount() const - { - return fThreadCount; - } - fileRequest* getNextRequest(); - void go(void); - void stop(); - FileBufferMgr& fileBufferManager() - { - return fIOMfbMgr; - } - config::Config* configPtr() - { - return fConfig; - } - BRM::LBID_t lbidLookup(BRM::LBID_t lbid, BRM::VER_t verid, bool vbFlag, BRM::OID_t& oid, uint32_t& offset); - void buildOidFileName(const BRM::OID_t oid, char* file_name); - - uint32_t getExtentSize() - { - return fdbrm.getExtentSize(); - } - - uint32_t blocksPerRead; - - bool IOTrace() const - { - return fIOTrace; - } - - private: - FileBufferMgr& fIOMfbMgr; - fileBlockRequestQueue& fIOMRequestQueue; - int fThreadCount; - pthread_t fPoppertid; - pthread_t fThreadArr[256]; - int createReaders(); - config::Config* fConfig; - BRM::DBRM fdbrm; - WriteEngine::FileOp fFileOp; - - // do not implement - ioManager(); - ioManager(const ioManager& iom); - const ioManager& operator=(const ioManager& iom); - bool fIOTrace; -}; - -} // namespace dbbc diff --git a/utils/testbc/logger.cpp b/utils/testbc/logger.cpp deleted file mode 100644 index a9912fb77..000000000 --- a/utils/testbc/logger.cpp +++ /dev/null @@ -1,63 +0,0 @@ -/* Copyright (C) 2014 InfiniDB, Inc. - - This program is free software; you can redistribute it and/or - modify it under the terms of the GNU General Public License - as published by the Free Software Foundation; version 2 of - the License. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU General Public License for more details. - - You should have received a copy of the GNU General Public License - along with this program; if not, write to the Free Software - Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, - MA 02110-1301, USA. */ - -/* - * $Id: logger.cpp 686 2008-08-21 22:08:33Z rdempsey $ - */ - -#include -using namespace boost; - -#include "messageobj.h" -#include "messageids.h" -#include "loggingid.h" -using namespace logging; - -#include "logger.h" - -namespace primitiveprocessor -{ -Logger::Logger() : fMl1(LoggingID(28)) -{ - fMsgMap[logging::M0000] = Message(logging::M0000); - fMsgMap[logging::M0016] = Message(logging::M0016); - fMsgMap[logging::M0045] = Message(logging::M0045); - fMsgMap[logging::M0053] = Message(logging::M0053); -} - -void Logger::logMessage(const Message::MessageID mid, const Message::Args& args, bool critical) -{ - mutex::scoped_lock lk(fLogLock); - MsgMap::iterator msgIter = fMsgMap.find(mid); - - if (msgIter == fMsgMap.end()) - msgIter = fMsgMap.find(logging::M0000); - - msgIter->second.reset(); - msgIter->second.format(args); - - if (critical) - { - fMl1.logCriticalMessage(msgIter->second); - } - else - { - fMl1.logWarningMessage(msgIter->second); - } -} - -} // namespace primitiveprocessor diff --git a/utils/testbc/logger.h b/utils/testbc/logger.h deleted file mode 100644 index 077ec09b7..000000000 --- a/utils/testbc/logger.h +++ /dev/null @@ -1,62 +0,0 @@ -/* Copyright (C) 2014 InfiniDB, Inc. - - This program is free software; you can redistribute it and/or - modify it under the terms of the GNU General Public License - as published by the Free Software Foundation; version 2 of - the License. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU General Public License for more details. - - You should have received a copy of the GNU General Public License - along with this program; if not, write to the Free Software - Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, - MA 02110-1301, USA. */ - -/* - * $Id: logger.h 706 2008-09-16 18:25:49Z bwelch $ - */ - -/** @file */ - -#pragma once - -#include -#include - -#include "messageobj.h" -#include "messagelog.h" - -namespace primitiveprocessor -{ -/** @brief message log wrapper class */ -class Logger -{ - public: - Logger(); - - void logMessage(const logging::Message::MessageID mid, const logging::Message::Args& args, - bool critical = false); - - void logMessage(const std::string& msg, bool critical = true, logging::Message::MessageID mid = 0) - { - logging::Message::Args args; - args.add(msg); - logMessage(mid, args, true); - } - - private: - // defaults okay - // Logger(const Logger& rhs); - // Logger& operator=(const Logger& rhs); - - typedef std::map MsgMap; - - MsgMap fMsgMap; - boost::mutex fLogLock; - logging::MessageLog fMl1; -}; - -} // namespace primitiveprocessor diff --git a/utils/testbc/stats.cpp b/utils/testbc/stats.cpp deleted file mode 100644 index bc5d1e7ff..000000000 --- a/utils/testbc/stats.cpp +++ /dev/null @@ -1,273 +0,0 @@ -/* Copyright (C) 2014 InfiniDB, Inc. - Copyright (C) 2016 MariaDB Corporation - - This program is free software; you can redistribute it and/or - modify it under the terms of the GNU General Public License - as published by the Free Software Foundation; version 2 of - the License. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU General Public License for more details. - - You should have received a copy of the GNU General Public License - along with this program; if not, write to the Free Software - Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, - MA 02110-1301, USA. */ - -/* - * $Id: stats.cpp 699 2008-09-09 19:44:18Z rdempsey $ - */ - -#include -#include -#include -#include -#include -#define NDEBUG -#include -#include -using namespace std; - -#include -#include -using namespace boost; - -#include "stats.h" -#include "messagelog.h" - -using namespace BRM; - -namespace -{ -void pause_(unsigned delay) -{ - struct timespec req; - struct timespec rem; - - req.tv_sec = delay; - req.tv_nsec = 0; - - rem.tv_sec = 0; - rem.tv_nsec = 0; - -again: - - if (nanosleep(&req, &rem) != 0) - if (rem.tv_sec > 0 || rem.tv_nsec > 0) - { - req = rem; - goto again; - } -} - -const string timestr() -{ - // Get a timestamp for output. - struct tm tm; - struct timeval tv; - - gettimeofday(&tv, 0); - localtime_r(&tv.tv_sec, &tm); - - ostringstream oss; - oss << setfill('0') << setw(2) << tm.tm_hour << ':' << setw(2) << tm.tm_min << ':' << setw(2) << tm.tm_sec - << '.' << setw(4) << tv.tv_usec / 100; - - return oss.str(); -} - -class TraceFile -{ - public: - TraceFile(uint32_t sessionID, const char* name) - { - if (sessionID > 0) - { - const char* outName; - - if (name == 0) - outName = "lbids"; - else - outName = name; - - ostringstream oss; - oss << MCSLOGDIR << "/trace/" << outName << '.' << sessionID; - oFile.reset(new ofstream()); - oFile->open(oss.str().c_str(), ios_base::out | ios_base::ate | ios_base::app); - } - } - - ~TraceFile() - { - } - - void close() - { - if (oFile) - { - oFile->close(); - } - } - - void log(OID_t oid, uint64_t lbid, pthread_t thdid, char event = '\0') - { - *oFile << oid << ' ' << timestr() << ' ' << lbid << ' ' << thdid; - - if (event != '\0') - *oFile << ' ' << event; - - *oFile << endl; - oFile->flush(); - } - - private: - // Compiler defaults okay - // TraceFile(const TraceFile& rhs); - // TraceFile operator=(const TraceFile& rhs); - shared_ptr oFile; -}; - -struct TraceFileInfo -{ - TraceFileInfo(uint32_t session = 0, const char* name = 0) : traceFile(session, name), lastTouched(0) - { - } - ~TraceFileInfo() - { - } - - void log(OID_t oid, uint64_t lbid, pthread_t thdid, char event = '\0') - { - traceFile.log(oid, lbid, thdid, event); - lastTouched = time(0); - } - - void close() - { - traceFile.close(); - } - - TraceFile traceFile; - time_t lastTouched; - - private: - // Compiler defaults okay - // TraceFileInfo(const TraceFileInfo& rhs); - // TraceFileInfo operator=(const TraceFileInfo& rhs); -}; - -// map a session id to a trace file -typedef map TraceFileMap_t; - -TraceFileMap_t traceFileMap; -// map mutex -mutex traceFileMapMutex; - -class StatMon -{ - public: - StatMon() - { - sigset_t sigset; - sigemptyset(&sigset); - sigaddset(&sigset, SIGPIPE); - sigaddset(&sigset, SIGUSR1); - sigaddset(&sigset, SIGUSR2); - pthread_sigmask(SIG_BLOCK, &sigset, 0); - } - void operator()() const - { - // struct timespec ts = { 60 * 1, 0 }; - mutex::scoped_lock lk(traceFileMapMutex); - TraceFileMap_t::iterator iter; - TraceFileMap_t::iterator end; - - for (;;) - { - lk.unlock(); - time_t beforeSleep = time(0); - // nanosleep(&ts, 0); - pause_(60); - lk.lock(); - iter = traceFileMap.begin(); - end = traceFileMap.end(); - - while (iter != end) - { - if (iter->second.lastTouched < beforeSleep) - { - // remove this session trace file - iter->second.close(); - traceFileMap.erase(iter++); - } - else - ++iter; - } - } - } - - private: - // Compiler defaults okay - // StatMon(const StatMon& rhs); - // StatMon operator=(const StatMon& rhs); -}; - -} // namespace - -namespace dbbc -{ -Stats::Stats() : fMonitorp(0) -{ - fMonitorp = new thread(StatMon()); -} - -Stats::Stats(const char* name) : fMonitorp(0), fName(name) -{ - fMonitorp = new thread(StatMon()); - // fName << name; -} - -Stats::~Stats() -{ - delete fMonitorp; -} - -void Stats::touchedLBID(uint64_t lbid, pthread_t thdid, uint32_t session) -{ - if (lbid < 0 || session == 0) - return; - - mutex::scoped_lock lk(traceFileMapMutex); - TraceFileMap_t::iterator iter = traceFileMap.find(session); - - if (iter == traceFileMap.end()) - { - traceFileMap[session] = TraceFileInfo(session); - iter = traceFileMap.find(session); - idbassert(iter != traceFileMap.end()); - } - - iter->second.log(lbid2oid(lbid), lbid, thdid); -} - -void Stats::markEvent(const uint64_t lbid, const pthread_t thdid, const uint32_t session, const char event) -{ - if (lbid < 0 || session == 0) - return; - - mutex::scoped_lock lk(traceFileMapMutex); - TraceFileMap_t::iterator iter = traceFileMap.find(session); - - if (iter == traceFileMap.end()) - { - traceFileMap[session] = TraceFileInfo(session, fName); - iter = traceFileMap.find(session); - idbassert(iter != traceFileMap.end()); - } - - iter->second.log(lbid2oid(lbid), lbid, thdid, event); -} - -} // namespace dbbc diff --git a/utils/testbc/stats.h b/utils/testbc/stats.h deleted file mode 100644 index 631241e9e..000000000 --- a/utils/testbc/stats.h +++ /dev/null @@ -1,63 +0,0 @@ -/* Copyright (C) 2014 InfiniDB, Inc. - - This program is free software; you can redistribute it and/or - modify it under the terms of the GNU General Public License - as published by the Free Software Foundation; version 2 of - the License. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU General Public License for more details. - - You should have received a copy of the GNU General Public License - along with this program; if not, write to the Free Software - Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, - MA 02110-1301, USA. */ - -/* - * $Id: stats.h 609 2008-06-11 12:40:07Z rdempsey $ - */ - -/** @file */ - -#pragma once - -#include -#include -#include -#include - -#include "brm.h" - -namespace dbbc -{ -class Stats -{ - public: - Stats(); - Stats(const char* name); - virtual ~Stats(); - - void touchedLBID(uint64_t lbid, pthread_t thdid, uint32_t session = 0); - void markEvent(const uint64_t lbid, const pthread_t thdid, const uint32_t session, const char event); - - inline BRM::OID_t lbid2oid(uint64_t lbid) - { - BRM::OID_t oid; - uint32_t fbo; - brm.lookup(lbid, 0, false, oid, fbo); - return oid; - } - - private: - Stats(const Stats& rhs); - Stats& operator=(const Stats& rhs); - - boost::thread* fMonitorp; - BRM::DBRM brm; - // ostringstream fName; - const char* fName; -}; - -} // namespace dbbc diff --git a/utils/testbc/testbc.cpp b/utils/testbc/testbc.cpp deleted file mode 100644 index 5be43c7d2..000000000 --- a/utils/testbc/testbc.cpp +++ /dev/null @@ -1,361 +0,0 @@ -/* Copyright (C) 2014 InfiniDB, Inc. - - This program is free software; you can redistribute it and/or - modify it under the terms of the GNU General Public License - as published by the Free Software Foundation; version 2 of - the License. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU General Public License for more details. - - You should have received a copy of the GNU General Public License - along with this program; if not, write to the Free Software - Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, - MA 02110-1301, USA. */ - -// -// C++ Implementation: bcTest -// -// Description: A simple Test driver for the Disk Block Buffer Cache -// -// Author: Jason Rodriguez , (C) 2007 -// -// - -#include -#include -#include -#include -#include -#include -#include - -#include "blockrequestprocessor.h" -#include "blockcacheclient.h" -#include "stats.h" -#include "brm.h" -#include "logger.h" -#include "iomanager.h" - -using namespace BRM; -using namespace dbbc; -using namespace std; -using namespace logging; -using namespace primitiveprocessor; - -Stats* gPMStatsPtr = NULL; -bool gPMProfOn = false; -uint32_t gSession = 0; -uint32_t lastRangeListIdx = 0; - -void timespec_sub(const struct timespec& tv1, const struct timespec& tv2, double& tm) -{ - tm = (double)(tv2.tv_sec - tv1.tv_sec) + 1.e-9 * (tv2.tv_nsec - tv1.tv_nsec); -} - -namespace primitiveprocessor -{ -Logger ml; -} - -class BCTest -{ - public: - struct OidRanges - { - OID_t oid; - HWM_t hwm; - LBIDRange_v ranges; - OidRanges(const OID_t o, const HWM_t h, const LBIDRange_v r) - { - oid = o; - hwm = h; - ranges = r; - } - }; // struct OidRanges - - BCTest(const int cacheSz = 64 * 1024, int readThr = 2, int readAhead = 1024); - - typedef OidRanges OidRanges_t; - typedef vector OidRangesList_t; - OidRangesList_t OidRangesList; - - DBRM dbrm; - uint32_t extentSize; - BRM::OID_t maxOid; - - int fCacheSz; - int fReadThr; - int fReadAhead; - uint32_t maxBlocksAvailable; - uint32_t fExtentSize; - - void setUp(); - int LoadOid(const OidRanges_t& o, uint32_t& loadCount); - void LoadLbid(const BRM::LBID_t lbid, const BRM::VER_t ver); - int ReadOidRanges(const OidRanges_t& v); - void ReadOidLbids(const BRM::LBID_t lbid, const BRM::VER_t ver); - - BlockRequestProcessor BRP; - -}; // class BCTest - -BCTest::BCTest(int cacheSz, int readThr, int readAhead) - : fCacheSz(cacheSz), fReadThr(readThr), fReadAhead(readAhead), BRP(fCacheSz, fReadThr, fReadAhead) -{ - setUp(); -} // ctor - -// -void BCTest::setUp() -{ - LBIDRange_v r; - HWM_t hwm; - OID_t oid = 1000; - extentSize = dbrm.getExtentSize(); - maxBlocksAvailable = 0; - int i = 0; - fExtentSize = dbrm.getExtentSize(); - - while (oid < 5000) - { - int ret = 0; - ret = dbrm.lookup(oid, r); - - if (ret == 0 && r.size() > 0) - { - dbrm.getHWM(oid, hwm); - maxBlocksAvailable += (r.size() * extentSize); - OidRanges_t oid_range(oid, hwm, r); - OidRangesList.push_back(oid_range); - // cout << "Setup i: " << i++ << " o: " << oid - // << " r: " << ret << " s: " << r.size() - // << " m: " << maxBlocksAvailable - // << endl; - hwm = 0; - r.clear(); - } - - oid++; - } - - // cout << "\t" << OidRangesList.size() << " oid ranges loaded " << endl << endl; - i = 0; -} // setUp() - -int BCTest::LoadOid(const OidRanges_t& o, uint32_t& loadCount) -{ - blockCacheClient bc(BRP); - uint32_t rCount = 0; - - for (uint32_t i = 0; i < o.ranges.size(); i++) - { - const InlineLBIDRange r = {o.ranges[i].start, o.ranges[i].size}; - - if (r.size > 0) - { - bc.check(r, 0, rCount); - // cout << "i: " << i << " c: " << rCount << " " << o.ranges[i].size << endl; - loadCount += rCount; - } - - rCount = 0; - } // for - - // cout << "hwm: " << o.hwm << " tot: " << loadCount << " " << o.ranges.size() << endl; - - return loadCount; - -} // LoadOid - -int BCTest::ReadOidRanges(const OidRanges_t& v) -{ - blockCacheClient bc(BRP); - int32_t readBlocks = 0; - int32_t missBlocks = 0; - - // int ret; - for (uint32_t i = 0; i < v.ranges.size(); i++) - { - FileBuffer fb(-1, -1); - const InlineLBIDRange r = {v.ranges[i].start, v.ranges[i].size}; - - for (int j = r.start; readBlocks < fCacheSz && j < r.start + r.size; j++) - { - // ret=0; - // ret=bc.read(j, 0, fb); - FileBuffer* ptr = bc.getBlockPtr(j, 0); - - if (ptr) - readBlocks++; - else - missBlocks++; - } - - // cout << " -- Read range idx: " << i << " hits: " << readBlocks << " miss: " << missBlocks << " hwm: " - // << v.hwm << endl; - } - - return readBlocks; - -} // ReadRange - -// add one block to block cache -// -void BCTest::LoadLbid(const BRM::LBID_t lbid, const BRM::VER_t ver) -{ - blockCacheClient bc(BRP); - bool b; - bc.check(lbid, ver, false, b); -} // LoadLbid - -// get one block out of block cache -// -void BCTest::ReadOidLbids(const BRM::LBID_t lbid, const BRM::VER_t ver) -{ - uint8_t d[8192] = {'\0'}; - blockCacheClient bc(BRP); - bc.read(lbid, ver, d); -} // ReadLbid - -struct loadThr -{ - loadThr::loadThr(BCTest& bc, int reps = 1) : fBC(bc), fReps(reps) - { - } - - void operator()() - { - uint32_t loadedBlocks = 0; - uint32_t oidBlocks; - uint32_t i = 0; - uint32_t rc = 0; - - clock_gettime(CLOCK_REALTIME, &tm1); - - for (uint32_t j = 0; j < fReps; j++) - for (i = 0; /*loadedBlocks " << endl; -} - -// -int main(int argc, char* argv[]) -{ - int cacheSz = 128; // K number of blocks - int thr = 1; - int ra = 1024; - int clients = 1; - int reps = 1; - - if (argc > 1 && atoi(argv[1]) > 0) - cacheSz = atoi(argv[1]) * 1024; - - if (argc > 2 && atoi(argv[2]) > 0) - thr = atoi(argv[2]); - - if (argc > 3 && atoi(argv[3]) > 0) - ra = atoi(argv[3]); - - if (argc > 4 && atoi(argv[4]) > 0) - clients = atoi(argv[4]); - - if (argc > 5 && atoi(argv[5]) > 0) - reps = atoi(argv[5]); - - BCTest bc(cacheSz, thr, ra); - - cout << "Cache Size: " << cacheSz << " read Threads: " << thr << " read Ahead: " << ra - << " clients: " << clients << " repetitions: " << reps << " max Blocks: " << bc.maxBlocksAvailable - << endl; - - // loader test - struct loadThr loader1(bc, reps); - vector v; - - for (int i = 0; i < clients; i++) - { - boost::thread* th1 = new boost::thread(loader1); - v.push_back(th1); - } - - for (int i = 0; i < clients; i++) - { - boost::thread* th1 = v[i]; - th1->join(); - delete th1; - } - - v.clear(); - - // reader test - - return 0; - -} // end main diff --git a/utils/testbc/testbc2.cpp b/utils/testbc/testbc2.cpp deleted file mode 100644 index 94407d6db..000000000 --- a/utils/testbc/testbc2.cpp +++ /dev/null @@ -1,354 +0,0 @@ -/* Copyright (C) 2014 InfiniDB, Inc. - - This program is free software; you can redistribute it and/or - modify it under the terms of the GNU General Public License - as published by the Free Software Foundation; version 2 of - the License. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU General Public License for more details. - - You should have received a copy of the GNU General Public License - along with this program; if not, write to the Free Software - Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, - MA 02110-1301, USA. */ - -// -// C++ Implementation: bcTest -// -// Description: A simple Test driver for the Disk Block Buffer Cache -// -// Author: Jason Rodriguez , (C) 2007 -// -// - -#include -#include -#include -#include -#include -#include -#include - -#include "blockrequestprocessor.h" -#include "blockcacheclient.h" -#include "stats.h" -#include "brm.h" -#include "logger.h" -#include "iomanager.h" - -using namespace BRM; -using namespace dbbc; -using namespace std; -using namespace logging; -using namespace primitiveprocessor; - -Stats* gPMStatsPtr = NULL; -bool gPMProfOn = false; -uint32_t gSession = 0; -uint32_t lastRangeListIdx = 0; -const uint32_t maxLoadBlocks(1024 * 1024); - -void timespec_sub(const struct timespec& tv1, const struct timespec& tv2, double& tm) -{ - tm = (double)(tv2.tv_sec - tv1.tv_sec) + 1.e-9 * (tv2.tv_nsec - tv1.tv_nsec); -} - -namespace primitiveprocessor -{ -Logger ml; -} - -class BCTest -{ - public: - struct OidRanges - { - OID_t oid; - HWM_t hwm; - LBIDRange_v ranges; - OidRanges(const OID_t o, const HWM_t h, const LBIDRange_v r) - { - oid = o; - hwm = h; - ranges = r; - } - }; // struct OidRanges - - BCTest(const int cacheSz = 64 * 1024, int readThr = 2, int readAhead = 1024); - - typedef OidRanges OidRanges_t; - typedef vector OidRangesList_t; - OidRangesList_t OidRangesList; - - DBRM dbrm; - uint32_t extentSize; - BRM::OID_t maxOid; - - int fCacheSz; - int fReadThr; - int fReadAhead; - uint32_t maxBlocksAvailable; - uint32_t fExtentSize; - - void setUp(); - int LoadOid(const OidRanges_t& o, uint32_t& loadCount); - void LoadLbid(const BRM::LBID_t lbid, const BRM::VER_t ver); - int ReadOidRanges(const OidRanges_t& v, uint32_t* hits, uint32_t* miss); - void ReadOidLbids(const BRM::LBID_t lbid, const BRM::VER_t ver); - - BlockRequestProcessor BRP; - -}; // class BCTest - -BCTest::BCTest(int cacheSz, int readThr, int readAhead) - : fCacheSz(cacheSz), fReadThr(readThr), fReadAhead(readAhead), BRP(fCacheSz, fReadThr, fReadAhead) -{ - setUp(); -} // ctor - -// -void BCTest::setUp() -{ - LBIDRange_v r; - HWM_t hwm; - OID_t oid = 1000; - extentSize = dbrm.getExtentSize(); - maxBlocksAvailable = 0; - int i = 0; - fExtentSize = dbrm.getExtentSize(); - - while (oid < 5000) - { - int ret = 0; - ret = dbrm.lookup(oid, r); - - if (ret == 0 && r.size() > 0) - { - dbrm.getHWM(oid, hwm); - maxBlocksAvailable += (r.size() * extentSize); - OidRanges_t oid_range(oid, hwm, r); - OidRangesList.push_back(oid_range); - // cout << "Setup i: " << i++ << " o: " << oid - // << " r: " << ret << " s: " << r.size() - // << " m: " << maxBlocksAvailable - // << endl; - hwm = 0; - r.clear(); - } - - oid++; - } - - // cout << "\t" << OidRangesList.size() << " oid ranges loaded " << endl << endl; - i = 0; -} // setUp() - -int BCTest::LoadOid(const OidRanges_t& o, uint32_t& loadCount) -{ - blockCacheClient bc(BRP); - uint32_t rCount = 0; - - for (uint32_t i = 0; i < o.ranges.size(); i++) - { - const InlineLBIDRange r = {o.ranges[i].start, o.ranges[i].size}; - - if (r.size > 0) - { - bc.check(r, 0, rCount); - // cout << "i: " << i << " c: " << rCount << " " << o.ranges[i].size << endl; - loadCount += rCount; - } - - rCount = 0; - } // for - - // cout << "hwm: " << o.hwm << " tot: " << loadCount << " " << o.ranges.size() << endl; - - return loadCount; - -} // LoadOid - -int BCTest::ReadOidRanges(const OidRanges_t& v, uint32_t* hits, uint32_t* miss) -{ - blockCacheClient bc(BRP); - uint8_t inBuff[8192]; - int32_t readBlocks = 0; - int32_t missBlocks = 0; - - for (uint32_t i = 0; i < v.ranges.size(); i++) - { - FileBuffer fb(-1, -1); - const InlineLBIDRange r = {v.ranges[i].start, v.ranges[i].size}; - - for (int j = r.start; readBlocks < fCacheSz && j < r.start + r.size; j++) - { - FileBuffer* ptr = bc.getBlockPtr(j, 0); - - if (ptr) - { - readBlocks++; - memcpy(inBuff, ptr->getData(), 8192); - } - else - missBlocks++; - } - - *hits += readBlocks; - *miss += missBlocks; - missBlocks = 0; - readBlocks = 0; - // cout << " -- Read range idx: " << i << " hits: " << readBlocks << " miss: " << missBlocks << " hwm: " - // << v.hwm << endl; - } - - return *hits; - -} // ReadRange - -// add one block to block cache -// -void BCTest::LoadLbid(const BRM::LBID_t lbid, const BRM::VER_t ver) -{ - blockCacheClient bc(BRP); - bool b; - bc.check(lbid, ver, false, b); -} // LoadLbid - -// get one block out of block cache -// -void BCTest::ReadOidLbids(const BRM::LBID_t lbid, const BRM::VER_t ver) -{ - uint8_t d[8192] = {'\0'}; - blockCacheClient bc(BRP); - bc.read(lbid, ver, d); -} // ReadLbid - -struct loadThr -{ - loadThr(BCTest& bc, int reps = 1) : fBC(bc), fReps(reps) - { - } - - void operator()() - { - uint32_t loadedBlocks = 0; - uint32_t readBlocks = 0; - uint32_t missBlocks = 0; - uint32_t oidBlocks; - uint32_t i = 0; - uint32_t rc = 0; - - clock_gettime(CLOCK_REALTIME, &tm1); - - for (uint32_t j = 0; j < fReps; j++) - for (i = 0; loadedBlocks < maxLoadBlocks && i < fBC.OidRangesList.size(); i++) - { - oidBlocks = 0; - rc = fBC.LoadOid(fBC.OidRangesList[i], oidBlocks); - /** - cout << "." - << "-- " << i << " " << fBC.OidRangesList[i].oid - << " h: " << fBC.OidRangesList[i].hwm - << "/" << oidBlocks - << endl; - **/ - loadedBlocks += oidBlocks; - readBlocks += fBC.ReadOidRanges(fBC.OidRangesList[i], &readBlocks, &missBlocks); - } - - clock_gettime(CLOCK_REALTIME, &tm2); - double tm3; - timespec_sub(tm1, tm2, tm3); - lastRangeListIdx = i; - - cout << "loadtest ld: " << loadedBlocks << " rd: " << readBlocks << "/" << missBlocks << " sz: " - << fBC.fCacheSz - //<< " last: " << lastRangeListIdx - << " tm: " << right << setw(10) << fixed << tm3 << endl; - - } // operator() - - BCTest& fBC; - uint32_t fReps; - struct timespec tm1; - struct timespec tm2; -}; - -struct readThr -{ - readThr(BCTest& bc, int reps = 1) : fBC(bc), fReps(reps) - { - } - - void operator()() - { - for (uint32_t k = 0; k < fReps; k++) - { - } - - } // operator() - - BCTest& fBC; - uint32_t fReps; -}; - -void usage() -{ - cout << "testbc " << endl; -} - -// -int main(int argc, char* argv[]) -{ - int cacheSz = 128; // K number of blocks - int thr = 1; - int ra = 1024; - int clients = 1; - int reps = 1; - - if (argc > 1 && atoi(argv[1]) > 0) - cacheSz = atoi(argv[1]) * 1024; - - if (argc > 2 && atoi(argv[2]) > 0) - thr = atoi(argv[2]); - - if (argc > 3 && atoi(argv[3]) > 0) - ra = atoi(argv[3]); - - if (argc > 4 && atoi(argv[4]) > 0) - clients = atoi(argv[4]); - - if (argc > 5 && atoi(argv[5]) > 0) - reps = atoi(argv[5]); - - BCTest bc(cacheSz, thr, ra); - - cout << "Cache Size: " << cacheSz << " read Threads: " << thr << " read Ahead: " << ra - << " clients: " << clients << " repetitions: " << reps << " max Blocks: " << bc.maxBlocksAvailable - << endl; - - // loader test - struct loadThr loader1(bc, reps); - vector v; - - for (int i = 0; i < clients; i++) - { - boost::thread* th1 = new boost::thread(loader1); - v.push_back(th1); - } - - for (int i = 0; i < clients; i++) - { - boost::thread* th1 = v[i]; - th1->join(); - delete th1; - } - - v.clear(); - - return 0; - -} // end main diff --git a/utils/testbc/testbc3.cpp b/utils/testbc/testbc3.cpp deleted file mode 100644 index ae5478450..000000000 --- a/utils/testbc/testbc3.cpp +++ /dev/null @@ -1,394 +0,0 @@ -/* Copyright (C) 2014 InfiniDB, Inc. - - This program is free software; you can redistribute it and/or - modify it under the terms of the GNU General Public License - as published by the Free Software Foundation; version 2 of - the License. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU General Public License for more details. - - You should have received a copy of the GNU General Public License - along with this program; if not, write to the Free Software - Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, - MA 02110-1301, USA. */ - -// -// C++ Implementation: bcTest -// -// Description: A simple Test driver for the Disk Block Buffer Cache -// -// Author: Jason Rodriguez , (C) 2007 -// -// - -#include -#include -#include -#include -#include -#include -#include - -#include "blockrequestprocessor.h" -#include "blockcacheclient.h" -#include "stats.h" -#include "brm.h" -#include "logger.h" -#include "iomanager.h" - -using namespace BRM; -using namespace dbbc; -using namespace std; -using namespace logging; -using namespace primitiveprocessor; - -Stats* gPMStatsPtr = NULL; -bool gPMProfOn = false; -uint32_t gSession = 0; -uint32_t lastRangeListIdx = 0; -const uint32_t maxLoadBlocks(1024 * 1024); - -void timespec_sub(const struct timespec& tv1, const struct timespec& tv2, double& tm) -{ - tm = (double)(tv2.tv_sec - tv1.tv_sec) + 1.e-9 * (tv2.tv_nsec - tv1.tv_nsec); -} - -namespace primitiveprocessor -{ -Logger ml; -} - -class BCTest -{ - public: - struct OidRanges - { - OID_t oid; - HWM_t hwm; - LBIDRange_v ranges; - OidRanges(const OID_t o, const HWM_t h, const LBIDRange_v r) - { - oid = o; - hwm = h; - ranges = r; - } - }; // struct OidRanges - - BCTest(const int cacheSz = 64 * 1024, int readThr = 2, int readAhead = 1024); - - typedef OidRanges OidRanges_t; - typedef vector OidRangesList_t; - OidRangesList_t OidRangesList; - - DBRM dbrm; - uint32_t extentSize; - BRM::OID_t maxOid; - - int fCacheSz; - int fReadThr; - int fReadAhead; - uint32_t maxBlocksAvailable; - uint32_t fExtentSize; - - void setUp(); - int LoadOid(const OidRanges_t& o, uint32_t& loadCount); - void LoadLbid(const BRM::LBID_t lbid, const BRM::VER_t ver); - int ReadOidRanges(const OidRanges_t& v, uint32_t* hits, uint32_t* miss); - void ReadOidLbids(const BRM::LBID_t lbid, const BRM::VER_t ver); - - BlockRequestProcessor BRP; - -}; // class BCTest - -BCTest::BCTest(int cacheSz, int readThr, int readAhead) - : fCacheSz(cacheSz), fReadThr(readThr), fReadAhead(readAhead), BRP(fCacheSz, fReadThr, fReadAhead) -{ - setUp(); -} // ctor - -// -void BCTest::setUp() -{ - LBIDRange_v r; - HWM_t hwm; - OID_t oid = 1000; - extentSize = dbrm.getExtentSize(); - maxBlocksAvailable = 0; - int i = 0; - fExtentSize = dbrm.getExtentSize(); - - while (oid < 5000) - { - int ret = 0; - ret = dbrm.lookup(oid, r); - - if (ret == 0 && r.size() > 0) - { - dbrm.getHWM(oid, hwm); - maxBlocksAvailable += (r.size() * extentSize); - OidRanges_t oid_range(oid, hwm, r); - OidRangesList.push_back(oid_range); - // cout << "Setup i: " << i++ << " o: " << oid - // << " r: " << ret << " s: " << r.size() - // << " m: " << maxBlocksAvailable - // << endl; - hwm = 0; - r.clear(); - } - - oid++; - } - - // cout << "\t" << OidRangesList.size() << " oid ranges loaded " << endl << endl; - i = 0; -} // setUp() - -int BCTest::LoadOid(const OidRanges_t& o, uint32_t& loadCount) -{ - blockCacheClient bc(BRP); - uint32_t rCount = 0; - - for (uint32_t i = 0; i < o.ranges.size(); i++) - { - const InlineLBIDRange r = {o.ranges[i].start, o.ranges[i].size}; - - if (r.size > 0) - { - bc.check(r, 0, rCount); - // cout << "i: " << i << " c: " << rCount << " " << o.ranges[i].size << endl; - loadCount += rCount; - } - - rCount = 0; - } // for - - // cout << "hwm: " << o.hwm << " tot: " << loadCount << " " << o.ranges.size() << endl; - - return loadCount; - -} // LoadOid - -int BCTest::ReadOidRanges(const OidRanges_t& v, uint32_t* hits, uint32_t* miss) -{ - blockCacheClient bc(BRP); - uint32_t readBlocks = 0; - uint32_t missBlocks = 0; - uint8_t inBuff[8192]; - int ret; - - for (uint32_t i = 0; i < v.ranges.size() && i < lastRangeListIdx; i++) - { - FileBuffer fb(-1, -1); - const InlineLBIDRange r = {v.ranges[i].start, v.ranges[i].size}; - - for (int j = r.start; j < r.start + r.size; j++) - { - ret = bc.read(j, 0, fb); - FileBuffer* ptr = bc.getBlockPtr(j, 0); - - if (ptr) - { - readBlocks++; - memcpy(inBuff, ptr->getData(), 8192); - } - else - missBlocks++; - } - - *hits += readBlocks; - *miss += missBlocks; - - // cout << " -- Read range idx: " << i << " hits: " << readBlocks << " miss: " << missBlocks << " hwm: " - // << v.hwm << endl; - } - - return readBlocks; - -} // ReadRange - -// add one block to block cache -// -void BCTest::LoadLbid(const BRM::LBID_t lbid, const BRM::VER_t ver) -{ - blockCacheClient bc(BRP); - bool b; - bc.check(lbid, ver, false, b); -} // LoadLbid - -// get one block out of block cache -// -void BCTest::ReadOidLbids(const BRM::LBID_t lbid, const BRM::VER_t ver) -{ - uint8_t d[8192] = {'\0'}; - blockCacheClient bc(BRP); - bc.read(lbid, ver, d); -} // ReadLbid - -struct loadThr -{ - loadThr(BCTest& bc, int reps = 1) : fBC(bc), fReps(reps) - { - } - - void operator()() - { - uint32_t loadedBlocks = 0; - uint32_t oidBlocks; - uint32_t i = 0; - uint32_t rc = 0; - struct timespec tm1; - struct timespec tm2; - - tm1.tv_sec = 0; - tm1.tv_nsec = 0; - tm2.tv_sec = 0; - tm2.tv_nsec = 0; - double tm3 = 0; - clock_gettime(CLOCK_REALTIME, &tm1); - - for (i = 0; (loadedBlocks + (fBC.OidRangesList[i].ranges.size() * fBC.extentSize)) < fBC.fCacheSz && - i < fBC.OidRangesList.size(); - i++) - { - oidBlocks = 0; - rc = fBC.LoadOid(fBC.OidRangesList[i], oidBlocks); - // cout - // << "-- Load " << i << " " << fBC.OidRangesList[i].oid - // << " h: " << fBC.OidRangesList[i].hwm - // << "/" << oidBlocks << " " << loadedBlocks - // << endl; - loadedBlocks += oidBlocks; - } // for (i... - - lastRangeListIdx = i; - - clock_gettime(CLOCK_REALTIME, &tm2); - timespec_sub(tm1, tm2, tm3); - - cout << "load ld: " << loadedBlocks << " sz: " << fBC.fCacheSz << " rng: " << lastRangeListIdx - << " tm: " << right << setw(10) << fixed << tm3 << endl; - - } // operator() - - BCTest& fBC; - uint32_t fReps; -}; - -struct readThr -{ - readThr(BCTest& bc, int reps = 1) : fBC(bc), fReps(reps) - { - } - - void operator()() - { - uint32_t rc = 0; - uint32_t readBlocks = 0; - uint32_t hits = 0; - uint32_t miss = 0; - uint32_t hitsTot = 0; - uint32_t missTot = 0; - struct timespec tm1; - struct timespec tm2; - tm1.tv_sec = 0; - tm1.tv_nsec = 0; - tm2.tv_sec = 0; - tm2.tv_nsec = 0; - double tm3 = 0; - clock_gettime(CLOCK_REALTIME, &tm1); - - for (uint32_t k = 0; k < fReps; k++) - { - for (uint32_t i = 0; i < lastRangeListIdx && i < fBC.OidRangesList.size(); i++) - { - rc = fBC.ReadOidRanges(fBC.OidRangesList[i], &hits, &miss); - // cout << "-- ReadTest " << fBC.OidRangesList[i].oid << " h: " << fBC.OidRangesList[i].hwm << "/" << - // rc << endl; - readBlocks += rc; - rc = 0; - } - - hitsTot += hits; - missTot += miss; - hits = 0; - miss = 0; - } - - clock_gettime(CLOCK_REALTIME, &tm2); - timespec_sub(tm1, tm2, tm3); - - cout << "readtest rd: " << hitsTot << "/" << missTot << " sz: " << fBC.fCacheSz << " tm: " << right - << setw(10) << fixed << tm3 << endl; - - } // operator() - - BCTest& fBC; - uint32_t fReps; -}; - -void usage() -{ - cout << "testbc " << endl; -} - -// -int main(int argc, char* argv[]) -{ - int cacheSz = 128; // K number of blocks - int thr = 1; - int ra = 1024; - int clients = 1; - int reps = 1; - - if (argc > 1 && atoi(argv[1]) > 0) - cacheSz = atoi(argv[1]) * 1024; - - if (argc > 2 && atoi(argv[2]) > 0) - thr = atoi(argv[2]); - - if (argc > 3 && atoi(argv[3]) > 0) - ra = atoi(argv[3]); - - if (argc > 4 && atoi(argv[4]) > 0) - clients = atoi(argv[4]); - - if (argc > 5 && atoi(argv[5]) > 0) - reps = atoi(argv[5]); - - BCTest bc(cacheSz, thr, ra); - - cout << "Cache Size: " << cacheSz << " read Threads: " << thr << " read Ahead: " << ra - << " clients: " << clients << " repetitions: " << reps << " max Blocks: " << bc.maxBlocksAvailable - << endl; - - // loader cache - struct loadThr loader1(bc, 1); - struct readThr reader1(bc, reps); - vector v; - - boost::thread* th1 = new boost::thread(loader1); - th1->join(); - - for (int i = 0; i < clients; i++) - { - boost::thread* rd1 = new boost::thread(reader1); - v.push_back(rd1); - } - - for (int i = 0; i < clients; i++) - { - boost::thread* rd1 = v[i]; - rd1->join(); - delete rd1; - } - - v.clear(); - - delete th1; - - return 0; - -} // end main diff --git a/utils/threadpool/CMakeLists.txt b/utils/threadpool/CMakeLists.txt index 90e919289..1502bb4b3 100644 --- a/utils/threadpool/CMakeLists.txt +++ b/utils/threadpool/CMakeLists.txt @@ -1,13 +1,7 @@ include_directories(${ENGINE_COMMON_INCLUDES}) -# ########## next target ############### - set(threadpool_LIB_SRCS weightedthreadpool.cpp threadpool.cpp prioritythreadpool.cpp fair_threadpool.cpp) columnstore_library(threadpool ${threadpool_LIB_SRCS}) -add_dependencies(threadpool loggingcpp external_boost) -target_link_libraries(threadpool boost_chrono) -install( - TARGETS threadpool - DESTINATION ${ENGINE_LIBDIR} - COMPONENT columnstore-engine -) +columnstore_link(threadpool PRIVATE boost_chrono loggingcpp) + +add_dependencies(threadpool external_boost) diff --git a/utils/udfsdk/CMakeLists.txt b/utils/udfsdk/CMakeLists.txt index 53986d56f..e13f07ab2 100755 --- a/utils/udfsdk/CMakeLists.txt +++ b/utils/udfsdk/CMakeLists.txt @@ -10,24 +10,10 @@ set(udfsdk_LIB_SRCS distinct_count.cpp ) -add_definitions(-DMYSQL_DYNAMIC_PLUGIN) - columnstore_library(udfsdk ${udfsdk_LIB_SRCS}) +columnstore_link(udfsdk PRIVATE loggingcpp) -add_dependencies(udfsdk loggingcpp) - -install( - TARGETS udfsdk - DESTINATION ${ENGINE_LIBDIR} - COMPONENT columnstore-engine -) - +# Do anyone use it? +add_definitions(-DMYSQL_DYNAMIC_PLUGIN) set(udf_mysql_LIB_SRCS udfmysql.cpp) - -add_library(udf_mysql SHARED ${udf_mysql_LIB_SRCS}) - -install( - TARGETS udf_mysql - DESTINATION ${MARIADB_PLUGINDIR} - COMPONENT columnstore-engine -) +columnstore_mysql_plugin_library(udf_mysql SHARED ${udf_mysql_LIB_SRCS}) diff --git a/utils/udfsdk/mcsv1_udaf.h b/utils/udfsdk/mcsv1_udaf.h index 53901267d..a82a8d402 100644 --- a/utils/udfsdk/mcsv1_udaf.h +++ b/utils/udfsdk/mcsv1_udaf.h @@ -129,7 +129,8 @@ class mcsv1Context; struct UserData { - UserData() : size(0), data(nullptr){}; + UserData() : size(0), data(nullptr) {}; + UserData(UserData&) = delete; explicit UserData(size_t sz) { size = sz; @@ -174,10 +175,6 @@ struct UserData // The default data store. You may or may not wish to use these fields. uint32_t size; uint8_t* data; - - private: - // For now, copy construction is unwanted - UserData(UserData&); }; // Flags to define the type and limitations of a UDA(n)F @@ -446,7 +443,7 @@ struct ColumnDatum std::string alias; // Only filled in for init() uint32_t charsetNumber; // For string collations ColumnDatum() - : dataType(execplan::CalpontSystemCatalog::UNDEFINED), scale(0), precision(-1), charsetNumber(8){}; + : dataType(execplan::CalpontSystemCatalog::UNDEFINED), scale(0), precision(-1), charsetNumber(8) {}; }; // Override mcsv1_UDAF to build your User Defined Aggregate (UDAF) and/or diff --git a/utils/windowfunction/CMakeLists.txt b/utils/windowfunction/CMakeLists.txt index 85e253c56..f3ddb4260 100755 --- a/utils/windowfunction/CMakeLists.txt +++ b/utils/windowfunction/CMakeLists.txt @@ -25,10 +25,4 @@ set(windowfunction_LIB_SRCS columnstore_library(windowfunction ${windowfunction_LIB_SRCS}) -add_dependencies(windowfunction loggingcpp) - -install( - TARGETS windowfunction - DESTINATION ${ENGINE_LIBDIR} - COMPONENT columnstore-engine -) +columnstore_link(windowfunction loggingcpp) diff --git a/versioning/BRM/CMakeLists.txt b/versioning/BRM/CMakeLists.txt index 9eaf8fbe9..4468c1c05 100644 --- a/versioning/BRM/CMakeLists.txt +++ b/versioning/BRM/CMakeLists.txt @@ -34,125 +34,64 @@ set(brm_LIB_SRCS ) columnstore_library(brm ${brm_LIB_SRCS}) - -add_dependencies(brm loggingcpp) - -install( - TARGETS brm - DESTINATION ${ENGINE_LIBDIR} - COMPONENT columnstore-engine -) +columnstore_link(brm loggingcpp datatypes) # ########## next target ############### set(controllernode_SRCS masternode.cpp masterdbrmnode.cpp ../../utils/common/crashtrace.cpp) -add_executable(controllernode ${controllernode_SRCS}) - -target_link_libraries(controllernode ${ENGINE_LDFLAGS} ${ENGINE_OAM_LIBS} ${ENGINE_EXEC_LIBS}) - -install( - TARGETS controllernode - DESTINATION ${ENGINE_BINDIR} - COMPONENT columnstore-engine -) +columnstore_executable(controllernode ${controllernode_SRCS}) +columnstore_link(controllernode ${ENGINE_LDFLAGS} ${ENGINE_OAM_LIBS} ${ENGINE_EXEC_LIBS}) # ########## next target ############### set(workernode_SRCS slavenode.cpp ../../utils/common/crashtrace.cpp) -add_executable(workernode ${workernode_SRCS}) - -target_link_libraries(workernode ${ENGINE_LDFLAGS} ${ENGINE_OAM_LIBS} ${ENGINE_EXEC_LIBS}) - -install( - TARGETS workernode - DESTINATION ${ENGINE_BINDIR} - COMPONENT columnstore-engine -) +columnstore_executable(workernode ${workernode_SRCS}) +columnstore_link(workernode ${ENGINE_LDFLAGS} ${ENGINE_OAM_LIBS} ${ENGINE_EXEC_LIBS}) # ########## next target ############### set(dbrmctl_SRCS dbrmctl.cpp) -add_executable(dbrmctl ${dbrmctl_SRCS}) - -target_link_libraries(dbrmctl ${ENGINE_LDFLAGS} ${ENGINE_OAM_LIBS} ${ENGINE_EXEC_LIBS}) - -install( - TARGETS dbrmctl - DESTINATION ${ENGINE_BINDIR} - COMPONENT columnstore-engine -) +columnstore_executable(dbrmctl ${dbrmctl_SRCS}) +columnstore_link(dbrmctl ${ENGINE_LDFLAGS} ${ENGINE_OAM_LIBS} ${ENGINE_EXEC_LIBS}) # ########## next target ############### set(reset_locks_SRCS reset_locks.cpp) -add_executable(reset_locks ${reset_locks_SRCS}) - -target_link_libraries(reset_locks ${ENGINE_LDFLAGS} ${ENGINE_OAM_LIBS} ${ENGINE_EXEC_LIBS} ${NETSNMP_LIBRARIES}) - -install( - TARGETS reset_locks - DESTINATION ${ENGINE_BINDIR} - COMPONENT columnstore-engine -) +columnstore_executable(reset_locks ${reset_locks_SRCS}) +columnstore_link(reset_locks ${ENGINE_LDFLAGS} ${ENGINE_OAM_LIBS} ${ENGINE_EXEC_LIBS} ${NETSNMP_LIBRARIES}) # ########## next target ############### set(rollback_SRCS rollback.cpp) -add_executable(rollback ${rollback_SRCS}) - -target_link_libraries(rollback ${ENGINE_LDFLAGS} ${ENGINE_OAM_LIBS} ${ENGINE_EXEC_LIBS} ${NETSNMP_LIBRARIES}) - -install( - TARGETS rollback - DESTINATION ${ENGINE_BINDIR} - COMPONENT columnstore-engine -) +columnstore_executable(rollback ${rollback_SRCS}) +columnstore_link(rollback ${ENGINE_LDFLAGS} ${ENGINE_OAM_LIBS} ${ENGINE_EXEC_LIBS} ${NETSNMP_LIBRARIES}) # ########## next target ############### set(save_brm_SRCS save_brm.cpp) -add_executable(save_brm ${save_brm_SRCS}) - -target_link_libraries(save_brm ${ENGINE_LDFLAGS} ${ENGINE_OAM_LIBS} ${ENGINE_EXEC_LIBS} ${NETSNMP_LIBRARIES}) - -install( - TARGETS save_brm - DESTINATION ${ENGINE_BINDIR} - COMPONENT columnstore-engine -) +columnstore_executable(save_brm ${save_brm_SRCS}) +columnstore_link(save_brm ${ENGINE_LDFLAGS} ${ENGINE_OAM_LIBS} ${ENGINE_EXEC_LIBS} ${NETSNMP_LIBRARIES}) # ########## next target ############### set(load_brm_SRCS load_brm.cpp) -add_executable(load_brm ${load_brm_SRCS}) +columnstore_executable(load_brm ${load_brm_SRCS}) +columnstore_link(load_brm ${ENGINE_LDFLAGS} ${ENGINE_OAM_LIBS} ${ENGINE_EXEC_LIBS} ${NETSNMP_LIBRARIES}) -target_link_libraries(load_brm ${ENGINE_LDFLAGS} ${ENGINE_OAM_LIBS} ${ENGINE_EXEC_LIBS} ${NETSNMP_LIBRARIES}) - -install( - TARGETS load_brm - DESTINATION ${ENGINE_BINDIR} - COMPONENT columnstore-engine -) - -add_executable(mcs-load-em load_em.cpp) -target_link_libraries( +columnstore_executable(mcs-load-em load_em.cpp) +columnstore_link( mcs-load-em ${ENGINE_LDFLAGS} ${MARIADB_CLIENT_LIBS} ${ENGINE_OAM_LIBS} ${ENGINE_EXEC_LIBS} ${NETSNMP_LIBRARIES} ) -install( - TARGETS mcs-load-em - DESTINATION ${ENGINE_BINDIR} - COMPONENT columnstore-engine -) -add_executable(mcs-load-brm-from-file load_brm_from_file.cpp) -target_link_libraries( +columnstore_executable(mcs-load-brm-from-file load_brm_from_file.cpp) +columnstore_link( mcs-load-brm-from-file ${ENGINE_LDFLAGS} ${MARIADB_CLIENT_LIBS} @@ -161,14 +100,9 @@ target_link_libraries( ${NETSNMP_LIBRARIES} boost_program_options ) -install( - TARGETS mcs-load-brm-from-file - DESTINATION ${ENGINE_BINDIR} - COMPONENT columnstore-engine -) -add_executable(mcs-shmem-locks shmem_locks.cpp) -target_link_libraries( +columnstore_executable(mcs-shmem-locks shmem_locks.cpp) +columnstore_link( mcs-shmem-locks ${ENGINE_LDFLAGS} ${MARIADB_CLIENT_LIBS} @@ -177,21 +111,6 @@ target_link_libraries( ${NETSNMP_LIBRARIES} boost_program_options ) -install( - TARGETS mcs-shmem-locks - DESTINATION ${ENGINE_BINDIR} - COMPONENT columnstore-engine -) -add_executable(mcs-oid-client oid-client.cpp) -target_link_libraries( - mcs-oid-client - ${ENGINE_LDFLAGS} - ${ENGINE_WRITE_LIBS} - boost_program_options -) - -install(TARGETS mcs-oid-client - DESTINATION ${ENGINE_BINDIR} - COMPONENT columnstore-engine -) +columnstore_executable(mcs-oid-client oid-client.cpp) +columnstore_link(mcs-oid-client ${ENGINE_LDFLAGS} ${ENGINE_WRITE_LIBS} boost_program_options) diff --git a/writeengine/bulk/CMakeLists.txt b/writeengine/bulk/CMakeLists.txt index a3dcef686..15068d037 100644 --- a/writeengine/bulk/CMakeLists.txt +++ b/writeengine/bulk/CMakeLists.txt @@ -26,11 +26,9 @@ set(we_bulk_STAT_SRCS ) add_definitions(-D_FILE_OFFSET_BITS=64) -add_library(we_bulk STATIC ${we_bulk_STAT_SRCS}) -add_dependencies(we_bulk loggingcpp) - -target_link_libraries(we_bulk ${NETSNMP_LIBRARIES}) +columnstore_static_library(we_bulk ${we_bulk_STAT_SRCS}) +columnstore_link(we_bulk ${NETSNMP_LIBRARIES} loggingcpp) remove_definitions(-D_FILE_OFFSET_BITS=64) @@ -38,9 +36,9 @@ remove_definitions(-D_FILE_OFFSET_BITS=64) set(cpimport.bin_SRCS cpimport.cpp) -add_executable(cpimport.bin ${cpimport.bin_SRCS}) +columnstore_executable(cpimport.bin ${cpimport.bin_SRCS}) add_dependencies(cpimport.bin marias3) -target_link_libraries( +columnstore_link( cpimport.bin ${ENGINE_LDFLAGS} ${NETSNMP_LIBRARIES} @@ -49,9 +47,3 @@ target_link_libraries( we_bulk we_xml ) - -install( - TARGETS cpimport.bin - DESTINATION ${ENGINE_BINDIR} - COMPONENT columnstore-engine -) diff --git a/writeengine/client/CMakeLists.txt b/writeengine/client/CMakeLists.txt index f7b64e7e0..52633ceca 100644 --- a/writeengine/client/CMakeLists.txt +++ b/writeengine/client/CMakeLists.txt @@ -8,10 +8,4 @@ columnstore_library(writeengineclient ${writeengineclient_LIB_SRCS}) add_dependencies(writeengineclient loggingcpp) -target_link_libraries(writeengineclient ${NETSNMP_LIBRARIES}) - -install( - TARGETS writeengineclient - DESTINATION ${ENGINE_LIBDIR} - COMPONENT columnstore-engine -) +columnstore_link(writeengineclient ${NETSNMP_LIBRARIES}) diff --git a/writeengine/redistribute/CMakeLists.txt b/writeengine/redistribute/CMakeLists.txt index 9b847bdda..29b8b94ee 100644 --- a/writeengine/redistribute/CMakeLists.txt +++ b/writeengine/redistribute/CMakeLists.txt @@ -10,12 +10,6 @@ columnstore_library(writeengineredistribute ${writeengineredistribute_LIB_SRCS}) add_dependencies(writeengineredistribute loggingcpp) -target_link_libraries(writeengineredistribute ${NETSNMP_LIBRARIES}) +columnstore_link(writeengineredistribute ${NETSNMP_LIBRARIES}) target_compile_definitions(writeengineredistribute PUBLIC BOOST_NO_CXX11_SCOPED_ENUMS) - -install( - TARGETS writeengineredistribute - DESTINATION ${ENGINE_LIBDIR} - COMPONENT columnstore-engine -) diff --git a/writeengine/server/CMakeLists.txt b/writeengine/server/CMakeLists.txt index 2d148cac5..9ebc5cd72 100644 --- a/writeengine/server/CMakeLists.txt +++ b/writeengine/server/CMakeLists.txt @@ -17,16 +17,10 @@ set(WriteEngineServer_SRCS ../../datatypes/mcs_datatype.cpp ) -add_executable(WriteEngineServer ${WriteEngineServer_SRCS}) +columnstore_executable(WriteEngineServer ${WriteEngineServer_SRCS}) add_dependencies(WriteEngineServer loggingcpp) -target_link_libraries( +columnstore_link( WriteEngineServer ${ENGINE_LDFLAGS} ${NETSNMP_LIBRARIES} ${ENGINE_WRITE_LIBS} threadpool writeengineredistribute ) - -install( - TARGETS WriteEngineServer - DESTINATION ${ENGINE_BINDIR} - COMPONENT columnstore-engine -) diff --git a/writeengine/splitter/CMakeLists.txt b/writeengine/splitter/CMakeLists.txt index d5a3aed50..96656adf9 100644 --- a/writeengine/splitter/CMakeLists.txt +++ b/writeengine/splitter/CMakeLists.txt @@ -14,11 +14,11 @@ set(cpimport_SRCS we_xmlgetter.cpp ) -add_executable(cpimport ${cpimport_SRCS}) +columnstore_executable(cpimport ${cpimport_SRCS}) add_dependencies(cpimport loggingcpp) -target_link_libraries( +columnstore_link( cpimport ${ENGINE_LDFLAGS} ${NETSNMP_LIBRARIES} @@ -27,9 +27,3 @@ target_link_libraries( threadpool marias3 ) - -install( - TARGETS cpimport - DESTINATION ${ENGINE_BINDIR} - COMPONENT columnstore-engine -) diff --git a/writeengine/wrapper/CMakeLists.txt b/writeengine/wrapper/CMakeLists.txt index 57d6b09ce..28a7b50f9 100644 --- a/writeengine/wrapper/CMakeLists.txt +++ b/writeengine/wrapper/CMakeLists.txt @@ -40,10 +40,4 @@ columnstore_library(writeengine ${writeengine_LIB_SRCS}) add_dependencies(writeengine loggingcpp) -target_link_libraries(writeengine ${NETSNMP_LIBRARIES}) - -install( - TARGETS writeengine - DESTINATION ${ENGINE_LIBDIR} - COMPONENT columnstore-engine -) +columnstore_link(writeengine ${NETSNMP_LIBRARIES}) diff --git a/writeengine/xml/CMakeLists.txt b/writeengine/xml/CMakeLists.txt index 4441b3238..98ab2fbb3 100644 --- a/writeengine/xml/CMakeLists.txt +++ b/writeengine/xml/CMakeLists.txt @@ -9,5 +9,3 @@ add_definitions(-D_FILE_OFFSET_BITS=64) add_library(we_xml STATIC ${we_xml_STAT_SRCS}) add_dependencies(we_xml loggingcpp) - -install(TARGETS we_xml DESTINATION ${ENGINE_LIBDIR})