Merge branch 'stable-23.10' into json_depth-rucha

.drone.jsonnet

@@ -127,6 +127,10 @@ local testPreparation(platform) =
   };
   platform_map[platform];
 
+local make_clickable_link(link) = "echo -e '\\e]8;;" +  link + "\\e\\\\" +  link + "\\e]8;;\\e\\\\'";
+local echo_running_on = ["echo running on ${DRONE_STAGE_MACHINE}",
+      make_clickable_link("https://us-east-1.console.aws.amazon.com/ec2/home?region=us-east-1#Instances:search=:${DRONE_STAGE_MACHINE};v=3;$case=tags:true%5C,client:false;$regex=tags:false%5C,client:false;sort=desc:launchTime")];
+
 local Pipeline(branch, platform, event, arch="amd64", server="10.6-enterprise", customBootstrapParams="", customBuildEnvCommandsMapKey="") = {
   local pkg_format = if (std.split(platform, ":")[0] == "rockylinux") then "rpm" else "deb",
   local init = if (pkg_format == "rpm") then "/usr/lib/systemd/systemd" else "systemd",
@@ -177,11 +181,12 @@ local Pipeline(branch, platform, event, arch="amd64", server="10.6-enterprise",
       },
     },
     commands: [
+      "ls " + result,
       '[ -z "$(ls -A "' + result + '")" ] && echo Nothing to publish! && exit 1',
 
       "aws s3 sync " + result + " s3://cspkg/" + branchp + eventp + "/" + server + "/" + arch + "/" + result + " --only-show-errors",
       'echo "Data uploaded to: ' + publish_pkg_url + '"',
-
+      make_clickable_link(publish_pkg_url),
       "rm -rf " + result + "/*",
     ],
   },
@@ -221,65 +226,27 @@ local Pipeline(branch, platform, event, arch="amd64", server="10.6-enterprise",
 
   local indexes(arr) = std.range(0, std.length(arr) - 1),
 
-  local execInnerDocker(command, dockerImage, flags="") =
+  local execInnerDocker(command, containerName, flags="") =
-    "docker exec " + flags + " -t " + dockerImage + " " + command,
+    "docker exec " + flags + " -t " + containerName + " " + command,
 
-  local execInnerDockerNoTTY(command, dockerImage, flags="") =
+  local execInnerDockerNoTTY(command, containerName, flags="") =
-    "docker exec " + flags + " " + dockerImage + " " + command,
+    "docker exec " + flags + " " + containerName + " " + command,
 
-  local installRpmDeb(pkg_format, rpmpackages, debpackages) =
+  local getContainerName(stepname) = stepname + "$${DRONE_BUILD_NUMBER}",
-    if (pkg_format == "rpm")
-    then ' bash -c "yum install -y ' + rpmpackages + '"'
-    else ' bash -c "apt update --yes && apt install -y ' + debpackages + '"',
 
+  local installCmapi(containerName, pkg_format) =
+    if (pkg_format == "deb") then execInnerDocker('bash -c "apt update -y && apt install -y mariadb-columnstore-cmapi"', containerName)
+    else execInnerDocker('bash -c "yum update -y && yum install -y MariaDB-columnstore-cmapi"', containerName),
 
-  local dockerImage(stepname) = stepname + "$${DRONE_BUILD_NUMBER}",
+  local prepareTestStage(containerName, result, do_setup) =
-  local installEngine(dockerImage, pkg_format) =
+    'sh -c "apk add bash && bash /mdb/' + builddir + "/storage/columnstore/columnstore/build/prepare_test_stage.sh" +
-    if (pkg_format == "deb") then execInnerDocker('bash -c "apt update && apt install -y mariadb-plugin-columnstore mariadb-test"', dockerImage)
+    " --container-name " + containerName +
-    else execInnerDocker('bash -c "yum install -y MariaDB-columnstore-engine MariaDB-test"', dockerImage),
+    " --docker-image " + img +
+    " --result-path " + result +
+    " --packages-url " + packages_url +
+    " --do-setup " + std.toString(do_setup) + '"',
 
-  local installCmapi(dockerImage, pkg_format) =
+  local reportTestStage(containerName, result, stage) = 'sh -c "apk add bash && bash /mdb/' + builddir + "/storage/columnstore/columnstore/build/report_test_stage.sh " + containerName + " " + result + " " + stage + '"',
-    if (pkg_format == "deb") then execInnerDocker('bash -c "apt update && apt install -y mariadb-columnstore-cmapi"', dockerImage)
-    else execInnerDocker('bash -c "yum install -y MariaDB-columnstore-cmapi"', dockerImage),
-
-  local prepareTestStage(dockerImage, pkg_format, result, do_setup) = [
-    "apk add bash && bash core_dumps/docker-awaiter.sh " + dockerImage,
-    if (pkg_format == "deb")
-    then execInnerDocker('sed -i "s/exit 101/exit 0/g" /usr/sbin/policy-rc.d', dockerImage),
-
-    'echo "Docker CGroups opts here"',
-    "ls -al /sys/fs/cgroup/cgroup.controllers || true ",
-    "ls -al /sys/fs/cgroup/ || true ",
-    "ls -al /sys/fs/cgroup/memory || true",
-    "docker ps --filter=name=" + dockerImage,
-
-    execInnerDocker('echo "Inner Docker CGroups opts here"', dockerImage),
-    execInnerDocker("ls -al /sys/fs/cgroup/cgroup.controllers || true", dockerImage),
-    execInnerDocker("ls -al /sys/fs/cgroup/ || true", dockerImage),
-    execInnerDocker("ls -al /sys/fs/cgroup/memory || true", dockerImage),
-
-
-    execInnerDocker("mkdir core", dockerImage),
-    execInnerDocker("chmod 777 core", dockerImage),
-    "docker cp core_dumps/. " + dockerImage + ":/",
-    "docker cp build/utils.sh " + dockerImage + ":/",
-    "docker cp setup-repo.sh " + dockerImage + ":/",
-    if (do_setup) then execInnerDocker("/setup-repo.sh", dockerImage),
-    execInnerDocker(installRpmDeb(pkg_format,
-                                  "cracklib-dicts diffutils elfutils epel-release findutils iproute gawk gcc-c++ gdb hostname lz4 patch perl procps-ng rsyslog sudo tar wget which",
-                                  "elfutils findutils iproute2 g++ gawk gdb hostname liblz4-tool patch procps rsyslog sudo tar wget"), dockerImage),
-    execInnerDocker('sysctl -w kernel.core_pattern="/core/%E_' + result + '_core_dump.%p"', dockerImage),
-  ],
-
-  local reportTestStage(dockerImage, result, stage) = [
-    execInnerDocker('bash -c "/logs.sh ' + stage + '"', dockerImage),
-    execInnerDocker('bash -c "/core_dump_check.sh core /core/ ' + stage + '"', dockerImage),
-    "docker cp " + dockerImage + ":/core/ /drone/src/" + result + "/",
-    "docker cp " + dockerImage + ":/unit_logs/ /drone/src/" + result + "/",
-    "ls -l /drone/src/" + result,
-    execInnerDocker('bash -c "/core_dump_drop.sh core"', dockerImage),
-    "docker stop " + dockerImage + " && docker rm " + dockerImage + ' || echo "cleanup ' + stage + ' failure"',
-  ],
 
   _volumes:: {
     mdb: {
@@ -295,41 +262,20 @@ local Pipeline(branch, platform, event, arch="amd64", server="10.6-enterprise",
     name: "smoke",
     depends_on: ["publish pkg"],
     image: "docker",
-    volumes: [pipeline._volumes.docker],
+    volumes: [pipeline._volumes.mdb, pipeline._volumes.docker],
     commands: [
-                "docker run --memory 3g --env OS=" + result + " --env PACKAGES_URL=" + packages_url + " --env DEBIAN_FRONTEND=noninteractive --env MCS_USE_S3_STORAGE=0 --name smoke$${DRONE_BUILD_NUMBER} --ulimit core=-1 --privileged --detach " + img + " " + init + " --unit=basic.target",
+      prepareTestStage(getContainerName("smoke"), result, true),
-              ]
+      "bash /mdb/" + builddir + "/storage/columnstore/columnstore/build/run_smoke.sh " + getContainerName("smoke"),
-              + prepareTestStage(dockerImage("smoke"), pkg_format, result, true) + [
-      installEngine(dockerImage("smoke"), pkg_format),
-      "sleep $${SMOKE_DELAY_SECONDS:-1s}",
-      // start mariadb and mariadb-columnstore services and run simple query
-      execInnerDocker("systemctl start mariadb", dockerImage("smoke")),
-      execInnerDocker("/usr/bin/mcsSetConfig SystemConfig CGroup just_no_group_use_local", dockerImage("smoke")),
-      execInnerDocker("systemctl restart mariadb-columnstore", dockerImage("smoke")),
-      execInnerDocker('mariadb -e "create database if not exists test; create table test.t1 (a int) engine=Columnstore; insert into test.t1 values (1); select * from test.t1"',
-                      dockerImage("smoke")),
-
-      // restart mariadb and mariadb-columnstore services and run simple query again
-      execInnerDocker("systemctl restart mariadb", dockerImage("smoke")),
-      execInnerDocker("systemctl restart mariadb-columnstore", dockerImage("smoke")),
-      "sleep 10",
-      execInnerDocker('mariadb -e "insert into test.t1 values (2); select * from test.t1"', dockerImage("smoke")),
     ],
   },
   smokelog:: {
     name: "smokelog",
     depends_on: ["smoke"],
     image: "docker",
-    volumes: [pipeline._volumes.docker],
+    volumes: [pipeline._volumes.docker, pipeline._volumes.mdb],
     commands: [
-      'echo "---------- start mariadb service logs ----------"',
+      reportTestStage(getContainerName("smoke"), result, "smoke"),
-      execInnerDocker('journalctl -u mariadb --no-pager || echo "mariadb service failure"', dockerImage("smoke")),
+    ],
-      'echo "---------- end mariadb service logs ----------"',
-      "echo",
-      'echo "---------- start columnstore debug log ----------"',
-      execInnerDocker('cat /var/log/mariadb/columnstore/debug.log || echo "missing columnstore debug.log"', dockerImage("smoke")),
-      'echo "---------- end columnstore debug log ----------"',
-    ] + reportTestStage(dockerImage("smoke"), result, "smoke"),
     when: {
       status: ["success", "failure"],
     },
@@ -345,26 +291,32 @@ local Pipeline(branch, platform, event, arch="amd64", server="10.6-enterprise",
       },
     },
     commands: [
-                // why do we mount cgroups here, but miss it on other steps?
+      // why do we mount cgroups here, but miss it on other steps?
-                "docker run --volume /sys/fs/cgroup:/sys/fs/cgroup:ro --env OS=" + result + " --env PACKAGES_URL=" + packages_url + " --env DEBIAN_FRONTEND=noninteractive --env MCS_USE_S3_STORAGE=0 --env UCF_FORCE_CONFNEW=1 --name upgrade$${DRONE_BUILD_NUMBER}" + version + " --ulimit core=-1 --privileged --detach " + img + " " + init + " --unit=basic.target",
+      prepareTestStage(getContainerName("upgrade") + version, result, false),
-              ]
-              + prepareTestStage(dockerImage("upgrade") + version, pkg_format, result, false) + [
       if (pkg_format == "deb")
       then execInnerDocker('bash -c "./upgrade_setup_deb.sh ' + version + " " + result + " " + arch + " " + repo_pkg_url_no_res + ' $${UPGRADE_TOKEN}"',
-                           dockerImage("upgrade") + version),
+                           getContainerName("upgrade") + version),
-      if (std.split(platform, ":")[0] == "rockylinux")
+      if (pkg_format == "rpm")
       then execInnerDocker('bash -c "./upgrade_setup_rpm.sh ' + version + " " + result + " " + arch + " " + repo_pkg_url_no_res + ' $${UPGRADE_TOKEN}"',
-                           dockerImage("upgrade") + version),
+                           getContainerName("upgrade") + version),
     ],
   },
   upgradelog:: {
     name: "upgradelog",
     depends_on: std.map(function(p) "upgrade-test from " + p, mdb_server_versions),
     image: "docker",
-    volumes: [pipeline._volumes.docker],
+    volumes: [pipeline._volumes.docker, pipeline._volumes.mdb],
-    commands: [
+    commands:
-      "echo",
+       ["echo"] +
-    ] + std.flatMap(function(ver) reportTestStage(dockerImage("upgrade") + ver, result, "upgrade_" + ver), mdb_server_versions),
+         std.map(
+           function(ver)
+             reportTestStage(
+               getContainerName("upgrade") + ver,
+               result,
+               "upgrade_" + ver
+             ),
+           mdb_server_versions
+         ),
     when: {
       status: ["success", "failure"],
     },
@@ -373,64 +325,53 @@ local Pipeline(branch, platform, event, arch="amd64", server="10.6-enterprise",
     name: "mtr",
     depends_on: ["smoke"],
     image: "docker:git",
-    volumes: [pipeline._volumes.docker],
+    volumes: [pipeline._volumes.docker, pipeline._volumes.mdb],
     environment: {
       MTR_SUITE_LIST: "${MTR_SUITE_LIST:-" + mtr_suite_list + "}",
       MTR_FULL_SUITE: "${MTR_FULL_SUITE:-false}",
     },
     commands: [
-                "docker run --shm-size=500m --memory 8g --env MYSQL_TEST_DIR=" + mtr_path + " --env OS=" + result + " --env PACKAGES_URL=" + packages_url + " --env DEBIAN_FRONTEND=noninteractive --env MCS_USE_S3_STORAGE=0 --name mtr$${DRONE_BUILD_NUMBER} --ulimit core=-1 --privileged --detach " + img + " " + init + " --unit=basic.target",
+      prepareTestStage(getContainerName("mtr"), result, true),
-              ]
-              + prepareTestStage("mtr$${DRONE_BUILD_NUMBER}", pkg_format, result, true) + [
-      installEngine(dockerImage("mtr"), pkg_format),
       "docker cp mysql-test/columnstore mtr$${DRONE_BUILD_NUMBER}:" + mtr_path + "/suite/",
-      execInnerDocker("chown -R mysql:mysql " + mtr_path, dockerImage("mtr")),
+      execInnerDocker("chown -R mysql:mysql " + mtr_path, getContainerName("mtr")),
       // disable systemd 'ProtectSystem' (we need to write to /usr/share/)
-      execInnerDocker("bash -c 'sed -i /ProtectSystem/d $(systemctl show --property FragmentPath mariadb | sed s/FragmentPath=//)'", dockerImage("mtr")),
+      execInnerDocker("bash -c 'sed -i /ProtectSystem/d $(systemctl show --property FragmentPath mariadb | sed s/FragmentPath=//)'", getContainerName("mtr")),
-      execInnerDocker("systemctl daemon-reload", dockerImage("mtr")),
+      execInnerDocker("systemctl daemon-reload", getContainerName("mtr")),
-      execInnerDocker("systemctl start mariadb", dockerImage("mtr")),
+      execInnerDocker("systemctl start mariadb", getContainerName("mtr")),
       // Set RAM consumption limits to avoid RAM contention b/w mtr and regression steps.
-      execInnerDocker("/usr/bin/mcsSetConfig SystemConfig CGroup just_no_group_use_local", dockerImage("mtr")),
+      execInnerDocker("/usr/bin/mcsSetConfig SystemConfig CGroup just_no_group_use_local", getContainerName("mtr")),
-      execInnerDocker('mariadb -e "create database if not exists test;"', dockerImage("mtr")),
+      execInnerDocker('mariadb -e "create database if not exists test;"', getContainerName("mtr")),
-      execInnerDocker("systemctl restart mariadb-columnstore", dockerImage("mtr")),
+      execInnerDocker("systemctl restart mariadb-columnstore", getContainerName("mtr")),
 
       // delay mtr for manual debugging on live instance
       "sleep $${MTR_DELAY_SECONDS:-1s}",
       'MTR_SUITE_LIST=$([ "$MTR_FULL_SUITE" == true ] && echo "' + mtr_full_set + '" || echo "$MTR_SUITE_LIST")',
       if (event == "custom" || event == "cron") then
         execInnerDocker('bash -c "wget -qO- https://cspkg.s3.amazonaws.com/mtr-test-data.tar.lz4 | lz4 -dc - | tar xf - -C /"',
-                        dockerImage("mtr")),
+                        getContainerName("mtr")),
       if (event == "custom" || event == "cron") then
         execInnerDocker('bash -c "cd ' + mtr_path + " && ./mtr --extern socket=" + socket_path + ' --force --print-core=detailed --print-method=gdb --max-test-fail=0 --suite=columnstore/setup"',
-                        dockerImage("mtr")),
+                        getContainerName("mtr")),
 
       if (event == "cron") then
         execInnerDocker('bash -c "cd ' + mtr_path + " && ./mtr --extern socket=" + socket_path +
                         " --force --print-core=detailed --print-method=gdb --max-test-fail=0 --suite="
                         + std.join(",", std.map(function(x) "columnstore/" + x, std.split(mtr_full_set, ","))),
-                        dockerImage("mtr")) + '"'
+                        getContainerName("mtr")) + '"'
       else
         execInnerDocker('bash -c "cd ' + mtr_path + " && ./mtr --extern socket=" + socket_path +
                         ' --force --print-core=detailed --print-method=gdb --max-test-fail=0 --suite=columnstore/$${MTR_SUITE_LIST//,/,columnstore/}"',
-                        dockerImage("mtr")),
+                        getContainerName("mtr")),
     ],
   },
   mtrlog:: {
     name: "mtrlog",
     depends_on: ["mtr"],
     image: "docker",
-    volumes: [pipeline._volumes.docker],
+    volumes: [pipeline._volumes.docker, pipeline._volumes.mdb],
     commands: [
-      'echo "---------- start mariadb service logs ----------"',
+      reportTestStage(getContainerName("mtr"), result, "mtr"),
-      execInnerDocker('journalctl -u mariadb --no-pager || echo "mariadb service failure"', dockerImage("mtr")),
+    ],
-      'echo "---------- end mariadb service logs ----------"',
-      "echo",
-      'echo "---------- start columnstore debug log ----------"',
-      execInnerDocker('cat /var/log/mariadb/columnstore/debug.log || echo "missing columnstore debug.log"', dockerImage("mtr")),
-      'echo "---------- end columnstore debug log ----------"',
-      'echo "---------- end columnstore debug log ----------"',
-      "docker cp mtr$${DRONE_BUILD_NUMBER}:" + mtr_path + "/var/log /drone/src/" + result + '/mtr-logs || echo "missing ' + mtr_path + '/var/log"',
-    ] + reportTestStage(dockerImage("mtr"), result, "mtr"),
     when: {
       status: ["success", "failure"],
     },
@@ -448,49 +389,46 @@ local Pipeline(branch, platform, event, arch="amd64", server="10.6-enterprise",
       REGRESSION_REF_AUX: branch_ref,
     },
     commands: [
-                // compute branch.
+      // compute branch.
-                'echo "$$REGRESSION_REF"',
+      'echo "$$REGRESSION_REF"',
-                'echo "$$REGRESSION_BRANCH_REF"',
+      'echo "$$REGRESSION_BRANCH_REF"',
-                // if REGRESSION_REF is empty, try to see whether regression repository has a branch named as one we PR.
+      // if REGRESSION_REF is empty, try to see whether regression repository has a branch named as one we PR.
-                'export REGRESSION_REF=$${REGRESSION_REF:-$$(git ls-remote https://github.com/mariadb-corporation/mariadb-columnstore-regression-test --h --sort origin "refs/heads/$$REGRESSION_BRANCH_REF" | grep -E -o "[^/]+$$")}',
+      'export REGRESSION_REF=$${REGRESSION_REF:-$$(git ls-remote https://github.com/mariadb-corporation/mariadb-columnstore-regression-test --h --sort origin "refs/heads/$$REGRESSION_BRANCH_REF" | grep -E -o "[^/]+$$")}',
-                'echo "$$REGRESSION_REF"',
+      'echo "$$REGRESSION_REF"',
-                // REGRESSION_REF can be empty if there is no appropriate branch in regression repository.
+      // REGRESSION_REF can be empty if there is no appropriate branch in regression repository.
-                // assign what is appropriate by default.
+      // assign what is appropriate by default.
-                "export REGRESSION_REF=$${REGRESSION_REF:-$$REGRESSION_REF_AUX}",
+      "export REGRESSION_REF=$${REGRESSION_REF:-$$REGRESSION_REF_AUX}",
-                'echo "$$REGRESSION_REF"',
+      'echo "$$REGRESSION_REF"',
-                // clone regression test repo
+      // clone regression test repo
-                "git clone --recurse-submodules --branch $$REGRESSION_REF --depth 1 https://github.com/mariadb-corporation/mariadb-columnstore-regression-test",
+      "git clone --recurse-submodules --branch $$REGRESSION_REF --depth 1 https://github.com/mariadb-corporation/mariadb-columnstore-regression-test",
-                // where are we now?
+      // where are we now?
-                "cd mariadb-columnstore-regression-test",
+      "cd mariadb-columnstore-regression-test",
-                "git rev-parse --abbrev-ref HEAD && git rev-parse HEAD",
+      "git rev-parse --abbrev-ref HEAD && git rev-parse HEAD",
-                "cd ..",
+      "cd ..",
-                "docker run --shm-size=500m --memory 12g --env OS=" + result + " --env PACKAGES_URL=" + packages_url + " --env DEBIAN_FRONTEND=noninteractive --env MCS_USE_S3_STORAGE=0 --name regression$${DRONE_BUILD_NUMBER} --ulimit core=-1 --privileged --detach " + img + " " + init + " --unit=basic.target",
+      prepareTestStage(getContainerName("regression"), result, true),
-              ]
-              + prepareTestStage(dockerImage("regression"), pkg_format, result, true) + [
 
       "docker cp mariadb-columnstore-regression-test regression$${DRONE_BUILD_NUMBER}:/",
       // list storage manager binary
       "ls -la /mdb/" + builddir + "/storage/columnstore/columnstore/storage-manager",
       "docker cp /mdb/" + builddir + "/storage/columnstore/columnstore/storage-manager regression$${DRONE_BUILD_NUMBER}:/",
       // check storage-manager unit test binary file
-      execInnerDocker("ls -l /storage-manager", dockerImage("regression")),
+      execInnerDocker("ls -l /storage-manager", getContainerName("regression")),
       // copy test data for regression test suite
-      execInnerDocker('bash -c "wget -qO- https://cspkg.s3.amazonaws.com/testData.tar.lz4 | lz4 -dc - | tar xf - -C mariadb-columnstore-regression-test/"', dockerImage("regression")),
+      execInnerDocker('bash -c "wget -qO- https://cspkg.s3.amazonaws.com/testData.tar.lz4 | lz4 -dc - | tar xf - -C mariadb-columnstore-regression-test/"', getContainerName("regression")),
-      installEngine(dockerImage("regression"), pkg_format),
 
       // set mariadb lower_case_table_names=1 config option
-      execInnerDocker('sed -i "/^.mariadb.$/a lower_case_table_names=1" ' + config_path_prefix + "server.cnf", dockerImage("regression")),
+      execInnerDocker('sed -i "/^.mariadb.$/a lower_case_table_names=1" ' + config_path_prefix + "server.cnf", getContainerName("regression")),
       // set default client character set to utf-8
-      execInnerDocker('sed -i "/^.client.$/a default-character-set=utf8" ' + config_path_prefix + "client.cnf", dockerImage("regression")),
+      execInnerDocker('sed -i "/^.client.$/a default-character-set=utf8" ' + config_path_prefix + "client.cnf", getContainerName("regression")),
 
       // Set RAM consumption limits to avoid RAM contention b/w mtr andregression steps.
-      execInnerDocker("/usr/bin/mcsSetConfig SystemConfig CGroup just_no_group_use_local", dockerImage("regression")),
+      execInnerDocker("/usr/bin/mcsSetConfig SystemConfig CGroup just_no_group_use_local", getContainerName("regression")),
 
-      execInnerDocker("systemctl start mariadb", dockerImage("regression")),
+      execInnerDocker("systemctl start mariadb", getContainerName("regression")),
-      execInnerDocker("systemctl restart mariadb-columnstore", dockerImage("regression")),
+      execInnerDocker("systemctl restart mariadb-columnstore", getContainerName("regression")),
       // delay regression for manual debugging on live instance
       "sleep $${REGRESSION_DELAY_SECONDS:-1s}",
-      execInnerDocker("/usr/bin/g++ /mariadb-columnstore-regression-test/mysql/queries/queryTester.cpp -O2 -o  /mariadb-columnstore-regression-test/mysql/queries/queryTester", dockerImage("regression")),
+      execInnerDocker("/usr/bin/g++ /mariadb-columnstore-regression-test/mysql/queries/queryTester.cpp -O2 -o  /mariadb-columnstore-regression-test/mysql/queries/queryTester", getContainerName("regression")),
     ],
   },
   regression(name, depends_on):: {
@@ -508,11 +446,11 @@ local Pipeline(branch, platform, event, arch="amd64", server="10.6-enterprise",
       },
     },
     commands: [
-      execInnerDocker("mkdir -p reg-logs", dockerImage("regression"), "--workdir /mariadb-columnstore-regression-test/mysql/queries/nightly/alltest"),
+      execInnerDocker("mkdir -p reg-logs", getContainerName("regression"), "--workdir /mariadb-columnstore-regression-test/mysql/queries/nightly/alltest"),
       execInnerDocker("bash -c 'sleep 4800 && bash /save_stack.sh /mariadb-columnstore-regression-test/mysql/queries/nightly/alltest/reg-logs/' & ",
-                      dockerImage("regresion")),
+                      getContainerName("regresion")),
       execInnerDockerNoTTY('bash -c "timeout -k 1m -s SIGKILL --preserve-status $${REGRESSION_TIMEOUT} ./go.sh --sm_unit_test_dir=/storage-manager --tests=' + name + " || ./regression_logs.sh " + name + '"',
-                           dockerImage("regression"),
+                           getContainerName("regression"),
                            "--env PRESERVE_LOGS=true --workdir /mariadb-columnstore-regression-test/mysql/queries/nightly/alltest"),
     ],
   },
@@ -520,24 +458,10 @@ local Pipeline(branch, platform, event, arch="amd64", server="10.6-enterprise",
     name: "regressionlog",
     depends_on: [regression_tests[std.length(regression_tests) - 1]],
     image: "docker",
-    volumes: [pipeline._volumes.docker],
+    volumes: [pipeline._volumes.docker, pipeline._volumes.mdb],
     commands: [
-      'echo "---------- start columnstore regression short report ----------"',
+      reportTestStage(getContainerName("regression"), result, "regression"),
-      execInnerDocker('cat go.log || echo "missing go.log"',
+    ],
-                      dockerImage("regression"),
-                      "--workdir /mariadb-columnstore-regression-test/mysql/queries/nightly/alltest"),
-
-      'echo "---------- end columnstore regression short report ----------"',
-      "echo",
-      "docker cp regression$${DRONE_BUILD_NUMBER}:/mariadb-columnstore-regression-test/mysql/queries/nightly/alltest/reg-logs/ /drone/src/" + result + "/",
-      "docker cp regression$${DRONE_BUILD_NUMBER}:/mariadb-columnstore-regression-test/mysql/queries/nightly/alltest/testErrorLogs.tgz /drone/src/" + result + '/ || echo "missing testErrorLogs.tgz"',
-      execInnerDocker('bash -c "tar czf regressionQueries.tgz /mariadb-columnstore-regression-test/mysql/queries/"', dockerImage("regression")),
-      execInnerDocker('bash -c "tar czf testErrorLogs2.tgz *.log /var/log/mariadb/columnstore" || echo "failed to grab regression results"',
-                      dockerImage("regression"),
-                      "--workdir /mariadb-columnstore-regression-test/mysql/queries/nightly/alltest"),
-      "docker cp regression$${DRONE_BUILD_NUMBER}:/mariadb-columnstore-regression-test/mysql/queries/nightly/alltest/testErrorLogs2.tgz /drone/src/" + result + '/ || echo "missing testErrorLogs.tgz"',
-      "docker cp regression$${DRONE_BUILD_NUMBER}:regressionQueries.tgz /drone/src/" + result + "/",
-    ] + reportTestStage(dockerImage("regression"), result, "regression"),
     when: {
       status: ["success", "failure"],
     },
@@ -598,47 +522,33 @@ local Pipeline(branch, platform, event, arch="amd64", server="10.6-enterprise",
     name: "cmapi test",
     depends_on: ["publish cmapi build"],
     image: "docker:git",
-    volumes: [pipeline._volumes.docker],
+    volumes: [pipeline._volumes.docker, pipeline._volumes.mdb],
     environment: {
       PYTHONPATH: "/usr/share/columnstore/cmapi/deps",
     },
     commands: [
-                "docker run --env OS=" + result + " --env PACKAGES_URL=" + packages_url + " --env DEBIAN_FRONTEND=noninteractive --env MCS_USE_S3_STORAGE=0 --env PYTHONPATH=$${PYTHONPATH} --name cmapi$${DRONE_BUILD_NUMBER} --ulimit core=-1 --privileged --detach " + img + " " + init + " --unit=basic.target",
+      prepareTestStage(getContainerName("cmapi"), result, true),
-              ] +
+      installCmapi(getContainerName("cmapi"), pkg_format),
-              prepareTestStage(dockerImage("cmapi"), pkg_format, result, true) + [
-      if (platform == "rockylinux:9") then execInnerDocker('bash -c "yum install -y libxcrypt-compat"', dockerImage("cmapi")),
-      installEngine(dockerImage("cmapi"), pkg_format),
-      installCmapi(dockerImage("cmapi"), pkg_format),
       "cd cmapi",
       "for i in mcs_node_control cmapi_server failover; do docker cp $${i}/test cmapi$${DRONE_BUILD_NUMBER}:" + cmapi_path + "/$${i}/; done",
       "docker cp run_tests.py cmapi$${DRONE_BUILD_NUMBER}:" + cmapi_path + "/",
-      execInnerDocker("systemctl start mariadb-columnstore-cmapi", dockerImage("cmapi")),
+      execInnerDocker("systemctl start mariadb-columnstore-cmapi", getContainerName("cmapi")),
       // set API key to /etc/columnstore/cmapi_server.conf
-      execInnerDocker('bash -c "mcs cluster set api-key --key somekey123"', dockerImage("cmapi")),
+      execInnerDocker('bash -c "mcs cluster set api-key --key somekey123"', getContainerName("cmapi")),
       // copy cmapi conf file for test purposes (there are api key already set inside)
-      execInnerDocker('bash -c "cp %s/cmapi_server.conf %s/cmapi_server/"' % [etc_path, cmapi_path], dockerImage("cmapi")),
+      execInnerDocker('bash -c "cp %s/cmapi_server.conf %s/cmapi_server/"' % [etc_path, cmapi_path], getContainerName("cmapi")),
-      execInnerDocker("systemctl stop mariadb-columnstore-cmapi", dockerImage("cmapi")),
+      execInnerDocker("systemctl stop mariadb-columnstore-cmapi", getContainerName("cmapi")),
-      execInnerDocker('bash -c "cd ' + cmapi_path + ' && python/bin/python3 run_tests.py"', dockerImage("cmapi")),
+      execInnerDocker('bash -c "cd ' + cmapi_path + ' && python/bin/python3 run_tests.py"', getContainerName("cmapi")),
     ],
   },
   cmapilog:: {
     name: "cmapilog",
     depends_on: ["cmapi test"],
     image: "docker",
-    volumes: [pipeline._volumes.docker],
+    volumes: [pipeline._volumes.docker, pipeline._volumes.mdb],
     commands: [
-                'echo "---------- start mariadb service logs ----------"',
+      reportTestStage(getContainerName("cmapi"), result, "cmapi"),
-                execInnerDocker('journalctl -u mariadb --no-pager || echo "mariadb service failure"', dockerImage("cmapi")),
+    ],
-                'echo "---------- end mariadb service logs ----------"',
-                "echo",
-                'echo "---------- start columnstore debug log ----------"',
-                execInnerDocker('cat /var/log/mariadb/columnstore/debug.log || echo "missing columnstore debug.log"', dockerImage("cmapi")),
-                'echo "---------- end columnstore debug log ----------"',
-                'echo "---------- start cmapi log ----------"',
-                execInnerDocker('cat /var/log/mariadb/columnstore/cmapi_server.log || echo "missing cmapi cmapi_server.log"', dockerImage("cmapi")),
-                'echo "---------- end cmapi log ----------"',
-              ]
-              + reportTestStage(dockerImage("cmapi"), result, "cmapi"),
     when: {
       status: ["success", "failure"],
     },
@@ -700,8 +610,9 @@ local Pipeline(branch, platform, event, arch="amd64", server="10.6-enterprise",
                SERVER_REMOTE: "${SERVER_REMOTE:-" + server_remote + "}",
                SERVER_SHA: "${SERVER_SHA:-" + server + "}",
              },
-             commands: [
+             commands: echo_running_on +
-               "echo $$SERVER_REF",
+             [
+              "echo $$SERVER_REF",
                "echo $$SERVER_REMOTE",
                "mkdir -p /mdb/" + builddir + " && cd /mdb/" + builddir,
                'git config --global url."https://github.com/".insteadOf git@github.com:',
@@ -720,9 +631,6 @@ local Pipeline(branch, platform, event, arch="amd64", server="10.6-enterprise",
              volumes: [pipeline._volumes.mdb],
              environment: {
                DEBIAN_FRONTEND: "noninteractive",
-               DEB_BUILD_OPTIONS: "parallel=4",
-               DH_BUILD_DDEBS: "1",
-               BUILDPACKAGE_FLAGS: "-b",  // Save time and produce only binary packages, not source
                AWS_ACCESS_KEY_ID: {
                  from_secret: "aws_access_key_id",
                },
@@ -742,10 +650,10 @@ local Pipeline(branch, platform, event, arch="amd64", server="10.6-enterprise",
                        ]
                        + customEnvCommands(customBuildEnvCommandsMapKey, builddir) +
                        [
-                         'bash -c "set -o pipefail && bash /mdb/' + builddir + "/storage/columnstore/columnstore/build/bootstrap_mcs.sh " +
+                        'bash -c "set -o pipefail && bash /mdb/' + builddir + "/storage/columnstore/columnstore/build/bootstrap_mcs.sh " +
                          "--build-type RelWithDebInfo " +
                          "--distro " + platform + " " +
-                         "--build-packages --sccache " +
+                         "--build-packages --install-deps --sccache " +
                          " " + customBootstrapParams +
                          " " + customBootstrapParamsForExisitingPipelines(platform) + " | " +
                          "/mdb/" + builddir + "/storage/columnstore/columnstore/build/ansi2txt.sh " +
@@ -822,19 +730,6 @@ local Pipeline(branch, platform, event, arch="amd64", server="10.6-enterprise",
          ] +
          [pipeline.publish("cmapi build")] +
          [pipeline.publish()] +
-         [
-           {
-             name: "publish pkg url",
-             depends_on: ["publish pkg"],
-             image: "alpine/git",
-             commands: [
-               "echo -e '\\e]8;;" + publish_pkg_url + "\\e\\\\" + publish_pkg_url + "\\e]8;;\\e\\\\'",
-               "echo 'for installation run:'",
-               "echo 'export OS=" + result + "'",
-               "echo 'export PACKAGES_URL=" + packages_url + "'",
-             ],
-           },
-         ] +
          (if (event == "cron") then [pipeline.publish("pkg latest", "latest")] else []) +
          [pipeline.smoke] +
          [pipeline.smokelog] +

.gitignore

@@ -153,6 +153,7 @@ oam/install_scripts/mcs-workernode.service
 oam/install_scripts/mcs-writeengineserver.service
 oam/install_scripts/mcs-loadbrm.py
 oam/install_scripts/mcs-savebrm.py
+oam/install_scripts/mcssavebrm.py
 oam/install_scripts/mcs-storagemanager.service
 oam/install_scripts/mcs_module_installer.sh
 oam/install_scripts/slave-rep-columnstore.sh
@@ -186,4 +187,3 @@ versioning/BRM/shmkeys.cpp
 obj/
 build/build
 
-

build/bootstrap_mcs.sh

@@ -19,6 +19,9 @@ CONFIG_DIR=$RPM_CONFIG_DIR
 
 SCRIPT_LOCATION=$(dirname "$0")
 MDB_SOURCE_PATH=$(realpath "$SCRIPT_LOCATION"/../../../..)
+COLUMSNTORE_SOURCE_PATH=$(realpath "$SCRIPT_LOCATION"/../)
+
+DEFAULT_MARIA_BUILD_PATH=$(realpath "$MDB_SOURCE_PATH"/../BuildOf_$(basename "$MDB_SOURCE_PATH"))
 
 BUILD_TYPE_OPTIONS=("Debug" "RelWithDebInfo")
 DISTRO_OPTIONS=("ubuntu:20.04" "ubuntu:22.04" "ubuntu:24.04" "debian:11" "debian:12" "rockylinux:8" "rockylinux:9")
@@ -28,21 +31,10 @@ MDB_CMAKE_FLAGS=()
 
 source "$SCRIPT_LOCATION"/utils.sh
 
-if [ "$EUID" -ne 0 ]; then
-    error "Please run script as root to install MariaDb to system paths"
-    exit 1
-fi
-
-cd $SCRIPT_LOCATION
-CURRENT_BRANCH=$(git rev-parse --abbrev-ref HEAD)
-message "Columnstore will be built from $color_yellow$CURRENT_BRANCH$color_cyan branch"
-cd - >/dev/null
-
 echo "Arguments received: $@"
-message "Building Mariadb Server from $color_yellow$MDB_SOURCE_PATH$color_normal"
 
 optparse.define short=A long=asan desc="Build with ASAN" variable=ASAN default=false value=true
-optparse.define short=a long=build-path variable=MARIA_BUILD_PATH default=$(realpath "$MDB_SOURCE_PATH"/../BuildOf_$(basename "$MDB_SOURCE_PATH"))
+optparse.define short=a long=build-path desc="Path for build output" variable=MARIA_BUILD_PATH default=$DEFAULT_MARIA_BUILD_PATH
 optparse.define short=B long=run-microbench desc="Compile and run microbenchmarks " variable=RUN_BENCHMARKS default=false value=true
 optparse.define short=c long=cloud desc="Enable cloud storage" variable=CLOUD_STORAGE_ENABLED default=false value=true
 optparse.define short=C long=force-cmake-reconfig desc="Force cmake reconfigure" variable=FORCE_CMAKE_CONFIG default=false value=true
@@ -68,10 +60,26 @@ optparse.define short=T long=tsan desc="Build with TSAN" variable=TSAN default=f
 optparse.define short=u long=skip-unit-tests desc="Skip UnitTests" variable=SKIP_UNIT_TESTS default=false value=true
 optparse.define short=U long=ubsan desc="Build with UBSAN" variable=UBSAN default=false value=true
 optparse.define short=v long=verbose desc="Verbose makefile commands" variable=MAKEFILE_VERBOSE default=false value=true
+optparse.define short=V long=add-branch-name-to-outdir desc="Add branch name to build output directory" variable=BRANCH_NAME_TO_OUTDIR default=false value=true
 optparse.define short=W long=without-core-dumps desc="Do not produce core dumps" variable=WITHOUT_COREDUMPS default=false value=true
 
 source $(optparse.build)
 
+message "Building MariaDB Server from $color_yellow$MDB_SOURCE_PATH$color_normal"
+
+cd $COLUMSNTORE_SOURCE_PATH
+COLUMNSTORE_BRANCH=$(git rev-parse --abbrev-ref HEAD)
+message "Columnstore will be built from $color_yellow$COLUMNSTORE_BRANCH$color_cyan branch"
+
+cd $MDB_SOURCE_PATH
+MARIADB_BRANCH=$(git rev-parse --abbrev-ref HEAD)
+message "MariaDB will be built from $color_yellow$MARIADB_BRANCH$color_cyan branch"
+cd - >/dev/null
+
+if [[ ${BRANCH_NAME_TO_OUTDIR} = true ]]; then
+    MARIA_BUILD_PATH="${MARIA_BUILD_PATH}_${MARIADB_BRANCH}_${COLUMNSTORE_BRANCH}"
+fi
+
 if [[ ! " ${BUILD_TYPE_OPTIONS[*]} " =~ " ${MCS_BUILD_TYPE} " ]]; then
     getChoice -q "Select your Build Type" -o BUILD_TYPE_OPTIONS
     MCS_BUILD_TYPE=$selectedChoice
@@ -104,7 +112,7 @@ install_deps() {
       cppunit-devel cmake3 libxcrypt-devel xz-devel zlib-devel libzstd-devel glibc-devel"
 
     DEB_BUILD_DEPS="apt-get -y update && apt-get -y install build-essential automake libboost-all-dev \
-      bison cmake libncurses5-dev libaio-dev libsystemd-dev libpcre2-dev libperl-dev libssl-dev libxml2-dev \
+      bison cmake libncurses5-dev python3 libaio-dev libsystemd-dev libpcre2-dev libperl-dev libssl-dev libxml2-dev \
       libkrb5-dev flex libpam-dev git libsnappy-dev libcurl4-openssl-dev libgtest-dev libcppunit-dev googletest \
       libjemalloc-dev liblz-dev liblzo2-dev liblzma-dev liblz4-dev libbz2-dev libbenchmark-dev libdistro-info-perl \
       graphviz devscripts ccache equivs eatmydata curl"
@@ -259,8 +267,9 @@ construct_cmake_flags() {
         -DWITH_WSREP=NO
     )
 
-    if [[ MAINTAINER_MODE = true ]]; then
+    if [[ $MAINTAINER_MODE = true ]]; then
         MDB_CMAKE_FLAGS+=(-DCOLUMNSTORE_MAINTAINER=YES)
+        message "Columnstore mainteiner mode on"
     else
         warn "Maintainer mode is disabled, be careful, alien"
     fi
@@ -406,9 +415,15 @@ generate_svgs() {
 }
 
 build_package() {
+    cd $MDB_SOURCE_PATH
+
     if [[ $pkg_format == "rpm" ]]; then
         command="cmake ${MDB_CMAKE_FLAGS[@]} && make -j\$(nproc) package"
     else
+        export DEBIAN_FRONTEND="noninteractive"
+        export DEB_BUILD_OPTIONS="parallel=$(nproc)"
+        export DH_BUILD_DDEBS="1"
+        export BUILDPACKAGE_FLAGS="-b"
         command="mk-build-deps debian/control -t 'apt-get -y -o Debug::pkgProblemResolver=yes --no-install-recommends' -r -i && \
        CMAKEFLAGS=\"${MDB_CMAKE_FLAGS[@]}\" debian/autobake-deb.sh"
     fi
@@ -420,6 +435,14 @@ build_package() {
     check_errorcode
 }
 
+check_debian_install_file() {
+    message "checking debian/mariadb-plugin-columnstore.install"
+    message_split
+    python3 $COLUMSNTORE_SOURCE_PATH/build/debian_install_file_compare.py \
+        ${COLUMSNTORE_SOURCE_PATH}/debian/mariadb-plugin-columnstore.install \
+        $MARIA_BUILD_PATH/mariadb-plugin-columnstore.install.generated
+}
+
 build_binary() {
     MARIA_BUILD_PATH=$(realpath "$MARIA_BUILD_PATH")
     message_split
@@ -438,7 +461,7 @@ build_binary() {
     message "Configuring cmake silently"
     ${CMAKE_BIN_NAME} "${MDB_CMAKE_FLAGS[@]}" -S"$MDB_SOURCE_PATH" -B"$MARIA_BUILD_PATH" | spinner
     message_split
-
+    check_debian_install_file
     generate_svgs
 
     ${CMAKE_BIN_NAME} --build "$MARIA_BUILD_PATH" -j "$CPUS" | onelinearizator &&
@@ -553,6 +576,12 @@ make_dir() {
 
 install() {
     if [[ $RECOMPILE_ONLY = false ]]; then
+
+        if [ "$EUID" -ne 0 ]; then
+            error "Please run script as root to install MariaDb to system paths"
+            exit 1
+        fi
+
         message_split
         message "Installing MariaDB"
         disable_plugins_for_bootstrap
@@ -636,7 +665,7 @@ smoke() {
     fi
 }
 
-if [[ $INSTALL_DEPS = true || $BUILD_PACKAGES = true ]]; then
+if [[ $INSTALL_DEPS = true ]]; then
     install_deps
 fi
 

build/debian_install_file_compare.py

@@ -0,0 +1,81 @@
+#!/usr/bin/env python3
+
+import argparse
+import fnmatch
+import sys
+
+def read_patterns(filename):
+    with open(filename, 'r') as f:
+        return [line.strip() for line in f if line.strip()]
+
+def read_paths_with_comments(filename):
+    paths = []
+    with open(filename, 'r') as f:
+        for line in f:
+            raw = line.rstrip('\n')
+            if not raw.strip():
+                continue
+            # Split into path and comment
+            if '#' in raw:
+                path, comment = raw.split('#', 1)
+                path = path.strip()
+                comment = comment.strip()
+            else:
+                path = raw.strip()
+                comment = ''
+            if path:
+                paths.append((raw, path, comment))
+    return paths
+
+def print_red(msg):
+    if sys.stdout.isatty():
+        print("\033[31m" + msg + "\033[0m")
+    else:
+        print(msg)
+
+def main():
+    parser = argparse.ArgumentParser(description="Compare wildcard patterns to file paths.")
+    parser.add_argument('patterns_file', help='File with wildcard patterns (one per line)')
+    parser.add_argument('paths_file', help='File with file paths (one per line)')
+    args = parser.parse_args()
+
+    patterns = read_patterns(args.patterns_file)
+    paths_with_comments = read_paths_with_comments(args.paths_file)
+
+    # Track which paths are matched (by index)
+    path_matched = [False] * len(paths_with_comments)
+
+    # Collect patterns that don't match any path
+    unmatched_patterns = []
+    for pattern in patterns:
+        matched = False
+        for i, (raw, path, comment) in enumerate(paths_with_comments):
+            if fnmatch.fnmatch(path, pattern):
+                matched = True
+                path_matched[i] = True
+        if not matched:
+            unmatched_patterns.append(pattern)
+
+    if unmatched_patterns:
+        print_red("The files declared in debian/mariadb-plugin-columnstore.install "
+                  " are not added to CMakeLists via columnstore_* statements see cmake/ColumnstoreLibrary.cmake")
+        for pattern in unmatched_patterns:
+            print(f"- {pattern}")
+
+    # Collect paths that weren't matched by any pattern
+    unmatched_paths = []
+    for (raw, path, comment), matched in zip(paths_with_comments, path_matched):
+        if not matched:
+            unmatched_paths.append((path, comment))
+
+    if unmatched_paths:
+        print_red("The files added via columnstore_* statements from cmake/ColumnstoreLibrary.cmake "
+                  "are missing in debian/mariadb-plugin-columnstore.install file")
+        for path, comment in unmatched_paths:
+            if comment:
+                print(f"- {path} {comment}")
+            else:
+                print(f"- {path}")
+
+if __name__ == "__main__":
+    main()

build/prepare_test_stage.sh

@@ -0,0 +1,127 @@
+#!/bin/bash
+
+# Should be executed by root
+set -eo pipefail
+
+SCRIPT_LOCATION=$(dirname "$0")
+COLUMNSTORE_SOURCE_PATH=$(realpath "$SCRIPT_LOCATION"/../)
+
+source "$SCRIPT_LOCATION"/utils.sh
+
+echo "Arguments received: $@"
+
+optparse.define short=c long=container-name desc="Name of the Docker container to run tests in" variable=CONTAINER_NAME
+optparse.define short=i long=docker-image      desc="Docker image name to start container from" variable=DOCKER_IMAGE
+optparse.define short=r long=result-path       desc="Name suffix used in core dump file path"   variable=RESULT
+optparse.define short=s long=do-setup          desc="Run setup-repo.sh inside the container"    variable=DO_SETUP
+optparse.define short=u long=packages-url      desc="Packages url"                              variable=PACKAGES_URL
+source $(optparse.build)
+
+if [[ "$EUID" -ne 0 ]]; then
+    error "Please run script as root"
+    exit 1
+fi
+
+if [[ -z "${CONTAINER_NAME:-}" || -z "${DOCKER_IMAGE:-}" || -z "${RESULT:-}" || -z "${DO_SETUP:-}" || -z "${PACKAGES_URL:-}" ]]; then
+  echo "Please provide --container-name, --docker-image, --result-path, --packages-url and --do-setup parameters, e.g. ./prepare_test_stage.sh --container-name smoke11212 --docker-image detravi/ubuntu:24.04 --result-path ubuntu24.04 --packages-url https://cspkg.s3.amazonaws.com/stable-23.10/pull_request/91/10.6-enterprise --do-setup true"
+  exit 1
+fi
+
+start_container() {
+    if [[ "$RESULT" == *rocky* ]]; then
+        SYSTEMD_PATH="/usr/lib/systemd/systemd"
+        MTR_PATH="/usr/share/mysql-test"
+    else
+        SYSTEMD_PATH="systemd"
+        MTR_PATH="/usr/share/mysql/mysql-test"
+    fi
+
+    docker_run_args=(
+        --env OS="$RESULT"
+        --env PACKAGES_URL="$PACKAGES_URL"
+        --env DEBIAN_FRONTEND=noninteractive
+        --env MCS_USE_S3_STORAGE=0
+        --name "$CONTAINER_NAME"
+        --ulimit core=-1
+        --privileged
+        --detach
+    )
+
+    if [[ "$CONTAINER_NAME" == *smoke* ]]; then
+        docker_run_args+=( --memory 3g )
+    elif [[ "$CONTAINER_NAME" == *mtr* ]]; then
+        docker_run_args+=( --shm-size=500m --memory 8g --env MYSQL_TEST_DIR="$MTR_PATH" )
+    elif [[ "$CONTAINER_NAME" == *cmapi* ]]; then
+        docker_run_args+=( --env PYTHONPATH="${PYTHONPATH}" )
+    elif [[ "$CONTAINER_NAME" == *upgrade* ]]; then
+        docker_run_args+=( --env UCF_FORCE_CONFNEW=1 --volume /sys/fs/cgroup:/sys/fs/cgroup:ro )
+    elif [[ "$CONTAINER_NAME" == *regression* ]]; then
+        docker_run_args+=( --shm-size=500m --memory 12g )
+    else
+        echo "Unknown container type: $CONTAINER_NAME"
+        exit 1
+    fi
+
+    docker_run_args+=( "$DOCKER_IMAGE" "$SYSTEMD_PATH" --unit=basic.target )
+
+    docker run "${docker_run_args[@]}"
+    sleep 5
+
+    apk add bash
+    bash "$COLUMNSTORE_SOURCE_PATH"/core_dumps/docker-awaiter.sh "$CONTAINER_NAME"
+
+    if ! docker ps -q --filter "name=${CONTAINER_NAME}" | grep -q .; then
+        error "Container '$CONTAINER_NAME' has not started!"
+        exit 1
+    fi
+}
+
+start_container
+
+if [[ "$RESULT" != *rocky* ]]; then
+    execInnerDocker 'sed -i "s/exit 101/exit 0/g" /usr/sbin/policy-rc.d' "$CONTAINER_NAME"
+fi
+
+#list_cgroups
+echo "Docker CGroups opts here"
+ls -al /sys/fs/cgroup/cgroup.controllers || true
+ls -al /sys/fs/cgroup/       || true
+ls -al /sys/fs/cgroup/memory || true
+
+execInnerDocker 'echo Inner Docker CGroups opts here'                            "$CONTAINER_NAME"
+execInnerDocker 'ls -al /sys/fs/cgroup/cgroup.controllers || true'               "$CONTAINER_NAME"
+execInnerDocker 'ls -al /sys/fs/cgroup/ || true'                                 "$CONTAINER_NAME"
+execInnerDocker 'ls -al /sys/fs/cgroup/memory || true'                           "$CONTAINER_NAME"
+
+# Prepare core dump directory inside container
+execInnerDocker 'mkdir -p core && chmod 777 core'           "$CONTAINER_NAME"
+docker cp "$COLUMNSTORE_SOURCE_PATH"/core_dumps/.           "$CONTAINER_NAME":/
+docker cp "$COLUMNSTORE_SOURCE_PATH"/build/utils.sh         "$CONTAINER_NAME":/
+docker cp "$COLUMNSTORE_SOURCE_PATH"/setup-repo.sh          "$CONTAINER_NAME":/
+
+if [[ "$DO_SETUP" == "true" ]]; then
+    execInnerDocker '/setup-repo.sh'                   "$CONTAINER_NAME"
+fi
+
+# install deps
+if [[ "$RESULT" == *rocky* ]]; then
+    execInnerDocker 'yum update -y && yum install -y cracklib-dicts diffutils elfutils epel-release findutils iproute gawk gcc-c++ gdb hostname lz4 patch perl procps-ng rsyslog sudo tar wget which'                   "$CONTAINER_NAME"
+    # sudo bypass
+    execInnerDocker "printf '%s\n' '#!/bin/sh' 'exec \"\$@\"' > /usr/bin/sudo && chmod +x /usr/bin/sudo" "$CONTAINER_NAME"
+else
+    execInnerDocker 'apt update -y && apt install -y elfutils findutils iproute2 g++ gawk gdb hostname liblz4-tool patch procps rsyslog sudo tar wget' "$CONTAINER_NAME"
+fi
+
+# Configure core dump naming pattern
+execInnerDocker 'sysctl -w kernel.core_pattern="/core/%E_${RESULT}_core_dump.%p"' "$CONTAINER_NAME"
+
+#Install columnstore in container
+echo "Installing columnstore..."
+if [[ "$RESULT" == *rocky* ]]; then
+    execInnerDocker 'yum install -y MariaDB-columnstore-engine MariaDB-test' "$CONTAINER_NAME"
+else
+    execInnerDocker 'apt update -y && apt install -y mariadb-plugin-columnstore mariadb-test' "$CONTAINER_NAME"
+fi
+
+sleep 5
+echo "PrepareTestStage completed in $CONTAINER_NAME"

build/report_test_stage.sh

@@ -0,0 +1,98 @@
+#!/bin/bash
+
+set -eo pipefail
+
+CONTAINER_NAME=$1
+RESULT=$2
+STAGE=$3
+
+SCRIPT_LOCATION=$(dirname "$0")
+source "$SCRIPT_LOCATION"/utils.sh
+
+echo "Arguments received: $@"
+
+if [[ "$EUID" -ne 0 ]]; then
+    error "Please run script as root"
+    exit 1
+fi
+
+if [[ -z $(docker ps -q --filter "name=${CONTAINER_NAME}") ]]; then
+    error "Container '${CONTAINER_NAME}' is not running."
+    exit 1
+fi
+
+if [[ "$RESULT" == *rocky* ]]; then
+    SYSTEMD_PATH="/usr/lib/systemd/systemd"
+    MTR_PATH="/usr/share/mysql-test"
+else
+    SYSTEMD_PATH="systemd"
+    MTR_PATH="/usr/share/mysql/mysql-test"
+fi
+
+echo "Reporting test stage: ${STAGE} executed in ${CONTAINER_NAME} container"
+
+if [[ "${CONTAINER_NAME}" == *smoke* ]] || [[ "${CONTAINER_NAME}" == *mtr* ]] || [[ "${CONTAINER_NAME}" == *cmapi* ]]; then
+    # common logs for smoke, mtr, cmapi
+    echo "---------- start mariadb service logs ----------"
+    execInnerDocker 'journalctl -u mariadb --no-pager || echo "mariadb service failure"' "$CONTAINER_NAME"
+    echo "---------- end mariadb service logs ----------"
+    echo
+    echo "---------- start columnstore debug log ----------"
+    execInnerDocker 'cat /var/log/mariadb/columnstore/debug.log || echo "missing columnstore debug.log"' "$CONTAINER_NAME"
+    echo "---------- end columnstore debug log ----------"
+
+    if [[ "${CONTAINER_NAME}" == *mtr* ]]; then
+        echo
+        docker cp "${CONTAINER_NAME}:${MTR_PATH}/var/log" "/drone/src/${RESULT}/mtr-logs" || echo "missing ${MTR_PATH}/var/log"
+    fi
+
+    if [[ "${CONTAINER_NAME}" == *cmapi* ]]; then
+        echo
+        echo "---------- start cmapi log ----------"
+        execInnerDocker 'cat /var/log/mariadb/columnstore/cmapi_server.log || echo "missing cmapi_server.log"' "$CONTAINER_NAME"
+        echo "---------- end cmapi log ----------"
+    fi
+
+elif [[ "${CONTAINER_NAME}" == *upgrade* ]]; then
+    # nothing to report here for upgrade
+    :
+
+elif [[ "${CONTAINER_NAME}" == *regression* ]]; then
+    echo "---------- start columnstore regression short report ----------"
+    execInnerDocker 'cat go.log || echo "missing go.log"' "$CONTAINER_NAME" '--workdir /mariadb-columnstore-regression-test/mysql/queries/nightly/alltest'
+    echo "---------- end columnstore regression short report ----------"
+    echo
+    docker cp "${CONTAINER_NAME}:/mariadb-columnstore-regression-test/mysql/queries/nightly/alltest/reg-logs/" "/drone/src/${RESULT}/" || echo "missing regression logs"
+    docker cp "${CONTAINER_NAME}:/mariadb-columnstore-regression-test/mysql/queries/nightly/alltest/testErrorLogs.tgz" "/drone/src/${RESULT}/" || echo "missing testErrorLogs.tgz"
+
+    execInnerDocker 'tar czf regressionQueries.tgz /mariadb-columnstore-regression-test/mysql/queries/' "$CONTAINER_NAME"
+    execInnerDocker 'tar czf testErrorLogs2.tgz *.log /var/log/mariadb/columnstore || echo "failed to grab regression results"' \
+                   "$CONTAINER_NAME" \
+                   '--workdir /mariadb-columnstore-regression-test/mysql/queries/nightly/alltest'
+    docker cp "${CONTAINER_NAME}:/mariadb-columnstore-regression-test/mysql/queries/nightly/alltest/testErrorLogs2.tgz" "/drone/src/${RESULT}/" || echo "missing testErrorLogs2.tgz"
+    docker cp "${CONTAINER_NAME}:regressionQueries.tgz" "/drone/src/${RESULT}/" || echo "missing regressionQueries.tgz"
+
+else
+    echo "Unknown stage's container provided: ${CONTAINER_NAME}"
+    exit 1
+fi
+
+execInnerDocker "/logs.sh ${STAGE}"                             "$CONTAINER_NAME"
+execInnerDocker "/core_dump_check.sh core /core/ ${STAGE}"      "$CONTAINER_NAME"
+
+docker cp "${CONTAINER_NAME}:/core/"      "/drone/src/${RESULT}/"
+docker cp "${CONTAINER_NAME}:/unit_logs/" "/drone/src/${RESULT}/"
+
+execInnerDocker "/core_dump_drop.sh core"                        "$CONTAINER_NAME"
+echo "Saved artifacts:"
+ls -R "/drone/src/${RESULT}/"
+echo "Done reporting ${STAGE}"
+
+cleanup() {
+    if [[ -n $(docker ps -q --filter "name=${CONTAINER_NAME}") ]]; then
+        echo "Cleaning up container ${CONTAINER_NAME}..."
+        docker rm -f "${CONTAINER_NAME}" || echo "Can't remove container ${CONTAINER_NAME}!"
+    fi
+}
+#Remove the container on exit
+trap cleanup EXIT

build/run_smoke.sh

@@ -0,0 +1,40 @@
+#!/bin/bash
+
+set -eo pipefail
+
+SCRIPT_LOCATION=$(dirname "$0")
+source "$SCRIPT_LOCATION"/utils.sh
+
+CONTAINER_NAME="$1"
+
+echo "Arguments received: $@"
+
+if [[ "$EUID" -ne 0 ]]; then
+    error "Please run script as root"
+    exit 1
+fi
+
+if [[ -z $(docker ps -q --filter "name=${CONTAINER_NAME}") ]]; then
+  error "Container '${CONTAINER_NAME}' is not running."
+  exit 1
+fi
+
+message "Running smoke checks..."
+
+# start mariadb and mariadb-columnstore services and run simple query
+execInnerDocker 'systemctl start mariadb'                                         "$CONTAINER_NAME"
+execInnerDocker '/usr/bin/mcsSetConfig SystemConfig CGroup just_no_group_use_local' "$CONTAINER_NAME"
+execInnerDocker 'systemctl restart mariadb-columnstore'                           "$CONTAINER_NAME"
+
+execInnerDocker 'mariadb -e "CREATE DATABASE IF NOT EXISTS test; \
+CREATE TABLE test.t1 (a INT) ENGINE=Columnstore; \
+INSERT INTO test.t1 VALUES (1); \
+SELECT * FROM test.t1;"'                                                       "$CONTAINER_NAME"
+
+# restart both services, wait a bit, then insert and select again
+execInnerDocker 'systemctl restart mariadb'                                      "$CONTAINER_NAME"
+execInnerDocker 'systemctl restart mariadb-columnstore'                           "$CONTAINER_NAME"
+
+sleep 10
+
+execInnerDocker 'mariadb -e "INSERT INTO test.t1 VALUES (2); SELECT * FROM test.t1;"' "$CONTAINER_NAME"

build/utils.sh

@@ -418,9 +418,23 @@ function optparse.define(){
         optparse_process="${optparse_process}#NL#TB#TB${shortname})#NL#TB#TB#TB${variable}=\"$val\";;"
 }
 
+function execInnerDocker() {
+    local cmd_str="$1"
+    local img="$2"
+    local flags="${3:-}"
+
+    docker exec $flags -t "$img" bash -c "$cmd_str"
+    local dockerCommandExitCode=$?
+
+    if [[ $dockerCommandExitCode -ne 0 ]]; then
+        echo "Command \"$cmd_str\" failed in container \"$img\""
+        exit $dockerCommandExitCode
+    fi
+}
+
 # -----------------------------------------------------------------------------------------------------------------------------
 function optparse.build(){
-        local build_file="$(mktemp -t "optparse-XXXXXX.tmp")"
+        local build_file="$(mktemp "${TMPDIR:-/tmp}/optparse-XXXXXX")"
 
         # Building getopts header here
 

cmake/ColumnstoreLibrary.cmake

@@ -1,8 +1,105 @@
-# Function to create either a static or shared library based on COLUMNSTORE_STATIC_LIBRARIES
+set(DEBIAN_INSTALL_FILE "${CMAKE_BINARY_DIR}/mariadb-plugin-columnstore.install.generated")
+file(WRITE ${DEBIAN_INSTALL_FILE} "#File is generated by ColumnstoreLibrary.cmake, do not edit\n")
+
+macro(add_to_debian_install_file file_path)
+    string(SUBSTRING "${file_path}" 1 -1 BINARY_ENTRY)
+    file(RELATIVE_PATH CMAKEFILE "${CMAKE_SOURCE_DIR}/storage/columnstore/columnstore" ${CMAKE_CURRENT_LIST_FILE})
+    string(STRIP "${BINARY_ENTRY}" BINARY_ENTRY)
+
+    file(APPEND ${DEBIAN_INSTALL_FILE} "${BINARY_ENTRY} # added in ${CMAKEFILE}\n")
+endmacro()
+
+function(get_target_output_filename TARGET_NAME OUTPUT_VAR)
+    # 1. Get the target's OUTPUT_NAME (falls back to TARGET_NAME)
+    get_target_property(OUTPUT_NAME ${TARGET_NAME} OUTPUT_NAME)
+    if(NOT OUTPUT_NAME)
+        set(OUTPUT_NAME "${TARGET_NAME}")
+    endif()
+
+    # 1. Get the correct suffix based on target type
+    get_target_property(TARGET_TYPE ${TARGET_NAME} TYPE)
+    if(TARGET_TYPE STREQUAL "EXECUTABLE")
+        set(SUFFIX "${CMAKE_EXECUTABLE_SUFFIX}")
+        set(PREFIX "") # No prefix for executables
+    elseif(TARGET_TYPE STREQUAL "SHARED_LIBRARY")
+        set(SUFFIX "${CMAKE_SHARED_LIBRARY_SUFFIX}")
+        set(PREFIX "lib")
+    elseif(TARGET_TYPE STREQUAL "STATIC_LIBRARY")
+        set(SUFFIX "${CMAKE_STATIC_LIBRARY_SUFFIX}")
+        set(PREFIX "lib")
+    else()
+        message(WARNING "Unknown target type for ${TARGET_NAME}")
+        set(SUFFIX "")
+    endif()
+
+    # 1. Combine into the final filename
+    set(${OUTPUT_VAR}
+        "${PREFIX}${OUTPUT_NAME}${SUFFIX}"
+        PARENT_SCOPE
+    )
+endfunction()
+
+macro(columnstore_install_target target destination)
+    install(
+        TARGETS ${target}
+        DESTINATION ${destination}
+        COMPONENT columnstore-engine
+    )
+    get_target_output_filename(${target} OUTPUT_FILENAME)
+    add_to_debian_install_file("${destination}/${OUTPUT_FILENAME}")
+endmacro()
+
+macro(columnstore_install_file file destination)
+    install(
+        FILES ${file}
+        DESTINATION ${destination}
+        COMPONENT columnstore-engine
+    )
+    get_filename_component(FILENAME ${file} NAME)
+    add_to_debian_install_file("${destination}/${FILENAME}")
+endmacro()
+
+macro(columnstore_install_program file destination)
+    install(
+        PROGRAMS ${file}
+        DESTINATION ${destination}
+        COMPONENT columnstore-engine
+    )
+    get_filename_component(FILENAME ${file} NAME)
+    add_to_debian_install_file(
+        "${destination}/${FILENAME}
+        "
+    )
+endmacro()
+
+macro(columnstore_static_library libname)
+    add_definitions(-fPIC -DPIC)
+    add_library(${libname} STATIC ${ARGN})
+endmacro()
+
+macro(columnstore_shared_library libname)
+    add_library(${libname} SHARED ${ARGN})
+    columnstore_install_target(${libname} ${ENGINE_LIBDIR})
+endmacro()
+
 macro(columnstore_library libname)
     if(COLUMNSTORE_STATIC_LIBRARIES)
-        add_library(${libname} STATIC ${ARGN})
+        columnstore_static_library(${libname} ${ARGN})
     else()
-        add_library(${libname} SHARED ${ARGN})
+        columnstore_shared_library(${libname} ${ARGN})
     endif()
 endmacro()
+
+macro(columnstore_mysql_plugin_library libname)
+    add_library(${libname} SHARED ${ARGN})
+    columnstore_install_target(${libname} ${MARIADB_PLUGINDIR})
+endmacro()
+
+macro(columnstore_link libname)
+    target_link_libraries(${libname} ${ARGN})
+endmacro()
+
+macro(columnstore_executable executable_name)
+    add_executable(${executable_name} ${ARGN})
+    columnstore_install_target(${executable_name} ${ENGINE_BINDIR})
+endmacro()

cmake/compiler_flags.cmake

@@ -51,6 +51,7 @@ set(COLUMNSTORE_MAINTAINER_FLAGS ${WERROR_FLAG})
 set(FLAGS_ALL
     -Wall
     -Wextra
+    -ggdb3
     -fno-omit-frame-pointer
     -fno-strict-aliasing
     -fsigned-char
@@ -59,9 +60,9 @@ set(FLAGS_ALL
     -DBOOST_BIND_GLOBAL_PLACEHOLDERS
 )
 
-set(FLAGS_RELEASE -g -O3 -DDBUG_OFF)
+set(FLAGS_RELEASE -O3 -DDBUG_OFF)
 
-set(FLAGS_DEBUG -ggdb3 -O0 -D_DEBUG)
+set(FLAGS_DEBUG -O0 -D_DEBUG)
 # } end Release, Debug and common flags
 
 # linker flags {

cmake/configureEngine.cmake

@@ -773,17 +773,10 @@ endif()
 
 # releasenum is used by external scripts for various tasks. Leave it alone.
 configure_file(${CMAKE_CURRENT_SOURCE_DIR}/build/releasenum.in ${CMAKE_CURRENT_BINARY_DIR}/build/releasenum IMMEDIATE)
-install(
+columnstore_install_file(${CMAKE_CURRENT_BINARY_DIR}/build/releasenum ${ENGINE_SUPPORTDIR})
-    FILES ${CMAKE_CURRENT_BINARY_DIR}/build/releasenum
+
-    DESTINATION ${ENGINE_SUPPORTDIR}
-    COMPONENT columnstore-engine
-)
 configure_file(${CMAKE_CURRENT_SOURCE_DIR}/columnstoreversion.h.in ${CMAKE_CURRENT_SOURCE_DIR}/columnstoreversion.h)
 configure_file(${CMAKE_CURRENT_SOURCE_DIR}/mcsconfig.h.in ${CMAKE_CURRENT_BINARY_DIR}/mcsconfig.h)
-
 configure_file(${CMAKE_CURRENT_SOURCE_DIR}/gitversionEngine.in ${CMAKE_CURRENT_BINARY_DIR}/gitversionEngine IMMEDIATE)
-install(
+
-    FILES ${CMAKE_CURRENT_BINARY_DIR}/gitversionEngine
+columnstore_install_file(${CMAKE_CURRENT_BINARY_DIR}/gitversionEngine ${ENGINE_SUPPORTDIR})
-    DESTINATION ${ENGINE_SUPPORTDIR}
-    COMPONENT columnstore-engine
-)

cmake/packages.cmake

@@ -14,7 +14,6 @@ if(NOT LEX_EXECUTABLE)
 endif()
 
 find_package(LibXml2)
-
 if(NOT LIBXML2_FOUND)
     message_once(CS_NO_LIBXML "Could not find a usable libxml2 development environment!")
     return()

datatypes/CMakeLists.txt

@@ -2,10 +2,5 @@ include_directories(${ENGINE_COMMON_INCLUDES})
 set(datatypes_LIB_SRCS mcs_int128.cpp mcs_decimal.cpp)
 
 columnstore_library(datatypes ${datatypes_LIB_SRCS})
-add_dependencies(datatypes loggingcpp external_boost)
+columnstore_link(datatypes PRIVATE loggingcpp)
-
+add_dependencies(datatypes external_boost)
-install(
-    TARGETS datatypes
-    DESTINATION ${ENGINE_LIBDIR}
-    COMPONENT columnstore-engine
-)

dbcon/ddlpackage/CMakeLists.txt

@@ -42,10 +42,4 @@ columnstore_library(
     ${FLEX_ddl_scan_OUTPUTS}
 )
 
-add_dependencies(ddlpackage loggingcpp)
+columnstore_link(ddlpackage loggingcpp)
-
-install(
-    TARGETS ddlpackage
-    DESTINATION ${ENGINE_LIBDIR}
-    COMPONENT columnstore-engine
-)

dbcon/ddlpackageproc/CMakeLists.txt

@@ -13,13 +13,4 @@ set(ddlpackageproc_LIB_SRCS
 )
 
 columnstore_library(ddlpackageproc ${ddlpackageproc_LIB_SRCS})
-
+columnstore_link(ddlpackageproc ${NETSNMP_LIBRARIES} loggingcpp)
-add_dependencies(ddlpackageproc loggingcpp)
-
-target_link_libraries(ddlpackageproc ${NETSNMP_LIBRARIES})
-
-install(
-    TARGETS ddlpackageproc
-    DESTINATION ${ENGINE_LIBDIR}
-    COMPONENT columnstore-engine
-)

dbcon/dmlpackage/CMakeLists.txt

@@ -41,10 +41,4 @@ columnstore_library(
     ${FLEX_dml_scan_OUTPUTS}
 )
 
-add_dependencies(dmlpackage loggingcpp)
+columnstore_link(dmlpackage loggingcpp)
-
-install(
-    TARGETS dmlpackage
-    DESTINATION ${ENGINE_LIBDIR}
-    COMPONENT columnstore-engine
-)

dbcon/dmlpackageproc/CMakeLists.txt

@@ -13,13 +13,4 @@ set(dmlpackageproc_LIB_SRCS
 )
 
 columnstore_library(dmlpackageproc ${dmlpackageproc_LIB_SRCS})
-
+columnstore_link(dmlpackageproc ${NETSNMP_LIBRARIES} loggingcpp)
-add_dependencies(dmlpackageproc loggingcpp)
-
-target_link_libraries(dmlpackageproc ${NETSNMP_LIBRARIES})
-
-install(
-    TARGETS dmlpackageproc
-    DESTINATION ${ENGINE_LIBDIR}
-    COMPONENT columnstore-engine
-)

dbcon/execplan/CMakeLists.txt

@@ -46,13 +46,4 @@ set(execplan_LIB_SRCS
 )
 
 columnstore_library(execplan ${execplan_LIB_SRCS})
-
+columnstore_link(execplan messageqcpp ${NETSNMP_LIBRARIES} ${ENGINE_DT_LIB} pron loggingcpp)
-add_dependencies(execplan loggingcpp)
-
-target_link_libraries(execplan messageqcpp ${NETSNMP_LIBRARIES} ${ENGINE_DT_LIB} pron)
-
-install(
-    TARGETS execplan
-    DESTINATION ${ENGINE_LIBDIR}
-    COMPONENT columnstore-engine
-)

dbcon/execplan/pseudocolumn.cpp

@@ -25,7 +25,7 @@
 #include <string>
 #include <sstream>
 using namespace std;
-
+#include "basic/string_utils.h"
 
 #include "bytestream.h"
 #include "collation.h"
@@ -138,8 +138,9 @@ string PseudoColumn::toCppCode(IncludeSet& includes) const
 {
   includes.insert("pseudocolumn.h");
   stringstream ss;
-  ss << "PseudoColumn(" << std::quoted(fSchemaName) << ", " << std::quoted(fTableName) << ", " << fisColumnStore << ", " << std::quoted(fColumnName)
+  ss << "PseudoColumn(" << std::quoted(fSchemaName) << ", " << std::quoted(fTableName) << ", "
-     << ", " << fPseudoType << ", " << sessionID() << ")";
+     << fisColumnStore << ", " << std::quoted(fColumnName) << ", " << fPseudoType << ", " << sessionID()
+     << ")";
 
   return ss.str();
 }

dbcon/execplan/windowfunctioncolumn.cpp

@@ -27,10 +27,13 @@
 #include <iostream>
 #include <sstream>
 using namespace std;
+#include "basic/string_utils.h"
 
 #include <boost/tokenizer.hpp>
 using namespace boost;
 
+#include "basic/string_utils.h"
+
 #include "bytestream.h"
 #include "windowfunctioncolumn.h"
 #include "constantcolumn.h"
@@ -398,23 +401,32 @@ void WindowFunctionColumn::adjustResultType()
 {
   if ((fResultType.colDataType == CalpontSystemCatalog::DECIMAL ||
        fResultType.colDataType == CalpontSystemCatalog::UDECIMAL) &&
-      !datatypes::ASCIIStringCaseInsensetiveEquals(fFunctionName, "COUNT") && !datatypes::ASCIIStringCaseInsensetiveEquals(fFunctionName, "COUNT(*)") &&
+      !datatypes::ASCIIStringCaseInsensetiveEquals(fFunctionName, "COUNT") &&
-      !datatypes::ASCIIStringCaseInsensetiveEquals(fFunctionName, "ROW_NUMBER") && !datatypes::ASCIIStringCaseInsensetiveEquals(fFunctionName, "RANK") &&
+      !datatypes::ASCIIStringCaseInsensetiveEquals(fFunctionName, "COUNT(*)") &&
-      !datatypes::ASCIIStringCaseInsensetiveEquals(fFunctionName, "PERCENT_RANK") && !datatypes::ASCIIStringCaseInsensetiveEquals(fFunctionName, "DENSE_RANK") &&
+      !datatypes::ASCIIStringCaseInsensetiveEquals(fFunctionName, "ROW_NUMBER") &&
-      !datatypes::ASCIIStringCaseInsensetiveEquals(fFunctionName, "CUME_DIST") && !datatypes::ASCIIStringCaseInsensetiveEquals(fFunctionName, "NTILE") &&
+      !datatypes::ASCIIStringCaseInsensetiveEquals(fFunctionName, "RANK") &&
+      !datatypes::ASCIIStringCaseInsensetiveEquals(fFunctionName, "PERCENT_RANK") &&
+      !datatypes::ASCIIStringCaseInsensetiveEquals(fFunctionName, "DENSE_RANK") &&
+      !datatypes::ASCIIStringCaseInsensetiveEquals(fFunctionName, "CUME_DIST") &&
+      !datatypes::ASCIIStringCaseInsensetiveEquals(fFunctionName, "NTILE") &&
       !datatypes::ASCIIStringCaseInsensetiveEquals(fFunctionName, "PERCENTILE") && !fFunctionParms.empty() &&
       fFunctionParms[0]->resultType().colDataType == CalpontSystemCatalog::DOUBLE)
     fResultType = fFunctionParms[0]->resultType();
 
-  if ((datatypes::ASCIIStringCaseInsensetiveEquals(fFunctionName, "LEAD") || datatypes::ASCIIStringCaseInsensetiveEquals(fFunctionName, "LAG") ||
+  if ((datatypes::ASCIIStringCaseInsensetiveEquals(fFunctionName, "LEAD") ||
-       datatypes::ASCIIStringCaseInsensetiveEquals(fFunctionName, "MIN") || datatypes::ASCIIStringCaseInsensetiveEquals(fFunctionName, "MAX") ||
+       datatypes::ASCIIStringCaseInsensetiveEquals(fFunctionName, "LAG") ||
-       datatypes::ASCIIStringCaseInsensetiveEquals(fFunctionName, "FIRST_VALUE") || datatypes::ASCIIStringCaseInsensetiveEquals(fFunctionName, "LAST_VALUE") ||
+       datatypes::ASCIIStringCaseInsensetiveEquals(fFunctionName, "MIN") ||
+       datatypes::ASCIIStringCaseInsensetiveEquals(fFunctionName, "MAX") ||
+       datatypes::ASCIIStringCaseInsensetiveEquals(fFunctionName, "FIRST_VALUE") ||
+       datatypes::ASCIIStringCaseInsensetiveEquals(fFunctionName, "LAST_VALUE") ||
        datatypes::ASCIIStringCaseInsensetiveEquals(fFunctionName, "NTH_VALUE")) &&
       !fFunctionParms.empty())
     fResultType = fFunctionParms[0]->resultType();
 
-  if (datatypes::ASCIIStringCaseInsensetiveEquals(fFunctionName, "SUM") || datatypes::ASCIIStringCaseInsensetiveEquals(fFunctionName, "AVG") ||
+  if (datatypes::ASCIIStringCaseInsensetiveEquals(fFunctionName, "SUM") ||
-      datatypes::ASCIIStringCaseInsensetiveEquals(fFunctionName, "AVG_DISTINCT") || datatypes::ASCIIStringCaseInsensetiveEquals(fFunctionName, "PERCENTILE"))
+      datatypes::ASCIIStringCaseInsensetiveEquals(fFunctionName, "AVG") ||
+      datatypes::ASCIIStringCaseInsensetiveEquals(fFunctionName, "AVG_DISTINCT") ||
+      datatypes::ASCIIStringCaseInsensetiveEquals(fFunctionName, "PERCENTILE"))
   {
     if (fFunctionParms[0]->resultType().colDataType == CalpontSystemCatalog::DECIMAL ||
         fFunctionParms[0]->resultType().colDataType == CalpontSystemCatalog::UDECIMAL)
@@ -531,7 +543,7 @@ void WindowFunctionColumn::evaluate(Row& row, bool& isNull)
             fResult.strVal.assign(str.unsafeStringRef());
 
           // stringColVal is padded with '\0' to colWidth so can't use str.length()
-          //if (strlen(fResult.strVal.str()) == 0)
+          // if (strlen(fResult.strVal.str()) == 0)
           //  isNull = true;
 
           break;

dbcon/joblist/CMakeLists.txt

@@ -57,8 +57,8 @@ set(joblist_LIB_SRCS
 
 # ########## next target ###############
 
-add_library(joblist_server STATIC ${joblist_LIB_SRCS})
+columnstore_static_library(joblist_server ${joblist_LIB_SRCS})
-add_dependencies(joblist_server loggingcpp)
+columnstore_link(joblist_server loggingcpp)
 
 # ########## next target ###############
 
@@ -68,20 +68,9 @@ columnstore_library(joblist ${joblist_LIB_SRCS})
 target_include_directories(
     joblist BEFORE PUBLIC ${OPENSSL_INCLUDE_DIR} ${LIBMARIADB_BININC_DIR} ${LIBMARIADB_SRCINC_DIR}
 )
-add_dependencies(joblist loggingcpp)
+columnstore_link(joblist loggingcpp)
-
-install(
-    TARGETS joblist
-    DESTINATION ${ENGINE_LIBDIR}
-    COMPONENT columnstore-engine
-)
 
 if(WITH_ORDERBY_UT)
-    add_executable(job_orderby_tests orderby-tests.cpp)
+    columnstore_executable(job_orderby_tests orderby-tests.cpp)
-    target_link_libraries(job_orderby_tests ${ENGINE_LDFLAGS} ${ENGINE_WRITE_LIBS} ${CPPUNIT_LIBRARIES} cppunit)
+    columnstore_link(job_orderby_tests ${ENGINE_LDFLAGS} ${ENGINE_WRITE_LIBS} ${CPPUNIT_LIBRARIES} cppunit)
-    install(
-        TARGETS job_orderby_tests
-        DESTINATION ${ENGINE_BINDIR}
-        COMPONENT columnstore-engine
-    )
 endif()

dbcon/mysql/CMakeLists.txt

@@ -71,16 +71,17 @@ if(COMMAND mysql_add_plugin)
         CONFIG
         columnstore.cnf
     )
+    add_to_debian_install_file(${MARIADB_MYCNFDIR}/columnstore.cnf)
+    add_to_debian_install_file("${MARIADB_PLUGINDIR}/ha_columnstore.so")
+
 else()
     message("Adding plugin via CMake")
-    add_library(ha_columnstore SHARED ${libcalmysql_SRCS})
+    columnstore_shared_library(ha_columnstore ${libcalmysql_SRCS})
     set_target_properties(ha_columnstore PROPERTIES PREFIX "")
 
-    add_dependencies(ha_columnstore loggingcpp)
-    add_dependencies(ha_columnstore marias3)
     add_definitions(-DMYSQL_DYNAMIC_PLUGIN -DPLUGIN_COLUMNSTORE_VERSION="${PACKAGE_VERSION}")
 
-    target_link_libraries(
+    columnstore_link(
         ha_columnstore
         ${S3API_DEPS}
         ${ENGINE_LDFLAGS}
@@ -88,33 +89,20 @@ else()
         ${NETSNMP_LIBRARIES}
         ${SERVER_BUILD_DIR}/libservices/libmysqlservices.a
         threadpool
+        loggingcpp
+        marias3
     )
-
-    install(
-        TARGETS ha_columnstore
-        DESTINATION ${MARIADB_PLUGINDIR}
-        COMPONENT columnstore-engine
-    )
-
     # define this dummy target for standalone builds (ie, when mysql_add_plugin doesn't exist)
     add_custom_target(columnstore DEPENDS ha_columnstore)
-    install(
+    columnstore_install_file(columnstore.cnf ${MARIADB_MYCNFDIR})
-        FILES columnstore.cnf
-        DESTINATION ${MARIADB_MYCNFDIR}
-        COMPONENT columnstore-engine
-    )
 endif()
 
 if(TARGET columnstore)
-    install(
+    columnstore_install_file(syscatalog_mysql.sql ${ENGINE_SUPPORTDIR})
-        FILES syscatalog_mysql.sql dumpcat_mysql.sql calsetuserpriority.sql calremoveuserpriority.sql
+    columnstore_install_file(dumpcat_mysql.sql ${ENGINE_SUPPORTDIR})
-              calshowprocesslist.sql columnstore_info.sql
+    columnstore_install_file(calsetuserpriority.sql ${ENGINE_SUPPORTDIR})
-        DESTINATION ${ENGINE_SUPPORTDIR}
+    columnstore_install_file(calremoveuserpriority.sql ${ENGINE_SUPPORTDIR})
-        COMPONENT columnstore-engine
+    columnstore_install_file(calshowprocesslist.sql ${ENGINE_SUPPORTDIR})
-    )
+    columnstore_install_file(columnstore_info.sql ${ENGINE_SUPPORTDIR})
-    install(
+    columnstore_install_program(install_mcs_mysql.sh ${ENGINE_SBINDIR})
-        PROGRAMS install_mcs_mysql.sh
-        DESTINATION ${ENGINE_SBINDIR}
-        COMPONENT columnstore-engine
-    )
 endif()

dbcon/mysql/ha_mcs.cpp

@@ -1808,6 +1808,8 @@ static handler* ha_mcs_cache_create_handler(handlerton* hton, TABLE_SHARE* table
   return new (mem_root) ha_mcs_cache(hton, table, mem_root);
 }
 
+bool get_innodb_queries_uses_mcs();
+
 /******************************************************************************
  ha_mcs Plugin code
 ******************************************************************************/
@@ -1826,6 +1828,14 @@ static int columnstore_init_func(void* p)
   fprintf(stderr, "Columnstore: Started; Version: %s-%s\n", columnstore_version.c_str(),
           columnstore_release.c_str());
 
+  LEX_CSTRING name = {STRING_WITH_LEN("INNODB")};
+  auto* plugin_innodb = ha_resolve_by_name(0, &name, 0);
+  if (!plugin_innodb || plugin_state(plugin_innodb) != PLUGIN_IS_READY)
+
+  {
+    DBUG_RETURN(HA_ERR_RETRY_INIT);
+  }
+
   strncpy(cs_version, columnstore_version.c_str(), sizeof(cs_version) - 1);
   cs_version[sizeof(cs_version) - 1] = 0;
 
@@ -1851,6 +1861,18 @@ static int columnstore_init_func(void* p)
   mcs_hton->create_unit = create_columnstore_unit_handler;
   mcs_hton->db_type = DB_TYPE_AUTOASSIGN;
 
+  if (get_innodb_queries_uses_mcs())
+  {
+    auto* innodb_hton = plugin_hton(plugin_innodb);
+    int error = innodb_hton == nullptr;  // Engine must exists!
+    if (error)
+    {
+      my_error(HA_ERR_INITIALIZATION, MYF(0), "Could not find storage engine %s", name.str);
+    }
+    innodb_hton->create_select = create_columnstore_select_handler;
+    innodb_hton->create_unit = create_columnstore_unit_handler;
+  }
+
 #ifdef HAVE_PSI_INTERFACE
   uint count = sizeof(all_mutexes) / sizeof(all_mutexes[0]);
   mysql_mutex_register("ha_mcs_cache", all_mutexes, count);

dbcon/mysql/ha_mcs_client_udfs.cpp

@@ -45,6 +45,7 @@ using namespace logging;
 
 #include "columnstoreversion.h"
 #include "ha_mcs_sysvars.h"
+#include "basic/string_utils.h"
 
 extern "C"
 {

dbcon/mysql/ha_mcs_common.h

@@ -0,0 +1,148 @@
+/* Copyright (C) 2016-2023 MariaDB Corporation
+
+   This program is free software; you can redistribute it and/or
+   modify it under the terms of the GNU General Public License
+   as published by the Free Software Foundation; version 2 of
+   the License.
+
+   This program is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+   GNU General Public License for more details.
+
+   You should have received a copy of the GNU General Public License
+   along with this program; if not, write to the Free Software
+   Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
+   MA 02110-1301, USA. */
+
+#pragma once
+
+#include "idb_mysql.h"
+
+namespace ha_mcs_common
+{
+
+inline bool isMCSTable(TABLE* table_ptr)
+{
+#if (defined(_MSC_VER) && defined(_DEBUG)) || defined(SAFE_MUTEX)
+
+  if (!(table_ptr->s && (*table_ptr->s->db_plugin)->name.str))
+#else
+  if (!(table_ptr->s && (table_ptr->s->db_plugin)->name.str))
+#endif
+    return true;
+
+#if (defined(_MSC_VER) && defined(_DEBUG)) || defined(SAFE_MUTEX)
+  std::string engineName = (*table_ptr->s->db_plugin)->name.str;
+#else
+  std::string engineName = table_ptr->s->db_plugin->name.str;
+#endif
+
+  if (engineName == "Columnstore" || engineName == "Columnstore_cache")
+    return true;
+  return false;
+}
+
+inline bool isMultiUpdateStatement(const enum_sql_command& command)
+{
+  return command == SQLCOM_UPDATE_MULTI;
+}
+
+inline bool isUpdateStatement(const enum_sql_command& command)
+{
+  return command == SQLCOM_UPDATE || isMultiUpdateStatement(command);
+}
+
+inline bool isDeleteStatement(const enum_sql_command& command)
+{
+  return (command == SQLCOM_DELETE) || (command == SQLCOM_DELETE_MULTI);
+}
+
+inline bool isUpdateOrDeleteStatement(const enum_sql_command& command)
+{
+  return isUpdateStatement(command) || isDeleteStatement(command);
+}
+
+inline bool isDMLStatement(const enum_sql_command& command)
+{
+  return (command == SQLCOM_INSERT || command == SQLCOM_INSERT_SELECT || command == SQLCOM_TRUNCATE ||
+          command == SQLCOM_LOAD || isUpdateOrDeleteStatement(command));
+}
+
+inline bool isForeignTableUpdate(THD* thd)
+{
+  LEX* lex = thd->lex;
+
+  if (!isUpdateStatement(lex->sql_command))
+    return false;
+
+  Item_field* item;
+  List_iterator_fast<Item> field_it(lex->first_select_lex()->item_list);
+
+  while ((item = (Item_field*)field_it++))
+  {
+    if (item->field && item->field->table && !isMCSTable(item->field->table))
+      return true;
+  }
+
+  return false;
+}
+
+// This function is different from isForeignTableUpdate()
+// above as it only checks if any of the tables involved
+// in the multi-table update statement is a foreign table,
+// irrespective of whether the update is performed on the
+// foreign table or not, as in isForeignTableUpdate().
+inline bool isUpdateHasForeignTable(THD* thd)
+{
+  LEX* lex = thd->lex;
+
+  if (!isUpdateStatement(lex->sql_command))
+    return false;
+
+  TABLE_LIST* table_ptr = lex->first_select_lex()->get_table_list();
+
+  for (; table_ptr; table_ptr = table_ptr->next_local)
+  {
+    if (table_ptr->table && !isMCSTable(table_ptr->table))
+      return true;
+  }
+
+  return false;
+}
+
+inline bool isMCSTableUpdate(THD* thd)
+{
+  LEX* lex = thd->lex;
+
+  if (!isUpdateStatement(lex->sql_command))
+    return false;
+
+  Item_field* item;
+  List_iterator_fast<Item> field_it(lex->first_select_lex()->item_list);
+
+  while ((item = (Item_field*)field_it++))
+  {
+    if (item->field && item->field->table && isMCSTable(item->field->table))
+      return true;
+  }
+
+  return false;
+}
+
+inline bool isMCSTableDelete(THD* thd)
+{
+  LEX* lex = thd->lex;
+
+  if (!isDeleteStatement(lex->sql_command))
+    return false;
+
+  TABLE_LIST* table_ptr = lex->first_select_lex()->get_table_list();
+
+  if (table_ptr && table_ptr->table && isMCSTable(table_ptr->table))
+    return true;
+
+  return false;
+}
+
+}  // namespace ha_mcs_common

dbcon/mysql/ha_mcs_impl.cpp

@@ -832,7 +832,7 @@ uint32_t doUpdateDelete(THD* thd, gp_walk_info& gwi, const std::vector<COND*>& c
     {
       Message::Args args;
 
-      if (isUpdateStatement(thd->lex->sql_command))
+      if (ha_mcs_common::isUpdateStatement(thd->lex->sql_command))
         args.add("Update");
 #if 0
             else if (thd->rgi_slave && thd->rgi_slave->m_table_map.count() != 0)
@@ -889,7 +889,7 @@ uint32_t doUpdateDelete(THD* thd, gp_walk_info& gwi, const std::vector<COND*>& c
   updateCP->isDML(true);
 
   //@Bug 2753. the memory already freed by destructor of UpdateSqlStatement
-  if (isUpdateStatement(thd->lex->sql_command))
+  if (ha_mcs_common::isUpdateStatement(thd->lex->sql_command))
   {
     ColumnAssignment* columnAssignmentPtr = nullptr;
     Item_field* item;
@@ -1184,7 +1184,7 @@ uint32_t doUpdateDelete(THD* thd, gp_walk_info& gwi, const std::vector<COND*>& c
   }
 
   // Exit early if there is nothing to update
-  if (colAssignmentListPtr->empty() && isUpdateStatement(thd->lex->sql_command))
+  if (colAssignmentListPtr->empty() && ha_mcs_common::isUpdateStatement(thd->lex->sql_command))
   {
     ci->affectedRows = 0;
     delete colAssignmentListPtr;
@@ -1198,7 +1198,7 @@ uint32_t doUpdateDelete(THD* thd, gp_walk_info& gwi, const std::vector<COND*>& c
   CalpontSystemCatalog::TableName aTableName;
   TABLE_LIST* first_table = 0;
 
-  if (isUpdateStatement(thd->lex->sql_command))
+  if (ha_mcs_common::isUpdateStatement(thd->lex->sql_command))
   {
     aTableName.schema = schemaName;
     aTableName.table = tableName;
@@ -1220,7 +1220,7 @@ uint32_t doUpdateDelete(THD* thd, gp_walk_info& gwi, const std::vector<COND*>& c
   IDEBUG(cout << "STMT: " << dmlStmt << " and sessionID " << thd->thread_id << endl);
   VendorDMLStatement dmlStatement(dmlStmt, sessionID);
 
-  if (isUpdateStatement(thd->lex->sql_command))
+  if (ha_mcs_common::isUpdateStatement(thd->lex->sql_command))
     dmlStatement.set_DMLStatementType(DML_UPDATE);
   else
     dmlStatement.set_DMLStatementType(DML_DELETE);
@@ -1229,7 +1229,7 @@ uint32_t doUpdateDelete(THD* thd, gp_walk_info& gwi, const std::vector<COND*>& c
   //@Bug 2753. To make sure the momory is freed.
   updateStmt.fColAssignmentListPtr = colAssignmentListPtr;
 
-  if (isUpdateStatement(thd->lex->sql_command))
+  if (ha_mcs_common::isUpdateStatement(thd->lex->sql_command))
   {
     TableName* qualifiedTablName = new TableName();
     qualifiedTablName->fName = tableName;
@@ -1318,7 +1318,7 @@ uint32_t doUpdateDelete(THD* thd, gp_walk_info& gwi, const std::vector<COND*>& c
   List<Item> items;
   SELECT_LEX select_lex;
 
-  if (isUpdateStatement(thd->lex->sql_command))
+  if (ha_mcs_common::isUpdateStatement(thd->lex->sql_command))
   {
     items = (thd->lex->first_select_lex()->item_list);
     thd->lex->first_select_lex()->item_list = thd->lex->value_list;
@@ -1467,7 +1467,7 @@ uint32_t doUpdateDelete(THD* thd, gp_walk_info& gwi, const std::vector<COND*>& c
       returnedCols.push_back((updateCP->columnMap()).begin()->second);
 
     //@Bug 6123. get the correct returned columnlist
-    if (isDeleteStatement(thd->lex->sql_command))
+    if (ha_mcs_common::isDeleteStatement(thd->lex->sql_command))
     {
       returnedCols.clear();
       // choose the smallest column to project
@@ -1518,7 +1518,7 @@ uint32_t doUpdateDelete(THD* thd, gp_walk_info& gwi, const std::vector<COND*>& c
 
     updateCP->returnedCols(returnedCols);
 
-    if (isUpdateStatement(thd->lex->sql_command))
+    if (ha_mcs_common::isUpdateStatement(thd->lex->sql_command))
     {
       const ParseTree* ptsub = updateCP->filters();
 
@@ -1979,7 +1979,7 @@ int ha_mcs_impl_analyze(THD* thd, TABLE* table)
   if (table->s->db.length && strcmp(table->s->db.str, "information_schema") == 0)
     return 0;
 
-  bool columnStore = (table ? isMCSTable(table) : true);
+  bool columnStore = (table ? ha_mcs_common::isMCSTable(table) : true);
   // Skip non columnstore tables.
   if (!columnStore)
     return 0;
@@ -2156,7 +2156,8 @@ int ha_mcs_impl_direct_update_delete_rows(bool execute, ha_rows* affected_rows,
   gwi.thd = thd;
   int rc = 0;
 
-  if (thd->slave_thread && !get_replication_slave(thd) && isDMLStatement(thd->lex->sql_command))
+  if (thd->slave_thread && !get_replication_slave(thd) &&
+      ha_mcs_common::isDMLStatement(thd->lex->sql_command))
   {
     if (affected_rows)
       *affected_rows = 0;
@@ -2189,7 +2190,7 @@ int ha_mcs::impl_rnd_init(TABLE* table, const std::vector<COND*>& condStack)
   gwi.thd = thd;
 
   if (thd->slave_thread && !get_replication_slave(thd) &&
-      (isDMLStatement(thd->lex->sql_command) || thd->lex->sql_command == SQLCOM_ALTER_TABLE))
+      (ha_mcs_common::isDMLStatement(thd->lex->sql_command) || thd->lex->sql_command == SQLCOM_ALTER_TABLE))
     return 0;
 
   // check whether the system is ready to process statement.
@@ -2241,7 +2242,7 @@ int ha_mcs::impl_rnd_init(TABLE* table, const std::vector<COND*>& condStack)
     UPDATE innotab1 SET a=100 WHERE a NOT IN (SELECT a FROM cstab1 WHERE a=1);
   */
   if (!isReadOnly() &&  // make sure the current table is being modified
-      isUpdateOrDeleteStatement(thd->lex->sql_command))
+      ha_mcs_common::isUpdateOrDeleteStatement(thd->lex->sql_command))
     return doUpdateDelete(thd, gwi, condStack);
 
   uint32_t sessionID = tid2sid(thd->thread_id);
@@ -2578,10 +2579,10 @@ int ha_mcs_impl_rnd_next(uchar* buf, TABLE* table, long timeZone)
   THD* thd = current_thd;
 
   if (thd->slave_thread && !get_replication_slave(thd) &&
-      (isDMLStatement(thd->lex->sql_command) || thd->lex->sql_command == SQLCOM_ALTER_TABLE))
+      (ha_mcs_common::isDMLStatement(thd->lex->sql_command) || thd->lex->sql_command == SQLCOM_ALTER_TABLE))
     return HA_ERR_END_OF_FILE;
 
-  if (isMCSTableUpdate(thd) || isMCSTableDelete(thd))
+  if (ha_mcs_common::isMCSTableUpdate(thd) || ha_mcs_common::isMCSTableDelete(thd))
     return HA_ERR_END_OF_FILE;
 
   // @bug 2547
@@ -2661,18 +2662,17 @@ int ha_mcs_impl_rnd_end(TABLE* table, bool is_pushdown_hand)
   THD* thd = current_thd;
 
   if (thd->slave_thread && !get_replication_slave(thd) &&
-      (isDMLStatement(thd->lex->sql_command) || thd->lex->sql_command == SQLCOM_ALTER_TABLE))
+      (ha_mcs_common::isDMLStatement(thd->lex->sql_command) || thd->lex->sql_command == SQLCOM_ALTER_TABLE))
-    return 0;
+    return rc;
 
   cal_connection_info* ci = nullptr;
-
   if (get_fe_conn_info_ptr() != NULL)
     ci = reinterpret_cast<cal_connection_info*>(get_fe_conn_info_ptr());
 
   if ((thd->lex)->sql_command == SQLCOM_ALTER_TABLE)
     return rc;
 
-  if (isMCSTableUpdate(thd) || isMCSTableDelete(thd))
+  if (ha_mcs_common::isMCSTableUpdate(thd) || ha_mcs_common::isMCSTableDelete(thd))
     return rc;
 
   if (!ci)
@@ -3764,7 +3764,7 @@ COND* ha_mcs_impl_cond_push(COND* cond, TABLE* table, std::vector<COND*>& condSt
 {
   THD* thd = current_thd;
 
-  if (isUpdateOrDeleteStatement(thd->lex->sql_command))
+  if (ha_mcs_common::isUpdateOrDeleteStatement(thd->lex->sql_command))
   {
     condStack.push_back(cond);
     return nullptr;
@@ -3848,7 +3848,7 @@ COND* ha_mcs_impl_cond_push(COND* cond, TABLE* table, std::vector<COND*>& condSt
 
 inline void disableBinlogForDML(THD* thd)
 {
-  if (isDMLStatement(thd->lex->sql_command) && (thd->variables.option_bits & OPTION_BIN_LOG))
+  if (ha_mcs_common::isDMLStatement(thd->lex->sql_command) && (thd->variables.option_bits & OPTION_BIN_LOG))
   {
     set_original_option_bits(thd->variables.option_bits, thd);
     thd->variables.option_bits &= ~OPTION_BIN_LOG;
@@ -3858,7 +3858,7 @@ inline void disableBinlogForDML(THD* thd)
 
 inline void restoreBinlogForDML(THD* thd)
 {
-  if (isDMLStatement(thd->lex->sql_command))
+  if (ha_mcs_common::isDMLStatement(thd->lex->sql_command))
   {
     ulonglong orig_option_bits = get_original_option_bits(thd);
 
@@ -4027,7 +4027,8 @@ int ha_mcs_impl_pushdown_init(mcs_handler_info* handler_info, TABLE* table, bool
   IDEBUG(cout << "pushdown_init for table " << endl);
   THD* thd = current_thd;
 
-  if (thd->slave_thread && !get_replication_slave(thd) && isDMLStatement(thd->lex->sql_command))
+  if (thd->slave_thread && !get_replication_slave(thd) &&
+      ha_mcs_common::isDMLStatement(thd->lex->sql_command))
     return 0;
 
   const char* timeZone = thd->variables.time_zone->get_name()->ptr();
@@ -4069,16 +4070,16 @@ int ha_mcs_impl_pushdown_init(mcs_handler_info* handler_info, TABLE* table, bool
 
   // MCOL-4023 We need to test this code path.
   // Update and delete code
-  if (isUpdateOrDeleteStatement(thd->lex->sql_command))
+  if (ha_mcs_common::isUpdateOrDeleteStatement(thd->lex->sql_command))
     return doUpdateDelete(thd, gwi, std::vector<COND*>());
 
   uint32_t sessionID = tid2sid(thd->thread_id);
   boost::shared_ptr<CalpontSystemCatalog> csc = CalpontSystemCatalog::makeCalpontSystemCatalog(sessionID);
   csc->identity(CalpontSystemCatalog::FE);
 
-  if (!get_fe_conn_info_ptr())
+  if (!get_fe_conn_info_ptr()) 
   {
-    set_fe_conn_info_ptr(reinterpret_cast<void*>(new cal_connection_info(), thd));
+    set_fe_conn_info_ptr((void*)new cal_connection_info());
     thd_set_ha_data(thd, mcs_hton, get_fe_conn_info_ptr());
   }
 
@@ -4493,7 +4494,8 @@ int ha_mcs_impl_select_next(uchar* buf, TABLE* table, long timeZone)
 {
   THD* thd = current_thd;
 
-  if (thd->slave_thread && !get_replication_slave(thd) && isDMLStatement(thd->lex->sql_command))
+  if (thd->slave_thread && !get_replication_slave(thd) &&
+      ha_mcs_common::isDMLStatement(thd->lex->sql_command))
     return HA_ERR_END_OF_FILE;
 
   int rc = HA_ERR_END_OF_FILE;
@@ -4506,7 +4508,7 @@ int ha_mcs_impl_select_next(uchar* buf, TABLE* table, long timeZone)
 
   cal_connection_info* ci = reinterpret_cast<cal_connection_info*>(get_fe_conn_info_ptr());
 
-  if (isUpdateOrDeleteStatement(thd->lex->sql_command))
+  if (ha_mcs_common::isUpdateOrDeleteStatement(thd->lex->sql_command))
     return rc;
 
   // @bug 2547

dbcon/mysql/ha_mcs_impl_if.h

@@ -26,7 +26,7 @@
 #include <boost/shared_ptr.hpp>
 #include <stack>
 #include <vector>
-
+#include "basic/string_utils.h"
 #include "idb_mysql.h"
 #include "ha_mcs_sysvars.h"
 
@@ -36,6 +36,8 @@ struct st_ha_create_information;
 class ha_columnstore_select_handler;
 class ha_columnstore_derived_handler;
 
+#include "basic/string_utils.h"
+
 #include "configcpp.h"
 #include "idberrorinfo.h"
 #include "calpontselectexecutionplan.h"
@@ -117,7 +119,8 @@ struct gp_walk_info
   std::vector<execplan::ReturnedColumn*> localCols;
   std::stack<execplan::ReturnedColumn*> rcWorkStack;
   std::stack<execplan::ParseTree*> ptWorkStack;
-  boost::shared_ptr<execplan::SimpleColumn> scsp; // while defined as SSCP, it is used as SRCP, nothing specific to SimpleColumn is used in use sites.
+  boost::shared_ptr<execplan::SimpleColumn> scsp;  // while defined as SSCP, it is used as SRCP, nothing
+                                                   // specific to SimpleColumn is used in use sites.
   uint32_t sessionid;
   bool fatalParseError;
   std::string parseErrorText;
@@ -227,7 +230,6 @@ struct gp_walk_info
   {
   }
   ~gp_walk_info();
-
 };
 
 struct SubQueryChainHolder;
@@ -237,7 +239,7 @@ struct ext_cond_info
   // circular dependency on header inclusion with ha_subquery.h.
   boost::shared_ptr<SubQueryChainHolder> chainHolder;
   gp_walk_info gwi;
-  ext_cond_info(long timeZone); // needs knowledge on SubQueryChainHolder, will be defined elsewhere
+  ext_cond_info(long timeZone);  // needs knowledge on SubQueryChainHolder, will be defined elsewhere
 };
 
 struct cal_table_info
@@ -401,11 +403,6 @@ void clearDeleteStacks(gp_walk_info& gwi);
 void parse_item(Item* item, std::vector<Item_field*>& field_vec, bool& hasNonSupportItem, uint16& parseInfo,
                 gp_walk_info* gwip = nullptr);
 const std::string bestTableName(const Item_field* ifp);
-bool isMCSTable(TABLE* table_ptr);
-bool isForeignTableUpdate(THD* thd);
-bool isUpdateHasForeignTable(THD* thd);
-bool isMCSTableUpdate(THD* thd);
-bool isMCSTableDelete(THD* thd);
 
 // execution plan util functions prototypes
 execplan::ReturnedColumn* buildReturnedColumn(Item* item, gp_walk_info& gwi, bool& nonSupport,
@@ -452,27 +449,6 @@ bool buildEqualityPredicate(execplan::ReturnedColumn* lhs, execplan::ReturnedCol
                             boost::shared_ptr<execplan::Operator>& sop, const Item_func::Functype& funcType,
                             const std::vector<Item*>& itemList, bool isInSubs = false);
 
-inline bool isUpdateStatement(const enum_sql_command& command)
-{
-  return ((command == SQLCOM_UPDATE) || (command == SQLCOM_UPDATE_MULTI));
-}
-
-inline bool isDeleteStatement(const enum_sql_command& command)
-{
-  return (command == SQLCOM_DELETE) || (command == SQLCOM_DELETE_MULTI);
-}
-
-inline bool isUpdateOrDeleteStatement(const enum_sql_command& command)
-{
-  return isUpdateStatement(command) || isDeleteStatement(command);
-}
-
-inline bool isDMLStatement(const enum_sql_command& command)
-{
-  return (command == SQLCOM_INSERT || command == SQLCOM_INSERT_SELECT || command == SQLCOM_TRUNCATE ||
-          command == SQLCOM_LOAD || isUpdateOrDeleteStatement(command));
-}
-
 #ifdef DEBUG_WALK_COND
 void debug_walk(const Item* item, void* arg);
 #endif

dbcon/mysql/ha_mcs_pushdown.cpp

@@ -17,8 +17,40 @@
 #include <typeinfo>
 #include <string>
 
+// This makes specific MDB classes' attributes public to implement
+// MCOL-4740 temporary solution. Search for MCOL-4740
+// to get the actual place where it is used.
+#define updated_leaves \
+  updated_leaves;      \
+                       \
+ public:
+
 #include "ha_mcs_pushdown.h"
 
+void update_counters_on_multi_update()
+{
+  if (ha_mcs_common::isMultiUpdateStatement(current_thd->lex->sql_command) &&
+      !ha_mcs_common::isForeignTableUpdate(current_thd))
+  {
+    SELECT_LEX_UNIT* unit = &current_thd->lex->unit;
+    SELECT_LEX* select_lex = unit->first_select();
+    auto* multi = (select_lex->join) ? reinterpret_cast<multi_update*>(select_lex->join->result) : nullptr;
+
+    if (multi)
+    {
+      multi->table_to_update = multi->update_tables ? multi->update_tables->table : 0;
+
+      cal_impl_if::cal_connection_info* ci =
+          reinterpret_cast<cal_impl_if::cal_connection_info*>(get_fe_conn_info_ptr());
+
+      if (ci)
+      {
+        multi->updated = multi->found = ci->affectedRows;
+      }
+    }
+  }
+}
+
 void check_walk(const Item* item, void* arg);
 
 void restore_query_state(ha_columnstore_select_handler* handler)
@@ -391,7 +423,7 @@ derived_handler* create_columnstore_derived_handler(THD* thd, TABLE_LIST* table_
 
   // MCOL-1482 Disable derived_handler if the multi-table update
   // statement contains a non-columnstore table.
-  if (cal_impl_if::isUpdateHasForeignTable(thd))
+  if (ha_mcs_common::isUpdateHasForeignTable(thd))
   {
     return handler;
   }
@@ -551,18 +583,18 @@ void ha_columnstore_derived_handler::print_error(int, unsigned long)
 
 /*@brief  create_columnstore_select_handler_- Creates handler
 ************************************************************
- * DESCRIPTION:
+* DESCRIPTION:
- * Creates a select handler if there is no non-equi JOIN, e.g
+* Creates a select handler if there is no non-equi JOIN, e.g
- * t1.c1 > t2.c2 and logical OR in the filter predicates.
+* t1.c1 > t2.c2 and logical OR in the filter predicates.
- * More details in server/sql/select_handler.h
+* More details in server/sql/select_handler.h
- * PARAMETERS:
+* PARAMETERS:
- *    thd - THD pointer.
+*    thd - THD pointer.
- *    sel_lex - SELECT_LEX* that describes the query.
+*    sel_lex - SELECT_LEX* that describes the query.
- *    sel_unit - SELECT_LEX_UNIT* that describes the query.
+*    sel_unit - SELECT_LEX_UNIT* that describes the query.
- * RETURN:
+* RETURN:
- *    select_handler if possible
+*    select_handler if possible
- *    NULL in other case
+*    NULL in other case
- ***********************************************************/
+***********************************************************/
 select_handler* create_columnstore_select_handler_(THD* thd, SELECT_LEX* sel_lex, SELECT_LEX_UNIT* sel_unit)
 {
   mcs_select_handler_mode_t select_handler_mode = get_select_handler_mode(thd);
@@ -578,7 +610,7 @@ select_handler* create_columnstore_select_handler_(THD* thd, SELECT_LEX* sel_lex
   // MCOL-1482 Disable select_handler for a multi-table update
   // with a non-columnstore table as the target table of the update
   // operation.
-  if (cal_impl_if::isForeignTableUpdate(thd))
+  if (ha_mcs_common::isForeignTableUpdate(thd))
   {
     return nullptr;
   }
@@ -646,15 +678,15 @@ select_handler* create_columnstore_select_handler_(THD* thd, SELECT_LEX* sel_lex
   // or unsupported feature.
   ha_columnstore_select_handler* handler;
 
-  if (sel_unit && sel_lex) // partial pushdown of the SELECT_LEX_UNIT
+  if (sel_unit && sel_lex)  // partial pushdown of the SELECT_LEX_UNIT
   {
     handler = new ha_columnstore_select_handler(thd, sel_lex, sel_unit);
   }
-  else if (sel_unit) // complete pushdown of the SELECT_LEX_UNIT
+  else if (sel_unit)  // complete pushdown of the SELECT_LEX_UNIT
   {
     handler = new ha_columnstore_select_handler(thd, sel_unit);
   }
-  else // Query only has a SELECT_LEX, no SELECT_LEX_UNIT
+  else  // Query only has a SELECT_LEX, no SELECT_LEX_UNIT
   {
     handler = new ha_columnstore_select_handler(thd, sel_lex);
   }
@@ -749,8 +781,7 @@ select_handler* create_columnstore_select_handler_(THD* thd, SELECT_LEX* sel_lex
           select_lex != select_lex->master_unit()->fake_select_lex)  // (2)
         thd->lex->set_limit_rows_examined();
 
-      if ((!sel_unit || sel_lex) && !join->tables_list &&
+      if ((!sel_unit || sel_lex) && !join->tables_list && (join->table_count || !select_lex->with_sum_func) &&
-          (join->table_count || !select_lex->with_sum_func) &&
           !select_lex->have_window_funcs())
       {
         if (!thd->is_error())
@@ -1010,6 +1041,11 @@ int ha_columnstore_select_handler::end_scan()
 
   scan_ended = true;
 
+  // MCOL-4740 multi_update::send_eof(), which outputs the affected
+  // number of rows to the client, is called after handler::rnd_end().
+  // So we set multi_update::updated and multi_update::found here.
+  update_counters_on_multi_update();
+
   int rc = ha_mcs_impl_rnd_end(table, true);
 
   DBUG_RETURN(rc);

dbcon/mysql/ha_mcs_pushdown.h

@@ -20,6 +20,7 @@
 #include "idb_mysql.h"
 #include "ha_mcs.h"
 #include "ha_mcs_sysvars.h"
+#include "ha_mcs_common.h"
 #define NEED_CALPONT_EXTERNS
 #include "ha_mcs_impl.h"
 #include "ha_mcs_impl_if.h"

dbcon/mysql/ha_mcs_sysvars.cpp

@@ -220,47 +220,54 @@ static MYSQL_THDVAR_ULONG(max_allowed_in_values, PLUGIN_VAR_RQCMDARG,
                           "The maximum length of the entries in the IN query clause.", NULL, NULL, 6000, 1,
                           ~0U, 1);
 
-st_mysql_sys_var* mcs_system_variables[] = {MYSQL_SYSVAR(compression_type),
+static my_bool innodb_queries_use_mcs;
-                                            MYSQL_SYSVAR(fe_conn_info_ptr),
+static MYSQL_SYSVAR_BOOL(innodb_queries_use_mcs, innodb_queries_use_mcs,
-                                            MYSQL_SYSVAR(original_optimizer_flags),
+                      PLUGIN_VAR_OPCMDARG | PLUGIN_VAR_READONLY,
-                                            MYSQL_SYSVAR(original_option_bits),
+                      "Direct all InnoDB-only queries into MCS via Select Handler.", NULL, NULL, FALSE);
-                                            MYSQL_SYSVAR(select_handler),
+    
-                                            MYSQL_SYSVAR(derived_handler),
+st_mysql_sys_var* mcs_system_variables[] = {
-                                            MYSQL_SYSVAR(select_handler_in_stored_procedures),
+    MYSQL_SYSVAR(compression_type),
-                                            MYSQL_SYSVAR(orderby_threads),
+    MYSQL_SYSVAR(fe_conn_info_ptr),
-                                            MYSQL_SYSVAR(decimal_scale),
+    MYSQL_SYSVAR(original_optimizer_flags),
-                                            MYSQL_SYSVAR(use_decimal_scale),
+    MYSQL_SYSVAR(original_option_bits),
-                                            MYSQL_SYSVAR(ordered_only),
+    MYSQL_SYSVAR(select_handler),
-                                            MYSQL_SYSVAR(string_scan_threshold),
+    MYSQL_SYSVAR(derived_handler),
-                                            MYSQL_SYSVAR(stringtable_threshold),
+    MYSQL_SYSVAR(select_handler_in_stored_procedures),
-                                            MYSQL_SYSVAR(diskjoin_smallsidelimit),
+    MYSQL_SYSVAR(orderby_threads),
-                                            MYSQL_SYSVAR(diskjoin_largesidelimit),
+    MYSQL_SYSVAR(decimal_scale),
-                                            MYSQL_SYSVAR(diskjoin_bucketsize),
+    MYSQL_SYSVAR(use_decimal_scale),
-                                            MYSQL_SYSVAR(diskjoin_max_partition_tree_depth),
+    MYSQL_SYSVAR(ordered_only),
-                                            MYSQL_SYSVAR(diskjoin_force_run),
+    MYSQL_SYSVAR(string_scan_threshold),
-                                            MYSQL_SYSVAR(max_pm_join_result_count),
+    MYSQL_SYSVAR(stringtable_threshold),
-                                            MYSQL_SYSVAR(um_mem_limit),
+    MYSQL_SYSVAR(diskjoin_smallsidelimit),
-                                            MYSQL_SYSVAR(double_for_decimal_math),
+    MYSQL_SYSVAR(diskjoin_largesidelimit),
-                                            MYSQL_SYSVAR(decimal_overflow_check),
+    MYSQL_SYSVAR(diskjoin_bucketsize),
-                                            MYSQL_SYSVAR(local_query),
+    MYSQL_SYSVAR(diskjoin_max_partition_tree_depth),
-                                            MYSQL_SYSVAR(use_import_for_batchinsert),
+    MYSQL_SYSVAR(diskjoin_force_run),
-                                            MYSQL_SYSVAR(import_for_batchinsert_delimiter),
+    MYSQL_SYSVAR(max_pm_join_result_count),
-                                            MYSQL_SYSVAR(import_for_batchinsert_enclosed_by),
+    MYSQL_SYSVAR(um_mem_limit),
-                                            MYSQL_SYSVAR(varbin_always_hex),
+    MYSQL_SYSVAR(double_for_decimal_math),
-                                            MYSQL_SYSVAR(replication_slave),
+    MYSQL_SYSVAR(decimal_overflow_check),
-                                            MYSQL_SYSVAR(cache_inserts),
+    MYSQL_SYSVAR(local_query),
-                                            MYSQL_SYSVAR(cache_use_import),
+    MYSQL_SYSVAR(use_import_for_batchinsert),
-                                            MYSQL_SYSVAR(cache_flush_threshold),
+    MYSQL_SYSVAR(import_for_batchinsert_delimiter),
-                                            MYSQL_SYSVAR(cmapi_host),
+    MYSQL_SYSVAR(import_for_batchinsert_enclosed_by),
-                                            MYSQL_SYSVAR(cmapi_port),
+    MYSQL_SYSVAR(varbin_always_hex),
-                                            MYSQL_SYSVAR(cmapi_version),
+    MYSQL_SYSVAR(replication_slave),
-                                            MYSQL_SYSVAR(cmapi_key),
+    MYSQL_SYSVAR(cache_inserts),
-                                            MYSQL_SYSVAR(s3_key),
+    MYSQL_SYSVAR(cache_use_import),
-                                            MYSQL_SYSVAR(s3_secret),
+    MYSQL_SYSVAR(cache_flush_threshold),
-                                            MYSQL_SYSVAR(s3_region),
+    MYSQL_SYSVAR(cmapi_host),
-                                            MYSQL_SYSVAR(pron),
+    MYSQL_SYSVAR(cmapi_port),
-                                            MYSQL_SYSVAR(max_allowed_in_values),
+    MYSQL_SYSVAR(cmapi_version),
-                                            NULL};
+    MYSQL_SYSVAR(cmapi_key),
+    MYSQL_SYSVAR(s3_key),
+    MYSQL_SYSVAR(s3_secret),
+    MYSQL_SYSVAR(s3_region),
+    MYSQL_SYSVAR(pron),
+    MYSQL_SYSVAR(max_allowed_in_values),
+    MYSQL_SYSVAR(innodb_queries_use_mcs),
+    NULL};
 
 st_mysql_show_var mcs_status_variables[] = {{"columnstore_version", (char*)&cs_version, SHOW_CHAR},
                                             {"columnstore_commit_hash", (char*)&cs_commit_hash, SHOW_CHAR},
@@ -654,4 +661,9 @@ ulong get_max_allowed_in_values(THD* thd)
 void set_max_allowed_in_values(THD* thd, ulong value)
 {
   THDVAR(thd, max_allowed_in_values) = value;
-}
+}
+
+bool get_innodb_queries_uses_mcs()
+{
+  return SYSVAR(innodb_queries_use_mcs);
+}

dbcon/mysql/ha_view.cpp

@@ -42,6 +42,7 @@ using namespace execplan;
 
 #include "ha_subquery.h"
 #include "ha_view.h"
+#include "ha_mcs_common.h"
 
 namespace cal_impl_if
 {
@@ -131,7 +132,7 @@ void View::transform()
       else
       {
         // check foreign engine tables
-        bool columnStore = (table_ptr->table ? isMCSTable(table_ptr->table) : true);
+        bool columnStore = (table_ptr->table ? ha_mcs_common::isMCSTable(table_ptr->table) : true);
 
         // trigger system catalog cache
         if (columnStore)

ddlproc/CMakeLists.txt

@@ -4,14 +4,6 @@ include_directories(${ENGINE_COMMON_INCLUDES})
 
 set(DDLProc_SRCS ddlproc.cpp ddlprocessor.cpp ../utils/common/crashtrace.cpp)
 
-add_executable(DDLProc ${DDLProc_SRCS})
+columnstore_executable(DDLProc ${DDLProc_SRCS})
 
-add_dependencies(DDLProc loggingcpp)
+columnstore_link(DDLProc ${ENGINE_LDFLAGS} ${ENGINE_WRITE_LIBS} ${NETSNMP_LIBRARIES} threadpool loggingcpp)
-
-target_link_libraries(DDLProc ${ENGINE_LDFLAGS} ${ENGINE_WRITE_LIBS} ${NETSNMP_LIBRARIES} threadpool)
-
-install(
-    TARGETS DDLProc
-    DESTINATION ${ENGINE_BINDIR}
-    COMPONENT columnstore-engine
-)

dmlproc/CMakeLists.txt

@@ -6,11 +6,9 @@ set(DMLProc_SRCS dmlproc.cpp dmlprocessor.cpp dmlresultbuffer.cpp batchinsertpro
                  ../utils/common/crashtrace.cpp
 )
 
-add_executable(DMLProc ${DMLProc_SRCS})
+columnstore_executable(DMLProc ${DMLProc_SRCS})
 
-add_dependencies(DMLProc loggingcpp)
+columnstore_link(
-
-target_link_libraries(
     DMLProc
     ${ENGINE_LDFLAGS}
     ${ENGINE_WRITE_LIBS}
@@ -18,10 +16,5 @@ target_link_libraries(
     threadpool
     ddlcleanuputil
     batchloader
-)
+    loggingcpp
-
-install(
-    TARGETS DMLProc
-    DESTINATION ${ENGINE_BINDIR}
-    COMPONENT columnstore-engine
 )

mysql-test/columnstore/bugfixes/mcol-4740.result

@@ -0,0 +1,80 @@
+DROP DATABASE IF EXISTS mcol_4740;
+CREATE DATABASE mcol_4740;
+USE mcol_4740;
+CREATE TABLE mcs_1 (a INT, b INT(11), c VARCHAR(100)) engine=columnstore;
+CREATE TABLE mcs_2 (a INT, b INT(11), c VARCHAR(100)) engine=columnstore;
+INSERT INTO mcs_1 VALUES (33, 99, 1);
+INSERT INTO mcs_1 VALUES (33, 99, 2);
+INSERT INTO mcs_1 VALUES (33, 99, 3);
+INSERT INTO mcs_2 VALUES (33, 11, 1);
+INSERT INTO mcs_2 VALUES (33, 11, 2);
+INSERT INTO mcs_2 VALUES (33, 11, 3);
+SELECT * FROM mcs_1;
+a	b	c
+33	99	1
+33	99	2
+33	99	3
+UPDATE mcs_1 A, mcs_2 B SET A.b = B.b WHERE A.c = B.c LIMIT 1;
+affected rows: 1
+info: Rows matched: 1  Changed: 1  Warnings: 0
+SELECT * FROM mcs_1;
+a	b	c
+33	11	1
+33	99	2
+33	99	3
+UPDATE mcs_1 A, mcs_2 B SET A.b = A.b + 1, B.b = B.b + 1;
+ERROR 42000: The storage engine for the table doesn't support MCS-1012: This version of Columnstore supports update of only one table at a time.
+CREATE TABLE `test1` (
+`id` text NOT NULL,
+`p_date` int(11) DEFAULT NULL,
+`con_bfr` text DEFAULT NULL,
+`org_bfr` text DEFAULT NULL,
+`pat_bfr` text DEFAULT NULL,
+`data_type` text DEFAULT NULL,
+`order_no` text DEFAULT NULL,
+`tra_bfr` text DEFAULT NULL,
+`load_bfr` text DEFAULT NULL,
+`status` int(11) DEFAULT NULL
+) ENGINE=Columnstore DEFAULT CHARSET=utf8;
+INSERT INTO test1 values ('index_srgt_id','0','consultation_date_bfr','organization_code_bfr','patient_id_bfr','data_type','order_no','transaction_time_bfr','load_timestamp_bfr','0');
+INSERT INTO test1 values ('adt-a55-9000000001','20190610','2.01906E+13','1116508857','PTNT-0001','ADT-01','511','2.01801E+13','2.01801E+13','0');
+INSERT INTO test1 values ('adt-a55-9000000002','20190610','2.01906E+13','1116508857','PTNT-0001','ADT-01','511','2.01801E+13','2.01801E+13','0');
+INSERT INTO test1 values ('adt-a55-9000000003','20190611','2.01906E+13','1116508857','PTNT-0001','ADT-01','511','2.01801E+13','2.01801E+13','0');
+INSERT INTO test1 values ('adt-a55-9000000004','20190612','2.01906E+13','1116508857','PTNT-0001','ADT-01','511','2.01801E+13','2.01801E+13','0');
+SELECT status FROM test1;
+status
+0
+0
+0
+0
+0
+UPDATE test1 main
+INNER JOIN (SELECT id,
+con_bfr,
+order_no,
+ROW_NUMBER() OVER (PARTITION BY IFNULL(p_date,''),
+IFNULL(con_bfr,''),
+IFNULL(org_bfr,''),
+IFNULL(pat_bfr,''),
+IFNULL(data_type,'')
+ORDER BY IFNULL(tra_bfr,'') DESC,
+IFNULL(load_bfr,'') DESC,
+id DESC
+) AS row_num
+FROM test1 tmp
+) tmp
+ON main.id = tmp.id
+AND main.con_bfr = tmp.con_bfr
+AND main.order_no = tmp.order_no
+AND tmp.row_num = 1
+SET main.status = 1;
+affected rows: 4
+info: Rows matched: 4  Changed: 4  Warnings: 0
+SELECT status FROM test1;
+status
+1
+0
+1
+1
+1
+DROP DATABASE mcol_4740;

mysql-test/columnstore/bugfixes/mcol-4740.test

@@ -0,0 +1,82 @@
+#
+# MCOL-4740 UPDATE involving multi-tables returns wrong "Rows matched"
+#
+
+--source ../include/have_columnstore.inc
+
+--disable_warnings
+DROP DATABASE IF EXISTS mcol_4740;
+--enable_warnings
+CREATE DATABASE mcol_4740;
+USE mcol_4740;
+
+# Test case 1 from MCOL
+CREATE TABLE mcs_1 (a INT, b INT(11), c VARCHAR(100)) engine=columnstore;
+CREATE TABLE mcs_2 (a INT, b INT(11), c VARCHAR(100)) engine=columnstore;
+
+INSERT INTO mcs_1 VALUES (33, 99, 1);
+INSERT INTO mcs_1 VALUES (33, 99, 2);
+INSERT INTO mcs_1 VALUES (33, 99, 3);
+INSERT INTO mcs_2 VALUES (33, 11, 1);
+INSERT INTO mcs_2 VALUES (33, 11, 2);
+INSERT INTO mcs_2 VALUES (33, 11, 3);
+
+SELECT * FROM mcs_1;
+
+--enable_info
+UPDATE mcs_1 A, mcs_2 B SET A.b = B.b WHERE A.c = B.c LIMIT 1;
+--disable_info
+
+SELECT * FROM mcs_1;
+
+--error ER_CHECK_NOT_IMPLEMENTED
+UPDATE mcs_1 A, mcs_2 B SET A.b = A.b + 1, B.b = B.b + 1;
+
+# Test case 2 from MCOL
+CREATE TABLE `test1` (
+  `id` text NOT NULL,
+  `p_date` int(11) DEFAULT NULL,
+  `con_bfr` text DEFAULT NULL,
+  `org_bfr` text DEFAULT NULL,
+  `pat_bfr` text DEFAULT NULL,
+  `data_type` text DEFAULT NULL,
+  `order_no` text DEFAULT NULL,
+  `tra_bfr` text DEFAULT NULL,
+  `load_bfr` text DEFAULT NULL,
+  `status` int(11) DEFAULT NULL
+) ENGINE=Columnstore DEFAULT CHARSET=utf8;
+
+INSERT INTO test1 values ('index_srgt_id','0','consultation_date_bfr','organization_code_bfr','patient_id_bfr','data_type','order_no','transaction_time_bfr','load_timestamp_bfr','0');
+INSERT INTO test1 values ('adt-a55-9000000001','20190610','2.01906E+13','1116508857','PTNT-0001','ADT-01','511','2.01801E+13','2.01801E+13','0');
+INSERT INTO test1 values ('adt-a55-9000000002','20190610','2.01906E+13','1116508857','PTNT-0001','ADT-01','511','2.01801E+13','2.01801E+13','0');
+INSERT INTO test1 values ('adt-a55-9000000003','20190611','2.01906E+13','1116508857','PTNT-0001','ADT-01','511','2.01801E+13','2.01801E+13','0');
+INSERT INTO test1 values ('adt-a55-9000000004','20190612','2.01906E+13','1116508857','PTNT-0001','ADT-01','511','2.01801E+13','2.01801E+13','0');
+
+SELECT status FROM test1;
+
+--enable_info
+UPDATE test1 main
+INNER JOIN (SELECT id,
+                   con_bfr,
+                   order_no,
+                   ROW_NUMBER() OVER (PARTITION BY IFNULL(p_date,''),
+                                                   IFNULL(con_bfr,''),
+                                                   IFNULL(org_bfr,''),
+                                                   IFNULL(pat_bfr,''),
+                                                   IFNULL(data_type,'')
+                                      ORDER BY IFNULL(tra_bfr,'') DESC,
+                                               IFNULL(load_bfr,'') DESC,
+                                               id DESC
+                                     ) AS row_num
+            FROM test1 tmp
+           ) tmp
+   ON main.id = tmp.id
+   AND main.con_bfr = tmp.con_bfr
+   AND main.order_no = tmp.order_no
+   AND tmp.row_num = 1
+SET main.status = 1;
+--disable_info
+
+SELECT status FROM test1;
+
+DROP DATABASE mcol_4740;

mysql-test/columnstore/oracle/func_concat_oracle.result

@@ -1,5 +1,6 @@
 SET sql_mode=ORACLE;
 SET default_storage_engine=ColumnStore;
+SET character_set_server = "utf8mb4";
 EXPLAIN EXTENDED SELECT 'a'||'b'||'c';
 id	select_type	table	type	possible_keys	key	key_len	ref	rows	filtered	Extra
 1	SIMPLE	NULL	NULL	NULL	NULL	NULL	NULL	NULL	NULL	No tables used
@@ -354,7 +355,7 @@ EXPLAIN EXTENDED SELECT c1 FROM t1 WHERE c1 LIKE '%'||'b' ORDER BY ord;
 id	select_type	table	type	possible_keys	key	key_len	ref	rows	filtered	Extra
 1	PUSHED SELECT	NULL	NULL	NULL	NULL	NULL	NULL	NULL	NULL	NULL
 Warnings:
-Note	1003	select "test"."t1"."c1" AS "c1" from "test"."t1" where "test"."t1"."c1" like convert(concat('%','b') using utf8mb3) order by "test"."t1"."ord"
+Note	1003	select "test"."t1"."c1" AS "c1" from "test"."t1" where "test"."t1"."c1" like convert(concat('%','b') using utf8mb4) order by "test"."t1"."ord"
 SELECT c1 FROM t1 WHERE c1 LIKE c2||'%'||'c' ORDER BY ord;
 c1
 abc

mysql-test/columnstore/oracle/func_concat_oracle.test

@@ -4,6 +4,7 @@
 
 SET sql_mode=ORACLE;
 SET default_storage_engine=ColumnStore;
+SET character_set_server = "utf8mb4";
 
 EXPLAIN EXTENDED SELECT 'a'||'b'||'c';
 EXPLAIN EXTENDED SELECT CONCAT('a'||'b'||'c');

oam/etc/CMakeLists.txt

@@ -1,5 +1 @@
-install(
+columnstore_install_file(Columnstore.xml ${ENGINE_SYSCONFDIR}/columnstore)
-    FILES Columnstore.xml
-    DESTINATION ${ENGINE_SYSCONFDIR}/columnstore
-    COMPONENT columnstore-engine
-)

oam/install_scripts/CMakeLists.txt

@@ -131,49 +131,37 @@ configure_file("${CMAKE_CURRENT_SOURCE_DIR}/mcs-savebrm.py.in" "${CMAKE_CURRENT_
 configure_file("${CMAKE_CURRENT_SOURCE_DIR}/mcs-savebrm.py.in" "${CMAKE_CURRENT_SOURCE_DIR}/mcssavebrm.py" @ONLY)
 configure_file("${CMAKE_CURRENT_SOURCE_DIR}/columnstoreSyslog.in" "${CMAKE_CURRENT_SOURCE_DIR}/columnstoreSyslog" @ONLY)
 
-install(
+columnstore_install_program(columnstore-post-install ${ENGINE_BINDIR})
-    PROGRAMS columnstore-post-install
+columnstore_install_program(columnstore-pre-uninstall ${ENGINE_BINDIR})
-             columnstore-pre-uninstall
+columnstore_install_program(columnstore_run.sh ${ENGINE_BINDIR})
-             columnstore_run.sh
+columnstore_install_program(post-mysql-install ${ENGINE_BINDIR})
-             post-mysql-install
+columnstore_install_program(post-mysqld-install ${ENGINE_BINDIR})
-             post-mysqld-install
+columnstore_install_program(columnstoreSyslogSetup.sh ${ENGINE_BINDIR})
-             columnstoreSyslogSetup.sh
+columnstore_install_program(mcs-stop-controllernode.sh ${ENGINE_BINDIR})
-             mcs-stop-controllernode.sh
+columnstore_install_program(mcs-loadbrm.py ${ENGINE_BINDIR})
-             mcs-loadbrm.py
+columnstore_install_program(mcs-savebrm.py ${ENGINE_BINDIR})
-             mcs-savebrm.py
+columnstore_install_program(mariadb-columnstore-start.sh ${ENGINE_BINDIR})
-             mariadb-columnstore-start.sh
+columnstore_install_program(mariadb-columnstore-stop.sh ${ENGINE_BINDIR})
-             mariadb-columnstore-stop.sh
+columnstore_install_program(loop_process_starter.sh ${ENGINE_BINDIR})
-             loop_process_starter.sh
-    DESTINATION ${ENGINE_BINDIR}
-    COMPONENT columnstore-engine
-)
 
-install(
+columnstore_install_file(mariadb-columnstore.service ${ENGINE_SUPPORTDIR})
-    FILES mariadb-columnstore.service
+columnstore_install_file(columnstoreAlias ${ENGINE_SUPPORTDIR})
-          columnstoreAlias
+columnstore_install_file(columnstoreSyslog ${ENGINE_SUPPORTDIR})
-          columnstoreSyslog
+columnstore_install_file(columnstoreSyslog7 ${ENGINE_SUPPORTDIR})
-          columnstoreSyslog7
+columnstore_install_file(columnstoreSyslog-ng ${ENGINE_SUPPORTDIR})
-          columnstoreSyslog-ng
+columnstore_install_file(columnstoreLogRotate ${ENGINE_SUPPORTDIR})
-          columnstoreLogRotate
+columnstore_install_file(myCnf-include-args.text ${ENGINE_SUPPORTDIR})
-          myCnf-include-args.text
+columnstore_install_file(myCnf-exclude-args.text ${ENGINE_SUPPORTDIR})
-          myCnf-exclude-args.text
+columnstore_install_file(mcs-workernode.service ${ENGINE_SUPPORTDIR})
-          mcs-workernode.service
+columnstore_install_file(mcs-controllernode.service ${ENGINE_SUPPORTDIR})
-          mcs-controllernode.service
+columnstore_install_file(mcs-primproc.service ${ENGINE_SUPPORTDIR})
-          mcs-primproc.service
+columnstore_install_file(mcs-writeengineserver.service ${ENGINE_SUPPORTDIR})
-          mcs-writeengineserver.service
+columnstore_install_file(mcs-dmlproc.service ${ENGINE_SUPPORTDIR})
-          mcs-dmlproc.service
+columnstore_install_file(mcs-ddlproc.service ${ENGINE_SUPPORTDIR})
-          mcs-ddlproc.service
+columnstore_install_file(mcs-loadbrm.service ${ENGINE_SUPPORTDIR})
-          mcs-loadbrm.service
+columnstore_install_file(mcs-storagemanager.service ${ENGINE_SUPPORTDIR})
-          mcs-storagemanager.service
-    DESTINATION ${ENGINE_SUPPORTDIR}
-    COMPONENT columnstore-engine
-)
 
-install(
+columnstore_install_file(module ${ENGINE_DATADIR}/local)
-    FILES module
-    DESTINATION ${ENGINE_DATADIR}/local
-    COMPONENT columnstore-engine
-)
 
 find_package(Python3 COMPONENTS Interpreter)
 if(Python3_FOUND)

oam/oamcpp/CMakeLists.txt

@@ -5,15 +5,6 @@ include_directories(${ENGINE_COMMON_INCLUDES})
 set(oamcpp_LIB_SRCS liboamcpp.cpp oamcache.cpp)
 
 columnstore_library(oamcpp ${oamcpp_LIB_SRCS})
-
+columnstore_link(oamcpp loggingcpp)
-add_dependencies(oamcpp loggingcpp)
-
-target_link_libraries(oamcpp)
 
 target_compile_options(oamcpp PRIVATE -Wno-unused-result)
-
-install(
-    TARGETS oamcpp
-    DESTINATION ${ENGINE_LIBDIR}
-    COMPONENT columnstore-engine
-)

oamapps/columnstoreDB/CMakeLists.txt

@@ -4,12 +4,6 @@ include_directories(${ENGINE_COMMON_INCLUDES})
 
 set(columnstoreDBWrite_SRCS columnstoreDB.cpp)
 
-add_executable(columnstoreDBWrite ${columnstoreDBWrite_SRCS})
+columnstore_executable(columnstoreDBWrite ${columnstoreDBWrite_SRCS})
 
-target_link_libraries(columnstoreDBWrite ${ENGINE_LDFLAGS} ncurses ${ENGINE_EXEC_LIBS})
+columnstore_link(columnstoreDBWrite ${ENGINE_LDFLAGS} ncurses ${ENGINE_EXEC_LIBS})
-
-install(
-    TARGETS columnstoreDBWrite
-    DESTINATION ${ENGINE_BINDIR}
-    COMPONENT columnstore-engine
-)

oamapps/columnstoreSupport/CMakeLists.txt

@@ -4,20 +4,13 @@ include_directories(${ENGINE_COMMON_INCLUDES})
 
 set(columnstoreSupport_SRCS columnstoreSupport.cpp mcsSupportUtil.cpp)
 
-add_executable(columnstoreSupport ${columnstoreSupport_SRCS})
+columnstore_executable(columnstoreSupport ${columnstoreSupport_SRCS})
-
 target_compile_options(columnstoreSupport PRIVATE -Wno-unused-result)
+columnstore_link(columnstoreSupport ${ENGINE_LDFLAGS} ncurses ${ENGINE_EXEC_LIBS})
 
-target_link_libraries(columnstoreSupport ${ENGINE_LDFLAGS} ncurses ${ENGINE_EXEC_LIBS})
+columnstore_install_program(dbmsReport.sh, ${ENGINE_BINDIR})
-
+columnstore_install_program(bulklogReport.sh, ${ENGINE_BINDIR})
-install(
+columnstore_install_program(configReport.sh, ${ENGINE_BINDIR})
-    TARGETS columnstoreSupport
+columnstore_install_program(hardwareReport.sh, ${ENGINE_BINDIR})
-    DESTINATION ${ENGINE_BINDIR}
+columnstore_install_program(logReport.sh, ${ENGINE_BINDIR})
-    COMPONENT columnstore-engine
+columnstore_install_program(resourceReport.sh, ${ENGINE_BINDIR})
-)
-
-install(
-    PROGRAMS dbmsReport.sh bulklogReport.sh configReport.sh hardwareReport.sh logReport.sh resourceReport.sh
-    DESTINATION ${ENGINE_BINDIR}
-    COMPONENT columnstore-engine
-)

oamapps/postConfigure/CMakeLists.txt

@@ -4,14 +4,6 @@ include_directories(${ENGINE_COMMON_INCLUDES})
 
 set(mycnfUpgrade_SRCS mycnfUpgrade.cpp)
 
-add_executable(mycnfUpgrade ${mycnfUpgrade_SRCS})
+columnstore_executable(mycnfUpgrade ${mycnfUpgrade_SRCS})
 
-target_compile_options(mycnfUpgrade PRIVATE -Wno-unused-result)
+columnstore_link(mycnfUpgrade ${ENGINE_LDFLAGS} ${ENGINE_READLINE_LIBRARY} ncurses ${ENGINE_EXEC_LIBS})
-
-target_link_libraries(mycnfUpgrade ${ENGINE_LDFLAGS} ${ENGINE_READLINE_LIBRARY} ncurses ${ENGINE_EXEC_LIBS})
-
-install(
-    TARGETS mycnfUpgrade
-    DESTINATION ${ENGINE_BINDIR}
-    COMPONENT columnstore-engine
-)

oamapps/postConfigure/mycnfUpgrade.cpp

@@ -46,6 +46,7 @@
 #include <stdio.h>
 #include <string>
 #include <unistd.h>
+#include <tuple>
 #include <vector>
 
 #include <sys/sysinfo.h>
@@ -282,7 +283,7 @@ int main(int argc, char* argv[])
     USER = p;
 
   string cmd = "chown " + USER + ":" + USER + " " + mycnfFile;
-  system(cmd.c_str());
+  std::ignore = std::system(cmd.c_str());
 
   exit(0);
 }

primitives/blockcache/CMakeLists.txt

@@ -13,13 +13,5 @@ set(dbbc_STAT_SRCS
     stats.cpp
     fsutils.cpp
 )
-
+columnstore_static_library(dbbc ${dbbc_STAT_SRCS})
-# libdbbc_a_CXXFLAGS = $(march_flags) $(AM_CXXFLAGS)
+columnstore_link(dbbc ${NETSNMP_LIBRARIES} loggingcpp)
-
-add_library(dbbc STATIC ${dbbc_STAT_SRCS})
-
-add_dependencies(dbbc loggingcpp)
-
-target_link_libraries(dbbc ${NETSNMP_LIBRARIES})
-
-install(TARGETS dbbc DESTINATION ${ENGINE_LIBDIR})

primitives/linux-port/CMakeLists.txt

@@ -4,10 +4,5 @@ include_directories(${ENGINE_COMMON_INCLUDES} ../blockcache ../primproc)
 
 set(processor_STAT_SRCS primitiveprocessor.cpp dictionary.cpp column.cpp)
 
-add_library(processor STATIC ${processor_STAT_SRCS})
+columnstore_static_library(processor ${processor_STAT_SRCS})
-
+columnstore_link(processor ${NETSNMP_LIBRARIES} loggingcpp)
-add_dependencies(processor loggingcpp)
-
-target_link_libraries(processor ${NETSNMP_LIBRARIES})
-
-install(TARGETS processor DESTINATION ${ENGINE_LIBDIR})

primitives/primproc/CMakeLists.txt

@@ -25,11 +25,10 @@ set(PrimProc_SRCS
     ../../utils/common/crashtrace.cpp
 )
 
-add_executable(PrimProc ${PrimProc_SRCS})
+columnstore_executable(PrimProc ${PrimProc_SRCS})
 
-add_dependencies(PrimProc loggingcpp)
 target_include_directories(PrimProc PRIVATE ${Boost_INCLUDE_DIRS})
-target_link_libraries(
+columnstore_link(
     PrimProc
     ${ENGINE_LDFLAGS}
     ${NETSNMP_LIBRARIES}
@@ -38,10 +37,5 @@ target_link_libraries(
     cacheutils
     dbbc
     processor
-)
+    loggingcpp
-
-install(
-    TARGETS PrimProc
-    DESTINATION ${ENGINE_BINDIR}
-    COMPONENT columnstore-engine
 )

storage-manager/CMakeLists.txt

@@ -64,11 +64,11 @@ configure_file(
 link_directories(${CMAKE_BINARY_DIR}/lib)
 set(CMAKE_INSTALL_RPATH $ORIGIN $ORIGIN/../lib)
 
-add_library(storagemanager SHARED ${storagemanager_SRCS})
+columnstore_library(storagemanager SHARED ${storagemanager_SRCS})
 add_dependencies(storagemanager marias3 external_boost)
 
 target_compile_definitions(storagemanager PUBLIC BOOST_NO_CXX11_SCOPED_ENUMS)
-target_link_libraries(
+columnstore_link(
     storagemanager
     boost_chrono
     boost_system
@@ -80,8 +80,8 @@ target_link_libraries(
 )
 target_include_directories(storagemanager PRIVATE ${Boost_INCLUDE_DIRS})
 
-add_executable(StorageManager src/main.cpp)
+columnstore_executable(StorageManager src/main.cpp)
-target_link_libraries(StorageManager storagemanager)
+columnstore_link(StorageManager storagemanager)
 set_property(TARGET StorageManager PROPERTY CXX_STANDARD 20)
 
 set(TMPDIR ${CMAKE_RUNTIME_OUTPUT_DIRECTORY})
@@ -89,11 +89,11 @@ set(CMAKE_RUNTIME_OUTPUT_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR})
 
 add_executable(unit_tests src/unit_tests.cpp)
 target_compile_definitions(unit_tests PUBLIC BOOST_NO_CXX11_SCOPED_ENUMS)
-target_link_libraries(unit_tests storagemanager)
+columnstore_link(unit_tests storagemanager)
 
-add_executable(testS3Connection src/testS3Connection.cpp)
+columnstore_executable(testS3Connection src/testS3Connection.cpp)
 target_compile_definitions(testS3Connection PUBLIC BOOST_NO_CXX11_SCOPED_ENUMS)
-target_link_libraries(testS3Connection storagemanager)
+columnstore_link(testS3Connection storagemanager)
 
 set(CMAKE_RUNTIME_OUTPUT_DIRECTORY ${TMPDIR})
 
@@ -112,42 +112,25 @@ add_custom_command(
 # The includes and lib linkages required to link against cloudio ... pretty crazy.  When lib dependencies are eventually
 # config'd right, change this to only include and link against cloudio.
 include_directories(${ENGINE_SRC_DIR}/utils/cloudio ${ENGINE_COMMON_INCLUDES})
-add_executable(smcat src/smcat.cpp)
+columnstore_executable(smcat src/smcat.cpp)
-target_link_libraries(smcat storagemanager cloudio ${ENGINE_LDFLAGS} ${ENGINE_EXEC_LIBS})
+columnstore_link(smcat storagemanager cloudio ${ENGINE_LDFLAGS} ${ENGINE_EXEC_LIBS})
 
-add_executable(smput src/smput.cpp)
+columnstore_executable(smput src/smput.cpp)
-target_link_libraries(smput storagemanager cloudio ${ENGINE_LDFLAGS} ${ENGINE_EXEC_LIBS})
+columnstore_link(smput storagemanager cloudio ${ENGINE_LDFLAGS} ${ENGINE_EXEC_LIBS})
 
-add_executable(smls src/smls.cpp)
+columnstore_executable(smls src/smls.cpp)
-target_link_libraries(smls storagemanager cloudio ${ENGINE_LDFLAGS} ${ENGINE_EXEC_LIBS})
+columnstore_link(smls storagemanager cloudio ${ENGINE_LDFLAGS} ${ENGINE_EXEC_LIBS})
 
-add_executable(smrm src/smrm.cpp)
+columnstore_executable(smrm src/smrm.cpp)
-target_link_libraries(smrm storagemanager cloudio ${ENGINE_LDFLAGS} ${ENGINE_EXEC_LIBS})
+columnstore_link(smrm storagemanager cloudio ${ENGINE_LDFLAGS} ${ENGINE_EXEC_LIBS})
 
-add_executable(smps src/smps.cpp)
+columnstore_executable(smps src/smps.cpp)
-target_link_libraries(smps storagemanager cloudio ${ENGINE_LDFLAGS} ${ENGINE_EXEC_LIBS})
+columnstore_link(smps storagemanager cloudio ${ENGINE_LDFLAGS} ${ENGINE_EXEC_LIBS})
 
-add_executable(smkill src/smkill.cpp)
+columnstore_executable(smkill src/smkill.cpp)
-target_link_libraries(smkill storagemanager cloudio ${ENGINE_LDFLAGS} ${ENGINE_EXEC_LIBS})
+columnstore_link(smkill storagemanager cloudio ${ENGINE_LDFLAGS} ${ENGINE_EXEC_LIBS})
-install(TARGETS storagemanager LIBRARY DESTINATION ${ENGINE_LIBDIR} COMPONENT columnstore-engine)
 
-install(
+columnstore_install_file(storagemanager.cnf ${ENGINE_SYSCONFDIR}/columnstore)
-    TARGETS StorageManager
-            smcat
-            smkill
-            smps
-            smput
-            smls
-            smrm
-            testS3Connection
-    RUNTIME DESTINATION ${ENGINE_BINDIR} COMPONENT columnstore-engine
-)
-
-install(
-    FILES storagemanager.cnf
-    DESTINATION ${ENGINE_SYSCONFDIR}/columnstore
-    COMPONENT columnstore-engine
-)
 
 install(
     FILES storagemanager.cnf

tests/CMakeLists.txt

@@ -30,82 +30,80 @@ if(WITH_UNITTESTS)
     add_executable(rowgroup_tests rowgroup-tests.cpp)
     target_compile_options(rowgroup_tests PRIVATE -Wno-sign-compare)
     add_dependencies(rowgroup_tests googletest)
-    target_link_libraries(rowgroup_tests ${ENGINE_LDFLAGS} ${GTEST_LIBRARIES} ${ENGINE_EXEC_LIBS})
+    columnstore_link(rowgroup_tests ${ENGINE_LDFLAGS} ${GTEST_LIBRARIES} ${ENGINE_EXEC_LIBS})
     gtest_add_tests(TARGET rowgroup_tests TEST_PREFIX columnstore:)
 
     add_executable(rewritetest rewritetest.cpp)
     add_dependencies(rewritetest googletest)
-    target_link_libraries(rewritetest ${ENGINE_LDFLAGS} ${GTEST_LIBRARIES} ${ENGINE_EXEC_LIBS})
+    columnstore_link(rewritetest ${ENGINE_LDFLAGS} ${GTEST_LIBRARIES} ${ENGINE_EXEC_LIBS})
     gtest_add_tests(TARGET rewritetest TEST_PREFIX columnstore:)
 
     add_executable(mcs_decimal_tests mcs_decimal-tests.cpp)
     add_dependencies(mcs_decimal_tests googletest)
-    target_link_libraries(mcs_decimal_tests ${ENGINE_LDFLAGS} ${GTEST_LIBRARIES} ${ENGINE_EXEC_LIBS})
+    columnstore_link(mcs_decimal_tests ${ENGINE_LDFLAGS} ${GTEST_LIBRARIES} ${ENGINE_EXEC_LIBS})
     gtest_add_tests(TARGET mcs_decimal_tests TEST_PREFIX columnstore:)
 
     add_executable(dataconvert_tests dataconvert-tests.cpp)
     add_dependencies(dataconvert_tests googletest)
-    target_link_libraries(dataconvert_tests ${ENGINE_LDFLAGS} ${GTEST_LIBRARIES} ${ENGINE_EXEC_LIBS})
+    columnstore_link(dataconvert_tests ${ENGINE_LDFLAGS} ${GTEST_LIBRARIES} ${ENGINE_EXEC_LIBS})
     gtest_add_tests(TARGET dataconvert_tests TEST_PREFIX columnstore:)
 
     add_executable(rebuild_em_tests rebuild-em-tests.cpp)
     add_dependencies(rebuild_em_tests googletest)
-    target_link_libraries(rebuild_em_tests ${ENGINE_LDFLAGS} ${GTEST_LIBRARIES} ${ENGINE_WRITE_LIBS})
+    columnstore_link(rebuild_em_tests ${ENGINE_LDFLAGS} ${GTEST_LIBRARIES} ${ENGINE_WRITE_LIBS})
     gtest_add_tests(TARGET rebuild_em_tests TEST_PREFIX columnstore:)
 
     add_executable(compression_tests compression-tests.cpp)
     add_dependencies(compression_tests googletest)
-    target_link_libraries(compression_tests ${ENGINE_LDFLAGS} ${GTEST_LIBRARIES} ${ENGINE_WRITE_LIBS})
+    columnstore_link(compression_tests ${ENGINE_LDFLAGS} ${GTEST_LIBRARIES} ${ENGINE_WRITE_LIBS})
     gtest_add_tests(TARGET compression_tests TEST_PREFIX columnstore:)
 
     add_executable(column_scan_filter_tests primitives_column_scan_and_filter.cpp)
     target_compile_options(column_scan_filter_tests PRIVATE -Wno-error -Wno-sign-compare)
     add_dependencies(column_scan_filter_tests googletest)
-    target_link_libraries(
+    columnstore_link(column_scan_filter_tests ${ENGINE_LDFLAGS} ${ENGINE_WRITE_LIBS} ${GTEST_LIBRARIES} processor dbbc)
-        column_scan_filter_tests ${ENGINE_LDFLAGS} ${ENGINE_WRITE_LIBS} ${GTEST_LIBRARIES} processor dbbc
-    )
     gtest_add_tests(TARGET column_scan_filter_tests TEST_PREFIX columnstore:)
 
     add_executable(simd_processors simd_processors.cpp)
     target_compile_options(simd_processors PRIVATE -Wno-error)
     add_dependencies(simd_processors googletest)
-    target_link_libraries(simd_processors ${ENGINE_LDFLAGS} ${ENGINE_WRITE_LIBS} ${GTEST_LIBRARIES} processor dbbc)
+    columnstore_link(simd_processors ${ENGINE_LDFLAGS} ${ENGINE_WRITE_LIBS} ${GTEST_LIBRARIES} processor dbbc)
     gtest_add_tests(TARGET simd_processors TEST_PREFIX columnstore:)
 
     add_executable(fair_threadpool_test fair_threadpool.cpp)
     add_dependencies(fair_threadpool_test googletest)
-    target_link_libraries(fair_threadpool_test ${ENGINE_LDFLAGS} ${ENGINE_WRITE_LIBS} ${GTEST_LIBRARIES} processor dbbc)
+    columnstore_link(fair_threadpool_test ${ENGINE_LDFLAGS} ${ENGINE_WRITE_LIBS} ${GTEST_LIBRARIES} processor dbbc)
     gtest_add_tests(TARGET fair_threadpool_test TEST_PREFIX columnstore:)
 
     add_executable(counting_allocator counting_allocator.cpp)
     target_compile_options(counting_allocator PRIVATE -Wno-sign-compare)
     add_dependencies(counting_allocator googletest)
-    target_link_libraries(counting_allocator ${ENGINE_LDFLAGS} ${ENGINE_WRITE_LIBS} ${GTEST_LIBRARIES})
+    columnstore_link(counting_allocator ${ENGINE_LDFLAGS} ${ENGINE_WRITE_LIBS} ${GTEST_LIBRARIES})
     gtest_add_tests(TARGET counting_allocator TEST_PREFIX columnstore:)
 
     add_executable(poolallocator poolallocator.cpp)
     target_compile_options(poolallocator PRIVATE -Wno-sign-compare)
     add_dependencies(poolallocator googletest)
-    target_link_libraries(poolallocator ${ENGINE_LDFLAGS} ${ENGINE_WRITE_LIBS} ${GTEST_LIBRARIES})
+    columnstore_link(poolallocator ${ENGINE_LDFLAGS} ${ENGINE_WRITE_LIBS} ${GTEST_LIBRARIES})
     gtest_add_tests(TARGET poolallocator TEST_PREFIX columnstore:)
 
     add_executable(stlpoolallocator stlpoolallocator.cpp)
     target_compile_options(stlpoolallocator PRIVATE -Wno-sign-compare)
     add_dependencies(stlpoolallocator googletest)
-    target_link_libraries(stlpoolallocator ${ENGINE_LDFLAGS} ${ENGINE_WRITE_LIBS} ${GTEST_LIBRARIES})
+    columnstore_link(stlpoolallocator ${ENGINE_LDFLAGS} ${ENGINE_WRITE_LIBS} ${GTEST_LIBRARIES})
     gtest_add_tests(TARGET stlpoolallocator TEST_PREFIX columnstore:)
 
     add_executable(comparators_tests comparators-tests.cpp)
-    target_link_libraries(comparators_tests ${ENGINE_LDFLAGS} ${ENGINE_WRITE_LIBS} ${CPPUNIT_LIBRARIES} cppunit)
+    columnstore_link(comparators_tests ${ENGINE_LDFLAGS} ${ENGINE_WRITE_LIBS} ${CPPUNIT_LIBRARIES} cppunit)
     add_test(NAME columnstore:comparators_tests COMMAND comparators_tests)
 
     add_executable(bytestream bytestream.cpp)
-    target_link_libraries(bytestream ${ENGINE_LDFLAGS} ${ENGINE_WRITE_LIBS} ${CPPUNIT_LIBRARIES} cppunit)
+    columnstore_link(bytestream ${ENGINE_LDFLAGS} ${ENGINE_WRITE_LIBS} ${CPPUNIT_LIBRARIES} cppunit)
     add_test(NAME columnstore:bytestream COMMAND bytestream)
 
     # standalone EM routines test
     add_executable(brm_em_standalone brm-em-standalone.cpp)
-    target_link_libraries(brm_em_standalone ${ENGINE_LDFLAGS} ${ENGINE_WRITE_LIBS} ${CPPUNIT_LIBRARIES} cppunit)
+    columnstore_link(brm_em_standalone ${ENGINE_LDFLAGS} ${ENGINE_WRITE_LIBS} ${CPPUNIT_LIBRARIES} cppunit)
     add_test(NAME columnstore:brm_em_standalone COMMAND brm_em_standalone)
     set_tests_properties(columnstore:brm_em_standalone PROPERTIES DISABLED True)
 endif()
@@ -116,7 +114,7 @@ if(WITH_MICROBENCHMARKS AND (NOT CMAKE_BUILD_TYPE STREQUAL "Debug"))
     target_include_directories(
         primitives_scan_bench PUBLIC ${ENGINE_COMMON_INCLUDES} ${ENGINE_BLOCKCACHE_INCLUDE} ${ENGINE_PRIMPROC_INCLUDE}
     )
-    target_link_libraries(
+    columnstore_link(
         primitives_scan_bench
         ${ENGINE_LDFLAGS}
         ${ENGINE_WRITE_LIBS}

tools/bincvt/li2bin.cpp

@@ -1,258 +0,0 @@
-#include <iostream>
-#include <inttypes.h>
-#include <string>
-#include <cstring>
-#include <cstdlib>
-//#define NDEBUG
-#include <cassert>
-using namespace std;
-
-#include <boost/tokenizer.hpp>
-#include <boost/static_assert.hpp>
-using namespace boost;
-
-
-// 1|155190|7706|1|17|21168.23|0.04|0.02|N|O|1996-03-13|1996-02-12|1996-03-22|DELIVER IN PERSON|TRUCK|egular
-// courts above the|
-
-#pragma pack(1)
-struct lineitem_image
-{
-  int32_t l_orderkey;
-  int32_t l_partkey;
-  int32_t l_suppkey;  // 12
-  int64_t l_linenumber;
-  int64_t l_quantity;
-  int64_t l_extendedprice;
-  int64_t l_discount;
-  int64_t l_tax;  // 40
-  char l_returnflag;
-  char l_linestatus;  // 2
-  int32_t l_shipdate;
-  int32_t l_commitdate;
-  int32_t l_receiptdate;  // 12
-  char l_shipinstruct[25];
-  char l_shipmode[10];
-  char l_comment[44];  // 79
-};
-BOOST_STATIC_ASSERT(sizeof(struct lineitem_image) == 145);
-
-// version of lineitem_image that treats some integer columns as unsigned ints.
-// Decimal, date, and date/time columns are still treated as integer.
-struct unsigned_lineitem_image
-{
-  uint32_t l_orderkey;
-  uint32_t l_partkey;
-  uint32_t l_suppkey;  // 12
-  uint64_t l_linenumber;
-  int64_t l_quantity;
-  int64_t l_extendedprice;
-  int64_t l_discount;
-  int64_t l_tax;  // 40
-  char l_returnflag;
-  char l_linestatus;  // 2
-  int32_t l_shipdate;
-  int32_t l_commitdate;
-  int32_t l_receiptdate;  // 12
-  char l_shipinstruct[25];
-  char l_shipmode[10];
-  char l_comment[44];  // 79
-};
-BOOST_STATIC_ASSERT(sizeof(struct unsigned_lineitem_image) == 145);
-
-struct Date
-{
-  unsigned spare : 6;
-  unsigned day : 6;
-  unsigned month : 4;
-  unsigned year : 16;
-};
-BOOST_STATIC_ASSERT(sizeof(struct Date) == 4);
-
-union date_image
-{
-  struct Date d;
-  int32_t i;
-};
-
-namespace
-{
-typedef boost::tokenizer<boost::char_separator<char> > tokenizer;
-
-int64_t idbbigdec2(const string& str)
-{
-  int64_t val = 0;
-  string::size_type idx = string::npos;
-  string tmp(str);
-  idx = tmp.find('.');
-
-  if (idx != string::npos)
-    tmp.erase(idx, 1);
-  else
-    tmp.append("00");
-
-  val = atoll(tmp.c_str());
-  return val;
-}
-
-int32_t idbdate(const string& str)
-{
-  date_image di;
-  di.i = 0;
-  boost::char_separator<char> sep("-");
-  tokenizer tokens(str, sep);
-  tokenizer::iterator tok_iter = tokens.begin();
-
-  // Note that the spare bits must be set, instead of leaving them
-  // initialized to 0; to be compatible with internal date format.
-  if (distance(tok_iter, tokens.end()) == 3)
-  {
-    di.d.spare = 0x3E;
-    di.d.year = atoi(tok_iter->c_str());
-    ++tok_iter;
-    di.d.month = atoi(tok_iter->c_str());
-    ++tok_iter;
-    di.d.day = atoi(tok_iter->c_str());
-    ++tok_iter;
-  }
-
-  return di.i;
-}
-
-int parseinto(lineitem_image& img, const string& buf)
-{
-  memset(&img, 0, sizeof(img));
-
-  boost::char_separator<char> sep("|");
-  tokenizer tokens(buf, sep);
-  tokenizer::iterator tok_iter = tokens.begin();
-
-  if (distance(tok_iter, tokens.end()) < 16)
-    return -1;
-
-  img.l_orderkey = atoi(tok_iter->c_str());
-  ++tok_iter;
-  img.l_partkey = atoi(tok_iter->c_str());
-  ++tok_iter;
-  img.l_suppkey = atoi(tok_iter->c_str());
-  ++tok_iter;
-  img.l_linenumber = atoll(tok_iter->c_str());
-  ++tok_iter;
-  img.l_quantity = idbbigdec2(tok_iter->c_str());
-  ++tok_iter;
-  img.l_extendedprice = idbbigdec2(tok_iter->c_str());
-  ++tok_iter;
-  img.l_discount = idbbigdec2(tok_iter->c_str());
-  ++tok_iter;
-  img.l_tax = idbbigdec2(tok_iter->c_str());
-  ++tok_iter;
-  img.l_returnflag = tok_iter->at(0);
-  ++tok_iter;
-  img.l_linestatus = tok_iter->at(0);
-  ++tok_iter;
-  img.l_shipdate = idbdate(tok_iter->c_str());
-  ++tok_iter;
-  img.l_commitdate = idbdate(tok_iter->c_str());
-  ++tok_iter;
-  img.l_receiptdate = idbdate(tok_iter->c_str());
-  ++tok_iter;
-  memcpy(&img.l_shipinstruct[0], tok_iter->c_str(), tok_iter->size());
-  ++tok_iter;
-  memcpy(&img.l_shipmode[0], tok_iter->c_str(), tok_iter->size());
-  ++tok_iter;
-  memcpy(&img.l_comment[0], tok_iter->c_str(), tok_iter->size());
-  ++tok_iter;
-
-  return 0;
-}
-
-int unsigned_parseinto(unsigned_lineitem_image& img, const string& buf)
-{
-  memset(&img, 0, sizeof(img));
-
-  boost::char_separator<char> sep("|");
-  tokenizer tokens(buf, sep);
-  tokenizer::iterator tok_iter = tokens.begin();
-
-  if (distance(tok_iter, tokens.end()) < 16)
-    return -1;
-
-  img.l_orderkey = strtoul(tok_iter->c_str(), 0, 10);
-  ++tok_iter;
-  img.l_partkey = strtoul(tok_iter->c_str(), 0, 10);
-  ++tok_iter;
-  img.l_suppkey = strtoul(tok_iter->c_str(), 0, 10);
-  ++tok_iter;
-  img.l_linenumber = strtoull(tok_iter->c_str(), 0, 10);
-  ++tok_iter;
-  img.l_quantity = idbbigdec2(tok_iter->c_str());
-  ++tok_iter;
-  img.l_extendedprice = idbbigdec2(tok_iter->c_str());
-  ++tok_iter;
-  img.l_discount = idbbigdec2(tok_iter->c_str());
-  ++tok_iter;
-  img.l_tax = idbbigdec2(tok_iter->c_str());
-  ++tok_iter;
-  img.l_returnflag = tok_iter->at(0);
-  ++tok_iter;
-  img.l_linestatus = tok_iter->at(0);
-  ++tok_iter;
-  img.l_shipdate = idbdate(tok_iter->c_str());
-  ++tok_iter;
-  img.l_commitdate = idbdate(tok_iter->c_str());
-  ++tok_iter;
-  img.l_receiptdate = idbdate(tok_iter->c_str());
-  ++tok_iter;
-  memcpy(&img.l_shipinstruct[0], tok_iter->c_str(), tok_iter->size());
-  ++tok_iter;
-  memcpy(&img.l_shipmode[0], tok_iter->c_str(), tok_iter->size());
-  ++tok_iter;
-  memcpy(&img.l_comment[0], tok_iter->c_str(), tok_iter->size());
-  ++tok_iter;
-
-  return 0;
-}
-
-}  // namespace
-
-int main(int argc, char** argv)
-{
-  if ((argc > 1) && (strcmp(argv[1], "-h") == 0))
-  {
-    std::cerr << "li2bin [-u]" << std::endl;
-    std::cerr << "  -u Create first 4 fields as unsigned integers" << std::endl;
-    return 0;
-  }
-
-  string input;
-
-
-  getline(cin, input);
-
-  if ((argc > 1) && (strcmp(argv[1], "-u") == 0))
-  {
-    unsigned_lineitem_image i;
-
-    while (!cin.eof())
-    {
-      if (unsigned_parseinto(i, input) == 0)
-        cout.write(reinterpret_cast<const char*>(&i), sizeof(i));
-
-      getline(cin, input);
-    }
-  }
-  else
-  {
-    lineitem_image i;
-
-    while (!cin.eof())
-    {
-      if (parseinto(i, input) == 0)
-        cout.write(reinterpret_cast<const char*>(&i), sizeof(i));
-
-      getline(cin, input);
-    }
-  }
-
-  return 0;
-}

tools/cfread/CMakeLists.txt

@@ -1,77 +0,0 @@
-include_directories(${KDE4_INCLUDES} ${KDE4_INCLUDE_DIR} ${QT_INCLUDES})
-
-# ########## next target ###############
-
-set(cfread_SRCS cfread.cpp)
-
-kde4_add_executable(cfread ${cfread_SRCS})
-
-target_link_libraries(
-    cfread
-    ${KDE4_KDECORE_LIBS}
-    pthread
-    writeengine
-    brm
-    rwlock
-    messageqcpp
-    dl
-    configcpp
-    xml2
-    loggingcpp
-    cacheutils
-    boost_idb
-    pthread
-    rt
-)
-
-install(TARGETS cfread ${INSTALL_TARGETS_DEFAULT_ARGS})
-
-# ########## next target ###############
-
-set(mtread_SRCS mtread.cpp)
-
-kde4_add_executable(mtread ${mtread_SRCS})
-
-target_link_libraries(
-    mtread
-    ${KDE4_KDECORE_LIBS}
-    pthread
-    writeengine
-    brm
-    rwlock
-    messageqcpp
-    dl
-    configcpp
-    xml2
-    loggingcpp
-    cacheutils
-    boost_idb
-    pthread
-    rt
-)
-
-install(TARGETS mtread ${INSTALL_TARGETS_DEFAULT_ARGS})
-
-# ########## install files ###############
-
-# original Makefile.am contents follow:
-
-# $Id: Makefile.am 333 2009-04-03 20:35:04Z rdempsey $ Process this file with automake to produce Makefile.in
-#
-# AM_CPPFLAGS = $(idb_cppflags) AM_CFLAGS = $(idb_cflags) AM_CXXFLAGS = $(idb_cxxflags) AM_LDFLAGS = $(idb_ldflags)
-# bin_PROGRAMS = cfread mtread cfread_SOURCES = cfread.cpp cfread_CPPFLAGS = @idb_common_includes@ $(AM_CPPFLAGS)
-# cfread_LDFLAGS = @idb_common_ldflags@ -lpthread -lwriteengine -lbrm -lrwlock -lmessageqcpp -ldl -lconfigcpp -lxml2
-# -lloggingcpp -lcacheutils -lboost_idb -lpthread -lrt $(AM_LDFLAGS) mtread_SOURCES = mtread.cpp mtread_CPPFLAGS =
-# @idb_common_includes@ $(AM_CPPFLAGS) mtread_LDFLAGS = @idb_common_ldflags@ -lpthread -lwriteengine -lbrm -lrwlock
-# -lmessageqcpp -ldl -lconfigcpp -lxml2 -lloggingcpp -lcacheutils -lboost_idb -lpthread -lrt $(AM_LDFLAGS)
-#
-# test:
-#
-# coverage:
-#
-# leakcheck:
-#
-# docs:
-#
-# bootstrap: install-data-am
-#

tools/cfread/cfread.cpp

@@ -1,183 +0,0 @@
-//
-
-#include <iostream>
-#include <iomanip>
-#include <boost/scoped_array.hpp>
-#include <errno.h>
-#define NDEBUG
-#include <cassert>
-#include "writeengine.h"
-
-using namespace std;
-
-void timespec_sub(const struct timespec& tv1, const struct timespec& tv2, struct timespec& diff)
-{
-  if (tv2.tv_nsec < tv1.tv_nsec)
-  {
-    diff.tv_sec = tv2.tv_sec - tv1.tv_sec - 1;
-    diff.tv_nsec = tv1.tv_nsec - tv2.tv_nsec;
-  }
-  else
-  {
-    diff.tv_sec = tv2.tv_sec - tv1.tv_sec;
-    diff.tv_nsec = tv2.tv_nsec - tv1.tv_nsec;
-  }
-}
-
-// TODO:
-// add getopt for CLAs
-// add threads for reading a file
-// add threads for reading multiple files
-
-// main()
-//
-int main(int argc, char** argv)
-{
-  uint64_t acc = 0;
-  uint64_t readBlocks = 0;  // read size ib blocks
-  uint64_t readSize = 0;    // read size ib bytes
-  uint64_t readBufferSz = 0;
-  const uint64_t blockSize = 8192;
-  char* alignedbuff = 0;
-  boost::scoped_array<char> realbuff;
-  const unsigned pageSize = 4096;  // getpagesize();
-  WriteEngine::FileOp fFileOp;
-  BRM::OID_t oid;
-  char fname[256];
-  struct timespec tm;
-  struct timespec tm2;
-  struct timespec tm3;
-  struct timespec starttm;
-  struct timespec endtm;
-  struct timespec tottm;
-  bool odirect = true;
-  int fd = 0;
-  char response = 'Y';
-
-  if (argc <= 1)
-  {
-    cerr << "usage: testread <oid> <buffer size in blocks>" << endl;
-    return -1;
-  }
-
-  oid = atoi(argv[1]);
-
-  if (oid <= 0)
-    exit(-1);
-
-  if (argc >= 2)
-  {
-    readBlocks = atoi(argv[2]);
-
-    if (readBlocks <= 0)
-      readBlocks = 8;
-  }
-
-  if (argc >= 4)
-  {
-    odirect = false;
-  }
-
-  readSize = readBlocks * blockSize;
-  readBufferSz = readSize + pageSize;
-
-  realbuff.reset(new char[readBufferSz]);
-
-  if (realbuff.get() == 0)
-  {
-    cerr << "thr_popper: Can't allocate space for a whole extent in memory" << endl;
-    return 0;
-  }
-
-  if (fFileOp.getFileName(oid, fname) != WriteEngine::NO_ERROR)
-  {
-    fname[0] = 0;
-    throw std::runtime_error("fileOp.getFileName failed");
-  }
-  else
-  {
-    cout << "Reading oid: " << oid << " od: " << odirect << " file: " << fname << endl;
-  }
-
-#if __LP64__
-  alignedbuff = (char*)((((ptrdiff_t)realbuff.get() >> 12) << 12) + pageSize);
-#else
-  alignedbuff = (char*)(((((ptrdiff_t)realbuff.get() >> 12) << 12) & 0xffffffff) + pageSize);
-#endif
-  idbassert(((ptrdiff_t)alignedbuff - (ptrdiff_t)realbuff.get()) < (ptrdiff_t)pageSize);
-  idbassert(((ptrdiff_t)alignedbuff % pageSize) == 0);
-
-  if (odirect)
-    fd = open(fname, O_RDONLY | O_DIRECT | O_LARGEFILE | O_NOATIME);
-  else
-    fd = open(fname, O_RDONLY | O_LARGEFILE | O_NOATIME);
-
-  if (fd < 0)
-  {
-    cerr << "Open failed" << endl;
-    perror("open");
-    throw runtime_error("Error opening file");
-  }
-
-  while (toupper(response) != 'N')
-  {
-    uint64_t i = 1;
-    uint64_t rCnt = 0;
-    clock_gettime(CLOCK_REALTIME, &starttm);
-
-    while (i != 0)
-    {
-      // clock_gettime(CLOCK_REALTIME, &tm);
-      i = pread(fd, alignedbuff, readSize, acc);
-      // clock_gettime(CLOCK_REALTIME, &tm2);
-      idbassert(i == 0 || i == readSize);
-      idbassert(i % pageSize == 0);
-      idbassert(acc % pageSize == 0);
-
-      if (i < 0 && errno == EINTR)
-      {
-        timespec_sub(tm, tm2, tm3);
-        cout << "* " << i << " " << right << setw(2) << setfill(' ') << tm3.tv_sec << "." << right << setw(9)
-             << setfill('0') << tm3.tv_nsec << endl;
-        continue;
-      }
-      else if (i < 0)
-      {
-        timespec_sub(tm, tm2, tm3);
-        cout << "* i: " << i << " sz: " << readSize << " acc: " << acc << right << setw(2) << setfill(' ')
-             << tm3.tv_sec << " " << right << tm3.tv_nsec << endl;
-        perror("pread");
-        // make loop exit
-        i = 0;
-      }
-
-      acc += i;
-
-      if (i > 0)
-        rCnt++;
-
-      // timespec_sub(tm, tm2, tm3);
-      // cout
-      //	<< i << " "
-      //	<< right << setw(2) << setfill(' ') << tm3.tv_sec << " "
-      //	<< right << tm3.tv_nsec
-      //	<< endl;
-
-    }  // while(acc...
-
-    clock_gettime(CLOCK_REALTIME, &endtm);
-    timespec_sub(starttm, endtm, tottm);
-
-    cout << "Total reads: " << rCnt << " sz: " << acc / (1024 * 1024) << "MB"
-         << " tm: " << tottm.tv_sec << "secs " << tottm.tv_nsec << "ns" << endl;
-
-    cout << "Repeat the last scan[Y,N]?" << endl;
-    cin >> response;
-    acc = 0;
-
-  }  // while response...
-
-  close(fd);
-  return 0;
-
-}  // main

tools/cfread/mtread.cpp

@@ -1,269 +0,0 @@
-//
-
-#include <iostream>
-#include <iomanip>
-#include <getopt.h>
-#include <errno.h>
-#include <boost/thread.hpp>
-#define NDEBUG
-#include <cassert>
-#include "writeengine.h"
-
-using namespace std;
-
-void timespec_sub(const struct timespec& tv1, const struct timespec& tv2, struct timespec& diff)
-{
-  if (tv2.tv_nsec < tv1.tv_nsec)
-  {
-    diff.tv_sec = tv2.tv_sec - tv1.tv_sec - 1;
-    diff.tv_nsec = tv1.tv_nsec - tv2.tv_nsec;
-  }
-  else
-  {
-    diff.tv_sec = tv2.tv_sec - tv1.tv_sec;
-    diff.tv_nsec = tv2.tv_nsec - tv1.tv_nsec;
-  }
-}
-
-// TODO:
-// add threads for reading a file
-// add threads for reading multiple files
-
-struct readThr
-{
- public:
-  readThr(const int oid, const int rSize)
-  {
-    fblockSize = 8192;
-    fpageSize = getpagesize();
-    freadBlocks = rSize;
-    freadSize = rSize * fblockSize;  // read size ib bytes
-    freadBufferSz = freadSize + fpageSize;
-    falignedbuff = 0;
-    foid = oid;
-    facc = 0;
-    memset(fname, 0, sizeof(fname));
-    memset((char*)&ftm, 0, sizeof(ftm));
-    memset((char*)&ftm2, 0, sizeof(ftm2));
-    memset((char*)&ftm3, 0, sizeof(ftm3));
-    memset((char*)&fstarttm, 0, sizeof(fstarttm));
-    memset((char*)&fendtm, 0, sizeof(fendtm));
-    memset((char*)&ftottm, 0, sizeof(ftottm));
-    fodirect = true;
-    fd = 0;
-    cout << "o: " << foid << " r: " << freadBlocks << " b: " << freadBufferSz << endl;
-  }
-
-  void operator()()
-  {
-    WriteEngine::FileOp fFileOp;
-    char frealbuff[freadBufferSz];
-    memset(frealbuff, 0, freadBufferSz);
-
-    if (frealbuff == 0)
-    {
-      cerr << "thr_popper: Can't allocate space for a whole extent in memory" << endl;
-      return;
-    }
-
-    if (fFileOp.getFileName(foid, fname) != WriteEngine::NO_ERROR)
-    {
-      fname[0] = 0;
-      throw std::runtime_error("fileOp.getFileName failed");
-    }
-    else
-    {
-      cout << "Reading oid: " << foid << " od: " << fodirect << " file: " << fname << endl;
-    }
-
-#if __LP64__
-    falignedbuff = (char*)((((ptrdiff_t)frealbuff >> 12) << 12) + fpageSize);
-#else
-    falignedbuff = (char*)(((((ptrdiff_t)frealbuff >> 12) << 12) & 0xffffffff) + fpageSize);
-#endif
-    idbassert(((ptrdiff_t)falignedbuff - (ptrdiff_t)frealbuff) < (ptrdiff_t)fpageSize);
-    idbassert(((ptrdiff_t)falignedbuff % fpageSize) == 0);
-
-    if (fodirect)
-      fd = open(fname, O_RDONLY | O_DIRECT | O_LARGEFILE | O_NOATIME);
-    else
-      fd = open(fname, O_RDONLY | O_LARGEFILE | O_NOATIME);
-
-    if (fd < 0)
-    {
-      cerr << "Open failed" << endl;
-      perror("open");
-      throw runtime_error("Error opening file");
-    }
-
-    uint64_t i = 1;
-    uint64_t rCnt = 0;
-
-    clock_gettime(CLOCK_REALTIME, &fstarttm);
-
-    while (i > 0)
-    {
-      clock_gettime(CLOCK_REALTIME, &ftm);
-      i = pread(fd, falignedbuff, freadSize, facc);
-      clock_gettime(CLOCK_REALTIME, &ftm2);
-
-      idbassert(i == 0 || i == freadSize);
-      idbassert(i % fpageSize == 0);
-      idbassert(facc % fpageSize == 0);
-
-      if (i < 0 && errno == EINTR)
-      {
-        timespec_sub(ftm, ftm2, ftm3);
-        cout << "* " << i << " " << right << setw(2) << setfill(' ') << ftm3.tv_sec << "." << right << setw(9)
-             << setfill('0') << ftm3.tv_nsec << endl;
-        continue;
-      }
-      else if (i < 0)
-      {
-        timespec_sub(ftm, ftm2, ftm3);
-        cout << "* i: " << i << " sz: " << freadSize << " acc: " << facc << right << setw(2) << setfill(' ')
-             << ftm3.tv_sec << " " << right << ftm3.tv_nsec << endl;
-        perror("pread");
-      }
-
-      facc += i;
-
-      if (i > 0)
-        rCnt++;
-
-      /**
-      timespec_sub(ftm, ftm2, ftm3);
-      cout
-          << rCnt << " " << facc/(1024*1024)
-          << right << setw(2) << setfill(' ') << ftm3.tv_sec << "."
-          << right << ftm3.tv_nsec << " i: " << i/(1024*1024)
-          << endl;
-      **/
-
-    }  // while(acc...
-
-    clock_gettime(CLOCK_REALTIME, &fendtm);
-    timespec_sub(fstarttm, fendtm, ftottm);
-
-    cout << "Total reads: " << rCnt << " sz: " << facc / (1024 * 1024) << "MB"
-         << " tm: " << ftottm.tv_sec << "secs " << ftottm.tv_nsec << "ns" << endl;
-
-    facc = 0;
-    close(fd);
-  }  // operator()
-
- public:
-  uint64_t facc;
-  uint64_t freadBlocks;  // read size ib blocks
-  uint64_t freadSize;    // read size ib bytes
-  uint64_t freadBufferSz;
-  uint64_t fblockSize;
-  char* falignedbuff;
-  unsigned fpageSize;
-  BRM::OID_t foid;
-  char fname[256];
-  struct timespec ftm;
-  struct timespec ftm2;
-  struct timespec ftm3;
-  struct timespec fstarttm;
-  struct timespec fendtm;
-  struct timespec ftottm;
-  bool fodirect;
-  int fd;
-
-};  // struct readThr
-
-//
-void usage()
-{
-  cerr << "usage: mtread -o <oid> -s <read size in blocks>" << endl;
-}
-
-// main()
-//
-int main(int argc, char** argv)
-{
-  int ch = 0;
-  int readBlocks = 0;
-  BRM::OID_t oid = 0;
-  std::vector<BRM::OID_t> oidList;
-
-  enum CLA_ENUM
-  {
-    OID = (int)0,
-    READSIZE
-  };
-
-  // longopt struct
-  // struct option {
-  // const char *name;
-  // int has_arg;
-  // int *flag;
-  // int val;
-  //};
-
-  static struct option long_options[] = {//{const char *name, 	int has_arg, 	int *flag,	int val},
-                                         {"oid", required_argument, NULL, OID},
-                                         {"rsize", required_argument, NULL, READSIZE},
-                                         {0, 0, 0, 0}};
-
-  if (argc <= 1)
-  {
-    return -1;
-  }
-
-  // process command line arguments
-  while ((ch = getopt_long_only(argc, argv, "o:s:", long_options, NULL)) != -1)
-  {
-    // pid_t pidId = getpid();
-    switch (ch)
-    {
-      case OID:
-      case 'o':
-        oid = atoi(optarg);
-        oidList.push_back(oid);
-        cout << "oid: " << optarg << endl;
-        break;
-
-      case READSIZE:
-      case 's':
-        readBlocks = atoi(optarg);
-        cout << "read size: " << optarg << endl;
-
-        if (readBlocks <= 0)
-          readBlocks = 1;
-
-        break;
-
-      case '?':
-      default:
-        cout << "optarg " << optarg << endl;
-        usage();
-        break;
-
-    }  // switch
-
-  }  // while...
-
-  uint32_t idx = 0;
-  std::vector<boost::thread*> thrList;
-
-  while (idx < oidList.size())
-  {
-    struct readThr rdr(oidList[idx++], readBlocks);
-    boost::thread* thr = new boost::thread(rdr);
-    thrList.push_back(thr);
-  }
-
-  idx = 0;
-
-  while (idx < thrList.size())
-  {
-    boost::thread* thr = thrList[idx++];
-    thr->join();
-    delete thr;
-  }
-
-  thrList.clear();
-
-}  // main

tools/clearShm/CMakeLists.txt

@@ -4,12 +4,6 @@ include_directories(${ENGINE_COMMON_INCLUDES})
 
 set(clearShm_SRCS main.cpp)
 
-add_executable(clearShm ${clearShm_SRCS})
+columnstore_executable(clearShm ${clearShm_SRCS})
 
-target_link_libraries(clearShm ${ENGINE_LDFLAGS} ${ENGINE_EXEC_LIBS})
+columnstore_link(clearShm ${ENGINE_LDFLAGS} ${ENGINE_EXEC_LIBS})
-
-install(
-    TARGETS clearShm
-    DESTINATION ${ENGINE_BINDIR}
-    COMPONENT columnstore-engine
-)

tools/cleartablelock/CMakeLists.txt

@@ -4,12 +4,6 @@ include_directories(${ENGINE_COMMON_INCLUDES})
 
 set(cleartablelock_SRCS cleartablelock.cpp cleartablelockthread.cpp)
 
-add_executable(cleartablelock ${cleartablelock_SRCS})
+columnstore_executable(cleartablelock ${cleartablelock_SRCS})
 
-target_link_libraries(cleartablelock ${ENGINE_LDFLAGS} ${NETSNMP_LIBRARIES} ${ENGINE_WRITE_LIBS})
+columnstore_link(cleartablelock ${ENGINE_LDFLAGS} ${NETSNMP_LIBRARIES} ${ENGINE_WRITE_LIBS})
-
-install(
-    TARGETS cleartablelock
-    DESTINATION ${ENGINE_BINDIR}
-    COMPONENT columnstore-engine
-)

tools/configMgt/CMakeLists.txt

@@ -5,7 +5,4 @@ include_directories(${ENGINE_COMMON_INCLUDES})
 set(autoConfigure_SRCS autoConfigure.cpp)
 
 add_executable(autoConfigure ${autoConfigure_SRCS})
-
+columnstore_link(autoConfigure ${ENGINE_LDFLAGS} ${NETSNMP_LIBRARIES} ${ENGINE_EXEC_LIBS})
-target_link_libraries(autoConfigure ${ENGINE_LDFLAGS} ${NETSNMP_LIBRARIES} ${ENGINE_EXEC_LIBS})
-
-# install(TARGETS autoConfigure DESTINATION ${ENGINE_BINDIR} COMPONENT columnstore-engine)

tools/cplogger/CMakeLists.txt

@@ -4,12 +4,6 @@ include_directories(${ENGINE_COMMON_INCLUDES})
 
 set(cplogger_SRCS main.cpp)
 
-add_executable(cplogger ${cplogger_SRCS})
+columnstore_executable(cplogger ${cplogger_SRCS})
 
-target_link_libraries(cplogger ${ENGINE_LDFLAGS} ${ENGINE_EXEC_LIBS})
+columnstore_link(cplogger ${ENGINE_LDFLAGS} ${ENGINE_EXEC_LIBS})
-
-install(
-    TARGETS cplogger
-    DESTINATION ${ENGINE_BINDIR}
-    COMPONENT columnstore-engine
-)

tools/dbbuilder/CMakeLists.txt

@@ -4,12 +4,6 @@ include_directories(${ENGINE_COMMON_INCLUDES})
 
 set(dbbuilder_SRCS dbbuilder.cpp systemcatalog.cpp)
 
-add_executable(dbbuilder ${dbbuilder_SRCS})
+columnstore_executable(dbbuilder ${dbbuilder_SRCS})
 
-target_link_libraries(dbbuilder ${ENGINE_LDFLAGS} ${NETSNMP_LIBRARIES} ${ENGINE_WRITE_LIBS})
+columnstore_link(dbbuilder ${ENGINE_LDFLAGS} ${NETSNMP_LIBRARIES} ${ENGINE_WRITE_LIBS})
-
-install(
-    TARGETS dbbuilder
-    DESTINATION ${ENGINE_BINDIR}
-    COMPONENT columnstore-engine
-)

tools/dbbuilder/dbbuilder.cpp

@@ -26,6 +26,7 @@
 #include <stdexcept>
 using namespace std;
 
+#include "basic/string_utils.h"
 #include <boost/algorithm/string.hpp>
 
 #include "mcsconfig.h"
@@ -202,7 +203,11 @@ int main(int argc, char* argv[])
             (iter->second).second = true;
             isUpgrade = true;
           }
-          messageHandler("", std::string("Upgrade flag is ") + std::to_string(isUpgrade) + std::string(" after checking upgrade candidate OID ") + oam.itoa(iter->first) + std::string(" "), false);
+          messageHandler("",
+                         std::string("Upgrade flag is ") + std::to_string(isUpgrade) +
+                             std::string(" after checking upgrade candidate OID ") + oam.itoa(iter->first) +
+                             std::string(" "),
+                         false);
         }
 
         if (!isUpgrade)

tools/dbloadxml/CMakeLists.txt

@@ -4,20 +4,14 @@ include_directories(${ENGINE_COMMON_INCLUDES})
 
 set(dbload_STAT_SRCS inputmgr.cpp)
 
-add_library(dbload STATIC ${dbload_STAT_SRCS})
+columnstore_static_library(dbload ${dbload_STAT_SRCS})
 
-add_dependencies(dbload loggingcpp)
+columnstore_link(dbload loggingcpp)
 
 # ########## next target ###############
 
 set(colxml_SRCS colxml.cpp)
 
-add_executable(colxml ${colxml_SRCS})
+columnstore_executable(colxml ${colxml_SRCS})
 
-target_link_libraries(colxml ${ENGINE_LDFLAGS} dbload ${ENGINE_WRITE_LIBS})
+columnstore_link(colxml ${ENGINE_LDFLAGS} dbload ${ENGINE_WRITE_LIBS})
-
-install(
-    TARGETS colxml
-    DESTINATION ${ENGINE_BINDIR}
-    COMPONENT columnstore-engine
-)

tools/ddlcleanup/CMakeLists.txt

@@ -4,12 +4,5 @@ include_directories(${ENGINE_COMMON_INCLUDES})
 
 set(ddlcleanup_SRCS ddlcleanup.cpp)
 
-add_executable(ddlcleanup ${ddlcleanup_SRCS})
+columnstore_executable(ddlcleanup ${ddlcleanup_SRCS})
-
+columnstore_link(ddlcleanup ${ENGINE_LDFLAGS} ${NETSNMP_LIBRARIES} ${ENGINE_WRITE_LIBS} ddlcleanuputil)
-target_link_libraries(ddlcleanup ${ENGINE_LDFLAGS} ${NETSNMP_LIBRARIES} ${ENGINE_WRITE_LIBS} ddlcleanuputil)
-
-install(
-    TARGETS ddlcleanup
-    DESTINATION ${ENGINE_BINDIR}
-    COMPONENT columnstore-engine
-)

tools/ddldriver/CMakeLists.txt

@@ -1,43 +0,0 @@
-include_directories(${KDE4_INCLUDES} ${KDE4_INCLUDE_DIR} ${QT_INCLUDES})
-
-# ########## next target ###############
-
-set(ddldriver_SRCS ddldriver.cpp)
-
-kde4_add_executable(ddldriver ${ddldriver_SRCS})
-
-target_link_libraries(ddldriver ${KDE4_KDECORE_LIBS})
-
-install(TARGETS ddldriver ${INSTALL_TARGETS_DEFAULT_ARGS})
-
-# ########## install files ###############
-
-# original Makefile.am contents follow:
-
-# Copyright (C) 2014 InfiniDB, Inc.
-#
-# This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public
-# License as published by the Free Software Foundation; version 2 of the License.
-#
-# This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied
-# warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License along with this program; if not, write to the Free
-# Software Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
-#
-# $Id: Makefile.am 333 2009-04-03 20:35:04Z rdempsey $ Process this file with automake to produce Makefile.in
-#
-# AM_CPPFLAGS = $(idb_cppflags) AM_CFLAGS = $(idb_cflags) AM_CXXFLAGS = $(idb_cxxflags) AM_LDFLAGS = $(idb_ldflags)
-# bin_PROGRAMS = ddldriver ddldriver_SOURCES = ddldriver.cpp ddldriver_CPPFLAGS = @idb_common_includes@ $(AM_CPPFLAGS)
-# ddldriver_LDFLAGS = @idb_common_ldflags@ @idb_write_libs@ @idb_common_libs@ @netsnmp_libs@ $(AM_LDFLAGS)
-#
-# test:
-#
-# coverage:
-#
-# leakcheck:
-#
-# docs:
-#
-# bootstrap: install-data-am
-#

tools/ddldriver/ddldriver.cpp

@@ -1,133 +0,0 @@
-/* Copyright (C) 2014 InfiniDB, Inc.
-
-   This program is free software; you can redistribute it and/or
-   modify it under the terms of the GNU General Public License
-   as published by the Free Software Foundation; version 2 of
-   the License.
-
-   This program is distributed in the hope that it will be useful,
-   but WITHOUT ANY WARRANTY; without even the implied warranty of
-   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-   GNU General Public License for more details.
-
-   You should have received a copy of the GNU General Public License
-   along with this program; if not, write to the Free Software
-   Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
-   MA 02110-1301, USA. */
-
-// $Id: ddldriver.cpp 2101 2013-01-21 14:12:52Z rdempsey $
-
-#include <unistd.h>
-#include <cstdio>
-#include <iostream>
-#include <cctype>
-#include <sstream>
-using namespace std;
-
-#include "ddlpkg.h"
-#include "sqlparser.h"
-using namespace ddlpackage;
-
-#include "bytestream.h"
-#include "messagequeue.h"
-using namespace messageqcpp;
-
-namespace
-{
-void usage()
-{
-  cout << "usage: ddlriver [-h] schema sql_text" << endl;
-}
-const string toupper_(const string& in)
-{
-  string::const_iterator iter = in.begin();
-  string::const_iterator end = in.end();
-  ostringstream oss;
-
-  while (iter != end)
-  {
-    oss << static_cast<char>(toupper(*iter));
-    ++iter;
-  }
-
-  return oss.str();
-}
-}  // namespace
-
-int main(int argc, char** argv)
-{
-  int c;
-
-  opterr = 0;
-
-  while ((c = getopt(argc, argv, "h")) != EOF)
-    switch (c)
-    {
-      case 'h':
-      case '?':
-      default:
-        usage();
-        return (c == 'h' ? 0 : 1);
-        break;
-    }
-
-  if (argc - optind < 2)
-  {
-    usage();
-    return 1;
-  }
-
-  string owner(toupper_(argv[optind++]));
-
-  SqlParser parser;
-  parser.setDefaultSchema(owner);
-
-  string stmtStr(toupper_(argv[optind++]));
-  parser.Parse(stmtStr.c_str());
-
-  if (!parser.Good())
-  {
-    cerr << "Failed to parse statement: " << stmtStr << endl;
-    return 1;
-  }
-
-  const ParseTree& ptree = parser.GetParseTree();
-  SqlStatement& stmt = *ptree.fList[0];
-
-  stmt.fSessionID = 1;
-  stmt.fSql = stmtStr;
-  stmt.fOwner = owner;
-
-  ByteStream bytestream;
-  bytestream << stmt.fSessionID;
-  stmt.serialize(bytestream);
-  MessageQueueClient mq("DDLProc");
-  ByteStream::byte b;
-  string errorMsg;
-
-  try
-  {
-    mq.write(bytestream);
-    bytestream = mq.read();
-    bytestream >> b;
-    bytestream >> errorMsg;
-  }
-  catch (runtime_error& rex)
-  {
-    cerr << "runtime_error in engine: " << rex.what() << endl;
-    return 1;
-  }
-  catch (...)
-  {
-    cerr << "uknown error in engine" << endl;
-    return 1;
-  }
-
-  if (b != 0)
-  {
-    cerr << "DDLProc error: " << errorMsg << endl;
-    return 1;
-  }
-
-  return 0;
-}

tools/dmldriver/CMakeLists.txt

@@ -1,44 +0,0 @@
-include_directories(${KDE4_INCLUDES} ${KDE4_INCLUDE_DIR} ${QT_INCLUDES})
-
-# ########## next target ###############
-
-set(dmldriver_SRCS dmldriver.cpp tpchrf2.cpp dmlif.cpp)
-
-kde4_add_executable(dmldriver ${dmldriver_SRCS})
-
-target_link_libraries(dmldriver ${KDE4_KDECORE_LIBS})
-
-install(TARGETS dmldriver ${INSTALL_TARGETS_DEFAULT_ARGS})
-
-# ########## install files ###############
-
-# original Makefile.am contents follow:
-
-# Copyright (C) 2014 InfiniDB, Inc.
-#
-# This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public
-# License as published by the Free Software Foundation; version 2 of the License.
-#
-# This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied
-# warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License along with this program; if not, write to the Free
-# Software Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
-#
-# $Id: Makefile.am 333 2009-04-03 20:35:04Z rdempsey $ Process this file with automake to produce Makefile.in
-#
-# AM_CPPFLAGS = $(idb_cppflags) AM_CFLAGS = $(idb_cflags) AM_CXXFLAGS = $(idb_cxxflags) AM_LDFLAGS = $(idb_ldflags)
-# bin_PROGRAMS = dmldriver dmldriver_SOURCES = dmldriver.cpp tpchrf2.cpp dmlif.cpp dmldriver_CPPFLAGS =
-# @idb_common_includes@ $(AM_CPPFLAGS) dmldriver_LDFLAGS = @idb_common_ldflags@ @idb_write_libs@ @idb_common_libs@
-# @netsnmp_libs@ $(AM_LDFLAGS)
-#
-# test:
-#
-# coverage:
-#
-# leakcheck:
-#
-# docs:
-#
-# bootstrap: install-data-am
-#

tools/dmldriver/dmldriver.cpp

@@ -1,205 +0,0 @@
-/* Copyright (C) 2014 InfiniDB, Inc.
-
-   This program is free software; you can redistribute it and/or
-   modify it under the terms of the GNU General Public License
-   as published by the Free Software Foundation; version 2 of
-   the License.
-
-   This program is distributed in the hope that it will be useful,
-   but WITHOUT ANY WARRANTY; without even the implied warranty of
-   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-   GNU General Public License for more details.
-
-   You should have received a copy of the GNU General Public License
-   along with this program; if not, write to the Free Software
-   Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
-   MA 02110-1301, USA. */
-
-// $Id: dmldriver.cpp 2101 2013-01-21 14:12:52Z rdempsey $
-
-//#define NDEBUG
-#include <cassert>
-#include <iostream>
-#include <string>
-#include <unistd.h>
-#include <cctype>
-#include <sstream>
-#include <fstream>
-#include <limits>
-using namespace std;
-
-#include <boost/scoped_array.hpp>
-#include <boost/tokenizer.hpp>
-using namespace boost;
-
-#include "tpchrf2.h"
-#include "dmlif.h"
-
-namespace
-{
-bool vflg;
-bool dflg;
-
-void usage()
-{
-  cout << "usage: dmldriver [-vhd] [-c intvl] [-f file] [-s sid] [-t flgs] [-r file] [-p cnt] [sql_text]"
-       << endl;
-  cout << "   -c intvl \tcommit every intvl statements" << endl;
-  cout << "   -f file  \tread statements from file (max 15KB/stmt)" << endl;
-  cout << "   -s sid   \tset sid as session id" << endl;
-  cout << "   -t flgs  \tset trace flags" << endl;
-  cout << "   -v       \tdisplay affected row count(s)" << endl;
-  cout << "   -d       \tdisplay debug info" << endl;
-  cout << "   -r file  \tread orderkeys from file for TPC-H RF2" << endl;
-  cout << "   -p cnt   \tpack cnt orderkeys into each delete stmt (only w/ -r)" << endl;
-  cout << "   -e schema\tset the schema name (only w/ -r)" << endl;
-  cout << "   -h       \tdisplay this help text" << endl;
-}
-}  // namespace
-
-int main(int argc, char** argv)
-{
-  int c;
-
-  opterr = 0;
-
-  vflg = false;
-  dflg = false;
-
-  bool fflg = false;
-  string infilename;
-
-  int cIntvl = numeric_limits<int>::max();
-
-  uint32_t sessionID = time(0) & 0x7fffffff;
-
-  bool rflg = false;
-
-  int packCnt = 1;
-
-  uint32_t tflg = 0;
-
-  string schema;
-
-  while ((c = getopt(argc, argv, "e:t:s:c:f:r:p:vhd")) != EOF)
-    switch (c)
-    {
-      case 'v': vflg = true; break;
-
-      case 'd': dflg = true; break;
-
-      case 'f':
-        fflg = true;
-        infilename = optarg;
-        break;
-
-      case 'c': cIntvl = static_cast<int>(strtol(optarg, 0, 0)); break;
-
-      case 's': sessionID = static_cast<uint32_t>(strtoul(optarg, 0, 0)); break;
-
-      case 't': tflg = static_cast<uint32_t>(strtoul(optarg, 0, 0)); break;
-
-      case 'r':
-        rflg = true;
-        infilename = optarg;
-        break;
-
-      case 'p': packCnt = static_cast<int>(strtol(optarg, 0, 0)); break;
-
-      case 'e': schema = optarg; break;
-
-      case 'h':
-      case '?':
-      default:
-        usage();
-        return (c == 'h' ? 0 : 1);
-        break;
-    }
-
-  if (!fflg && !rflg && ((argc - optind) < 1))
-  {
-    usage();
-    return 1;
-  }
-
-  if (fflg && rflg)
-  {
-    cout << "-f and -r are mutually exclusive!" << endl << endl;
-    usage();
-    return 1;
-  }
-
-  if (!schema.empty() && !rflg)
-  {
-    cout << "-e requires -r!" << endl << endl;
-    usage();
-    return 1;
-  }
-
-  string stmtStr;
-
-  if (!fflg && !rflg)
-    stmtStr = argv[optind++];
-
-  int rc = 0;
-
-  dmlif::DMLIF dmlif(sessionID, tflg, dflg, vflg);
-
-  if (fflg)
-  {
-    ifstream ifs(infilename.c_str());
-
-    if (!ifs.good())
-    {
-      cerr << "Error accessing file " << infilename << endl;
-      return 1;
-    }
-
-    const streamsize ilinelen = 15 * 1024;
-    scoped_array<char> iline(new char[ilinelen]);
-    int cnt = 0;
-
-    for (;;)
-    {
-      ifs.getline(iline.get(), ilinelen);
-
-      if (ifs.eof())
-        break;
-
-      rc = dmlif.sendOne(iline.get());
-
-      if (rc != 0)
-        break;
-
-      cnt++;
-
-      if ((cnt % cIntvl) == 0)
-        dmlif.sendOne("COMMIT;");
-    }
-  }
-  else if (rflg)
-  {
-    ifstream ifs(infilename.c_str());
-
-    if (!ifs.good())
-    {
-      cerr << "Error accessing file " << infilename << endl;
-      return 1;
-    }
-
-    if (schema.empty())
-      schema = "tpch";
-
-    tpch::RF2 rf2(schema, sessionID, tflg, cIntvl, packCnt, dflg, vflg);
-    rc = rf2.run(ifs);
-  }
-  else
-  {
-    rc = dmlif.sendOne(stmtStr);
-  }
-
-  if (rc == 0)
-    dmlif.sendOne("COMMIT;");
-
-  return 0;
-}

tools/dmldriver/dmlif.cpp

@@ -1,384 +0,0 @@
-/* Copyright (C) 2014 InfiniDB, Inc.
-
-   This program is free software; you can redistribute it and/or
-   modify it under the terms of the GNU General Public License
-   as published by the Free Software Foundation; version 2 of
-   the License.
-
-   This program is distributed in the hope that it will be useful,
-   but WITHOUT ANY WARRANTY; without even the implied warranty of
-   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-   GNU General Public License for more details.
-
-   You should have received a copy of the GNU General Public License
-   along with this program; if not, write to the Free Software
-   Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
-   MA 02110-1301, USA. */
-
-// $Id: dmlif.cpp 2101 2013-01-21 14:12:52Z rdempsey $
-
-//#define NDEBUG
-#include <cassert>
-#include <string>
-#include <sstream>
-using namespace std;
-
-#include <boost/tokenizer.hpp>
-using namespace boost;
-
-#include "vendordmlstatement.h"
-#include "calpontdmlpackage.h"
-#include "calpontdmlfactory.h"
-using namespace dmlpackage;
-
-#include "bytestream.h"
-#include "messagequeue.h"
-using namespace messageqcpp;
-
-#include "simplecolumn.h"
-#include "calpontselectexecutionplan.h"
-#include "sessionmanager.h"
-#include "simplefilter.h"
-#include "constantcolumn.h"
-#include "constantfilter.h"
-using namespace execplan;
-
-#include "brmtypes.h"
-
-#include "dmlif.h"
-using namespace dmlif;
-
-namespace dmlif
-{
-DMLIF::DMLIF(uint32_t sessionid, uint32_t tflg, bool dflg, bool vflg)
- : fSessionID(sessionid), fTflg(tflg), fDflg(dflg), fVflg(vflg), fOPt(0), fLPt(0)
-{
-  fMqp.reset(new MessageQueueClient("DMLProc"));
-}
-
-DMLIF::~DMLIF()
-{
-}
-
-int DMLIF::sendOne(const string& stmt)
-{
-  int rc;
-
-  string tStmt(stmt);
-
-  if (*tStmt.rbegin() != ';')
-    tStmt += ";";
-
-  VendorDMLStatement dmlStmt(tStmt, fSessionID);
-  CalpontDMLPackage* pDMLPackage = CalpontDMLFactory::makeCalpontDMLPackage(dmlStmt);
-
-  if (pDMLPackage == 0)
-  {
-    cerr << "Failed to parse statement: " << tStmt << endl;
-    return -1;
-  }
-
-  string queryString = pDMLPackage->get_QueryString();
-
-  if (fDflg)
-    cout << "qs: >" << queryString << '<' << endl;
-
-  string dmlStatement = pDMLPackage->get_DMLStatement();
-
-  if (fDflg)
-    cout << "DML: " << dmlStatement << endl;
-
-  bool isDML = true;
-
-  if (dmlStatement == "COMMIT" || dmlStatement == "ROLLBACK")
-  {
-    isDML = false;
-  }
-
-  if (isDML)
-  {
-    char_separator<char> sep(" ");
-    tokenizer<char_separator<char> > tok(queryString, sep);
-    tokenizer<char_separator<char> >::iterator iter = tok.begin();
-    idbassert(iter != tok.end());
-    string where = *iter;
-    ++iter;
-    idbassert(iter != tok.end());
-    string col1 = *iter;
-    ++iter;
-    idbassert(iter != tok.end());
-    string op = *iter;
-    ++iter;
-    idbassert(iter != tok.end());
-    string col2 = *iter;
-    ++iter;
-    idbassert(iter == tok.end());
-
-    if (fDflg)
-      cout << "SQL: " << pDMLPackage->get_SQLStatement() << endl;
-
-    if (fDflg)
-      cout << "hf: " << pDMLPackage->HasFilter() << endl;
-
-    DMLTable* tp = pDMLPackage->get_Table();
-
-    if (fDflg)
-      cout << "sn: " << tp->get_SchemaName() << " tn: " << tp->get_TableName() << endl;
-
-    if (fDflg)
-      cout << "row count: " << tp->get_RowList().size() << endl;
-
-    SRCP srcp(new SimpleColumn(tp->get_SchemaName(), tp->get_TableName(), col1, fSessionID));
-    CalpontSelectExecutionPlan::ColumnMap cm;
-    cm.insert(make_pair(col1, srcp));
-    pDMLPackage->get_ExecutionPlan()->columnMap(cm);
-    CalpontSelectExecutionPlan::ReturnedColumnList rcl;
-    rcl.push_back(srcp);
-    pDMLPackage->get_ExecutionPlan()->returnedCols(rcl);
-    pDMLPackage->get_ExecutionPlan()->sessionID(fSessionID);
-    pDMLPackage->get_ExecutionPlan()->traceFlags(fTflg);
-    SessionManager sm;
-    BRM::TxnID txnid = sm.getTxnID(fSessionID);
-
-    if (!txnid.valid)
-      txnid = sm.newTxnID(fSessionID);
-
-    pDMLPackage->get_ExecutionPlan()->txnID(txnid.id);
-    pDMLPackage->get_ExecutionPlan()->verID(sm.verID());
-    ParseTree* pt = new ParseTree();
-    ReturnedColumn* rc1 = srcp->clone();
-    ReturnedColumn* rc2 = new ConstantColumn(col2, ConstantColumn::NUM);
-    SOP sop(new Operator(op));
-    SimpleFilter* sf = new SimpleFilter(sop, rc1, rc2);
-    pt->data(sf);
-    pDMLPackage->get_ExecutionPlan()->filters(pt);
-
-    if (fDflg)
-      cout << "ep: " << *pDMLPackage->get_ExecutionPlan() << endl;
-  }
-
-  ByteStream bytestream;
-  pDMLPackage->write(bytestream);
-  delete pDMLPackage;
-  ByteStream::octbyte rows;
-
-  rc = DMLSend(bytestream, rows);
-
-  if (isDML && fVflg)
-    cout << rows << " rows affected" << endl;
-
-  return rc;
-}
-
-int DMLIF::DMLSend(ByteStream& bytestream, ByteStream::octbyte& rows)
-{
-  ByteStream::byte b;
-  string errorMsg;
-
-  try
-  {
-    fMqp->connect();
-    fMqp->write(bytestream);
-    bytestream = fMqp->read();
-    fMqp->shutdown();
-
-    if (fDflg)
-      cout << "read " << bytestream.length() << " bytes from DMLProc" << endl;
-
-    bytestream >> b;
-
-    if (fDflg)
-      cout << "b = " << (int)b << endl;
-
-    bytestream >> rows;
-
-    if (fDflg)
-      cout << "rows = " << rows << endl;
-
-    bytestream >> errorMsg;
-
-    if (fDflg)
-      cout << "errorMsg = " << errorMsg << endl;
-  }
-  catch (runtime_error& rex)
-  {
-    cerr << "runtime_error in engine: " << rex.what() << endl;
-    return -1;
-  }
-  catch (...)
-  {
-    cerr << "uknown error in engine" << endl;
-    return -1;
-  }
-
-  if (b != 0)
-  {
-    cerr << "DMLProc error: " << errorMsg << endl;
-    return -1;
-  }
-
-  return 0;
-}
-
-void DMLIF::rf2Start(const string& sn)
-{
-  fSchema = sn;
-  fOFilterStr = "";
-  fLFilterStr = "";
-  fOPt = 0;
-  fLPt = 0;
-}
-
-void DMLIF::rf2Add(int64_t okey)
-{
-  ostringstream oss;
-  oss << okey;
-  string okeyStr(oss.str());
-
-  if (fOFilterStr.empty())
-  {
-    fOFilterStr = "o_orderkey=" + okeyStr;
-    ReturnedColumn* rc1 = new SimpleColumn(fSchema, "orders", "o_orderkey", fSessionID);
-    ReturnedColumn* rc2 = new ConstantColumn(okeyStr, ConstantColumn::NUM);
-    SOP sop(new Operator("="));
-    ConstantFilter* cf = new ConstantFilter(sop, rc1, rc2);
-    sop.reset(new Operator("or"));
-    cf->op(sop);
-    fOPt = new ParseTree(cf);
-  }
-  else
-  {
-    fOFilterStr += " or o_orderkey=" + okeyStr;
-    ReturnedColumn* rc1 = new SimpleColumn(fSchema, "orders", "o_orderkey", fSessionID);
-    ReturnedColumn* rc2 = new ConstantColumn(okeyStr, ConstantColumn::NUM);
-    SOP sop(new Operator("="));
-    ConstantFilter* cf = dynamic_cast<ConstantFilter*>(fOPt->data());
-    cf->pushFilter(new SimpleFilter(sop, rc1, rc2));
-  }
-
-  if (fLFilterStr.empty())
-  {
-    fLFilterStr = "l_orderkey=" + okeyStr;
-    ReturnedColumn* rc1 = new SimpleColumn(fSchema, "lineitem", "l_orderkey", fSessionID);
-    ReturnedColumn* rc2 = new ConstantColumn(okeyStr, ConstantColumn::NUM);
-    SOP sop(new Operator("="));
-    ConstantFilter* cf = new ConstantFilter(sop, rc1, rc2);
-    sop.reset(new Operator("or"));
-    cf->op(sop);
-    fLPt = new ParseTree(cf);
-  }
-  else
-  {
-    fLFilterStr += " or l_orderkey=" + okeyStr;
-    ReturnedColumn* rc1 = new SimpleColumn(fSchema, "lineitem", "l_orderkey", fSessionID);
-    ReturnedColumn* rc2 = new ConstantColumn(okeyStr, ConstantColumn::NUM);
-    SOP sop(new Operator("="));
-    ConstantFilter* cf = dynamic_cast<ConstantFilter*>(fLPt->data());
-    cf->pushFilter(new SimpleFilter(sop, rc1, rc2));
-  }
-}
-
-int DMLIF::rf2Send()
-{
-  if (fOFilterStr.empty())
-    return -1;
-
-  int rc = 0;
-  string dmlstr;
-  dmlstr = "delete from " + fSchema + ".orders where " + fOFilterStr + ';';
-
-  if (fDflg)
-    cout << dmlstr << endl;
-
-  VendorDMLStatement dmlStmt(dmlstr, fSessionID);
-  CalpontDMLPackage* pDMLPackage = CalpontDMLFactory::makeCalpontDMLPackage(dmlStmt);
-
-  if (pDMLPackage == 0)
-  {
-    cerr << "Failed to parse statement: " << dmlstr << endl;
-    return -1;
-  }
-
-  SRCP srcp(new SimpleColumn(fSchema, "orders", "o_orderkey", fSessionID));
-  CalpontSelectExecutionPlan::ColumnMap cm;
-  cm.insert(make_pair("o_orderkey", srcp));
-  pDMLPackage->get_ExecutionPlan()->columnMap(cm);
-  CalpontSelectExecutionPlan::ReturnedColumnList rcl;
-  rcl.push_back(srcp);
-  pDMLPackage->get_ExecutionPlan()->returnedCols(rcl);
-  pDMLPackage->get_ExecutionPlan()->sessionID(fSessionID);
-  pDMLPackage->get_ExecutionPlan()->traceFlags(fTflg);
-  SessionManager sm;
-  BRM::TxnID txnid = sm.getTxnID(fSessionID);
-
-  if (!txnid.valid)
-    txnid = sm.newTxnID(fSessionID);
-
-  pDMLPackage->get_ExecutionPlan()->txnID(txnid.id);
-  pDMLPackage->get_ExecutionPlan()->verID(sm.verID());
-  pDMLPackage->get_ExecutionPlan()->filters(fOPt);
-
-  if (fDflg)
-    cout << "ep: " << *pDMLPackage->get_ExecutionPlan() << endl;
-
-  ByteStream bytestream;
-  pDMLPackage->write(bytestream);
-  delete pDMLPackage;
-  pDMLPackage = 0;
-  ByteStream::octbyte rows = 0;
-
-  rc = DMLSend(bytestream, rows);
-
-  if (fVflg)
-    cout << rows << " rows affected" << endl;
-
-  dmlstr = "delete from " + fSchema + ".lineitem where " + fLFilterStr + ';';
-
-  if (fDflg)
-    cout << dmlstr << endl;
-
-  VendorDMLStatement dmlStmt1(dmlstr, fSessionID);
-  pDMLPackage = CalpontDMLFactory::makeCalpontDMLPackage(dmlStmt1);
-
-  if (pDMLPackage == 0)
-  {
-    cerr << "Failed to parse statement: " << dmlstr << endl;
-    return -1;
-  }
-
-  srcp.reset(new SimpleColumn(fSchema, "lineitem", "l_orderkey", fSessionID));
-  cm.clear();
-  cm.insert(make_pair("l_orderkey", srcp));
-  pDMLPackage->get_ExecutionPlan()->columnMap(cm);
-  rcl.clear();
-  rcl.push_back(srcp);
-  pDMLPackage->get_ExecutionPlan()->returnedCols(rcl);
-  pDMLPackage->get_ExecutionPlan()->sessionID(fSessionID);
-  pDMLPackage->get_ExecutionPlan()->traceFlags(fTflg);
-  txnid = sm.getTxnID(fSessionID);
-
-  if (!txnid.valid)
-    txnid = sm.newTxnID(fSessionID);
-
-  pDMLPackage->get_ExecutionPlan()->txnID(txnid.id);
-  pDMLPackage->get_ExecutionPlan()->verID(sm.verID());
-  pDMLPackage->get_ExecutionPlan()->filters(fLPt);
-
-  if (fDflg)
-    cout << "ep: " << *pDMLPackage->get_ExecutionPlan() << endl;
-
-  bytestream.reset();
-  pDMLPackage->write(bytestream);
-  delete pDMLPackage;
-  pDMLPackage = 0;
-  rows = 0;
-
-  rc = DMLSend(bytestream, rows);
-
-  if (fVflg)
-    cout << rows << " rows affected" << endl;
-
-  return 0;
-}
-
-}  // namespace dmlif

tools/dmldriver/dmlif.h

@@ -1,65 +0,0 @@
-/* Copyright (C) 2014 InfiniDB, Inc.
-
-   This program is free software; you can redistribute it and/or
-   modify it under the terms of the GNU General Public License
-   as published by the Free Software Foundation; version 2 of
-   the License.
-
-   This program is distributed in the hope that it will be useful,
-   but WITHOUT ANY WARRANTY; without even the implied warranty of
-   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-   GNU General Public License for more details.
-
-   You should have received a copy of the GNU General Public License
-   along with this program; if not, write to the Free Software
-   Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
-   MA 02110-1301, USA. */
-
-// $Id: dmlif.h 2101 2013-01-21 14:12:52Z rdempsey $
-
-#pragma once
-
-#include <string>
-#include <cstddef>
-#include <boost/scoped_ptr.hpp>
-
-#include "messagequeue.h"
-#include "expressionparser.h"
-
-namespace dmlif
-{
-class DMLIF
-{
- public:
-  DMLIF(uint32_t sessionid, uint32_t tflg = 0, bool dflg = false, bool vflg = false);
-  ~DMLIF();
-
-  int sendOne(const std::string& stmt);
-
-  void rf2Start(const std::string& sn);
-  void rf2Add(int64_t okey);
-  int rf2Send();
-
- protected:
-  int DMLSend(messageqcpp::ByteStream& bytestream, messageqcpp::ByteStream::octbyte& rows);
-
- private:
-  // DMLIF(const DMLIF& rhs);
-  // DMLIF& operator=(const DMLIF& rhs);
-
-  uint32_t fSessionID;
-  uint32_t fTflg;
-  bool fDflg;
-  bool fVflg;
-
-  boost::scoped_ptr<messageqcpp::MessageQueueClient> fMqp;
-
-  std::string fSchema;
-  std::string fOFilterStr;
-  std::string fLFilterStr;
-
-  execplan::ParseTree* fOPt;
-  execplan::ParseTree* fLPt;
-};
-
-}  // namespace dmlif

tools/dmldriver/tpchrf2.cpp

@@ -1,86 +0,0 @@
-/* Copyright (C) 2014 InfiniDB, Inc.
-
-   This program is free software; you can redistribute it and/or
-   modify it under the terms of the GNU General Public License
-   as published by the Free Software Foundation; version 2 of
-   the License.
-
-   This program is distributed in the hope that it will be useful,
-   but WITHOUT ANY WARRANTY; without even the implied warranty of
-   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-   GNU General Public License for more details.
-
-   You should have received a copy of the GNU General Public License
-   along with this program; if not, write to the Free Software
-   Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
-   MA 02110-1301, USA. */
-
-// $Id: tpchrf2.cpp 2101 2013-01-21 14:12:52Z rdempsey $
-
-#include <istream>
-using namespace std;
-
-#include <boost/scoped_array.hpp>
-#include <boost/tokenizer.hpp>
-using namespace boost;
-
-#include "tpchrf2.h"
-#include "dmlif.h"
-
-namespace tpch
-{
-RF2::RF2(const string& sn, uint32_t sid, uint32_t tflg, int c, int p, bool d, bool v)
- : fSchema(sn), fSessionID(sid), fTflg(tflg), fIntvl(c), fPack(p), fDflg(d), fVflg(v)
-{
-}
-
-RF2::~RF2()
-{
-}
-
-int RF2::run(istream& in)
-{
-  const streamsize ilinelen = 1024;
-  scoped_array<char> iline(new char[ilinelen]);
-  int cnt = 0;
-  dmlif::DMLIF dmlif(fSessionID, fTflg, fDflg, fVflg);
-
-  for (;;)
-  {
-    dmlif.rf2Start(fSchema);
-
-    for (int i = 0; i < fPack; i++)
-    {
-      in.getline(iline.get(), ilinelen);
-
-      if (in.eof())
-        break;
-
-      typedef char_separator<char> cs;
-      typedef tokenizer<cs> tk;
-      cs sep("|");
-      tk tok(string(iline.get()), sep);
-      tk::iterator iter = tok.begin();
-      idbassert(iter != tok.end());
-      string keystr = *iter;
-      ++iter;
-      // idbassert(iter == tok.end());
-      int64_t okey = strtol(keystr.c_str(), 0, 0);
-      dmlif.rf2Add(okey);
-    }
-
-    dmlif.rf2Send();
-    cnt++;
-
-    if ((cnt % fIntvl) == 0)
-      dmlif.sendOne("COMMIT;");
-
-    if (in.eof())
-      break;
-  }
-
-  dmlif.sendOne("COMMIT;");
-  return 0;
-}
-
-}  // namespace tpch

tools/dmldriver/tpchrf2.h

@@ -1,52 +0,0 @@
-/* Copyright (C) 2014 InfiniDB, Inc.
-
-   This program is free software; you can redistribute it and/or
-   modify it under the terms of the GNU General Public License
-   as published by the Free Software Foundation; version 2 of
-   the License.
-
-   This program is distributed in the hope that it will be useful,
-   but WITHOUT ANY WARRANTY; without even the implied warranty of
-   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-   GNU General Public License for more details.
-
-   You should have received a copy of the GNU General Public License
-   along with this program; if not, write to the Free Software
-   Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
-   MA 02110-1301, USA. */
-
-// $Id: tpchrf2.h 2101 2013-01-21 14:12:52Z rdempsey $
-
-#pragma once
-
-#include <string>
-#include <istream>
-#include <cstddef>
-#include <limits>
-#include <stdint.h>
-
-namespace tpch
-{
-class RF2
-{
- public:
-  RF2(const std::string& sn, uint32_t sid, uint32_t tflg = 0, int c = std::numeric_limits<int>::max(),
-      int p = 1, bool d = false, bool v = false);
-  ~RF2();
-
-  int run(std::istream& in);
-
- private:
-  // RF2(const RF2& rhs);
-  // RF2& operator=(const RF2& rhs);
-
-  std::string fSchema;
-  uint32_t fSessionID;
-  uint32_t fTflg;
-  int fIntvl;
-  int fPack;
-  bool fDflg;
-  bool fVflg;
-};
-
-}  // namespace tpch

tools/editem/CMakeLists.txt

@@ -4,12 +4,5 @@ include_directories(${ENGINE_COMMON_INCLUDES})
 
 set(editem_SRCS editem.cpp)
 
-add_executable(editem ${editem_SRCS})
+columnstore_executable(editem ${editem_SRCS})
-
+columnstore_link(editem ${ENGINE_LDFLAGS} ${NETSNMP_LIBRARIES} ${ENGINE_EXEC_LIBS})
-target_link_libraries(editem ${ENGINE_LDFLAGS} ${NETSNMP_LIBRARIES} ${ENGINE_EXEC_LIBS})
-
-install(
-    TARGETS editem
-    DESTINATION ${ENGINE_BINDIR}
-    COMPONENT columnstore-engine
-)

tools/fixdate/fixdate.cpp

@@ -1,173 +0,0 @@
-/* Copyright (C) 2014 InfiniDB, Inc.
-
-   This program is free software; you can redistribute it and/or
-   modify it under the terms of the GNU General Public License
-   as published by the Free Software Foundation; version 2 of
-   the License.
-
-   This program is distributed in the hope that it will be useful,
-   but WITHOUT ANY WARRANTY; without even the implied warranty of
-   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-   GNU General Public License for more details.
-
-   You should have received a copy of the GNU General Public License
-   along with this program; if not, write to the Free Software
-   Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
-   MA 02110-1301, USA. */
-
-/*
- * $Id: fixdate.cpp 2101 2013-01-21 14:12:52Z rdempsey $
- *
- */
-
-#include <iostream>
-#include <fstream>
-#include <cassert>
-#include <vector>
-#include <algorithm>
-#include <iterator>
-using namespace std;
-
-#include <unistd.h>
-
-#include <boost/format.hpp>
-#include <boost/scoped_array.hpp>
-using namespace boost;
-
-#include "bytestream.h"
-using namespace messageqcpp;
-
-#include "dmlpackageprocessor.h"
-using namespace dmlpackageprocessor;
-
-namespace
-{
-const streamsize blkSz = 8192;
-
-void usage()
-{
-}
-
-u_int64_t fixDate(ByteStream& bs, ostream& out)
-{
-  ByteStream fixed;
-  ByteStream::quadbyte o;
-  u_int64_t cnt = 0;
-#if 0
-    DMLPackageProcessor::Date minDate;
-    DMLPackageProcessor::Date maxDate;
-    minDate.year = 1992;
-    minDate.month = 1;
-    minDate.day = 2;
-    maxDate.year = 1998;
-    maxDate.month = 12;
-    maxDate.day = 25;
-    ByteStream::quadbyte mxd;
-    ByteStream::quadbyte mnd;
-    mxd = *(reinterpret_cast<ByteStream::quadbyte*>(&maxDate));
-    mnd = *(reinterpret_cast<ByteStream::quadbyte*>(&minDate));
-#endif
-  DMLPackageProcessor::Date fixDate;
-  fixDate.spare = 0;
-  ByteStream::quadbyte f;
-
-  while (bs.length() > 0)
-  {
-    bs >> o;
-
-    if (o >= 0xfffffffe)
-    {
-      fixed << o;
-      continue;
-    }
-
-    f = o & 0xffff;
-    fixDate.year = f;
-    o >>= 16;
-    f = o & 0xf;
-    fixDate.month = f;
-    o >>= 4;
-    f = o & 0x3f;
-    fixDate.day = f;
-    // o >>= 6;
-    o = *(reinterpret_cast<ByteStream::quadbyte*>(&fixDate));
-    fixed << o;
-#if 0
-        cout << DMLPackageProcessor::dateToString(o) << endl;
-        idbassert(o >= mnd && o <= mxd);
-        cnt++;
-#endif
-  }
-
-  out << fixed;
-  return cnt;
-}
-
-}  // namespace
-
-int main(int argc, char* argv[])
-{
-  int c;
-
-  opterr = 0;
-
-  while ((c = getopt(argc, argv, "h")) != EOF)
-    switch (c)
-    {
-      case 'h':
-        usage();
-        return 0;
-        break;
-
-      default:
-        usage();
-        return 1;
-        break;
-    }
-
-  if ((argc - optind) < 1)
-  {
-    usage();
-    return 1;
-  }
-
-  ByteStream bs;
-
-  ifstream ifs(argv[optind + 0]);
-  ByteStream::byte inbuf[blkSz];
-  streampos fLen;
-  u_int64_t blkNo = 0;
-
-  ifs.seekg(0, ios_base::end);
-  fLen = ifs.tellg();
-  ifs.seekg(0, ios_base::beg);
-
-  idbassert((fLen % blkSz) == 0);
-  u_int64_t numBlks = fLen / blkSz;
-  cout << numBlks << " blocks to fix..." << endl;
-
-  ofstream ofs("fixdate.cdf");
-
-  cout << "pct done:    " << setw(3);
-
-  for (;;)
-  {
-    ifs.read(reinterpret_cast<char*>(inbuf), blkSz);
-
-    if (ifs.eof())
-      break;
-
-    bs.load(inbuf, blkSz);
-
-    fixDate(bs, ofs);
-    cout << "\b\b\b" << setw(3) << (u_int64_t)(blkNo * 100 / numBlks);
-    // cout << setw(3) << (u_int64_t)(blkNo * 100 / numBlks) << endl;
-
-    blkNo++;
-  }
-
-  cout << "\b\b\b" << setw(3) << 100 << endl;
-  // cout << setw(3) << 100 << endl;
-
-  return 0;
-}

tools/getConfig/CMakeLists.txt

@@ -4,12 +4,6 @@ include_directories(${ENGINE_COMMON_INCLUDES})
 
 set(getConfig_SRCS main.cpp)
 
-add_executable(mcsGetConfig ${getConfig_SRCS})
+columnstore_executable(mcsGetConfig ${getConfig_SRCS})
 
-target_link_libraries(mcsGetConfig ${ENGINE_LDFLAGS} ${ENGINE_EXEC_LIBS})
+columnstore_link(mcsGetConfig ${ENGINE_LDFLAGS} ${ENGINE_EXEC_LIBS})
-
-install(
-    TARGETS mcsGetConfig
-    DESTINATION ${ENGINE_BINDIR}
-    COMPONENT columnstore-engine
-)

tools/idb_comp/main.cpp

@@ -1,265 +0,0 @@
-#include <unistd.h>
-#include <sys/types.h>
-#include <sys/stat.h>
-#include <fcntl.h>
-#include <cerrno>
-//#define NDEBUG
-#include <cassert>
-#include <cstring>
-
-#include <lzo/lzoconf.h>
-#include <lzo/lzo1x.h>
-
-#include <iostream>
-#include <iomanip>
-#include <sstream>
-#include <inttypes.h>
-#include <cstdlib>
-using namespace std;
-
-#include <boost/scoped_array.hpp>
-
-namespace
-{
-size_t exp_buf_size;
-unsigned vflg;
-
-const string lzo_rctos(int r)
-{
-  switch (r)
-  {
-    case LZO_E_INPUT_NOT_CONSUMED: return "LZO_E_INPUT_NOT_CONSUMED";
-
-    default: break;
-  }
-
-  return "Unknown Error!";
-}
-
-void usage()
-{
-  cout << "usage: idb_comp [-e size][-h] file(s)" << endl
-       << "\t-e size size (in KB) of expansion buffer" << endl
-       << "\t-h      display this help" << endl;
-}
-
-int doit(const string& infile)
-{
-  int rc = 0;
-  int ifd = open(infile.c_str(), O_RDONLY | O_LARGEFILE | O_DIRECT);
-
-  if (ifd < 0)
-  {
-    cerr << infile << ": open: " << strerror(errno) << endl;
-    return 1;
-  }
-
-  struct stat istatbuf;
-
-  fstat(ifd, &istatbuf);
-
-  string outname(infile);
-
-  string::size_type ptr;
-
-  ptr = outname.find_last_of('.');
-
-  if (ptr != string::npos)
-    outname.erase(ptr);
-
-  ptr = outname.find_last_of('/');
-
-  if (ptr != string::npos)
-    outname.erase(0, ptr + 1);
-
-  outname = "./" + outname + ".cmp";
-
-  int ofd = open(outname.c_str(), O_WRONLY | O_CREAT | O_TRUNC | O_LARGEFILE | O_DIRECT, 0644);
-
-  if (ofd < 0)
-  {
-    cerr << outname << ": open: " << strerror(errno) << endl;
-    close(ifd);
-    return 1;
-  }
-
-  lzo_init();
-
-  ssize_t nread = -1;
-  ssize_t nwritten = -1;
-  lzo_bytep ibuf;
-  lzo_bytep tibuf;
-  lzo_bytep cbuf;
-  lzo_bytep tcbuf;
-  lzo_bytep wkmem;
-  lzo_uint32 ibuf_len = 0;
-  lzo_uint cbuf_len = 0;
-
-  ibuf_len = 512 * 1024 * 8;
-  tibuf = new lzo_byte[ibuf_len + 4095];
-
-  if ((ptrdiff_t)tibuf & 0xfffULL)
-    ibuf = (lzo_bytep)((ptrdiff_t)tibuf & 0xfffffffffffff000ULL) + 4096;
-  else
-    ibuf = tibuf;
-
-  cbuf_len = 512 * 1024 * 8;
-  tcbuf = new lzo_byte[cbuf_len + 4095 + exp_buf_size * 1024];
-
-  if ((ptrdiff_t)tcbuf & 0xfff)
-    cbuf = (lzo_bytep)((ptrdiff_t)tcbuf & 0xfffffffffffff000ULL) + 4096;
-  else
-    cbuf = tcbuf;
-
-  wkmem = new lzo_byte[LZO1X_1_15_MEM_COMPRESS];
-
-  int r = LZO_E_OK;
-
-  const int TOTAL_HDR_LEN = 4096 * 2;
-  char* hdrbuf = new char[TOTAL_HDR_LEN + 4095];
-  memset(hdrbuf, 0, TOTAL_HDR_LEN + 4095);
-  char* hdrbufp = 0;
-
-  if ((ptrdiff_t)hdrbuf & 0xfff)
-    hdrbufp = (char*)((ptrdiff_t)hdrbuf & 0xfffffffffffff000ULL) + 4096;
-  else
-    hdrbufp = hdrbuf;
-
-  struct compHdr
-  {
-    uint64_t ptrs[512];
-  };
-
-  idbassert(sizeof(compHdr) <= 4096);
-
-  compHdr* hdrptr1 = (compHdr*)hdrbufp;
-  compHdr* hdrptr = hdrptr1 + 1;  // advance to 2nd hdr to store compression ptrs
-  lseek(ofd, TOTAL_HDR_LEN, SEEK_SET);
-
-  nread = read(ifd, ibuf, ibuf_len);
-
-  int idx = 0;
-  off_t cmpoff = TOTAL_HDR_LEN;
-
-  while (nread > 0)
-  {
-    cbuf_len = 512 * 1024 * 8;
-    memset(cbuf, 0, cbuf_len);
-    r = lzo1x_1_15_compress(ibuf, nread, cbuf, &cbuf_len, wkmem);
-
-    if (r != LZO_E_OK)
-    {
-      cerr << "compression failed!: " << r << endl;
-      rc = 1;
-      goto out;
-    }
-
-    if (cbuf_len > (unsigned)nread)
-    {
-      cerr << "WARNING: expansion detected! (output grew by " << (cbuf_len - nread) << " bytes)" << endl;
-      idbassert((cbuf_len - nread) <= exp_buf_size * 1024);
-    }
-
-    if (cbuf_len & 0xfff)
-      cbuf_len = (cbuf_len & 0xfffffffffffff000ULL) + 4096;
-
-    // cbuf_len = 512 * 1024 * 8;
-    nwritten = write(ofd, cbuf, cbuf_len);
-
-    if (nwritten < 0 || (unsigned)nwritten != cbuf_len)
-    {
-      cerr << outname << ": write: " << strerror(errno) << " (" << nwritten << ')' << endl;
-      rc = 1;
-      goto out;
-    }
-
-    if (vflg > 0)
-    {
-      lzo_bytep tbuf;
-      lzo_uint tbuflen = 4 * 1024 * 1024 + 4;
-      boost::scoped_array<lzo_byte> tbuf_sa(new lzo_byte[tbuflen]);
-      tbuf = tbuf_sa.get();
-      cout << "idx: " << idx << " off: " << cmpoff << " size: " << cbuf_len;
-      r = lzo1x_decompress(cbuf, cbuf_len, tbuf, &tbuflen, 0);
-      cout << " r: " << lzo_rctos(r) << " size: " << tbuflen << endl;
-    }
-
-    hdrptr->ptrs[idx] = cmpoff;
-    idx++;
-    cmpoff += cbuf_len;
-
-    nread = read(ifd, ibuf, ibuf_len);
-  }
-
-  if (nread < 0)
-  {
-    cerr << infile << ": read: " << strerror(errno) << endl;
-    rc = 1;
-    goto out;
-  }
-
-  hdrptr->ptrs[idx] = cmpoff;
-  idbassert(idx <= 64);
-
-  // Fill in meta-data information in first header
-  hdrptr1->ptrs[0] = 0xfdc119a384d0778eULL;
-  hdrptr1->ptrs[1] = 1;
-  hdrptr1->ptrs[2] = 1;
-
-  nwritten = pwrite(ofd, hdrbufp, TOTAL_HDR_LEN, 0);
-  idbassert(nwritten == TOTAL_HDR_LEN);
-
-out:
-  delete[] wkmem;
-  delete[] tcbuf;
-  delete[] tibuf;
-  fsync(ofd);
-  struct stat ostatbuf;
-  fstat(ofd, &ostatbuf);
-  idbassert(ostatbuf.st_size == (signed)hdrptr->ptrs[idx]);
-  delete[] hdrbuf;
-  cout << infile << ": Input Size: " << istatbuf.st_size << " Output size: " << ostatbuf.st_size
-       << " Compression: " << (100LL - (ostatbuf.st_size * 100LL / istatbuf.st_size)) << '%' << endl;
-  close(ofd);
-  close(ifd);
-
-  return rc;
-}
-
-}  // namespace
-
-int main(int argc, char** argv)
-{
-  opterr = 0;
-  int c;
-  exp_buf_size = 128;
-  vflg = 0;
-
-  while ((c = getopt(argc, argv, "e:vh")) != -1)
-    switch (c)
-    {
-      case 'e': exp_buf_size = atoi(optarg); break;
-
-      case 'v': vflg++; break;
-
-      case 'h':
-      default:
-        usage();
-        return (c == 'h' ? 0 : 1);
-        break;
-    }
-
-  if ((argc - optind) < 1)
-  {
-    usage();
-    return 1;
-  }
-
-  int rc = 0;
-
-  for (int i = optind; i < argc; i++)
-    if (doit(argv[i]))
-      rc = 1;
-
-  return rc;
-}

tools/idbmeminfo/CMakeLists.txt

@@ -4,12 +4,5 @@ include_directories(${ENGINE_COMMON_INCLUDES})
 
 set(idbmeminfo_SRCS idbmeminfo.cpp)
 
-add_executable(idbmeminfo ${idbmeminfo_SRCS})
+columnstore_executable(idbmeminfo ${idbmeminfo_SRCS})
-
+columnstore_link(idbmeminfo ${ENGINE_LDFLAGS})
-target_link_libraries(idbmeminfo ${ENGINE_LDFLAGS})
-
-install(
-    TARGETS idbmeminfo
-    DESTINATION ${ENGINE_BINDIR}
-    COMPONENT columnstore-engine
-)

tools/passwd/CMakeLists.txt

@@ -4,21 +4,10 @@ include_directories(${ENGINE_COMMON_INCLUDES} ${ENGINE_UTILS_COMMON_INCLUDE})
 set(cspasswd_SRCS cspasswd.cpp secrets.cpp)
 set(cskeys_SRCS cskeys.cpp secrets.cpp)
 
-add_executable(cspasswd ${cspasswd_SRCS})
+columnstore_executable(cspasswd ${cspasswd_SRCS})
-add_executable(cskeys ${cskeys_SRCS})
+columnstore_executable(cskeys ${cskeys_SRCS})
 
 target_include_directories(cspasswd BEFORE PUBLIC ${OPENSSL_INCLUDE_DIR})
 target_include_directories(cskeys BEFORE PUBLIC ${OPENSSL_INCLUDE_DIR})
-target_link_libraries(cspasswd ${ENGINE_LDFLAGS} ${ENGINE_EXEC_LIBS} ${SSL_LIBRARIES})
+columnstore_link(cspasswd ${ENGINE_LDFLAGS} ${ENGINE_EXEC_LIBS} ${SSL_LIBRARIES})
-target_link_libraries(cskeys ${ENGINE_LDFLAGS} ${ENGINE_EXEC_LIBS} ${SSL_LIBRARIES})
+columnstore_link(cskeys ${ENGINE_LDFLAGS} ${ENGINE_EXEC_LIBS} ${SSL_LIBRARIES})
-
-install(
-    TARGETS cspasswd
-    DESTINATION ${ENGINE_BINDIR}
-    COMPONENT columnstore-engine
-)
-install(
-    TARGETS cskeys
-    DESTINATION ${ENGINE_BINDIR}
-    COMPONENT columnstore-engine
-)

tools/qfe/CMakeLists.txt

@@ -1,55 +0,0 @@
-include_directories(${KDE4_INCLUDES} ${KDE4_INCLUDE_DIR} ${QT_INCLUDES})
-
-# ########## next target ###############
-
-set(QFE_SRCS
-    cseputils.cpp
-    ddlstmts.cpp
-    parsequery.cpp
-    returnedrows.cpp
-    sendcsep.cpp
-    server.cpp
-    socketio.cpp
-    qfelexer.lpp
-    qfeparser.ypp
-)
-
-kde4_add_executable(QFE ${QFE_SRCS})
-
-target_link_libraries(QFE ${KDE4_KDECORE_LIBS})
-
-install(TARGETS QFE ${INSTALL_TARGETS_DEFAULT_ARGS})
-
-# ########## install files ###############
-
-# original Makefile.am contents follow:
-
-# Copyright (C) 2014 InfiniDB, Inc.
-#
-# This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public
-# License as published by the Free Software Foundation; version 2 of the License.
-#
-# This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied
-# warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License along with this program; if not, write to the Free
-# Software Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
-#
-# $Id$ Process this file with automake to produce Makefile.in
-#
-# AM_CPPFLAGS = $(idb_cppflags) AM_CFLAGS = $(idb_cflags) AM_CXXFLAGS = $(idb_cxxflags) AM_LDFLAGS = $(idb_ldflags)
-# AM_YFLAGS = -d -p qfe AM_LFLAGS = -i -Pqfe -olex.yy.c bin_PROGRAMS = QFE QFE_SOURCES = \ cseputils.cpp \ ddlstmts.cpp
-# \ parsequery.cpp \ returnedrows.cpp \ sendcsep.cpp \ server.cpp \ socketio.cpp \ qfelexer.lpp \ qfeparser.ypp
-# QFE_CPPFLAGS = @idb_common_includes@ $(AM_CPPFLAGS) QFE_LDFLAGS = @idb_common_ldflags@ @idb_write_libs@
-# @idb_exec_libs@ $(AM_LDFLAGS) BUILT_SOURCES = qfeparser.cpp qfelexer.cpp qfeparser.h
-#
-# test:
-#
-# coverage:
-#
-# leakcheck:
-#
-# docs:
-#
-# bootstrap: install-data-am
-#

tools/qfe/cli.cpp

@@ -1,148 +0,0 @@
-#include <poll.h>
-#include <sys/socket.h>
-#include <netinet/in.h>
-#include <arpa/inet.h>
-#include <netdb.h>
-#include <iostream>
-#include <string>
-#include <sstream>
-#include <iomanip>
-#include <sys/types.h>
-#include <sys/stat.h>
-#include <fcntl.h>
-#include <unistd.h>
-#include <cerrno>
-#include <cstring>
-#include <cstdlib>
-using namespace std;
-
-#include "socktype.h"
-#include "socketio.h"
-using namespace qfe;
-
-namespace
-{
-void usage()
-{
-  cout << "usage: cli [-h] [-s serverip] [-p port] [-c schema] <query>" << endl;
-  cout << '\t' << "-s serverip connect to serverip instead of 127.0.0.1" << endl;
-  cout << '\t' << "-p port     connect on port instead of 9198" << endl;
-  cout << '\t' << "-c schema   use schema as default instead of tpch1" << endl;
-  cout << '\t' << "-h          display this help" << endl;
-}
-}  // namespace
-
-int main(int argc, char** argv)
-{
-  opterr = 0;
-  int c;
-  string serverip("127.0.0.1");
-  string schema("tpch1");
-  short PortNo = 9198;
-
-  while ((c = getopt(argc, argv, "s:c:p:h")) != -1)
-    switch (c)
-    {
-      case 's': serverip = optarg; break;
-
-      case 'c': schema = optarg; break;
-
-      case 'p':
-        PortNo = atoi(optarg);
-
-        if (PortNo == 0)
-        {
-          usage();
-          return 1;
-        }
-
-        break;
-
-      case 'h':
-      case '?':
-      default:
-        usage();
-        return 0;
-        break;
-    }
-
-  if (argc - optind < 1)
-  {
-    usage();
-    return 1;
-  }
-
-  string query(argv[optind + 0]);
-
-  SockType fd = -1;
-
-  fd = ::socket(PF_INET, SOCK_STREAM, IPPROTO_TCP);
-
-  if (fd < 0)
-  {
-    cerr << "socket create error: " << strerror(errno) << endl;
-    return 1;
-  }
-
-  int rc = 0;
-  struct sockaddr_in serv_addr;
-  struct addrinfo hints;
-  struct addrinfo* res = 0;
-  memset(&hints, 0, sizeof(struct addrinfo));
-  hints.ai_family = PF_INET;
-  hints.ai_socktype = SOCK_STREAM;
-  hints.ai_protocol = IPPROTO_TCP;
-  rc = getaddrinfo(serverip.c_str(), 0, &hints, &res);
-
-  if (rc != 0)
-  {
-    cerr << "Error resolving '" << serverip << "': " << gai_strerror(rc) << endl;
-    return 1;
-  }
-
-  sockaddr_in* sain = 0;
-  sain = reinterpret_cast<sockaddr_in*>(res->ai_addr);
-  memset(&serv_addr, 0, sizeof(serv_addr));
-  serv_addr.sin_family = PF_INET;
-  serv_addr.sin_addr = sain->sin_addr;
-  serv_addr.sin_port = htons(PortNo);
-  freeaddrinfo(res);
-
-  rc = ::connect(fd, (sockaddr*)&serv_addr, sizeof(serv_addr));
-
-  if (rc < 0)
-  {
-    cerr << "socket connect error: " << strerror(errno) << endl;
-    return 1;
-  }
-
-  socketio::writeString(fd, schema);
-  socketio::writeString(fd, query);
-
-  uint32_t flag = 0;
-  string row;
-  row = socketio::readString(fd);
-
-  if (row != "OK")
-  {
-    cerr << "query failed: " << row << endl;
-    goto bailout;
-  }
-
-  row = socketio::readString(fd);
-
-  while (!row.empty())
-  {
-    cout << row << endl;
-    row = socketio::readString(fd);
-  }
-
-  flag = 0;
-  SockWriteFcn(fd, &flag, 4);
-bailout:
-  ::shutdown(fd, SHUT_RDWR);
-  ::close(fd);
-  fd = -1;
-
-  return 0;
-}

tools/qfe/cseputils.cpp

@@ -1,141 +0,0 @@
-#include <unistd.h>
-#include <string>
-using namespace std;
-
-#include "constantcolumn.h"
-#include "simplefilter.h"
-#include "calpontsystemcatalog.h"
-#include "parsetree.h"
-#include "simplecolumn.h"
-#include "calpontselectexecutionplan.h"
-using namespace execplan;
-
-#include "cseputils.h"
-
-namespace qfe
-{
-extern string DefaultSchema;
-
-namespace utils
-{
-ConstantColumn* createConstCol(const string& valstr)
-{
-  ConstantColumn* cc = new ConstantColumn(valstr);
-  cc->alias(valstr);
-  return cc;
-}
-
-template <typename T>
-ConstantColumn* createConstCol(const string& valstr, T val)
-{
-  ConstantColumn* cc = new ConstantColumn(valstr, val);
-  cc->alias(valstr);
-  return cc;
-}
-
-SimpleFilter* createSimpleFilter(boost::shared_ptr<CalpontSystemCatalog>& csc,
-                                 const CalpontSystemCatalog::TableColName& tcn, const string& opstr,
-                                 ConstantColumn* cc)
-{
-  SimpleFilter* lsf = new SimpleFilter();
-
-  Operator* op = new Operator();
-  op->data(opstr);
-  CalpontSystemCatalog::ColType ccct;
-  ccct = op->resultType();
-  ccct.colDataType = cc->resultType().colDataType;
-  op->operationType(ccct);
-
-  SOP sop(op);
-  lsf->op(sop);
-
-  CalpontSystemCatalog::OID oid = csc->lookupOID(tcn);
-  CalpontSystemCatalog::ColType ct = csc->colType(oid);
-
-  SimpleColumn* sc = new SimpleColumn();
-  sc->schemaName(tcn.schema);
-  sc->tableName(tcn.table);
-  sc->tableAlias(tcn.table);
-  sc->columnName(tcn.column);
-  sc->oid(oid);
-  sc->resultType(ct);
-  sc->alias(tcn.toString());
-
-  lsf->lhs(sc);
-  lsf->rhs(cc);
-
-  return lsf;
-}
-
-void appendSimpleFilter(ParseTree*& ptree, SimpleFilter* filter)
-{
-  if (ptree->data() == 0)
-  {
-    // degenerate case, this filter goes at this node
-    ptree->data(filter);
-  }
-  else if (ptree->right() == 0 && ptree->left() == 0)
-  {
-    // this will be the case when there is a single node in the tree
-    // that contains a filter.  Here we want to make the root node an
-    // 'and' operator, push the existing down to the lhs and make a
-    // new node for the new filter
-    ParseTree* newLhs = new ParseTree(ptree->data());
-    ParseTree* newRhs = new ParseTree(filter);
-
-    Operator* op = new Operator();
-    op->data("and");
-
-    ptree->data(op);
-    ptree->left(newLhs);
-    ptree->right(newRhs);
-  }
-  else
-  {
-    // this will be the case once we have a tree with an 'and' at the
-    // root node, a filter in the lhs, and an arbitrary height tree
-    // with the same properties on the rhs.  Because all operators
-    // are guaranteed to be and for now we simply insert a new rhs
-    // node and "push down" the existing tree
-    Operator* op = new Operator();
-    op->data("and");
-
-    ParseTree* newRhs = new ParseTree(op);
-    newRhs->left(new ParseTree(filter));
-    newRhs->right(ptree->right());
-    ptree->right(newRhs);
-  }
-}
-
-void updateParseTree(boost::shared_ptr<execplan::CalpontSystemCatalog>& csc,
-                     execplan::CalpontSelectExecutionPlan*& csep, execplan::SimpleColumn* sc,
-                     const std::string& relop, pair<int, string> cval)
-{
-  execplan::ConstantColumn* cc = 0;
-
-  if (cval.first == 0)
-    cc = createConstCol(cval.second, static_cast<int64_t>(atoll(cval.second.c_str())));
-  else
-    cc = createConstCol(cval.second);
-
-  if (sc->schemaName() == "infinidb_unknown" && !DefaultSchema.empty())
-    sc->schemaName(DefaultSchema);
-
-  execplan::SimpleFilter* sf = 0;
-  sf = createSimpleFilter(csc, execplan::make_tcn(sc->schemaName(), sc->tableName(), sc->columnName()), relop,
-                          cc);
-  execplan::ParseTree* ptp = 0;
-  ptp = csep->filters();
-
-  if (ptp == 0)
-    ptp = new execplan::ParseTree();
-
-  appendSimpleFilter(ptp, sf);
-  csep->filters(ptp);
-}
-
-// template instantiations
-template ConstantColumn* createConstCol<int64_t>(const string& valstr, int64_t val);
-
-}  // namespace utils
-}  // namespace qfe

tools/qfe/cseputils.h

@@ -1,34 +0,0 @@
-#pragma once
-
-#include <string>
-
-namespace execplan
-{
-class ConstantColumn;
-class SimpleFilter;
-class ParseTree;
-class SimpleColumn;
-}  // namespace execplan
-#include "calpontsystemcatalog.h"
-
-namespace qfe
-{
-namespace utils
-{
-execplan::ConstantColumn* createConstCol(const std::string& valstr);
-
-template <typename T>
-execplan::ConstantColumn* createConstCol(const std::string& valstr, T val);
-
-execplan::SimpleFilter* createSimpleFilter(boost::shared_ptr<execplan::CalpontSystemCatalog>& csc,
-                                           const execplan::CalpontSystemCatalog::TableColName& tcn,
-                                           const std::string& opstr, execplan::ConstantColumn* cc);
-
-void appendSimpleFilter(execplan::ParseTree*& ptree, execplan::SimpleFilter* filter);
-
-void updateParseTree(boost::shared_ptr<execplan::CalpontSystemCatalog>&,
-                     execplan::CalpontSelectExecutionPlan*&, execplan::SimpleColumn*, const std::string&,
-                     pair<int, string>);
-
-}  // namespace utils
-}  // namespace qfe

tools/qfe/ddlstmts.cpp

@@ -1,80 +0,0 @@
-#include <unistd.h>
-#include <string>
-#include <stdexcept>
-using namespace std;
-
-#include <boost/scoped_ptr.hpp>
-using namespace boost;
-
-#include "ddlpkg.h"
-#include "sqlparser.h"
-using namespace ddlpackage;
-
-#include "bytestream.h"
-#include "messagequeue.h"
-using namespace messageqcpp;
-
-#include "ddlstmts.h"
-
-namespace qfe
-{
-extern string DefaultSchema;
-}
-
-namespace
-{
-using namespace qfe;
-
-int processDDLStmt(const string& stmt, uint32_t sid)
-{
-  MessageQueueClient* mq = 0;
-  SqlParser parser;
-
-  parser.setDefaultSchema(DefaultSchema);
-  parser.Parse(stmt.c_str());
-
-  if (parser.Good())
-  {
-    const ddlpackage::ParseTree& ptree = parser.GetParseTree();
-    SqlStatement& ddlstmt = *ptree.fList[0];
-    ddlstmt.fSessionID = sid;
-    ddlstmt.fSql = stmt;
-    ddlstmt.fOwner = DefaultSchema;
-    ByteStream bytestream;
-    bytestream << ddlstmt.fSessionID;
-    ddlstmt.serialize(bytestream);
-    mq = new MessageQueueClient("DDLProc");
-    scoped_ptr<MessageQueueClient> smq(mq);
-    ByteStream::byte b = 0;
-    mq->write(bytestream);
-    bytestream = mq->read();
-    bytestream >> b;
-    string emsg;
-    bytestream >> emsg;
-
-    if (b != 0)
-      throw runtime_error(emsg);
-  }
-  else
-  {
-    throw runtime_error("syntax error");
-  }
-
-  return 0;
-}
-
-}  // namespace
-
-namespace qfe
-{
-void processCreateStmt(const string& stmt, uint32_t sid)
-{
-  processDDLStmt(stmt, sid);
-}
-
-void processDropStmt(const string& stmt, uint32_t sid)
-{
-  processDDLStmt(stmt, sid);
-}
-
-}  // namespace qfe

tools/qfe/ddlstmts.h

@@ -1,10 +0,0 @@
-#pragma once
-
-#include <unistd.h>
-#include <string>
-
-namespace qfe
-{
-void processCreateStmt(const std::string&, uint32_t);
-void processDropStmt(const std::string&, uint32_t);
-}  // namespace qfe

tools/qfe/parsequery.cpp

@@ -1,97 +0,0 @@
-#include <unistd.h>
-#include <string>
-#include <memory>
-using namespace std;
-
-#include <boost/thread/mutex.hpp>
-#include <boost/scoped_ptr.hpp>
-using namespace boost;
-
-#include "cseputils.h"
-#include "parsequery.h"
-
-#include "calpontselectexecutionplan.h"
-#include "simplecolumn.h"
-#include "simplefilter.h"
-#include "constantcolumn.h"
-#include "calpontsystemcatalog.h"
-#include "sessionmanager.h"
-using namespace execplan;
-
-#include "brmtypes.h"
-using namespace BRM;
-
-extern int qfeparse();
-extern int qfedebug;
-
-struct yy_buffer_state;
-extern yy_buffer_state* qfe_scan_string(const char*);
-extern void qfe_delete_buffer(yy_buffer_state*);
-extern CalpontSelectExecutionPlan* ParserCSEP;
-extern boost::shared_ptr<CalpontSystemCatalog> ParserCSC;
-
-namespace
-{
-mutex ParserMutex;
-}  // namespace
-
-namespace qfe
-{
-extern string DefaultSchema;
-
-CalpontSelectExecutionPlan* parseQuery(const string& query, const uint32_t sid)
-{
-  // We're going to make parsing the query single-threaded for now. This makes it a lot
-  // easier to interface with the parser and doesn;t materially affect overall query
-  // performance (I think)
-  mutex::scoped_lock lk(ParserMutex);
-
-  boost::shared_ptr<CalpontSystemCatalog> csc = CalpontSystemCatalog::makeCalpontSystemCatalog(sid);
-  CalpontSelectExecutionPlan* csep = 0;
-  csep = new CalpontSelectExecutionPlan();
-  // we use an auto_ptr here with some trepidation. We only want auto delete on an execption.
-  // If the parseing and plan build succeed, we want the ptr to stay around. boost::scoped_ptr<>
-  // doesn't have an API to release ownership, so we use auto_ptr...
-  auto_ptr<CalpontSelectExecutionPlan> scsep(csep);
-
-  yy_buffer_state* ybs = 0;
-  ybs = qfe_scan_string(query.c_str());
-
-  if (ybs != 0)
-  {
-    ParserCSEP = csep;
-    ParserCSC = csc;
-
-    if (qfeparse() != 0)
-      throw runtime_error("syntax error");
-
-    qfe_delete_buffer(ybs);
-  }
-  else
-    throw runtime_error("Internal parser memory error");
-
-  csep->data(query);
-
-  SessionManager sm;
-  TxnID txnID;
-  txnID = sm.getTxnID(sid);
-
-  if (!txnID.valid)
-  {
-    txnID.id = 0;
-    txnID.valid = true;
-  }
-
-  QueryContext verID;
-  verID = sm.verID();
-
-  csep->txnID(txnID.id);
-  csep->verID(verID);
-  csep->sessionID(sid);
-
-  // cout << *csep << endl;
-  scsep.release();
-  return csep;
-}
-
-}  // namespace qfe

tools/qfe/parsequery.h

@@ -1,12 +0,0 @@
-#pragma once
-
-#include <unistd.h>
-#include <string>
-
-#include "calpontselectexecutionplan.h"
-
-namespace qfe
-{
-execplan::CalpontSelectExecutionPlan* parseQuery(const std::string&, uint32_t);
-
-}

tools/qfe/qfe-cli.py

@@ -1,51 +0,0 @@
-#!/usr/bin/python
-'''
-Created on May 1, 2013
-
-@author: rtw
-'''
-
-import struct
-import binascii
-import socket
-import sys
-
-QFE_HOST = 'localhost'
-#QFE_HOST = 'winsvr2008r2.calpont.com'
-QFE_SOCK = 9198
-
-def do_query(query):
-    client_socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
-    client_socket.connect((QFE_HOST, QFE_SOCK))
-
-    defschem = "tpch1";
-    lenstruct = struct.pack("I", len(defschem))
-    client_socket.send(lenstruct)
-    client_socket.send(defschem)
-    
-    lenstruct = struct.pack("I", len(query))
-    client_socket.send(lenstruct)
-    client_socket.send(query)
-    
-    rowlen = 1
-    while rowlen:
-        rowlen = client_socket.recv(4)
-        if len(rowlen) < 4:
-            break
-        rowval = struct.unpack("I",rowlen)[0]
-	if rowval == 0:
-		lenstruct = struct.pack("I", 0);
-		client_socket.send(lenstruct)
-		break
-        row = client_socket.recv(rowval)
-        print row
-
-    client_socket.close()
-        
-if __name__ == '__main__':
-    if len(sys.argv) < 2:
-        query = raw_input ( "QFE-cli > " )
-    else:
-        query = sys.argv[1]
-        
-    sys.exit(do_query(query))

tools/qfe/qfelexer.lpp

@@ -1,65 +0,0 @@
-%{
-#include <utility>
-#include <string>
-#include "returnedcolumn.h"
-#ifndef _MSC_VER
-#include "qfeparser.h"
-#else
-#include "bison-win.h"
-#endif
-
-extern int qfeerror(const char *s);
-
-extern std::string* newstr(const char* cp);
-
-%}
-
-%option noyywrap
-%option yylineno
-%option nounput
-
-%x CCONST
-
-digit		[0-9]
-int_const	[+-]?{digit}+
-letter		[a-zA-Z_]
-objname		{letter}({letter}|{digit})*
-
-%%
-
-"SELECT"	{ return QFEP_SELECT; }
-"FROM"		{ return QFEP_FROM; }
-"WHERE"		{ return QFEP_WHERE; }
-"GROUP BY"	{ return GROUPBY; }
-"AND"		{ qfelval.cp = newstr(qfetext); return LOGICOP; }
-"OR"		{ qfelval.cp = newstr(qfetext); return LOGICOP; }
-"LIMIT"		{ return LIMIT; }
-"ORDER BY"	{ return ORDERBY; }
-"ASC"		{ return ASC; }
-"DESC"		{ return DESC; }
-"AS"		{ return AS; }
-"SUM"		{ qfelval.cp = newstr(qfetext); return FUNC; }
-"MIN"		{ qfelval.cp = newstr(qfetext); return FUNC; }
-"MAX"		{ qfelval.cp = newstr(qfetext); return FUNC; }
-"AVG"		{ qfelval.cp = newstr(qfetext); return FUNC; }
-
-{objname}	{ qfelval.cp = newstr(qfetext); return OBJNAME; }
-
-"'"		{ BEGIN(CCONST); }
-<CCONST>[^']*	{ qfelval.cp = newstr(qfetext); return CHAR_CONST; }
-<CCONST>"'"	{ BEGIN(INITIAL); }
-
-{int_const}	{ qfelval.cp = newstr(qfetext); return INT_CONST; }
-
-">"		{ qfelval.cp = newstr(qfetext); return RELOP; }
-">="		{ qfelval.cp = newstr(qfetext); return RELOP; }
-"<"		{ qfelval.cp = newstr(qfetext); return RELOP; }
-"<="		{ qfelval.cp = newstr(qfetext); return RELOP; }
-"<>"		{ qfelval.cp = newstr(qfetext); return RELOP; }
-"="		{ qfelval.cp = newstr(qfetext); return RELOP; }
-
-[ \t]*		{}
-[\n]		{ qfelineno++; }
-
-.		{ return qfetext[0]; }
-

tools/qfe/qfeparser.ypp

@@ -1,306 +0,0 @@
-%{
-#include <iostream>
-#include <string>
-#include <utility>
-using namespace std;
-
-namespace qfe
-{
-extern string DefaultSchema;
-}
-
-int qfeerror(const char *s);
-int qfelex(void);
-
-#include "calpontselectexecutionplan.h"
-#include "simplecolumn.h"
-#include "calpontsystemcatalog.h"
-#include "aggregatecolumn.h"
-execplan::CalpontSelectExecutionPlan* ParserCSEP;
-execplan::CalpontSystemCatalog* ParserCSC;
-namespace execplan
-{
-class ReturnedColumn;
-}
-
-#include "cseputils.h"
-
-%}
-
-%union
-{
-	execplan::ReturnedColumn* rcp;
-	std::string* cp;
-	std::pair<int, std::string>* cvp;
-}
-
-//Some of these defines cause conflicts in calpontselectexecutionplan.h, so we prepend QFEP_
-%token CHAR_CONST RELOP LOGICOP QFEP_SELECT QFEP_FROM QFEP_WHERE GROUPBY OBJNAME INT_CONST LIMIT ORDERBY ASC
-%token DESC AS FUNC
-
-%type <cp> select_column_list table_name filter_predicate_list obj_name_aux opt_alias opt_as
-%type <cp> opt_limit_clause
-%type <cp> OBJNAME CHAR_CONST INT_CONST RELOP LOGICOP FUNC AS
-
-%type <rcp> aliased_column_name select_column_name obj_name func_col groupby_item
-
-%type <cvp> constant
-
-%destructor { delete $$; } OBJNAME CHAR_CONST INT_CONST RELOP LOGICOP FUNC AS
-%destructor { delete $$; } aliased_column_name select_column_name obj_name func_col groupby_item
-%destructor { delete $$; } constant
-
-%%
-
-statement:	QFEP_SELECT select_column_list
-			QFEP_FROM table_spec
-			opt_where_clause
-			opt_groupby_clause
-			opt_orderby_clause
-			opt_limit_clause ';'
-		;
-
-select_column_list:	aliased_column_name
-			{
-				execplan::SRCP rcp($1);
-				ParserCSEP->returnedCols().push_back(rcp);
-				$$ = 0;
-			}
-			| select_column_list ',' aliased_column_name
-			{
-				execplan::SRCP rcp($3);
-				ParserCSEP->returnedCols().push_back(rcp);
-				$$ = 0;
-			}
-			;
-
-select_column_name:	obj_name
-			{
-				execplan::SimpleColumn* sc = dynamic_cast<execplan::SimpleColumn*>($1);
-				if (sc->schemaName() == "infinidb_unknown" && !qfe::DefaultSchema.empty())
-					sc->schemaName(qfe::DefaultSchema);
-				sc->setOID();
-				sc->tableAlias(sc->tableName());
-				sc->alias(execplan::make_tcn(sc->schemaName(), sc->tableName(), sc->columnName()).toString());
-				$$ = $1;
-			}
-			| func_col
-			;
-
-func_col:	FUNC '(' select_column_name ')'
-		{
-			execplan::AggregateColumn* ag = new execplan::AggregateColumn(execplan::AggregateColumn::agname2num(*($1)), $3);
-			delete $1;
-			$$ = ag;
-		}
-		;
-
-aliased_column_name:	select_column_name opt_alias
-			;
-
-opt_alias:	/* empty */
-		{ $$ = 0; }
-		| opt_as OBJNAME
-		{
-			delete $2;
-			$$ = $1;
-		}
-		| opt_as CHAR_CONST
-		{
-			delete $2;
-			$$ = $1;
-		}
-		;
-
-opt_as:	/* empty */
-	{ $$ = 0; }
-	| AS
-	;
-
-table_spec:	aliased_table_name
-		;
-
-/* this is too loose for a table name, but it's easier to parse */
-table_name:	obj_name
-		{
-			execplan::SimpleColumn* sc=0;
-			sc = dynamic_cast<execplan::SimpleColumn*>($1);
-			execplan::CalpontSystemCatalog::TableAliasName tan;
-			tan.schema = sc->tableName();
-			if (tan.schema == "infinidb_unknown" && !qfe::DefaultSchema.empty())
-				tan.schema = qfe::DefaultSchema;
-			if (qfe::DefaultSchema.empty())
-				qfe::DefaultSchema = tan.schema;
-			tan.table = sc->columnName();
-			tan.alias = tan.table;
-			execplan::CalpontSelectExecutionPlan::TableList tl;
-			tl.push_back(tan);
-			ParserCSEP->tableList(tl);
-			delete sc;
-			$$ = 0;
-		}
-		;
-
-aliased_table_name:	table_name opt_alias
-			;
-
-opt_where_clause:	/* empty */
-			| QFEP_WHERE filter_predicate_list
-			;
-
-filter_predicate_list:	obj_name RELOP constant
-			{
-				pair<int, string> cval = *($3);
-				delete $3;
-				execplan::SimpleColumn* sc=0;
-				sc = dynamic_cast<execplan::SimpleColumn*>($1);
-				qfe::utils::updateParseTree(ParserCSC, ParserCSEP,
-					sc, *($2), cval);
-				delete sc;
-				delete $2;
-				$$ = 0;
-			}
-			| filter_predicate_list LOGICOP obj_name RELOP constant
-			{
-				//string logicop = *($2);
-				delete $2;
-				pair<int, string> cval = *($5);
-				delete $5;
-				execplan::SimpleColumn* sc=0;
-				sc = dynamic_cast<execplan::SimpleColumn*>($3);
-				qfe::utils::updateParseTree(ParserCSC, ParserCSEP,
-					sc, *($4), cval);
-				delete sc;
-				delete $4;
-				$$ = 0;
-			}
-			;
-
-obj_name:	obj_name_aux
-		{
-			//This is possibly a table name, but we shove it into a SimpleColumn. We'll
-			// fix this in the table_spec production
-			execplan::SimpleColumn* sc = new execplan::SimpleColumn("infinidb_unknown", "infinidb_unknown", *($1));
-			sc->tableAlias("infinidb_unknown");
-			sc->alias(execplan::make_tcn("infinidb_unknown", "infinidb_unknown", *($1)).toString());
-			//cerr << "inside parser: " << *sc << endl;
-			delete $1;
-			$$ = sc;
-		}
-		| obj_name_aux '.' obj_name_aux
-		{
-			//This is possibly a table name, but we shove it into a SimpleColumn. We'll
-			// fix this in the table_spec production
-			execplan::SimpleColumn* sc = new execplan::SimpleColumn("infinidb_unknown", *($1), *($3));
-			sc->tableAlias(*($1));
-			sc->alias(execplan::make_tcn("infinidb_unknown", *($1), *($3)).toString());
-			//cerr << "inside parser: " << *sc << endl;
-			delete $1;
-			delete $3;
-			$$ = sc;
-		}
-		| obj_name_aux '.' obj_name_aux '.' obj_name_aux
-		{
-			if (qfe::DefaultSchema.empty())
-				qfe::DefaultSchema = *($1);
-			execplan::SimpleColumn* sc = new execplan::SimpleColumn(*($1), *($3), *($5));
-			sc->tableAlias(*($3));
-			sc->alias(execplan::make_tcn(*($1), *($3), *($5)).toString());
-			//cerr << "inside parser: " << *sc << endl;
-			delete $1;
-			delete $3;
-			delete $5;
-			$$ = sc;
-		}
-		;
-
-obj_name_aux:	OBJNAME
-		| '`' OBJNAME '`'
-		{
-			$$ = $2;
-		}
-		;
-
-constant:	CHAR_CONST
-		{
-			pair<int, string>* p = new pair<int, string>(1, *($1));
-			delete $1;
-			$$ = p;
-		}
-		| INT_CONST
-		{
-			pair<int, string>* p = new pair<int, string>(0, *($1));
-			delete $1;
-			$$ = p;
-		}
-		;
-
-opt_groupby_clause:	/* empty */
-			| GROUPBY groupby_items_list
-			{ YYERROR; }
-			;
-
-groupby_items_list:	groupby_item
-			| groupby_items_list ',' groupby_item
-			{ delete $3; }
-			;
-
-groupby_item:	obj_name
-		| INT_CONST
-		{
-			delete $1;
-			$$ = 0;
-		}
-		;
-
-opt_orderby_clause:	/* empty */
-			| ORDERBY groupby_items_list opt_direction
-			{ YYERROR; }
-			;
-
-opt_direction:	/* empty */
-		| ASC
-		| DESC
-		;
-
-opt_limit_clause:	/* empty */
-			{ $$ = 0; }
-			| LIMIT INT_CONST
-			{
-				delete $2;
-				YYERROR;
-				$$ = 0;
-			}
-			| LIMIT INT_CONST ',' INT_CONST
-			{
-				delete $2;
-				delete $4;
-				YYERROR;
-				$$ = 0;
-			}
-			;
-
-%%
-
-int qfeerror(const string& s)
-{
-  extern int qfelineno;	// defined and maintained in lex.c
-  extern char *qfetext;	// defined and maintained in lex.c
-  
-  cerr << "ERROR: " << s << " at symbol \"" << qfetext;
-  cerr << "\" on line " << qfelineno << endl;
-  return -1;
-}
-
-int qfeerror(const char *s)
-{
-  return qfeerror(string(s));
-}
-
-string* newstr(const char* cp)
-{
-	string* strp;
-	strp = new string(cp);
-	return strp;
-}
-

tools/qfe/returnedrows.cpp

@@ -1,95 +0,0 @@
-#include <unistd.h>
-#include <iostream>
-#include <string>
-#include <cerrno>
-using namespace std;
-
-#include <boost/scoped_ptr.hpp>
-#include <boost/scoped_array.hpp>
-using namespace boost;
-
-#include "mcsconfig.h"
-
-#include "socktype.h"
-
-#include "exceptclasses.h"
-
-#include "socketio.h"
-
-#include "bytestream.h"
-#include "messagequeue.h"
-using namespace messageqcpp;
-
-#include "rowgroup.h"
-using namespace rowgroup;
-
-namespace qfe
-{
-void processReturnedRows(MessageQueueClient* mqc, SockType fd)
-{
-  scoped_ptr<MessageQueueClient> cleaner(mqc);
-  SBS sbs;
-  sbs = mqc->read();
-  // cerr << "got a bs of " << sbs->length() << " bytes" << endl;
-
-  RowGroup rg;
-  rg.deserialize(*sbs);
-
-  // cerr << "got a base rowgroup with rows of " << rg.getRowSize() << " bytes" << endl;
-  // cerr << rg.toString() << endl;
-
-  ByteStream bs;
-  ByteStream::quadbyte tableOID = 100;
-  bs.reset();
-  bs << tableOID;
-  mqc->write(bs);
-
-  sbs = mqc->read();
-  // cerr << "got a bs of " << sbs->length() << " bytes" << endl;
-  RGData rgd;
-  rgd.deserialize(*sbs, true);
-  rg.setData(&rgd);
-  // cerr << "got a rowgroup with: " << rg.getRowCount() << " rows" << endl;
-
-  socketio::writeString(fd, "OK");
-  Row r;
-
-  while (rg.getRowCount() > 0)
-  {
-    rg.initRow(&r);
-    rg.getRow(0, &r);
-    string csv;
-    bs.reset();
-
-    for (unsigned i = 0; i < rg.getRowCount(); i++)
-    {
-      csv = r.toCSV();
-      bs << csv;
-      r.nextRow();
-    }
-
-    // cerr << "writing " << bs.length() << " bytes back to client" << endl;
-    SockWriteFcn(fd, bs.buf(), bs.length());
-
-    bs.reset();
-    bs << tableOID;
-    mqc->write(bs);
-
-    sbs = mqc->read();
-    // cerr << "got a bs of " << sbs->length() << " bytes" << endl;
-    rgd.deserialize(*sbs, true);
-    rg.setData(&rgd);
-    // cerr << "got a rowgroup with: " << rg.getRowCount() << " rows" << endl;
-  }
-
-  tableOID = 0;
-  bs.reset();
-  bs << tableOID;
-  mqc->write(bs);
-
-  // sync with the client on end-of-results
-  SockWriteFcn(fd, &tableOID, 4);
-  SockReadFcn(fd, &tableOID, 4);
-}
-
-}  // namespace qfe

tools/qfe/returnedrows.h

@@ -1,10 +0,0 @@
-#pragma once
-
-#include "socktype.h"
-#include "messagequeue.h"
-
-namespace qfe
-{
-void processReturnedRows(messageqcpp::MessageQueueClient*, SockType);
-
-}

tools/qfe/sendcsep.cpp

@@ -1,53 +0,0 @@
-#include <unistd.h>
-#include <stdexcept>
-#include <memory>
-using namespace std;
-
-#include <boost/scoped_ptr.hpp>
-using namespace boost;
-
-#include "calpontselectexecutionplan.h"
-using namespace execplan;
-#include "bytestream.h"
-#include "messagequeue.h"
-using namespace messageqcpp;
-
-namespace qfe
-{
-MessageQueueClient* sendCSEP(CalpontSelectExecutionPlan* csep)
-{
-  scoped_ptr<CalpontSelectExecutionPlan> cleaner(csep);
-
-  ByteStream bs;
-
-  MessageQueueClient* mqc = 0;
-
-  mqc = new MessageQueueClient("ExeMgr1");
-  auto_ptr<MessageQueueClient> smqc(mqc);
-
-  bs.reset();
-  ByteStream::quadbyte wantTuples = 4;
-  bs << wantTuples;
-  mqc->write(bs);
-
-  bs.reset();
-  csep->serialize(bs);
-  mqc->write(bs);
-
-  SBS sbs;
-  sbs = mqc->read();
-  *sbs >> wantTuples;
-  // cerr << "got flag: " << wantTuples << endl;
-  string msg;
-  sbs = mqc->read();
-  *sbs >> msg;
-  // cerr << "got msg: " << msg << endl;
-
-  if (wantTuples != 0)
-    throw runtime_error(msg);
-
-  smqc.release();
-  return mqc;
-}
-
-}  // namespace qfe

tools/qfe/sendcsep.h

@@ -1,12 +0,0 @@
-#pragma once
-
-#include "messagequeue.h"
-#include "calpontselectexecutionplan.h"
-
-namespace qfe
-{
-// Takes ownership of the alloc'd ptr
-// Returns an alloc'd mqc
-messageqcpp::MessageQueueClient* sendCSEP(execplan::CalpontSelectExecutionPlan*);
-
-}  // namespace qfe

tools/qfe/server.cpp

@@ -1,429 +0,0 @@
-/* Copyright (C) 2014 InfiniDB, Inc.
-
-   This program is free software; you can redistribute it and/or
-   modify it under the terms of the GNU General Public License
-   as published by the Free Software Foundation; version 2 of
-   the License.
-
-   This program is distributed in the hope that it will be useful,
-   but WITHOUT ANY WARRANTY; without even the implied warranty of
-   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-   GNU General Public License for more details.
-
-   You should have received a copy of the GNU General Public License
-   along with this program; if not, write to the Free Software
-   Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
-   MA 02110-1301, USA. */
-
-/*
-        Protocol definition:
-*/
-
-#include <poll.h>
-#include <sys/socket.h>
-#include <netinet/in.h>
-#include <arpa/inet.h>
-#include <iostream>
-#include <string>
-#include <stdexcept>
-#include <sstream>
-#include <iomanip>
-#include <sys/types.h>
-#include <sys/stat.h>
-#include <fcntl.h>
-#include <unistd.h>
-#include <signal.h>
-#include <cerrno>
-using namespace std;
-
-#include <boost/thread.hpp>
-#include <boost/scoped_ptr.hpp>
-#include <boost/scoped_array.hpp>
-#include <boost/tokenizer.hpp>
-#include <boost/algorithm/string.hpp>
-using namespace boost;
-
-#include "mcsconfig.h"
-
-#include "socktype.h"
-#include "parsequery.h"
-#include "sendcsep.h"
-#include "returnedrows.h"
-#include "socketio.h"
-#include "ddlstmts.h"
-
-#include "exceptclasses.h"  //brings in idbassert_s macro
-
-#include "messagequeue.h"
-using namespace messageqcpp;
-
-#include "atomicops.h"
-
-namespace execplan
-{
-class CalpontSelectExecutionPlan;
-}
-
-#define SINGLE_THREADED
-
-namespace qfe
-{
-string DefaultSchema;
-}
-
-namespace
-{
-using namespace qfe;
-
-enum StmtType
-{
-  UNKNOWN,
-  QUERY,
-  CREATE,
-  DROP,
-  SHOW,
-};
-
-volatile uint32_t SystemSID;
-
-void log(const string& s)
-{
-  cerr << s << endl;
-}
-
-struct QueryMessage
-{
-  QueryMessage() : isValid(false)
-  {
-  }
-  ~QueryMessage()
-  {
-  }
-
-  string toString() const;
-
-  bool isValid;
-  string queryText;
-  string defaultSchema;
-};
-
-string QueryMessage::toString() const
-{
-  ostringstream oss;
-  oss << "valid: " << boolalpha << isValid << ", "
-      << "queryText: " << queryText << ", "
-      << "defaultSchema: " << defaultSchema;
-  return oss.str();
-}
-
-ostream& operator<<(ostream& os, const QueryMessage& rhs)
-{
-  os << rhs.toString();
-  return os;
-}
-
-class ThreadFunc
-{
- public:
-  ThreadFunc(SockType fd) : fFd(fd)
-  {
-  }
-  ~ThreadFunc()
-  {
-  }
-
-  void run();
-  void operator()()
-  {
-    run();
-  }
-
- private:
-  ThreadFunc(const ThreadFunc& rhs);
-  ThreadFunc& operator=(const ThreadFunc& rhs);
-
-  SockType fFd;
-};
-
-bool serverInit()
-{
-
-  setsid();
-
-  // Handle certain signals (we want these to return EINTR so we can throw)
-  // SIGPIPE
-  // I don't think we'll get any of these from init (except possibly HUP, but that's an indication
-  // of bad things anyway)
-  // SIGHUP?
-  // SIGUSR1?
-  // SIGUSR2?
-  // SIGPOLL?
-  struct sigaction sa;
-  memset(&sa, 0, sizeof(struct sigaction));
-  sa.sa_handler = SIG_IGN;
-  sigaction(SIGPIPE, &sa, 0);
-  sigaction(SIGHUP, &sa, 0);
-  sigaction(SIGUSR1, &sa, 0);
-  sigaction(SIGUSR2, &sa, 0);
-#ifndef __FreeBSD__
-  sigaction(SIGPOLL, &sa, 0);
-#endif
-#if 0
-    int fd;
-    close(2);
-    fd = open("/tmp/qfe.err", O_CREAT | O_TRUNC | O_WRONLY, 0644);
-
-    if (fd >= 0 && fd != 2)
-    {
-        dup2(fd, 2);
-        close(fd);
-    }
-
-#endif
-  return true;
-}
-
-SockType initListenSock(short portNo)
-{
-  SockType listenSock = -1;
-  listenSock = socket(PF_INET, SOCK_STREAM, IPPROTO_TCP);
-  idbassert_s(listenSock >= 0, string("socket create error: ") + strerror(errno));
-  // if (listenSock < 0) throw runtime_error(string("socket create error: ") + strerror(errno));
-  int optval = 1;
-  setsockopt(listenSock, SOL_SOCKET, SO_REUSEADDR, reinterpret_cast<const char*>(&optval), sizeof(optval));
-  int rc = 0;
-  struct sockaddr_in serv_addr;
-  memset(&serv_addr, 0, sizeof(serv_addr));
-  serv_addr.sin_family = AF_INET;
-  serv_addr.sin_addr.s_addr = INADDR_ANY;
-  serv_addr.sin_port = htons(portNo);
-  const int MaxTries = 5 * 60 / 10;
-  int tries = 0;
-again:
-  rc = ::bind(listenSock, (sockaddr*)&serv_addr, sizeof(serv_addr));
-
-  if (rc < 0)
-  {
-    if (errno == EADDRINUSE)
-    {
-      // cerr << "Addr in use..." << endl;
-      if (++tries >= MaxTries)
-      {
-        log("Waited too long for socket to bind...giving up");
-        // cerr << "Waited too long for socket to bind...giving up" << endl;
-        exit(1);
-      }
-
-      sleep(10);
-      goto again;
-    }
-
-    idbassert_s(0, string("socket bind error: ") + strerror(errno));
-    // throw runtime_error(string("socket bind error: ") + strerror(errno));
-  }
-
-  rc = listen(listenSock, 16);
-  idbassert_s(rc >= 0, string("socket listen error") + strerror(errno));
-  // if (rc < 0) throw runtime_error(string("socket listen error") + strerror(errno));
-
-  return listenSock;
-}
-
-QueryMessage getNextMsg(SockType fd)
-{
-  QueryMessage msg;
-
-  try
-  {
-    msg.defaultSchema = socketio::readString(fd);
-    msg.queryText = socketio::readString(fd);
-    msg.isValid = true;
-  }
-  catch (runtime_error& rex)
-  {
-    cerr << "re reading ctl msg: " << rex.what() << endl;
-    msg.queryText = "";
-  }
-  catch (...)
-  {
-    cerr << "ex reading ctl msg" << endl;
-    msg.queryText = "";
-  }
-
-  return msg;
-}
-
-StmtType guessStatementType(const string& stmt)
-{
-  typedef boost::tokenizer<boost::char_separator<char> > tokenizer;
-  char_separator<char> sep;
-  tokenizer tokens(stmt, sep);
-  tokenizer::iterator tok_iter = tokens.begin();
-  string first_word;
-  first_word = *tok_iter;
-  algorithm::to_lower(first_word);
-
-  if (first_word == "select")
-    return QUERY;
-
-  if (first_word == "create")
-    return CREATE;
-
-  if (first_word == "drop")
-    return DROP;
-
-  if (first_word == "show")
-    return SHOW;
-
-  return UNKNOWN;
-}
-
-struct ScopedCleaner
-{
-  ScopedCleaner(SockType fd = -1) : fFd(fd)
-  {
-  }
-  ~ScopedCleaner()
-  {
-    if (fFd >= 0)
-      shutdown(fFd, SHUT_RDWR);
-
-    close(fFd);
-  }
-
-  SockType fFd;
-};
-
-void ThreadFunc::run()
-{
-  QueryMessage m;
-  execplan::CalpontSelectExecutionPlan* csep = 0;
-  MessageQueueClient* msgqcl;
-
-  ScopedCleaner cleaner(fFd);
-
-  uint32_t sid = 1;
-  sid = atomicops::atomicInc(&SystemSID);
-
-  try
-  {
-    m = getNextMsg(fFd);
-
-    if (m.isValid)
-    {
-      DefaultSchema = m.defaultSchema;
-      StmtType st = guessStatementType(m.queryText);
-
-      switch (st)
-      {
-        case QUERY:
-          csep = parseQuery(m.queryText, sid);
-          // sendCSEP takes ownership of the ptr from parseQuery
-          msgqcl = sendCSEP(csep);
-          // processReturnedRows takes ownership of the ptr from sendCSEP
-          processReturnedRows(msgqcl, fFd);
-          break;
-
-        case CREATE: processCreateStmt(m.queryText, sid); break;
-
-        case DROP: processDropStmt(m.queryText, sid); break;
-
-        case SHOW:
-        {
-          ostringstream oss;
-          oss << "select calpontsys.systable.tablename from calpontsys.systable where "
-                 "calpontsys.systable.schema='"
-              << m.defaultSchema << "';";
-          csep = parseQuery(oss.str(), sid);
-          msgqcl = sendCSEP(csep);
-          processReturnedRows(msgqcl, fFd);
-          break;
-        }
-
-        default: throw runtime_error("couldn't guess the statement type"); break;
-      }
-    }
-  }
-  catch (std::exception& ex)
-  {
-    socketio::writeString(fFd, ex.what());
-    throw;  // in a multi-threaded server this will simply cause this thread to exit
-  }
-  catch (...)
-  {
-    socketio::writeString(fFd, "internal query processing error");
-    throw;
-  }
-}
-
-}  // namespace
-
-int main(int argc, char** argv)
-{
-  int c;
-  SockType listenSock;
-  short portNo;
-
-  portNo = 0;
-  char* p = getenv("IDB_QFE_PORT");
-
-  if (p && *p)
-    portNo = atoi(p);
-
-  if (portNo <= 0)
-    portNo = 9198;
-
-  listenSock = -1;
-  opterr = 0;
-
-  while ((c = getopt(argc, argv, "p:")) != -1)
-    switch (c)
-    {
-      case 'p': portNo = atoi(optarg); break;
-
-      case '?':
-      default: break;
-    }
-
-  if (!serverInit())
-  {
-    log("Could not initialize the QFE Server!");
-    cerr << "Could not initialize the QFE Server!" << endl;
-    return 1;
-  }
-
-  listenSock = initListenSock(portNo);
-
-  SystemSID = 0;
-
-  for (;;)
-  {
-    int querySock = -1;
-    querySock = accept(listenSock, 0, 0);
-    idbassert_s(querySock >= 0, string("socket accept error: ") + strerror(errno));
-
-    // ThreadFunc now owns querySock and is responsible for cleaning it up
-    ThreadFunc tf(querySock);
-
-#ifdef SINGLE_THREADED
-
-    try
-    {
-      tf.run();
-    }
-    catch (std::exception& ex)
-    {
-      cerr << "ThreadFunc run threw an exception: " << ex.what() << endl;
-    }
-    catch (...)
-    {
-      cerr << "ThreadFunc run threw an exception" << endl;
-    }
-
-#else
-    thread t(tf);
-#endif
-  }
-
-  return 0;
-}

tools/qfe/socketio.cpp

@@ -1,133 +0,0 @@
-#include <unistd.h>
-#include <poll.h>
-#include <sys/socket.h>
-#include <netinet/in.h>
-#include <arpa/inet.h>
-#include <cerrno>
-#include <sstream>
-using namespace std;
-
-#include <boost/scoped_array.hpp>
-using namespace boost;
-
-#include "mcsconfig.h"
-
-#include "exceptclasses.h"
-
-#include "socketio.h"
-
-namespace qfe
-{
-namespace socketio
-{
-void readn(int fd, void* buf, const size_t wanted)
-{
-  size_t needed = wanted;
-  size_t sofar = 0;
-  char* p = static_cast<char*>(buf);
-  ssize_t rrc = -1;
-  pollfd fds[1];
-  int en = 0;
-
-  fds[0].fd = fd;
-  fds[0].events = POLLIN;
-
-  while (wanted > sofar)
-  {
-    fds[0].revents = 0;
-    poll(fds, 1, -1);
-    errno = 0;
-    rrc = read(fd, (p + sofar), needed);
-    en = errno;
-
-    if (rrc < 0)
-    {
-      if (en == EAGAIN || en == EINTR || en == 512)
-        continue;
-
-      ostringstream oss;
-      oss << "qfe: readn: read() returned " << rrc << " (" << strerror(en) << ")";
-      idbassert_s(0, oss.str());
-    }
-
-    needed -= rrc;
-    sofar += rrc;
-  }
-}
-
-size_t writen(int fd, const void* data, const size_t nbytes)
-{
-  size_t nleft;
-  ssize_t nwritten;
-  const char* bufp = static_cast<const char*>(data);
-  nleft = nbytes;
-
-  while (nleft > 0)
-  {
-    // the O_NONBLOCK flag is not set, this is a blocking I/O.
-    if ((nwritten = ::write(fd, bufp, nleft)) < 0)
-    {
-      if (errno == EINTR)
-        nwritten = 0;
-      else
-      {
-        // save the error no first
-        int e = errno;
-        string errorMsg = "qfe: writen: write() error: ";
-        scoped_array<char> buf(new char[80]);
-#if STRERROR_R_CHAR_P
-        const char* p;
-
-        if ((p = strerror_r(e, buf.get(), 80)) != 0)
-          errorMsg += p;
-
-#else
-        int p;
-
-        if ((p = strerror_r(e, buf.get(), 80)) == 0)
-          errorMsg += buf.get();
-
-#endif
-        idbassert_s(0, errorMsg);
-      }
-    }
-
-    nleft -= nwritten;
-    bufp += nwritten;
-  }
-
-  return nbytes;
-}
-
-uint32_t readNumber32(SockType fd)
-{
-  uint32_t np;
-  SockReadFcn(fd, &np, 4);
-  return np;
-}
-
-string readString(SockType fd)
-{
-  string s;
-  uint32_t len = readNumber32(fd);
-
-  if (len > 0)
-  {
-    scoped_array<char> buf(new char[len + 1]);
-    SockReadFcn(fd, buf.get(), len);
-    buf[len] = 0;
-    s = buf.get();
-  }
-
-  return s;
-}
-
-void writeString(SockType fd, const string& data)
-{
-  uint32_t len = data.length();
-  SockWriteFcn(fd, &len, 4);
-  SockWriteFcn(fd, data.c_str(), len);
-}
-
-}  // namespace socketio
-}  // namespace qfe

tools/qfe/socketio.h

@@ -1,20 +0,0 @@
-#pragma once
-
-#include <stdint.h>
-#include <unistd.h>
-#include <string>
-
-#include "socktype.h"
-
-namespace qfe
-{
-namespace socketio
-{
-void readn(int fd, void* buf, const size_t wanted);
-size_t writen(int fd, const void* data, const size_t nbytes);
-uint32_t readNumber32(SockType);
-std::string readString(SockType);
-void writeString(SockType, const std::string&);
-
-}  // namespace socketio
-}  // namespace qfe

tools/qfe/socktype.h

@@ -1,6 +0,0 @@
-#pragma once
-
-
-typedef int SockType;
-#define SockReadFcn qfe::socketio::readn
-#define SockWriteFcn qfe::socketio::writen

tools/rebuildEM/CMakeLists.txt

@@ -1,10 +1,5 @@
 include_directories(${ENGINE_COMMON_INCLUDES})
 
 set(rebuildEM_SRCS main.cpp rebuildEM.cpp)
-add_executable(mcsRebuildEM ${rebuildEM_SRCS})
+columnstore_executable(mcsRebuildEM ${rebuildEM_SRCS})
-target_link_libraries(mcsRebuildEM ${ENGINE_LDFLAGS} ${ENGINE_WRITE_LIBS} boost_system boost_filesystem)
+columnstore_link(mcsRebuildEM ${ENGINE_LDFLAGS} ${ENGINE_WRITE_LIBS} boost_system boost_filesystem)
-install(
-    TARGETS mcsRebuildEM
-    DESTINATION ${ENGINE_BINDIR}
-    COMPONENT columnstore-engine
-)

tools/rgprint/CMakeLists.txt

@@ -4,12 +4,6 @@ include_directories(${ENGINE_COMMON_INCLUDES})
 
 set(rgprint_SRCS rgprint.cpp)
 
-add_executable(rgprint ${rgprint_SRCS})
+columnstore_executable(rgprint ${rgprint_SRCS})
 
-target_link_libraries(rgprint ${ENGINE_LDFLAGS} ${NETSNMP_LIBRARIES} ${ENGINE_WRITE_LIBS})
+columnstore_link(rgprint ${ENGINE_LDFLAGS} ${NETSNMP_LIBRARIES} ${ENGINE_WRITE_LIBS})
-
-install(
-    TARGETS rgprint
-    DESTINATION ${ENGINE_BINDIR}
-    COMPONENT columnstore-engine
-)

tools/sendPlan/CMakeLists.txt

@@ -1,32 +0,0 @@
-include_directories(${KDE4_INCLUDES} ${KDE4_INCLUDE_DIR} ${QT_INCLUDES})
-
-# ########## next target ###############
-
-set(sendPlan_SRCS sendplan.cpp)
-
-kde4_add_executable(sendPlan ${sendPlan_SRCS})
-
-target_link_libraries(sendPlan ${KDE4_KDECORE_LIBS})
-
-install(TARGETS sendPlan ${INSTALL_TARGETS_DEFAULT_ARGS})
-
-# ########## install files ###############
-
-# original Makefile.am contents follow:
-
-# $Id: Makefile.am 333 2009-04-03 20:35:04Z rdempsey $ Process this file with automake to produce Makefile.in
-#
-# AM_CPPFLAGS = $(idb_cppflags) AM_CFLAGS = $(idb_cflags) AM_CXXFLAGS = $(idb_cxxflags) AM_LDFLAGS = $(idb_ldflags)
-# bin_PROGRAMS = sendPlan sendPlan_SOURCES = sendplan.cpp sendPlan_CPPFLAGS = @idb_common_includes@ $(AM_CPPFLAGS)
-# sendPlan_LDFLAGS = @idb_common_ldflags@ @idb_exec_libs@ $(AM_LDFLAGS)
-#
-# test:
-#
-# coverage:
-#
-# leakcheck:
-#
-# docs:
-#
-# bootstrap: install-data-am
-#

tools/sendPlan/sendplan.cpp

@@ -1,392 +0,0 @@
-// $Id: sendplan.cpp 1739 2012-03-22 12:57:59Z pleblanc $
-#include <iostream>
-#include <fstream>
-#include <sstream>
-#include <unistd.h>
-#include <sys/time.h>
-#include <iomanip>
-using namespace std;
-
-#include "bytestream.h"
-using namespace messageqcpp;
-
-#include "calpontselectexecutionplan.h"
-#include "sessionmanager.h"
-using namespace execplan;
-
-#include "calpontsystemcatalog.h"
-using namespace execplan;
-
-#include "tableband.h"
-#include "joblist.h"
-#include "joblistfactory.h"
-using namespace joblist;
-
-#include "configcpp.h"
-#include "errorcodes.h"
-
-#include "rowgroup.h"
-using namespace rowgroup;
-
-typedef CalpontSystemCatalog::OID OID;
-
-namespace
-{
-bool vflg;
-
-void usage()
-{
-  cout << "usage: sendPlan [-v|h|d|B] [-t lvl] [-s sid] plan_file ..." << endl;
-  cout << "-v verbose output" << endl;
-  cout << "-t lvl set trace level to lvl" << endl;
-  cout << "-s sid set session id to sid" << endl;
-  cout << "-d display the query, but don't run it" << endl;
-  cout << "-B Bob's personal preferred output format" << endl;
-  cout << "-h display this help" << endl;
-}
-
-double tm_diff(const struct timeval* st, const struct timeval* et)
-{
-  double sd = (double)st->tv_sec + (double)st->tv_usec / 1000000.0;
-  double ed = (double)et->tv_sec + (double)et->tv_usec / 1000000.0;
-
-  return (ed - sd);
-}
-
-//...Extract the stats string to be printed and the runtime start time from
-//...the full stats string we receive from ExeMgr.
-void parseStatsString(const string& fullStatsString, string& printStatsString,
-                      struct timeval& queryRunStartTime)
-{
-  string::size_type delimPos = fullStatsString.find('|');
-  printStatsString = fullStatsString.substr(0, delimPos);
-
-  istringstream startTimeString(fullStatsString.substr(delimPos + 1));
-  startTimeString >> queryRunStartTime.tv_sec >> queryRunStartTime.tv_usec;
-}
-
-}  // namespace
-
-int main(int argc, char** argv)
-{
-  vflg = false;
-  uint32_t tlvl = 0;
-  bool dflg = false;
-  int c;
-  int32_t sid = -1;
-  bool Bflg = false;
-
-  opterr = 0;
-
-  while ((c = getopt(argc, argv, "vt:ds:Bh")) != EOF)
-    switch (c)
-    {
-      case 't': tlvl = static_cast<uint32_t>(strtoul(optarg, 0, 0)); break;
-
-      case 'v': vflg = true; break;
-
-      case 'd': dflg = true; break;
-
-      case 's': sid = static_cast<int32_t>(strtol(optarg, 0, 0)); break;
-
-      case 'B': Bflg = true; break;
-
-      case 'h':
-      case '?':
-      default:
-        usage();
-        return (c == 'h' ? 0 : 1);
-        break;
-    }
-
-  if (dflg)
-    vflg = true;
-
-  if ((argc - optind) < 1)
-  {
-    usage();
-    return 1;
-  }
-
-  ifstream inputf;
-  ByteStream bs;
-  ByteStream dbs;
-  ByteStream eoq;
-  ByteStream tbs;
-  ByteStream statsStream;
-  ByteStream::quadbyte q = 0;
-  eoq << q;
-  uint32_t sessionid;
-  time_t t;
-  SJLP jl;
-  DeliveredTableMap tm;
-  DeliveredTableMap::iterator iter;
-  DeliveredTableMap::iterator end;
-  CalpontSelectExecutionPlan csep;
-  struct timeval start_time;
-  struct timeval end_time;
-
-  MessageQueueClient* mqc = 0;
-
-  if (!dflg)
-    mqc = new MessageQueueClient("ExeMgr1");
-
-  if (sid == -1)
-  {
-    time(&t);
-    sessionid = static_cast<uint32_t>(t);
-  }
-  else
-  {
-    sessionid = static_cast<uint32_t>(sid);
-  }
-
-  sessionid &= 0x7fffffff;
-  logging::ErrorCodes errorCodes;
-
-  for (; optind < argc; optind++)
-  {
-    inputf.open(argv[optind]);
-
-    if (!inputf.good())
-    {
-      cerr << "error opening plan stream " << argv[optind] << endl;
-      return 1;
-    }
-
-    bs.reset();
-    inputf >> bs;
-
-    inputf.close();
-
-    csep.unserialize(bs);
-
-    csep.sessionID(sessionid);
-    SessionManager sm;
-    csep.verID(sm.verID());
-
-    csep.traceFlags(0);
-    ResourceManager* rm = ResourceManager::instance();
-    jl = JobListFactory::makeJobList(&csep, rm);
-    csep.traceFlags(tlvl);
-
-    if (vflg)
-    {
-      if (dflg)
-        cout << endl << "Query:" << endl;
-      else
-      {
-        cout << endl << "Session: " << sessionid << ", Sending Query";
-
-        if (Bflg)
-          cout << " (" << argv[optind] << ')';
-
-        cout << ':' << endl;
-      }
-
-      if (!Bflg)
-        cout << csep.data() << endl << endl;
-    }
-
-    if (dflg)
-      continue;
-
-    try
-    {
-      dbs.reset();
-      csep.serialize(dbs);
-
-      gettimeofday(&start_time, 0);
-
-      // try tuples first, but expect the worst...
-      bool expectTuples = false;
-      ByteStream tbs;
-      ByteStream::quadbyte tqb = 4;
-      tbs << tqb;
-      mqc->write(tbs);
-
-      // send the CSEP
-      mqc->write(dbs);
-
-      // read the response to the tuple request
-      tbs = mqc->read();
-      idbassert(tbs.length() == 4);
-      tbs >> tqb;
-
-      if (tqb == 4)
-        expectTuples = true;
-
-      if (!expectTuples)
-        cout << "Using TableBand I/F" << endl;
-      else
-        cout << "Using tuple I/F" << endl;
-
-      tm = jl->deliveredTables();
-
-      iter = tm.begin();
-      end = tm.end();
-
-      OID toid;
-      uint64_t rowTot;
-      bool reported = false;
-      bool needRGCtor = true;
-
-      while (iter != end)
-      {
-        toid = iter->first;
-        q = static_cast<ByteStream::quadbyte>(toid);
-        tbs.reset();
-        tbs << q;
-        mqc->write(tbs);
-
-        ByteStream tbbs;
-        TableBand tb;
-        RowGroup rg;
-        rowTot = 0;
-        uint16_t status = 0;
-        TableBand::VBA::size_type rc;
-        ofstream out;
-
-        for (;;)
-        {
-          tbbs = mqc->read();
-#if 0
-                    cout << tbbs.length() << endl;
-                    out.open("bs1.dat");
-                    idbassert(out.good());
-                    out << tbbs;
-                    out.close();
-                    tbbs = mqc->read();
-                    cout << tbbs.length() << endl;
-                    out.open("bs2.dat");
-                    idbassert(out.good());
-                    out << tbbs;
-                    out.close();
-                    tbbs = mqc->read();
-                    cout << tbbs.length() << endl;
-                    out.open("bs3.dat");
-                    idbassert(out.good());
-                    out << tbbs;
-                    out.close();
-#endif
-
-          if (tbbs.length())
-          {
-            if (!expectTuples)
-              tb.unserialize(tbbs);
-            else
-            {
-              if (needRGCtor)
-              {
-                rg.deserialize(tbbs);
-                needRGCtor = false;
-                tbbs = mqc->read();
-              }
-
-              rg.setData((uint8_t*)tbbs.buf());
-            }
-          }
-          else
-          {
-            //@bug 1346
-            if (!status)
-              status = logging::makeJobListErr;
-
-            break;
-          }
-
-          if (!expectTuples)
-          {
-            rc = tb.getRowCount();
-            status = tb.getStatus();
-          }
-          else
-          {
-            rc = rg.getRowCount();
-            status = rg.getStatus();
-
-            if (rc == 0)
-              status = 0;
-          }
-
-          if (rc == 0)
-            break;
-
-          rowTot += rc;
-        }
-
-        BatchPrimitive* step = dynamic_cast<BatchPrimitive*>(iter->second.get());
-
-        if (vflg && step)
-        {
-          cout << "For table " << step->tableName();
-
-          if (!Bflg)
-            cout << " " << toid;
-
-          cout << ": read " << rowTot << " rows" << endl;
-        }
-
-        if (status && !reported)
-        {
-          cout << "### Query failed: " << errorCodes.errorString(status) << "  Check crit.log\n";
-          reported = true;
-        }
-
-        if (!step && !reported)
-        {
-          cout << "### Query failed: Did not return project BatchPrimitive. Check crit.log\n";
-          reported = true;
-        }
-
-        ++iter;
-      }
-
-      if (vflg)
-      {
-        gettimeofday(&end_time, 0);
-        cout << "Query time: " << fixed << setprecision(1) << tm_diff(&start_time, &end_time) << " secs"
-             << endl;
-
-        //...Ask for query stats through special table id of 3
-        const OID TABLE_ID_TO_GET_QUERY_STATS = 3;
-
-        if (!Bflg)
-          cout << "Retrieving stats..." << endl;
-
-        toid = TABLE_ID_TO_GET_QUERY_STATS;
-        q = static_cast<ByteStream::quadbyte>(toid);
-        statsStream.reset();
-        statsStream << q;
-        mqc->write(statsStream);
-
-        ByteStream bs_statsString;
-        bs_statsString = mqc->read();
-        string statsString;
-        bs_statsString >> statsString;
-
-        string printStatsString;
-        struct timeval startRunTime;
-        parseStatsString(statsString, printStatsString, startRunTime);
-        cout << printStatsString << "; QuerySetupTime-" << tm_diff(&start_time, &startRunTime) << "secs"
-             << endl;
-      }
-
-      //...Close this query/session
-      mqc->write(eoq);
-      jl.reset();
-    }
-    catch (const exception& ex)
-    {
-      cout << "### SendPlan caught an exception: " << ex.what() << endl;
-    }
-  }
-
-  // 	jl.reset();
-  CalpontSystemCatalog::removeCalpontSystemCatalog(sessionid);
-  config::Config::deleteInstanceMap();
-
-  delete mqc;
-
-  return 0;
-}