database/mariadb-columnstore-engine
1
0
Fork 0
mirror of https://github.com/mariadb-corporation/mariadb-columnstore-engine.git synced 2025-08-05 16:15:50 +03:00
Code Activity

Merge branch 'stable-23.10' into json_depth-rucha

Browse Source
This commit is contained in:
Rucha Deodhar
2025-06-03 16:29:29 +05:30
committed by GitHub
parent 53870ac06f b2c0106c93
commit 313b4c2212
200 changed files with 2722 additions and 20812 deletions
Download Patch File Download Diff File Expand all files Collapse all files

283
.drone.jsonnet
View File

@@ -127,6 +127,10 @@ local testPreparation(platform) =
};
platform_map[platform];
local make_clickable_link(link) = "echo -e '\\e]8;;" + link + "\\e\\\\" + link + "\\e]8;;\\e\\\\'";
local echo_running_on = ["echo running on ${DRONE_STAGE_MACHINE}",
make_clickable_link("https://us-east-1.console.aws.amazon.com/ec2/home?region=us-east-1#Instances:search=:${DRONE_STAGE_MACHINE};v=3;$case=tags:true%5C,client:false;$regex=tags:false%5C,client:false;sort=desc:launchTime")];
local Pipeline(branch, platform, event, arch="amd64", server="10.6-enterprise", customBootstrapParams="", customBuildEnvCommandsMapKey="") = {
local pkg_format = if (std.split(platform, ":")[0] == "rockylinux") then "rpm" else "deb",
local init = if (pkg_format == "rpm") then "/usr/lib/systemd/systemd" else "systemd",
@@ -177,11 +181,12 @@ local Pipeline(branch, platform, event, arch="amd64", server="10.6-enterprise",
},
},
commands: [
"ls " + result,
'[ -z "$(ls -A "' + result + '")" ] && echo Nothing to publish! && exit 1',
"aws s3 sync " + result + " s3://cspkg/" + branchp + eventp + "/" + server + "/" + arch + "/" + result + " --only-show-errors",
'echo "Data uploaded to: ' + publish_pkg_url + '"',
make_clickable_link(publish_pkg_url),
"rm -rf " + result + "/*",
],
},
@@ -221,65 +226,27 @@ local Pipeline(branch, platform, event, arch="amd64", server="10.6-enterprise",
local indexes(arr) = std.range(0, std.length(arr) - 1),
local execInnerDocker(command, dockerImage, flags="") =
"docker exec " + flags + " -t " + dockerImage + " " + command,
local execInnerDocker(command, containerName, flags="") =
"docker exec " + flags + " -t " + containerName + " " + command,
local execInnerDockerNoTTY(command, dockerImage, flags="") =
"docker exec " + flags + " " + dockerImage + " " + command,
local execInnerDockerNoTTY(command, containerName, flags="") =
"docker exec " + flags + " " + containerName + " " + command,
local installRpmDeb(pkg_format, rpmpackages, debpackages) =
if (pkg_format == "rpm")
then ' bash -c "yum install -y ' + rpmpackages + '"'
else ' bash -c "apt update --yes && apt install -y ' + debpackages + '"',
local getContainerName(stepname) = stepname + "$${DRONE_BUILD_NUMBER}",
local installCmapi(containerName, pkg_format) =
if (pkg_format == "deb") then execInnerDocker('bash -c "apt update -y && apt install -y mariadb-columnstore-cmapi"', containerName)
else execInnerDocker('bash -c "yum update -y && yum install -y MariaDB-columnstore-cmapi"', containerName),
local dockerImage(stepname) = stepname + "$${DRONE_BUILD_NUMBER}",
local installEngine(dockerImage, pkg_format) =
if (pkg_format == "deb") then execInnerDocker('bash -c "apt update && apt install -y mariadb-plugin-columnstore mariadb-test"', dockerImage)
else execInnerDocker('bash -c "yum install -y MariaDB-columnstore-engine MariaDB-test"', dockerImage),
local prepareTestStage(containerName, result, do_setup) =
'sh -c "apk add bash && bash /mdb/' + builddir + "/storage/columnstore/columnstore/build/prepare_test_stage.sh" +
" --container-name " + containerName +
" --docker-image " + img +
" --result-path " + result +
" --packages-url " + packages_url +
" --do-setup " + std.toString(do_setup) + '"',
local installCmapi(dockerImage, pkg_format) =
if (pkg_format == "deb") then execInnerDocker('bash -c "apt update && apt install -y mariadb-columnstore-cmapi"', dockerImage)
else execInnerDocker('bash -c "yum install -y MariaDB-columnstore-cmapi"', dockerImage),
local prepareTestStage(dockerImage, pkg_format, result, do_setup) = [
"apk add bash && bash core_dumps/docker-awaiter.sh " + dockerImage,
if (pkg_format == "deb")
then execInnerDocker('sed -i "s/exit 101/exit 0/g" /usr/sbin/policy-rc.d', dockerImage),
'echo "Docker CGroups opts here"',
"ls -al /sys/fs/cgroup/cgroup.controllers || true ",
"ls -al /sys/fs/cgroup/ || true ",
"ls -al /sys/fs/cgroup/memory || true",
"docker ps --filter=name=" + dockerImage,
execInnerDocker('echo "Inner Docker CGroups opts here"', dockerImage),
execInnerDocker("ls -al /sys/fs/cgroup/cgroup.controllers || true", dockerImage),
execInnerDocker("ls -al /sys/fs/cgroup/ || true", dockerImage),
execInnerDocker("ls -al /sys/fs/cgroup/memory || true", dockerImage),
execInnerDocker("mkdir core", dockerImage),
execInnerDocker("chmod 777 core", dockerImage),
"docker cp core_dumps/. " + dockerImage + ":/",
"docker cp build/utils.sh " + dockerImage + ":/",
"docker cp setup-repo.sh " + dockerImage + ":/",
if (do_setup) then execInnerDocker("/setup-repo.sh", dockerImage),
execInnerDocker(installRpmDeb(pkg_format,
"cracklib-dicts diffutils elfutils epel-release findutils iproute gawk gcc-c++ gdb hostname lz4 patch perl procps-ng rsyslog sudo tar wget which",
"elfutils findutils iproute2 g++ gawk gdb hostname liblz4-tool patch procps rsyslog sudo tar wget"), dockerImage),
execInnerDocker('sysctl -w kernel.core_pattern="/core/%E_' + result + '_core_dump.%p"', dockerImage),
],
local reportTestStage(dockerImage, result, stage) = [
execInnerDocker('bash -c "/logs.sh ' + stage + '"', dockerImage),
execInnerDocker('bash -c "/core_dump_check.sh core /core/ ' + stage + '"', dockerImage),
"docker cp " + dockerImage + ":/core/ /drone/src/" + result + "/",
"docker cp " + dockerImage + ":/unit_logs/ /drone/src/" + result + "/",
"ls -l /drone/src/" + result,
execInnerDocker('bash -c "/core_dump_drop.sh core"', dockerImage),
"docker stop " + dockerImage + " && docker rm " + dockerImage + ' || echo "cleanup ' + stage + ' failure"',
],
local reportTestStage(containerName, result, stage) = 'sh -c "apk add bash && bash /mdb/' + builddir + "/storage/columnstore/columnstore/build/report_test_stage.sh " + containerName + " " + result + " " + stage + '"',
_volumes:: {
mdb: {
@@ -295,41 +262,20 @@ local Pipeline(branch, platform, event, arch="amd64", server="10.6-enterprise",
name: "smoke",
depends_on: ["publish pkg"],
image: "docker",
volumes: [pipeline._volumes.docker],
volumes: [pipeline._volumes.mdb, pipeline._volumes.docker],
commands: [
"docker run --memory 3g --env OS=" + result + " --env PACKAGES_URL=" + packages_url + " --env DEBIAN_FRONTEND=noninteractive --env MCS_USE_S3_STORAGE=0 --name smoke$${DRONE_BUILD_NUMBER} --ulimit core=-1 --privileged --detach " + img + " " + init + " --unit=basic.target",
]
+ prepareTestStage(dockerImage("smoke"), pkg_format, result, true) + [
installEngine(dockerImage("smoke"), pkg_format),
"sleep $${SMOKE_DELAY_SECONDS:-1s}",
// start mariadb and mariadb-columnstore services and run simple query
execInnerDocker("systemctl start mariadb", dockerImage("smoke")),
execInnerDocker("/usr/bin/mcsSetConfig SystemConfig CGroup just_no_group_use_local", dockerImage("smoke")),
execInnerDocker("systemctl restart mariadb-columnstore", dockerImage("smoke")),
execInnerDocker('mariadb -e "create database if not exists test; create table test.t1 (a int) engine=Columnstore; insert into test.t1 values (1); select * from test.t1"',
dockerImage("smoke")),
// restart mariadb and mariadb-columnstore services and run simple query again
execInnerDocker("systemctl restart mariadb", dockerImage("smoke")),
execInnerDocker("systemctl restart mariadb-columnstore", dockerImage("smoke")),
"sleep 10",
execInnerDocker('mariadb -e "insert into test.t1 values (2); select * from test.t1"', dockerImage("smoke")),
prepareTestStage(getContainerName("smoke"), result, true),
"bash /mdb/" + builddir + "/storage/columnstore/columnstore/build/run_smoke.sh " + getContainerName("smoke"),
],
},
smokelog:: {
name: "smokelog",
depends_on: ["smoke"],
image: "docker",
volumes: [pipeline._volumes.docker],
volumes: [pipeline._volumes.docker, pipeline._volumes.mdb],
commands: [
'echo "---------- start mariadb service logs ----------"',
execInnerDocker('journalctl -u mariadb --no-pager || echo "mariadb service failure"', dockerImage("smoke")),
'echo "---------- end mariadb service logs ----------"',
"echo",
'echo "---------- start columnstore debug log ----------"',
execInnerDocker('cat /var/log/mariadb/columnstore/debug.log || echo "missing columnstore debug.log"', dockerImage("smoke")),
'echo "---------- end columnstore debug log ----------"',
] + reportTestStage(dockerImage("smoke"), result, "smoke"),
reportTestStage(getContainerName("smoke"), result, "smoke"),
],
when: {
status: ["success", "failure"],
},
@@ -346,25 +292,31 @@ local Pipeline(branch, platform, event, arch="amd64", server="10.6-enterprise",
},
commands: [
// why do we mount cgroups here, but miss it on other steps?
"docker run --volume /sys/fs/cgroup:/sys/fs/cgroup:ro --env OS=" + result + " --env PACKAGES_URL=" + packages_url + " --env DEBIAN_FRONTEND=noninteractive --env MCS_USE_S3_STORAGE=0 --env UCF_FORCE_CONFNEW=1 --name upgrade$${DRONE_BUILD_NUMBER}" + version + " --ulimit core=-1 --privileged --detach " + img + " " + init + " --unit=basic.target",
]
+ prepareTestStage(dockerImage("upgrade") + version, pkg_format, result, false) + [
prepareTestStage(getContainerName("upgrade") + version, result, false),
if (pkg_format == "deb")
then execInnerDocker('bash -c "./upgrade_setup_deb.sh ' + version + " " + result + " " + arch + " " + repo_pkg_url_no_res + ' $${UPGRADE_TOKEN}"',
dockerImage("upgrade") + version),
if (std.split(platform, ":")[0] == "rockylinux")
getContainerName("upgrade") + version),
if (pkg_format == "rpm")
then execInnerDocker('bash -c "./upgrade_setup_rpm.sh ' + version + " " + result + " " + arch + " " + repo_pkg_url_no_res + ' $${UPGRADE_TOKEN}"',
dockerImage("upgrade") + version),
getContainerName("upgrade") + version),
],
},
upgradelog:: {
name: "upgradelog",
depends_on: std.map(function(p) "upgrade-test from " + p, mdb_server_versions),
image: "docker",
volumes: [pipeline._volumes.docker],
commands: [
"echo",
] + std.flatMap(function(ver) reportTestStage(dockerImage("upgrade") + ver, result, "upgrade_" + ver), mdb_server_versions),
volumes: [pipeline._volumes.docker, pipeline._volumes.mdb],
commands:
["echo"] +
std.map(
function(ver)
reportTestStage(
getContainerName("upgrade") + ver,
result,
"upgrade_" + ver
),
mdb_server_versions
),
when: {
status: ["success", "failure"],
},
@@ -373,64 +325,53 @@ local Pipeline(branch, platform, event, arch="amd64", server="10.6-enterprise",
name: "mtr",
depends_on: ["smoke"],
image: "docker:git",
volumes: [pipeline._volumes.docker],
volumes: [pipeline._volumes.docker, pipeline._volumes.mdb],
environment: {
MTR_SUITE_LIST: "${MTR_SUITE_LIST:-" + mtr_suite_list + "}",
MTR_FULL_SUITE: "${MTR_FULL_SUITE:-false}",
},
commands: [
"docker run --shm-size=500m --memory 8g --env MYSQL_TEST_DIR=" + mtr_path + " --env OS=" + result + " --env PACKAGES_URL=" + packages_url + " --env DEBIAN_FRONTEND=noninteractive --env MCS_USE_S3_STORAGE=0 --name mtr$${DRONE_BUILD_NUMBER} --ulimit core=-1 --privileged --detach " + img + " " + init + " --unit=basic.target",
]
+ prepareTestStage("mtr$${DRONE_BUILD_NUMBER}", pkg_format, result, true) + [
installEngine(dockerImage("mtr"), pkg_format),
prepareTestStage(getContainerName("mtr"), result, true),
"docker cp mysql-test/columnstore mtr$${DRONE_BUILD_NUMBER}:" + mtr_path + "/suite/",
execInnerDocker("chown -R mysql:mysql " + mtr_path, dockerImage("mtr")),
execInnerDocker("chown -R mysql:mysql " + mtr_path, getContainerName("mtr")),
// disable systemd 'ProtectSystem' (we need to write to /usr/share/)
execInnerDocker("bash -c 'sed -i /ProtectSystem/d $(systemctl show --property FragmentPath mariadb | sed s/FragmentPath=//)'", dockerImage("mtr")),
execInnerDocker("systemctl daemon-reload", dockerImage("mtr")),
execInnerDocker("systemctl start mariadb", dockerImage("mtr")),
execInnerDocker("bash -c 'sed -i /ProtectSystem/d $(systemctl show --property FragmentPath mariadb | sed s/FragmentPath=//)'", getContainerName("mtr")),
execInnerDocker("systemctl daemon-reload", getContainerName("mtr")),
execInnerDocker("systemctl start mariadb", getContainerName("mtr")),
// Set RAM consumption limits to avoid RAM contention b/w mtr and regression steps.
execInnerDocker("/usr/bin/mcsSetConfig SystemConfig CGroup just_no_group_use_local", dockerImage("mtr")),
execInnerDocker('mariadb -e "create database if not exists test;"', dockerImage("mtr")),
execInnerDocker("systemctl restart mariadb-columnstore", dockerImage("mtr")),
execInnerDocker("/usr/bin/mcsSetConfig SystemConfig CGroup just_no_group_use_local", getContainerName("mtr")),
execInnerDocker('mariadb -e "create database if not exists test;"', getContainerName("mtr")),
execInnerDocker("systemctl restart mariadb-columnstore", getContainerName("mtr")),
// delay mtr for manual debugging on live instance
"sleep $${MTR_DELAY_SECONDS:-1s}",
'MTR_SUITE_LIST=$([ "$MTR_FULL_SUITE" == true ] && echo "' + mtr_full_set + '" || echo "$MTR_SUITE_LIST")',
if (event == "custom" || event == "cron") then
execInnerDocker('bash -c "wget -qO- https://cspkg.s3.amazonaws.com/mtr-test-data.tar.lz4 | lz4 -dc - | tar xf - -C /"',
dockerImage("mtr")),
getContainerName("mtr")),
if (event == "custom" || event == "cron") then
execInnerDocker('bash -c "cd ' + mtr_path + " && ./mtr --extern socket=" + socket_path + ' --force --print-core=detailed --print-method=gdb --max-test-fail=0 --suite=columnstore/setup"',
dockerImage("mtr")),
getContainerName("mtr")),
if (event == "cron") then
execInnerDocker('bash -c "cd ' + mtr_path + " && ./mtr --extern socket=" + socket_path +
" --force --print-core=detailed --print-method=gdb --max-test-fail=0 --suite="
+ std.join(",", std.map(function(x) "columnstore/" + x, std.split(mtr_full_set, ","))),
dockerImage("mtr")) + '"'
getContainerName("mtr")) + '"'
else
execInnerDocker('bash -c "cd ' + mtr_path + " && ./mtr --extern socket=" + socket_path +
' --force --print-core=detailed --print-method=gdb --max-test-fail=0 --suite=columnstore/$${MTR_SUITE_LIST//,/,columnstore/}"',
dockerImage("mtr")),
getContainerName("mtr")),
],
},
mtrlog:: {
name: "mtrlog",
depends_on: ["mtr"],
image: "docker",
volumes: [pipeline._volumes.docker],
volumes: [pipeline._volumes.docker, pipeline._volumes.mdb],
commands: [
'echo "---------- start mariadb service logs ----------"',
execInnerDocker('journalctl -u mariadb --no-pager || echo "mariadb service failure"', dockerImage("mtr")),
'echo "---------- end mariadb service logs ----------"',
"echo",
'echo "---------- start columnstore debug log ----------"',
execInnerDocker('cat /var/log/mariadb/columnstore/debug.log || echo "missing columnstore debug.log"', dockerImage("mtr")),
'echo "---------- end columnstore debug log ----------"',
'echo "---------- end columnstore debug log ----------"',
"docker cp mtr$${DRONE_BUILD_NUMBER}:" + mtr_path + "/var/log /drone/src/" + result + '/mtr-logs || echo "missing ' + mtr_path + '/var/log"',
] + reportTestStage(dockerImage("mtr"), result, "mtr"),
reportTestStage(getContainerName("mtr"), result, "mtr"),
],
when: {
status: ["success", "failure"],
},
@@ -464,33 +405,30 @@ local Pipeline(branch, platform, event, arch="amd64", server="10.6-enterprise",
"cd mariadb-columnstore-regression-test",
"git rev-parse --abbrev-ref HEAD && git rev-parse HEAD",
"cd ..",
"docker run --shm-size=500m --memory 12g --env OS=" + result + " --env PACKAGES_URL=" + packages_url + " --env DEBIAN_FRONTEND=noninteractive --env MCS_USE_S3_STORAGE=0 --name regression$${DRONE_BUILD_NUMBER} --ulimit core=-1 --privileged --detach " + img + " " + init + " --unit=basic.target",
]
+ prepareTestStage(dockerImage("regression"), pkg_format, result, true) + [
prepareTestStage(getContainerName("regression"), result, true),
"docker cp mariadb-columnstore-regression-test regression$${DRONE_BUILD_NUMBER}:/",
// list storage manager binary
"ls -la /mdb/" + builddir + "/storage/columnstore/columnstore/storage-manager",
"docker cp /mdb/" + builddir + "/storage/columnstore/columnstore/storage-manager regression$${DRONE_BUILD_NUMBER}:/",
// check storage-manager unit test binary file
execInnerDocker("ls -l /storage-manager", dockerImage("regression")),
execInnerDocker("ls -l /storage-manager", getContainerName("regression")),
// copy test data for regression test suite
execInnerDocker('bash -c "wget -qO- https://cspkg.s3.amazonaws.com/testData.tar.lz4 | lz4 -dc - | tar xf - -C mariadb-columnstore-regression-test/"', dockerImage("regression")),
installEngine(dockerImage("regression"), pkg_format),
execInnerDocker('bash -c "wget -qO- https://cspkg.s3.amazonaws.com/testData.tar.lz4 | lz4 -dc - | tar xf - -C mariadb-columnstore-regression-test/"', getContainerName("regression")),
// set mariadb lower_case_table_names=1 config option
execInnerDocker('sed -i "/^.mariadb.$/a lower_case_table_names=1" ' + config_path_prefix + "server.cnf", dockerImage("regression")),
execInnerDocker('sed -i "/^.mariadb.$/a lower_case_table_names=1" ' + config_path_prefix + "server.cnf", getContainerName("regression")),
// set default client character set to utf-8
execInnerDocker('sed -i "/^.client.$/a default-character-set=utf8" ' + config_path_prefix + "client.cnf", dockerImage("regression")),
execInnerDocker('sed -i "/^.client.$/a default-character-set=utf8" ' + config_path_prefix + "client.cnf", getContainerName("regression")),
// Set RAM consumption limits to avoid RAM contention b/w mtr andregression steps.
execInnerDocker("/usr/bin/mcsSetConfig SystemConfig CGroup just_no_group_use_local", dockerImage("regression")),
execInnerDocker("/usr/bin/mcsSetConfig SystemConfig CGroup just_no_group_use_local", getContainerName("regression")),
execInnerDocker("systemctl start mariadb", dockerImage("regression")),
execInnerDocker("systemctl restart mariadb-columnstore", dockerImage("regression")),
execInnerDocker("systemctl start mariadb", getContainerName("regression")),
execInnerDocker("systemctl restart mariadb-columnstore", getContainerName("regression")),
// delay regression for manual debugging on live instance
"sleep $${REGRESSION_DELAY_SECONDS:-1s}",
execInnerDocker("/usr/bin/g++ /mariadb-columnstore-regression-test/mysql/queries/queryTester.cpp -O2 -o /mariadb-columnstore-regression-test/mysql/queries/queryTester", dockerImage("regression")),
execInnerDocker("/usr/bin/g++ /mariadb-columnstore-regression-test/mysql/queries/queryTester.cpp -O2 -o /mariadb-columnstore-regression-test/mysql/queries/queryTester", getContainerName("regression")),
],
},
regression(name, depends_on):: {
@@ -508,11 +446,11 @@ local Pipeline(branch, platform, event, arch="amd64", server="10.6-enterprise",
},
},
commands: [
execInnerDocker("mkdir -p reg-logs", dockerImage("regression"), "--workdir /mariadb-columnstore-regression-test/mysql/queries/nightly/alltest"),
execInnerDocker("mkdir -p reg-logs", getContainerName("regression"), "--workdir /mariadb-columnstore-regression-test/mysql/queries/nightly/alltest"),
execInnerDocker("bash -c 'sleep 4800 && bash /save_stack.sh /mariadb-columnstore-regression-test/mysql/queries/nightly/alltest/reg-logs/' & ",
dockerImage("regresion")),
getContainerName("regresion")),
execInnerDockerNoTTY('bash -c "timeout -k 1m -s SIGKILL --preserve-status $${REGRESSION_TIMEOUT} ./go.sh --sm_unit_test_dir=/storage-manager --tests=' + name + " || ./regression_logs.sh " + name + '"',
dockerImage("regression"),
getContainerName("regression"),
"--env PRESERVE_LOGS=true --workdir /mariadb-columnstore-regression-test/mysql/queries/nightly/alltest"),
],
},
@@ -520,24 +458,10 @@ local Pipeline(branch, platform, event, arch="amd64", server="10.6-enterprise",
name: "regressionlog",
depends_on: [regression_tests[std.length(regression_tests) - 1]],
image: "docker",
volumes: [pipeline._volumes.docker],
volumes: [pipeline._volumes.docker, pipeline._volumes.mdb],
commands: [
'echo "---------- start columnstore regression short report ----------"',
execInnerDocker('cat go.log || echo "missing go.log"',
dockerImage("regression"),
"--workdir /mariadb-columnstore-regression-test/mysql/queries/nightly/alltest"),
'echo "---------- end columnstore regression short report ----------"',
"echo",
"docker cp regression$${DRONE_BUILD_NUMBER}:/mariadb-columnstore-regression-test/mysql/queries/nightly/alltest/reg-logs/ /drone/src/" + result + "/",
"docker cp regression$${DRONE_BUILD_NUMBER}:/mariadb-columnstore-regression-test/mysql/queries/nightly/alltest/testErrorLogs.tgz /drone/src/" + result + '/ || echo "missing testErrorLogs.tgz"',
execInnerDocker('bash -c "tar czf regressionQueries.tgz /mariadb-columnstore-regression-test/mysql/queries/"', dockerImage("regression")),
execInnerDocker('bash -c "tar czf testErrorLogs2.tgz *.log /var/log/mariadb/columnstore" || echo "failed to grab regression results"',
dockerImage("regression"),
"--workdir /mariadb-columnstore-regression-test/mysql/queries/nightly/alltest"),
"docker cp regression$${DRONE_BUILD_NUMBER}:/mariadb-columnstore-regression-test/mysql/queries/nightly/alltest/testErrorLogs2.tgz /drone/src/" + result + '/ || echo "missing testErrorLogs.tgz"',
"docker cp regression$${DRONE_BUILD_NUMBER}:regressionQueries.tgz /drone/src/" + result + "/",
] + reportTestStage(dockerImage("regression"), result, "regression"),
reportTestStage(getContainerName("regression"), result, "regression"),
],
when: {
status: ["success", "failure"],
},
@@ -598,47 +522,33 @@ local Pipeline(branch, platform, event, arch="amd64", server="10.6-enterprise",
name: "cmapi test",
depends_on: ["publish cmapi build"],
image: "docker:git",
volumes: [pipeline._volumes.docker],
volumes: [pipeline._volumes.docker, pipeline._volumes.mdb],
environment: {
PYTHONPATH: "/usr/share/columnstore/cmapi/deps",
},
commands: [
"docker run --env OS=" + result + " --env PACKAGES_URL=" + packages_url + " --env DEBIAN_FRONTEND=noninteractive --env MCS_USE_S3_STORAGE=0 --env PYTHONPATH=$${PYTHONPATH} --name cmapi$${DRONE_BUILD_NUMBER} --ulimit core=-1 --privileged --detach " + img + " " + init + " --unit=basic.target",
] +
prepareTestStage(dockerImage("cmapi"), pkg_format, result, true) + [
if (platform == "rockylinux:9") then execInnerDocker('bash -c "yum install -y libxcrypt-compat"', dockerImage("cmapi")),
installEngine(dockerImage("cmapi"), pkg_format),
installCmapi(dockerImage("cmapi"), pkg_format),
prepareTestStage(getContainerName("cmapi"), result, true),
installCmapi(getContainerName("cmapi"), pkg_format),
"cd cmapi",
"for i in mcs_node_control cmapi_server failover; do docker cp $${i}/test cmapi$${DRONE_BUILD_NUMBER}:" + cmapi_path + "/$${i}/; done",
"docker cp run_tests.py cmapi$${DRONE_BUILD_NUMBER}:" + cmapi_path + "/",
execInnerDocker("systemctl start mariadb-columnstore-cmapi", dockerImage("cmapi")),
execInnerDocker("systemctl start mariadb-columnstore-cmapi", getContainerName("cmapi")),
// set API key to /etc/columnstore/cmapi_server.conf
execInnerDocker('bash -c "mcs cluster set api-key --key somekey123"', dockerImage("cmapi")),
execInnerDocker('bash -c "mcs cluster set api-key --key somekey123"', getContainerName("cmapi")),
// copy cmapi conf file for test purposes (there are api key already set inside)
execInnerDocker('bash -c "cp %s/cmapi_server.conf %s/cmapi_server/"' % [etc_path, cmapi_path], dockerImage("cmapi")),
execInnerDocker("systemctl stop mariadb-columnstore-cmapi", dockerImage("cmapi")),
execInnerDocker('bash -c "cd ' + cmapi_path + ' && python/bin/python3 run_tests.py"', dockerImage("cmapi")),
execInnerDocker('bash -c "cp %s/cmapi_server.conf %s/cmapi_server/"' % [etc_path, cmapi_path], getContainerName("cmapi")),
execInnerDocker("systemctl stop mariadb-columnstore-cmapi", getContainerName("cmapi")),
execInnerDocker('bash -c "cd ' + cmapi_path + ' && python/bin/python3 run_tests.py"', getContainerName("cmapi")),
],
},
cmapilog:: {
name: "cmapilog",
depends_on: ["cmapi test"],
image: "docker",
volumes: [pipeline._volumes.docker],
volumes: [pipeline._volumes.docker, pipeline._volumes.mdb],
commands: [
'echo "---------- start mariadb service logs ----------"',
execInnerDocker('journalctl -u mariadb --no-pager || echo "mariadb service failure"', dockerImage("cmapi")),
'echo "---------- end mariadb service logs ----------"',
"echo",
'echo "---------- start columnstore debug log ----------"',
execInnerDocker('cat /var/log/mariadb/columnstore/debug.log || echo "missing columnstore debug.log"', dockerImage("cmapi")),
'echo "---------- end columnstore debug log ----------"',
'echo "---------- start cmapi log ----------"',
execInnerDocker('cat /var/log/mariadb/columnstore/cmapi_server.log || echo "missing cmapi cmapi_server.log"', dockerImage("cmapi")),
'echo "---------- end cmapi log ----------"',
]
+ reportTestStage(dockerImage("cmapi"), result, "cmapi"),
reportTestStage(getContainerName("cmapi"), result, "cmapi"),
],
when: {
status: ["success", "failure"],
},
@@ -700,7 +610,8 @@ local Pipeline(branch, platform, event, arch="amd64", server="10.6-enterprise",
SERVER_REMOTE: "${SERVER_REMOTE:-" + server_remote + "}",
SERVER_SHA: "${SERVER_SHA:-" + server + "}",
},
commands: [
commands: echo_running_on +
[
"echo $$SERVER_REF",
"echo $$SERVER_REMOTE",
"mkdir -p /mdb/" + builddir + " && cd /mdb/" + builddir,
@@ -720,9 +631,6 @@ local Pipeline(branch, platform, event, arch="amd64", server="10.6-enterprise",
volumes: [pipeline._volumes.mdb],
environment: {
DEBIAN_FRONTEND: "noninteractive",
DEB_BUILD_OPTIONS: "parallel=4",
DH_BUILD_DDEBS: "1",
BUILDPACKAGE_FLAGS: "-b", // Save time and produce only binary packages, not source
AWS_ACCESS_KEY_ID: {
from_secret: "aws_access_key_id",
},
@@ -745,7 +653,7 @@ local Pipeline(branch, platform, event, arch="amd64", server="10.6-enterprise",
'bash -c "set -o pipefail && bash /mdb/' + builddir + "/storage/columnstore/columnstore/build/bootstrap_mcs.sh " +
"--build-type RelWithDebInfo " +
"--distro " + platform + " " +
"--build-packages --sccache " +
"--build-packages --install-deps --sccache " +
" " + customBootstrapParams +
" " + customBootstrapParamsForExisitingPipelines(platform) + " | " +
"/mdb/" + builddir + "/storage/columnstore/columnstore/build/ansi2txt.sh " +
@@ -822,19 +730,6 @@ local Pipeline(branch, platform, event, arch="amd64", server="10.6-enterprise",
] +
[pipeline.publish("cmapi build")] +
[pipeline.publish()] +
[
{
name: "publish pkg url",
depends_on: ["publish pkg"],
image: "alpine/git",
commands: [
"echo -e '\\e]8;;" + publish_pkg_url + "\\e\\\\" + publish_pkg_url + "\\e]8;;\\e\\\\'",
"echo 'for installation run:'",
"echo 'export OS=" + result + "'",
"echo 'export PACKAGES_URL=" + packages_url + "'",
],
},
] +
(if (event == "cron") then [pipeline.publish("pkg latest", "latest")] else []) +
[pipeline.smoke] +
[pipeline.smokelog] +

2
.gitignore vendored
View File

@@ -153,6 +153,7 @@ oam/install_scripts/mcs-workernode.service
oam/install_scripts/mcs-writeengineserver.service
oam/install_scripts/mcs-loadbrm.py
oam/install_scripts/mcs-savebrm.py
oam/install_scripts/mcssavebrm.py
oam/install_scripts/mcs-storagemanager.service
oam/install_scripts/mcs_module_installer.sh
oam/install_scripts/slave-rep-columnstore.sh
@@ -186,4 +187,3 @@ versioning/BRM/shmkeys.cpp
obj/
build/build

61
build/bootstrap_mcs.sh
View File

@@ -19,6 +19,9 @@ CONFIG_DIR=$RPM_CONFIG_DIR
SCRIPT_LOCATION=$(dirname "$0")
MDB_SOURCE_PATH=$(realpath "$SCRIPT_LOCATION"/../../../..)
COLUMSNTORE_SOURCE_PATH=$(realpath "$SCRIPT_LOCATION"/../)
DEFAULT_MARIA_BUILD_PATH=$(realpath "$MDB_SOURCE_PATH"/../BuildOf_$(basename "$MDB_SOURCE_PATH"))
BUILD_TYPE_OPTIONS=("Debug" "RelWithDebInfo")
DISTRO_OPTIONS=("ubuntu:20.04" "ubuntu:22.04" "ubuntu:24.04" "debian:11" "debian:12" "rockylinux:8" "rockylinux:9")
@@ -28,21 +31,10 @@ MDB_CMAKE_FLAGS=()
source "$SCRIPT_LOCATION"/utils.sh
if [ "$EUID" -ne 0 ]; then
error "Please run script as root to install MariaDb to system paths"
exit 1
fi
cd $SCRIPT_LOCATION
CURRENT_BRANCH=$(git rev-parse --abbrev-ref HEAD)
message "Columnstore will be built from $color_yellow$CURRENT_BRANCH$color_cyan branch"
cd - >/dev/null
echo "Arguments received: $@"
message "Building Mariadb Server from $color_yellow$MDB_SOURCE_PATH$color_normal"
optparse.define short=A long=asan desc="Build with ASAN" variable=ASAN default=false value=true
optparse.define short=a long=build-path variable=MARIA_BUILD_PATH default=$(realpath "$MDB_SOURCE_PATH"/../BuildOf_$(basename "$MDB_SOURCE_PATH"))
optparse.define short=a long=build-path desc="Path for build output" variable=MARIA_BUILD_PATH default=$DEFAULT_MARIA_BUILD_PATH
optparse.define short=B long=run-microbench desc="Compile and run microbenchmarks " variable=RUN_BENCHMARKS default=false value=true
optparse.define short=c long=cloud desc="Enable cloud storage" variable=CLOUD_STORAGE_ENABLED default=false value=true
optparse.define short=C long=force-cmake-reconfig desc="Force cmake reconfigure" variable=FORCE_CMAKE_CONFIG default=false value=true
@@ -68,10 +60,26 @@ optparse.define short=T long=tsan desc="Build with TSAN" variable=TSAN default=f
optparse.define short=u long=skip-unit-tests desc="Skip UnitTests" variable=SKIP_UNIT_TESTS default=false value=true
optparse.define short=U long=ubsan desc="Build with UBSAN" variable=UBSAN default=false value=true
optparse.define short=v long=verbose desc="Verbose makefile commands" variable=MAKEFILE_VERBOSE default=false value=true
optparse.define short=V long=add-branch-name-to-outdir desc="Add branch name to build output directory" variable=BRANCH_NAME_TO_OUTDIR default=false value=true
optparse.define short=W long=without-core-dumps desc="Do not produce core dumps" variable=WITHOUT_COREDUMPS default=false value=true
source $(optparse.build)
message "Building MariaDB Server from $color_yellow$MDB_SOURCE_PATH$color_normal"
cd $COLUMSNTORE_SOURCE_PATH
COLUMNSTORE_BRANCH=$(git rev-parse --abbrev-ref HEAD)
message "Columnstore will be built from $color_yellow$COLUMNSTORE_BRANCH$color_cyan branch"
cd $MDB_SOURCE_PATH
MARIADB_BRANCH=$(git rev-parse --abbrev-ref HEAD)
message "MariaDB will be built from $color_yellow$MARIADB_BRANCH$color_cyan branch"
cd - >/dev/null
if [[ ${BRANCH_NAME_TO_OUTDIR} = true ]]; then
MARIA_BUILD_PATH="${MARIA_BUILD_PATH}_${MARIADB_BRANCH}_${COLUMNSTORE_BRANCH}"
fi
if [[ ! " ${BUILD_TYPE_OPTIONS[*]} " =~ " ${MCS_BUILD_TYPE} " ]]; then
getChoice -q "Select your Build Type" -o BUILD_TYPE_OPTIONS
MCS_BUILD_TYPE=$selectedChoice
@@ -104,7 +112,7 @@ install_deps() {
cppunit-devel cmake3 libxcrypt-devel xz-devel zlib-devel libzstd-devel glibc-devel"
DEB_BUILD_DEPS="apt-get -y update && apt-get -y install build-essential automake libboost-all-dev \
bison cmake libncurses5-dev libaio-dev libsystemd-dev libpcre2-dev libperl-dev libssl-dev libxml2-dev \
bison cmake libncurses5-dev python3 libaio-dev libsystemd-dev libpcre2-dev libperl-dev libssl-dev libxml2-dev \
libkrb5-dev flex libpam-dev git libsnappy-dev libcurl4-openssl-dev libgtest-dev libcppunit-dev googletest \
libjemalloc-dev liblz-dev liblzo2-dev liblzma-dev liblz4-dev libbz2-dev libbenchmark-dev libdistro-info-perl \
graphviz devscripts ccache equivs eatmydata curl"
@@ -259,8 +267,9 @@ construct_cmake_flags() {
-DWITH_WSREP=NO
)
if [[ MAINTAINER_MODE = true ]]; then
if [[ $MAINTAINER_MODE = true ]]; then
MDB_CMAKE_FLAGS+=(-DCOLUMNSTORE_MAINTAINER=YES)
message "Columnstore mainteiner mode on"
else
warn "Maintainer mode is disabled, be careful, alien"
fi
@@ -406,9 +415,15 @@ generate_svgs() {
}
build_package() {
cd $MDB_SOURCE_PATH
if [[ $pkg_format == "rpm" ]]; then
command="cmake ${MDB_CMAKE_FLAGS[@]} && make -j\$(nproc) package"
else
export DEBIAN_FRONTEND="noninteractive"
export DEB_BUILD_OPTIONS="parallel=$(nproc)"
export DH_BUILD_DDEBS="1"
export BUILDPACKAGE_FLAGS="-b"
command="mk-build-deps debian/control -t 'apt-get -y -o Debug::pkgProblemResolver=yes --no-install-recommends' -r -i && \
CMAKEFLAGS=\"${MDB_CMAKE_FLAGS[@]}\" debian/autobake-deb.sh"
fi
@@ -420,6 +435,14 @@ build_package() {
check_errorcode
}
check_debian_install_file() {
message "checking debian/mariadb-plugin-columnstore.install"
message_split
python3 $COLUMSNTORE_SOURCE_PATH/build/debian_install_file_compare.py \
${COLUMSNTORE_SOURCE_PATH}/debian/mariadb-plugin-columnstore.install \
$MARIA_BUILD_PATH/mariadb-plugin-columnstore.install.generated
}
build_binary() {
MARIA_BUILD_PATH=$(realpath "$MARIA_BUILD_PATH")
message_split
@@ -438,7 +461,7 @@ build_binary() {
message "Configuring cmake silently"
${CMAKE_BIN_NAME} "${MDB_CMAKE_FLAGS[@]}" -S"$MDB_SOURCE_PATH" -B"$MARIA_BUILD_PATH" | spinner
message_split
check_debian_install_file
generate_svgs
${CMAKE_BIN_NAME} --build "$MARIA_BUILD_PATH" -j "$CPUS" | onelinearizator &&
@@ -553,6 +576,12 @@ make_dir() {
install() {
if [[ $RECOMPILE_ONLY = false ]]; then
if [ "$EUID" -ne 0 ]; then
error "Please run script as root to install MariaDb to system paths"
exit 1
fi
message_split
message "Installing MariaDB"
disable_plugins_for_bootstrap
@@ -636,7 +665,7 @@ smoke() {
fi
}
if [[ $INSTALL_DEPS = true || $BUILD_PACKAGES = true ]]; then
if [[ $INSTALL_DEPS = true ]]; then
install_deps
fi

81
build/debian_install_file_compare.py Normal file
View File

@@ -0,0 +1,81 @@
#!/usr/bin/env python3
import argparse
import fnmatch
import sys
def read_patterns(filename):
with open(filename, 'r') as f:
return [line.strip() for line in f if line.strip()]
def read_paths_with_comments(filename):
paths = []
with open(filename, 'r') as f:
for line in f:
raw = line.rstrip('\n')
if not raw.strip():
continue
# Split into path and comment
if '#' in raw:
path, comment = raw.split('#', 1)
path = path.strip()
comment = comment.strip()
else:
path = raw.strip()
comment = ''
if path:
paths.append((raw, path, comment))
return paths
def print_red(msg):
if sys.stdout.isatty():
print("\033[31m" + msg + "\033[0m")
else:
print(msg)
def main():
parser = argparse.ArgumentParser(description="Compare wildcard patterns to file paths.")
parser.add_argument('patterns_file', help='File with wildcard patterns (one per line)')
parser.add_argument('paths_file', help='File with file paths (one per line)')
args = parser.parse_args()
patterns = read_patterns(args.patterns_file)
paths_with_comments = read_paths_with_comments(args.paths_file)
# Track which paths are matched (by index)
path_matched = [False] * len(paths_with_comments)
# Collect patterns that don't match any path
unmatched_patterns = []
for pattern in patterns:
matched = False
for i, (raw, path, comment) in enumerate(paths_with_comments):
if fnmatch.fnmatch(path, pattern):
matched = True
path_matched[i] = True
if not matched:
unmatched_patterns.append(pattern)
if unmatched_patterns:
print_red("The files declared in debian/mariadb-plugin-columnstore.install "
" are not added to CMakeLists via columnstore_* statements see cmake/ColumnstoreLibrary.cmake")
for pattern in unmatched_patterns:
print(f"- {pattern}")
# Collect paths that weren't matched by any pattern
unmatched_paths = []
for (raw, path, comment), matched in zip(paths_with_comments, path_matched):
if not matched:
unmatched_paths.append((path, comment))
if unmatched_paths:
print_red("The files added via columnstore_* statements from cmake/ColumnstoreLibrary.cmake "
"are missing in debian/mariadb-plugin-columnstore.install file")
for path, comment in unmatched_paths:
if comment:
print(f"- {path} {comment}")
else:
print(f"- {path}")
if __name__ == "__main__":
main()

127
build/prepare_test_stage.sh Executable file
View File

@@ -0,0 +1,127 @@
#!/bin/bash
# Should be executed by root
set -eo pipefail
SCRIPT_LOCATION=$(dirname "$0")
COLUMNSTORE_SOURCE_PATH=$(realpath "$SCRIPT_LOCATION"/../)
source "$SCRIPT_LOCATION"/utils.sh
echo "Arguments received: $@"
optparse.define short=c long=container-name desc="Name of the Docker container to run tests in" variable=CONTAINER_NAME
optparse.define short=i long=docker-image desc="Docker image name to start container from" variable=DOCKER_IMAGE
optparse.define short=r long=result-path desc="Name suffix used in core dump file path" variable=RESULT
optparse.define short=s long=do-setup desc="Run setup-repo.sh inside the container" variable=DO_SETUP
optparse.define short=u long=packages-url desc="Packages url" variable=PACKAGES_URL
source $(optparse.build)
if [[ "$EUID" -ne 0 ]]; then
error "Please run script as root"
exit 1
fi
if [[ -z "${CONTAINER_NAME:-}" || -z "${DOCKER_IMAGE:-}" || -z "${RESULT:-}" || -z "${DO_SETUP:-}" || -z "${PACKAGES_URL:-}" ]]; then
echo "Please provide --container-name, --docker-image, --result-path, --packages-url and --do-setup parameters, e.g. ./prepare_test_stage.sh --container-name smoke11212 --docker-image detravi/ubuntu:24.04 --result-path ubuntu24.04 --packages-url https://cspkg.s3.amazonaws.com/stable-23.10/pull_request/91/10.6-enterprise --do-setup true"
exit 1
fi
start_container() {
if [[ "$RESULT" == *rocky* ]]; then
SYSTEMD_PATH="/usr/lib/systemd/systemd"
MTR_PATH="/usr/share/mysql-test"
else
SYSTEMD_PATH="systemd"
MTR_PATH="/usr/share/mysql/mysql-test"
fi
docker_run_args=(
--env OS="$RESULT"
--env PACKAGES_URL="$PACKAGES_URL"
--env DEBIAN_FRONTEND=noninteractive
--env MCS_USE_S3_STORAGE=0
--name "$CONTAINER_NAME"
--ulimit core=-1
--privileged
--detach
)
if [[ "$CONTAINER_NAME" == *smoke* ]]; then
docker_run_args+=( --memory 3g )
elif [[ "$CONTAINER_NAME" == *mtr* ]]; then
docker_run_args+=( --shm-size=500m --memory 8g --env MYSQL_TEST_DIR="$MTR_PATH" )
elif [[ "$CONTAINER_NAME" == *cmapi* ]]; then
docker_run_args+=( --env PYTHONPATH="${PYTHONPATH}" )
elif [[ "$CONTAINER_NAME" == *upgrade* ]]; then
docker_run_args+=( --env UCF_FORCE_CONFNEW=1 --volume /sys/fs/cgroup:/sys/fs/cgroup:ro )
elif [[ "$CONTAINER_NAME" == *regression* ]]; then
docker_run_args+=( --shm-size=500m --memory 12g )
else
echo "Unknown container type: $CONTAINER_NAME"
exit 1
fi
docker_run_args+=( "$DOCKER_IMAGE" "$SYSTEMD_PATH" --unit=basic.target )
docker run "${docker_run_args[@]}"
sleep 5
apk add bash
bash "$COLUMNSTORE_SOURCE_PATH"/core_dumps/docker-awaiter.sh "$CONTAINER_NAME"
if ! docker ps -q --filter "name=${CONTAINER_NAME}" | grep -q .; then
error "Container '$CONTAINER_NAME' has not started!"
exit 1
fi
}
start_container
if [[ "$RESULT" != *rocky* ]]; then
execInnerDocker 'sed -i "s/exit 101/exit 0/g" /usr/sbin/policy-rc.d' "$CONTAINER_NAME"
fi
#list_cgroups
echo "Docker CGroups opts here"
ls -al /sys/fs/cgroup/cgroup.controllers || true
ls -al /sys/fs/cgroup/ || true
ls -al /sys/fs/cgroup/memory || true
execInnerDocker 'echo Inner Docker CGroups opts here' "$CONTAINER_NAME"
execInnerDocker 'ls -al /sys/fs/cgroup/cgroup.controllers || true' "$CONTAINER_NAME"
execInnerDocker 'ls -al /sys/fs/cgroup/ || true' "$CONTAINER_NAME"
execInnerDocker 'ls -al /sys/fs/cgroup/memory || true' "$CONTAINER_NAME"
# Prepare core dump directory inside container
execInnerDocker 'mkdir -p core && chmod 777 core' "$CONTAINER_NAME"
docker cp "$COLUMNSTORE_SOURCE_PATH"/core_dumps/. "$CONTAINER_NAME":/
docker cp "$COLUMNSTORE_SOURCE_PATH"/build/utils.sh "$CONTAINER_NAME":/
docker cp "$COLUMNSTORE_SOURCE_PATH"/setup-repo.sh "$CONTAINER_NAME":/
if [[ "$DO_SETUP" == "true" ]]; then
execInnerDocker '/setup-repo.sh' "$CONTAINER_NAME"
fi
# install deps
if [[ "$RESULT" == *rocky* ]]; then
execInnerDocker 'yum update -y && yum install -y cracklib-dicts diffutils elfutils epel-release findutils iproute gawk gcc-c++ gdb hostname lz4 patch perl procps-ng rsyslog sudo tar wget which' "$CONTAINER_NAME"
# sudo bypass
execInnerDocker "printf '%s\n' '#!/bin/sh' 'exec \"\$@\"' > /usr/bin/sudo && chmod +x /usr/bin/sudo" "$CONTAINER_NAME"
else
execInnerDocker 'apt update -y && apt install -y elfutils findutils iproute2 g++ gawk gdb hostname liblz4-tool patch procps rsyslog sudo tar wget' "$CONTAINER_NAME"
fi
# Configure core dump naming pattern
execInnerDocker 'sysctl -w kernel.core_pattern="/core/%E_${RESULT}_core_dump.%p"' "$CONTAINER_NAME"
#Install columnstore in container
echo "Installing columnstore..."
if [[ "$RESULT" == *rocky* ]]; then
execInnerDocker 'yum install -y MariaDB-columnstore-engine MariaDB-test' "$CONTAINER_NAME"
else
execInnerDocker 'apt update -y && apt install -y mariadb-plugin-columnstore mariadb-test' "$CONTAINER_NAME"
fi
sleep 5
echo "PrepareTestStage completed in $CONTAINER_NAME"

98
build/report_test_stage.sh Executable file
View File

@@ -0,0 +1,98 @@
#!/bin/bash
set -eo pipefail
CONTAINER_NAME=$1
RESULT=$2
STAGE=$3
SCRIPT_LOCATION=$(dirname "$0")
source "$SCRIPT_LOCATION"/utils.sh
echo "Arguments received: $@"
if [[ "$EUID" -ne 0 ]]; then
error "Please run script as root"
exit 1
fi
if [[ -z $(docker ps -q --filter "name=${CONTAINER_NAME}") ]]; then
error "Container '${CONTAINER_NAME}' is not running."
exit 1
fi
if [[ "$RESULT" == *rocky* ]]; then
SYSTEMD_PATH="/usr/lib/systemd/systemd"
MTR_PATH="/usr/share/mysql-test"
else
SYSTEMD_PATH="systemd"
MTR_PATH="/usr/share/mysql/mysql-test"
fi
echo "Reporting test stage: ${STAGE} executed in ${CONTAINER_NAME} container"
if [[ "${CONTAINER_NAME}" == *smoke* ]] || [[ "${CONTAINER_NAME}" == *mtr* ]] || [[ "${CONTAINER_NAME}" == *cmapi* ]]; then
# common logs for smoke, mtr, cmapi
echo "---------- start mariadb service logs ----------"
execInnerDocker 'journalctl -u mariadb --no-pager || echo "mariadb service failure"' "$CONTAINER_NAME"
echo "---------- end mariadb service logs ----------"
echo
echo "---------- start columnstore debug log ----------"
execInnerDocker 'cat /var/log/mariadb/columnstore/debug.log || echo "missing columnstore debug.log"' "$CONTAINER_NAME"
echo "---------- end columnstore debug log ----------"
if [[ "${CONTAINER_NAME}" == *mtr* ]]; then
echo
docker cp "${CONTAINER_NAME}:${MTR_PATH}/var/log" "/drone/src/${RESULT}/mtr-logs" || echo "missing ${MTR_PATH}/var/log"
fi
if [[ "${CONTAINER_NAME}" == *cmapi* ]]; then
echo
echo "---------- start cmapi log ----------"
execInnerDocker 'cat /var/log/mariadb/columnstore/cmapi_server.log || echo "missing cmapi_server.log"' "$CONTAINER_NAME"
echo "---------- end cmapi log ----------"
fi
elif [[ "${CONTAINER_NAME}" == *upgrade* ]]; then
# nothing to report here for upgrade
:
elif [[ "${CONTAINER_NAME}" == *regression* ]]; then
echo "---------- start columnstore regression short report ----------"
execInnerDocker 'cat go.log || echo "missing go.log"' "$CONTAINER_NAME" '--workdir /mariadb-columnstore-regression-test/mysql/queries/nightly/alltest'
echo "---------- end columnstore regression short report ----------"
echo
docker cp "${CONTAINER_NAME}:/mariadb-columnstore-regression-test/mysql/queries/nightly/alltest/reg-logs/" "/drone/src/${RESULT}/" || echo "missing regression logs"
docker cp "${CONTAINER_NAME}:/mariadb-columnstore-regression-test/mysql/queries/nightly/alltest/testErrorLogs.tgz" "/drone/src/${RESULT}/" || echo "missing testErrorLogs.tgz"
execInnerDocker 'tar czf regressionQueries.tgz /mariadb-columnstore-regression-test/mysql/queries/' "$CONTAINER_NAME"
execInnerDocker 'tar czf testErrorLogs2.tgz *.log /var/log/mariadb/columnstore || echo "failed to grab regression results"' \
"$CONTAINER_NAME" \
'--workdir /mariadb-columnstore-regression-test/mysql/queries/nightly/alltest'
docker cp "${CONTAINER_NAME}:/mariadb-columnstore-regression-test/mysql/queries/nightly/alltest/testErrorLogs2.tgz" "/drone/src/${RESULT}/" || echo "missing testErrorLogs2.tgz"
docker cp "${CONTAINER_NAME}:regressionQueries.tgz" "/drone/src/${RESULT}/" || echo "missing regressionQueries.tgz"
else
echo "Unknown stage's container provided: ${CONTAINER_NAME}"
exit 1
fi
execInnerDocker "/logs.sh ${STAGE}" "$CONTAINER_NAME"
execInnerDocker "/core_dump_check.sh core /core/ ${STAGE}" "$CONTAINER_NAME"
docker cp "${CONTAINER_NAME}:/core/" "/drone/src/${RESULT}/"
docker cp "${CONTAINER_NAME}:/unit_logs/" "/drone/src/${RESULT}/"
execInnerDocker "/core_dump_drop.sh core" "$CONTAINER_NAME"
echo "Saved artifacts:"
ls -R "/drone/src/${RESULT}/"
echo "Done reporting ${STAGE}"
cleanup() {
if [[ -n $(docker ps -q --filter "name=${CONTAINER_NAME}") ]]; then
echo "Cleaning up container ${CONTAINER_NAME}..."
docker rm -f "${CONTAINER_NAME}" || echo "Can't remove container ${CONTAINER_NAME}!"
fi
}
#Remove the container on exit
trap cleanup EXIT

40
build/run_smoke.sh Executable file
View File

@@ -0,0 +1,40 @@
#!/bin/bash
set -eo pipefail
SCRIPT_LOCATION=$(dirname "$0")
source "$SCRIPT_LOCATION"/utils.sh
CONTAINER_NAME="$1"
echo "Arguments received: $@"
if [[ "$EUID" -ne 0 ]]; then
error "Please run script as root"
exit 1
fi
if [[ -z $(docker ps -q --filter "name=${CONTAINER_NAME}") ]]; then
error "Container '${CONTAINER_NAME}' is not running."
exit 1
fi
message "Running smoke checks..."
# start mariadb and mariadb-columnstore services and run simple query
execInnerDocker 'systemctl start mariadb' "$CONTAINER_NAME"
execInnerDocker '/usr/bin/mcsSetConfig SystemConfig CGroup just_no_group_use_local' "$CONTAINER_NAME"
execInnerDocker 'systemctl restart mariadb-columnstore' "$CONTAINER_NAME"
execInnerDocker 'mariadb -e "CREATE DATABASE IF NOT EXISTS test; \
CREATE TABLE test.t1 (a INT) ENGINE=Columnstore; \
INSERT INTO test.t1 VALUES (1); \
SELECT * FROM test.t1;"' "$CONTAINER_NAME"
# restart both services, wait a bit, then insert and select again
execInnerDocker 'systemctl restart mariadb' "$CONTAINER_NAME"
execInnerDocker 'systemctl restart mariadb-columnstore' "$CONTAINER_NAME"
sleep 10
execInnerDocker 'mariadb -e "INSERT INTO test.t1 VALUES (2); SELECT * FROM test.t1;"' "$CONTAINER_NAME"

16
build/utils.sh
View File

@@ -418,9 +418,23 @@ function optparse.define(){
optparse_process="${optparse_process}#NL#TB#TB${shortname})#NL#TB#TB#TB${variable}=\"$val\";;"
}
function execInnerDocker() {
local cmd_str="$1"
local img="$2"
local flags="${3:-}"
docker exec $flags -t "$img" bash -c "$cmd_str"
local dockerCommandExitCode=$?
if [[ $dockerCommandExitCode -ne 0 ]]; then
echo "Command \"$cmd_str\" failed in container \"$img\""
exit $dockerCommandExitCode
fi
}
# -----------------------------------------------------------------------------------------------------------------------------
function optparse.build(){
local build_file="$(mktemp -t "optparse-XXXXXX.tmp")"
local build_file="$(mktemp "${TMPDIR:-/tmp}/optparse-XXXXXX")"
# Building getopts header here

103
cmake/ColumnstoreLibrary.cmake
View File

@@ -1,8 +1,105 @@
# Function to create either a static or shared library based on COLUMNSTORE_STATIC_LIBRARIES
set(DEBIAN_INSTALL_FILE "${CMAKE_BINARY_DIR}/mariadb-plugin-columnstore.install.generated")
file(WRITE ${DEBIAN_INSTALL_FILE} "#File is generated by ColumnstoreLibrary.cmake, do not edit\n")
macro(add_to_debian_install_file file_path)
string(SUBSTRING "${file_path}" 1 -1 BINARY_ENTRY)
file(RELATIVE_PATH CMAKEFILE "${CMAKE_SOURCE_DIR}/storage/columnstore/columnstore" ${CMAKE_CURRENT_LIST_FILE})
string(STRIP "${BINARY_ENTRY}" BINARY_ENTRY)
file(APPEND ${DEBIAN_INSTALL_FILE} "${BINARY_ENTRY} # added in ${CMAKEFILE}\n")
endmacro()
function(get_target_output_filename TARGET_NAME OUTPUT_VAR)
# 1. Get the target's OUTPUT_NAME (falls back to TARGET_NAME)
get_target_property(OUTPUT_NAME ${TARGET_NAME} OUTPUT_NAME)
if(NOT OUTPUT_NAME)
set(OUTPUT_NAME "${TARGET_NAME}")
endif()
# 1. Get the correct suffix based on target type
get_target_property(TARGET_TYPE ${TARGET_NAME} TYPE)
if(TARGET_TYPE STREQUAL "EXECUTABLE")
set(SUFFIX "${CMAKE_EXECUTABLE_SUFFIX}")
set(PREFIX "") # No prefix for executables
elseif(TARGET_TYPE STREQUAL "SHARED_LIBRARY")
set(SUFFIX "${CMAKE_SHARED_LIBRARY_SUFFIX}")
set(PREFIX "lib")
elseif(TARGET_TYPE STREQUAL "STATIC_LIBRARY")
set(SUFFIX "${CMAKE_STATIC_LIBRARY_SUFFIX}")
set(PREFIX "lib")
else()
message(WARNING "Unknown target type for ${TARGET_NAME}")
set(SUFFIX "")
endif()
# 1. Combine into the final filename
set(${OUTPUT_VAR}
"${PREFIX}${OUTPUT_NAME}${SUFFIX}"
PARENT_SCOPE
)
endfunction()
macro(columnstore_install_target target destination)
install(
TARGETS ${target}
DESTINATION ${destination}
COMPONENT columnstore-engine
)
get_target_output_filename(${target} OUTPUT_FILENAME)
add_to_debian_install_file("${destination}/${OUTPUT_FILENAME}")
endmacro()
macro(columnstore_install_file file destination)
install(
FILES ${file}
DESTINATION ${destination}
COMPONENT columnstore-engine
)
get_filename_component(FILENAME ${file} NAME)
add_to_debian_install_file("${destination}/${FILENAME}")
endmacro()
macro(columnstore_install_program file destination)
install(
PROGRAMS ${file}
DESTINATION ${destination}
COMPONENT columnstore-engine
)
get_filename_component(FILENAME ${file} NAME)
add_to_debian_install_file(
"${destination}/${FILENAME}
"
)
endmacro()
macro(columnstore_static_library libname)
add_definitions(-fPIC -DPIC)
add_library(${libname} STATIC ${ARGN})
endmacro()
macro(columnstore_shared_library libname)
add_library(${libname} SHARED ${ARGN})
columnstore_install_target(${libname} ${ENGINE_LIBDIR})
endmacro()
macro(columnstore_library libname)
if(COLUMNSTORE_STATIC_LIBRARIES)
add_library(${libname} STATIC ${ARGN})
columnstore_static_library(${libname} ${ARGN})
else()
add_library(${libname} SHARED ${ARGN})
columnstore_shared_library(${libname} ${ARGN})
endif()
endmacro()
macro(columnstore_mysql_plugin_library libname)
add_library(${libname} SHARED ${ARGN})
columnstore_install_target(${libname} ${MARIADB_PLUGINDIR})
endmacro()
macro(columnstore_link libname)
target_link_libraries(${libname} ${ARGN})
endmacro()
macro(columnstore_executable executable_name)
add_executable(${executable_name} ${ARGN})
columnstore_install_target(${executable_name} ${ENGINE_BINDIR})
endmacro()

5
cmake/compiler_flags.cmake
View File

@@ -51,6 +51,7 @@ set(COLUMNSTORE_MAINTAINER_FLAGS ${WERROR_FLAG})
set(FLAGS_ALL
-Wall
-Wextra
-ggdb3
-fno-omit-frame-pointer
-fno-strict-aliasing
-fsigned-char
@@ -59,9 +60,9 @@ set(FLAGS_ALL
-DBOOST_BIND_GLOBAL_PLACEHOLDERS
)
set(FLAGS_RELEASE -g -O3 -DDBUG_OFF)
set(FLAGS_RELEASE -O3 -DDBUG_OFF)
set(FLAGS_DEBUG -ggdb3 -O0 -D_DEBUG)
set(FLAGS_DEBUG -O0 -D_DEBUG)
# } end Release, Debug and common flags
# linker flags {

15
cmake/configureEngine.cmake
View File

@@ -773,17 +773,10 @@ endif()
# releasenum is used by external scripts for various tasks. Leave it alone.
configure_file(${CMAKE_CURRENT_SOURCE_DIR}/build/releasenum.in ${CMAKE_CURRENT_BINARY_DIR}/build/releasenum IMMEDIATE)
install(
FILES ${CMAKE_CURRENT_BINARY_DIR}/build/releasenum
DESTINATION ${ENGINE_SUPPORTDIR}
COMPONENT columnstore-engine
)
columnstore_install_file(${CMAKE_CURRENT_BINARY_DIR}/build/releasenum ${ENGINE_SUPPORTDIR})
configure_file(${CMAKE_CURRENT_SOURCE_DIR}/columnstoreversion.h.in ${CMAKE_CURRENT_SOURCE_DIR}/columnstoreversion.h)
configure_file(${CMAKE_CURRENT_SOURCE_DIR}/mcsconfig.h.in ${CMAKE_CURRENT_BINARY_DIR}/mcsconfig.h)
configure_file(${CMAKE_CURRENT_SOURCE_DIR}/gitversionEngine.in ${CMAKE_CURRENT_BINARY_DIR}/gitversionEngine IMMEDIATE)
install(
FILES ${CMAKE_CURRENT_BINARY_DIR}/gitversionEngine
DESTINATION ${ENGINE_SUPPORTDIR}
COMPONENT columnstore-engine
)
columnstore_install_file(${CMAKE_CURRENT_BINARY_DIR}/gitversionEngine ${ENGINE_SUPPORTDIR})

1
cmake/packages.cmake
View File

@@ -14,7 +14,6 @@ if(NOT LEX_EXECUTABLE)
endif()
find_package(LibXml2)
if(NOT LIBXML2_FOUND)
message_once(CS_NO_LIBXML "Could not find a usable libxml2 development environment!")
return()

9
datatypes/CMakeLists.txt
View File

@@ -2,10 +2,5 @@ include_directories(${ENGINE_COMMON_INCLUDES})
set(datatypes_LIB_SRCS mcs_int128.cpp mcs_decimal.cpp)
columnstore_library(datatypes ${datatypes_LIB_SRCS})
add_dependencies(datatypes loggingcpp external_boost)
install(
TARGETS datatypes
DESTINATION ${ENGINE_LIBDIR}
COMPONENT columnstore-engine
)
columnstore_link(datatypes PRIVATE loggingcpp)
add_dependencies(datatypes external_boost)

8
dbcon/ddlpackage/CMakeLists.txt
View File

@@ -42,10 +42,4 @@ columnstore_library(
${FLEX_ddl_scan_OUTPUTS}
)
add_dependencies(ddlpackage loggingcpp)
install(
TARGETS ddlpackage
DESTINATION ${ENGINE_LIBDIR}
COMPONENT columnstore-engine
)
columnstore_link(ddlpackage loggingcpp)

11
dbcon/ddlpackageproc/CMakeLists.txt
View File

@@ -13,13 +13,4 @@ set(ddlpackageproc_LIB_SRCS
)
columnstore_library(ddlpackageproc ${ddlpackageproc_LIB_SRCS})
add_dependencies(ddlpackageproc loggingcpp)
target_link_libraries(ddlpackageproc ${NETSNMP_LIBRARIES})
install(
TARGETS ddlpackageproc
DESTINATION ${ENGINE_LIBDIR}
COMPONENT columnstore-engine
)
columnstore_link(ddlpackageproc ${NETSNMP_LIBRARIES} loggingcpp)

8
dbcon/dmlpackage/CMakeLists.txt
View File

@@ -41,10 +41,4 @@ columnstore_library(
${FLEX_dml_scan_OUTPUTS}
)
add_dependencies(dmlpackage loggingcpp)
install(
TARGETS dmlpackage
DESTINATION ${ENGINE_LIBDIR}
COMPONENT columnstore-engine
)
columnstore_link(dmlpackage loggingcpp)

11
dbcon/dmlpackageproc/CMakeLists.txt
View File

@@ -13,13 +13,4 @@ set(dmlpackageproc_LIB_SRCS
)
columnstore_library(dmlpackageproc ${dmlpackageproc_LIB_SRCS})
add_dependencies(dmlpackageproc loggingcpp)
target_link_libraries(dmlpackageproc ${NETSNMP_LIBRARIES})
install(
TARGETS dmlpackageproc
DESTINATION ${ENGINE_LIBDIR}
COMPONENT columnstore-engine
)
columnstore_link(dmlpackageproc ${NETSNMP_LIBRARIES} loggingcpp)

11
dbcon/execplan/CMakeLists.txt
View File

@@ -46,13 +46,4 @@ set(execplan_LIB_SRCS
)
columnstore_library(execplan ${execplan_LIB_SRCS})
add_dependencies(execplan loggingcpp)
target_link_libraries(execplan messageqcpp ${NETSNMP_LIBRARIES} ${ENGINE_DT_LIB} pron)
install(
TARGETS execplan
DESTINATION ${ENGINE_LIBDIR}
COMPONENT columnstore-engine
)
columnstore_link(execplan messageqcpp ${NETSNMP_LIBRARIES} ${ENGINE_DT_LIB} pron loggingcpp)

7
dbcon/execplan/pseudocolumn.cpp
View File

@@ -25,7 +25,7 @@
#include <string>
#include <sstream>
using namespace std;
#include "basic/string_utils.h"
#include "bytestream.h"
#include "collation.h"
@@ -138,8 +138,9 @@ string PseudoColumn::toCppCode(IncludeSet& includes) const
{
includes.insert("pseudocolumn.h");
stringstream ss;
ss << "PseudoColumn(" << std::quoted(fSchemaName) << ", " << std::quoted(fTableName) << ", " << fisColumnStore << ", " << std::quoted(fColumnName)
<< ", " << fPseudoType << ", " << sessionID() << ")";
ss << "PseudoColumn(" << std::quoted(fSchemaName) << ", " << std::quoted(fTableName) << ", "
<< fisColumnStore << ", " << std::quoted(fColumnName) << ", " << fPseudoType << ", " << sessionID()
<< ")";
return ss.str();
}

32
dbcon/execplan/windowfunctioncolumn.cpp
View File

@@ -27,10 +27,13 @@
#include <iostream>
#include <sstream>
using namespace std;
#include "basic/string_utils.h"
#include <boost/tokenizer.hpp>
using namespace boost;
#include "basic/string_utils.h"
#include "bytestream.h"
#include "windowfunctioncolumn.h"
#include "constantcolumn.h"
@@ -398,23 +401,32 @@ void WindowFunctionColumn::adjustResultType()
{
if ((fResultType.colDataType == CalpontSystemCatalog::DECIMAL ||
fResultType.colDataType == CalpontSystemCatalog::UDECIMAL) &&
!datatypes::ASCIIStringCaseInsensetiveEquals(fFunctionName, "COUNT") && !datatypes::ASCIIStringCaseInsensetiveEquals(fFunctionName, "COUNT(*)") &&
!datatypes::ASCIIStringCaseInsensetiveEquals(fFunctionName, "ROW_NUMBER") && !datatypes::ASCIIStringCaseInsensetiveEquals(fFunctionName, "RANK") &&
!datatypes::ASCIIStringCaseInsensetiveEquals(fFunctionName, "PERCENT_RANK") && !datatypes::ASCIIStringCaseInsensetiveEquals(fFunctionName, "DENSE_RANK") &&
!datatypes::ASCIIStringCaseInsensetiveEquals(fFunctionName, "CUME_DIST") && !datatypes::ASCIIStringCaseInsensetiveEquals(fFunctionName, "NTILE") &&
!datatypes::ASCIIStringCaseInsensetiveEquals(fFunctionName, "COUNT") &&
!datatypes::ASCIIStringCaseInsensetiveEquals(fFunctionName, "COUNT(*)") &&
!datatypes::ASCIIStringCaseInsensetiveEquals(fFunctionName, "ROW_NUMBER") &&
!datatypes::ASCIIStringCaseInsensetiveEquals(fFunctionName, "RANK") &&
!datatypes::ASCIIStringCaseInsensetiveEquals(fFunctionName, "PERCENT_RANK") &&
!datatypes::ASCIIStringCaseInsensetiveEquals(fFunctionName, "DENSE_RANK") &&
!datatypes::ASCIIStringCaseInsensetiveEquals(fFunctionName, "CUME_DIST") &&
!datatypes::ASCIIStringCaseInsensetiveEquals(fFunctionName, "NTILE") &&
!datatypes::ASCIIStringCaseInsensetiveEquals(fFunctionName, "PERCENTILE") && !fFunctionParms.empty() &&
fFunctionParms[0]->resultType().colDataType == CalpontSystemCatalog::DOUBLE)
fResultType = fFunctionParms[0]->resultType();
if ((datatypes::ASCIIStringCaseInsensetiveEquals(fFunctionName, "LEAD") || datatypes::ASCIIStringCaseInsensetiveEquals(fFunctionName, "LAG") ||
datatypes::ASCIIStringCaseInsensetiveEquals(fFunctionName, "MIN") || datatypes::ASCIIStringCaseInsensetiveEquals(fFunctionName, "MAX") ||
datatypes::ASCIIStringCaseInsensetiveEquals(fFunctionName, "FIRST_VALUE") || datatypes::ASCIIStringCaseInsensetiveEquals(fFunctionName, "LAST_VALUE") ||
if ((datatypes::ASCIIStringCaseInsensetiveEquals(fFunctionName, "LEAD") ||
datatypes::ASCIIStringCaseInsensetiveEquals(fFunctionName, "LAG") ||
datatypes::ASCIIStringCaseInsensetiveEquals(fFunctionName, "MIN") ||
datatypes::ASCIIStringCaseInsensetiveEquals(fFunctionName, "MAX") ||
datatypes::ASCIIStringCaseInsensetiveEquals(fFunctionName, "FIRST_VALUE") ||
datatypes::ASCIIStringCaseInsensetiveEquals(fFunctionName, "LAST_VALUE") ||
datatypes::ASCIIStringCaseInsensetiveEquals(fFunctionName, "NTH_VALUE")) &&
!fFunctionParms.empty())
fResultType = fFunctionParms[0]->resultType();
if (datatypes::ASCIIStringCaseInsensetiveEquals(fFunctionName, "SUM") || datatypes::ASCIIStringCaseInsensetiveEquals(fFunctionName, "AVG") ||
datatypes::ASCIIStringCaseInsensetiveEquals(fFunctionName, "AVG_DISTINCT") || datatypes::ASCIIStringCaseInsensetiveEquals(fFunctionName, "PERCENTILE"))
if (datatypes::ASCIIStringCaseInsensetiveEquals(fFunctionName, "SUM") ||
datatypes::ASCIIStringCaseInsensetiveEquals(fFunctionName, "AVG") ||
datatypes::ASCIIStringCaseInsensetiveEquals(fFunctionName, "AVG_DISTINCT") ||
datatypes::ASCIIStringCaseInsensetiveEquals(fFunctionName, "PERCENTILE"))
{
if (fFunctionParms[0]->resultType().colDataType == CalpontSystemCatalog::DECIMAL ||
fFunctionParms[0]->resultType().colDataType == CalpontSystemCatalog::UDECIMAL)
@@ -531,7 +543,7 @@ void WindowFunctionColumn::evaluate(Row& row, bool& isNull)
fResult.strVal.assign(str.unsafeStringRef());
// stringColVal is padded with '\0' to colWidth so can't use str.length()
//if (strlen(fResult.strVal.str()) == 0)
// if (strlen(fResult.strVal.str()) == 0)
// isNull = true;
break;

21
dbcon/joblist/CMakeLists.txt
View File

@@ -57,8 +57,8 @@ set(joblist_LIB_SRCS
# ########## next target ###############
add_library(joblist_server STATIC ${joblist_LIB_SRCS})
add_dependencies(joblist_server loggingcpp)
columnstore_static_library(joblist_server ${joblist_LIB_SRCS})
columnstore_link(joblist_server loggingcpp)
# ########## next target ###############
@@ -68,20 +68,9 @@ columnstore_library(joblist ${joblist_LIB_SRCS})
target_include_directories(
joblist BEFORE PUBLIC ${OPENSSL_INCLUDE_DIR} ${LIBMARIADB_BININC_DIR} ${LIBMARIADB_SRCINC_DIR}
)
add_dependencies(joblist loggingcpp)
install(
TARGETS joblist
DESTINATION ${ENGINE_LIBDIR}
COMPONENT columnstore-engine
)
columnstore_link(joblist loggingcpp)
if(WITH_ORDERBY_UT)
add_executable(job_orderby_tests orderby-tests.cpp)
target_link_libraries(job_orderby_tests ${ENGINE_LDFLAGS} ${ENGINE_WRITE_LIBS} ${CPPUNIT_LIBRARIES} cppunit)
install(
TARGETS job_orderby_tests
DESTINATION ${ENGINE_BINDIR}
COMPONENT columnstore-engine
)
columnstore_executable(job_orderby_tests orderby-tests.cpp)
columnstore_link(job_orderby_tests ${ENGINE_LDFLAGS} ${ENGINE_WRITE_LIBS} ${CPPUNIT_LIBRARIES} cppunit)
endif()

42
dbcon/mysql/CMakeLists.txt
View File

@@ -71,16 +71,17 @@ if(COMMAND mysql_add_plugin)
CONFIG
columnstore.cnf
)
add_to_debian_install_file(${MARIADB_MYCNFDIR}/columnstore.cnf)
add_to_debian_install_file("${MARIADB_PLUGINDIR}/ha_columnstore.so")
else()
message("Adding plugin via CMake")
add_library(ha_columnstore SHARED ${libcalmysql_SRCS})
columnstore_shared_library(ha_columnstore ${libcalmysql_SRCS})
set_target_properties(ha_columnstore PROPERTIES PREFIX "")
add_dependencies(ha_columnstore loggingcpp)
add_dependencies(ha_columnstore marias3)
add_definitions(-DMYSQL_DYNAMIC_PLUGIN -DPLUGIN_COLUMNSTORE_VERSION="${PACKAGE_VERSION}")
target_link_libraries(
columnstore_link(
ha_columnstore
${S3API_DEPS}
${ENGINE_LDFLAGS}
@@ -88,33 +89,20 @@ else()
${NETSNMP_LIBRARIES}
${SERVER_BUILD_DIR}/libservices/libmysqlservices.a
threadpool
loggingcpp
marias3
)
install(
TARGETS ha_columnstore
DESTINATION ${MARIADB_PLUGINDIR}
COMPONENT columnstore-engine
)
# define this dummy target for standalone builds (ie, when mysql_add_plugin doesn't exist)
add_custom_target(columnstore DEPENDS ha_columnstore)
install(
FILES columnstore.cnf
DESTINATION ${MARIADB_MYCNFDIR}
COMPONENT columnstore-engine
)
columnstore_install_file(columnstore.cnf ${MARIADB_MYCNFDIR})
endif()
if(TARGET columnstore)
install(
FILES syscatalog_mysql.sql dumpcat_mysql.sql calsetuserpriority.sql calremoveuserpriority.sql
calshowprocesslist.sql columnstore_info.sql
DESTINATION ${ENGINE_SUPPORTDIR}
COMPONENT columnstore-engine
)
install(
PROGRAMS install_mcs_mysql.sh
DESTINATION ${ENGINE_SBINDIR}
COMPONENT columnstore-engine
)
columnstore_install_file(syscatalog_mysql.sql ${ENGINE_SUPPORTDIR})
columnstore_install_file(dumpcat_mysql.sql ${ENGINE_SUPPORTDIR})
columnstore_install_file(calsetuserpriority.sql ${ENGINE_SUPPORTDIR})
columnstore_install_file(calremoveuserpriority.sql ${ENGINE_SUPPORTDIR})
columnstore_install_file(calshowprocesslist.sql ${ENGINE_SUPPORTDIR})
columnstore_install_file(columnstore_info.sql ${ENGINE_SUPPORTDIR})
columnstore_install_program(install_mcs_mysql.sh ${ENGINE_SBINDIR})
endif()

22
dbcon/mysql/ha_mcs.cpp
View File

@@ -1808,6 +1808,8 @@ static handler* ha_mcs_cache_create_handler(handlerton* hton, TABLE_SHARE* table
return new (mem_root) ha_mcs_cache(hton, table, mem_root);
}
bool get_innodb_queries_uses_mcs();
/******************************************************************************
ha_mcs Plugin code
******************************************************************************/
@@ -1826,6 +1828,14 @@ static int columnstore_init_func(void* p)
fprintf(stderr, "Columnstore: Started; Version: %s-%s\n", columnstore_version.c_str(),
columnstore_release.c_str());
LEX_CSTRING name = {STRING_WITH_LEN("INNODB")};
auto* plugin_innodb = ha_resolve_by_name(0, &name, 0);
if (!plugin_innodb || plugin_state(plugin_innodb) != PLUGIN_IS_READY)
{
DBUG_RETURN(HA_ERR_RETRY_INIT);
}
strncpy(cs_version, columnstore_version.c_str(), sizeof(cs_version) - 1);
cs_version[sizeof(cs_version) - 1] = 0;
@@ -1851,6 +1861,18 @@ static int columnstore_init_func(void* p)
mcs_hton->create_unit = create_columnstore_unit_handler;
mcs_hton->db_type = DB_TYPE_AUTOASSIGN;
if (get_innodb_queries_uses_mcs())
{
auto* innodb_hton = plugin_hton(plugin_innodb);
int error = innodb_hton == nullptr; // Engine must exists!
if (error)
{
my_error(HA_ERR_INITIALIZATION, MYF(0), "Could not find storage engine %s", name.str);
}
innodb_hton->create_select = create_columnstore_select_handler;
innodb_hton->create_unit = create_columnstore_unit_handler;
}
#ifdef HAVE_PSI_INTERFACE
uint count = sizeof(all_mutexes) / sizeof(all_mutexes[0]);
mysql_mutex_register("ha_mcs_cache", all_mutexes, count);

1
dbcon/mysql/ha_mcs_client_udfs.cpp
View File

@@ -45,6 +45,7 @@ using namespace logging;
#include "columnstoreversion.h"
#include "ha_mcs_sysvars.h"
#include "basic/string_utils.h"
extern "C"
{

148
dbcon/mysql/ha_mcs_common.h Normal file
View File

@@ -0,0 +1,148 @@
/* Copyright (C) 2016-2023 MariaDB Corporation
This program is free software; you can redistribute it and/or
modify it under the terms of the GNU General Public License
as published by the Free Software Foundation; version 2 of
the License.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program; if not, write to the Free Software
Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
MA 02110-1301, USA. */
#pragma once
#include "idb_mysql.h"
namespace ha_mcs_common
{
inline bool isMCSTable(TABLE* table_ptr)
{
#if (defined(_MSC_VER) && defined(_DEBUG)) || defined(SAFE_MUTEX)
if (!(table_ptr->s && (*table_ptr->s->db_plugin)->name.str))
#else
if (!(table_ptr->s && (table_ptr->s->db_plugin)->name.str))
#endif
return true;
#if (defined(_MSC_VER) && defined(_DEBUG)) || defined(SAFE_MUTEX)
std::string engineName = (*table_ptr->s->db_plugin)->name.str;
#else
std::string engineName = table_ptr->s->db_plugin->name.str;
#endif
if (engineName == "Columnstore" || engineName == "Columnstore_cache")
return true;
return false;
}
inline bool isMultiUpdateStatement(const enum_sql_command& command)
{
return command == SQLCOM_UPDATE_MULTI;
}
inline bool isUpdateStatement(const enum_sql_command& command)
{
return command == SQLCOM_UPDATE || isMultiUpdateStatement(command);
}
inline bool isDeleteStatement(const enum_sql_command& command)
{
return (command == SQLCOM_DELETE) || (command == SQLCOM_DELETE_MULTI);
}
inline bool isUpdateOrDeleteStatement(const enum_sql_command& command)
{
return isUpdateStatement(command) || isDeleteStatement(command);
}
inline bool isDMLStatement(const enum_sql_command& command)
{
return (command == SQLCOM_INSERT || command == SQLCOM_INSERT_SELECT || command == SQLCOM_TRUNCATE ||
command == SQLCOM_LOAD || isUpdateOrDeleteStatement(command));
}
inline bool isForeignTableUpdate(THD* thd)
{
LEX* lex = thd->lex;
if (!isUpdateStatement(lex->sql_command))
return false;
Item_field* item;
List_iterator_fast<Item> field_it(lex->first_select_lex()->item_list);
while ((item = (Item_field*)field_it++))
{
if (item->field && item->field->table && !isMCSTable(item->field->table))
return true;
}
return false;
}
// This function is different from isForeignTableUpdate()
// above as it only checks if any of the tables involved
// in the multi-table update statement is a foreign table,
// irrespective of whether the update is performed on the
// foreign table or not, as in isForeignTableUpdate().
inline bool isUpdateHasForeignTable(THD* thd)
{
LEX* lex = thd->lex;
if (!isUpdateStatement(lex->sql_command))
return false;
TABLE_LIST* table_ptr = lex->first_select_lex()->get_table_list();
for (; table_ptr; table_ptr = table_ptr->next_local)
{
if (table_ptr->table && !isMCSTable(table_ptr->table))
return true;
}
return false;
}
inline bool isMCSTableUpdate(THD* thd)
{
LEX* lex = thd->lex;
if (!isUpdateStatement(lex->sql_command))
return false;
Item_field* item;
List_iterator_fast<Item> field_it(lex->first_select_lex()->item_list);
while ((item = (Item_field*)field_it++))
{
if (item->field && item->field->table && isMCSTable(item->field->table))
return true;
}
return false;
}
inline bool isMCSTableDelete(THD* thd)
{
LEX* lex = thd->lex;
if (!isDeleteStatement(lex->sql_command))
return false;
TABLE_LIST* table_ptr = lex->first_select_lex()->get_table_list();
if (table_ptr && table_ptr->table && isMCSTable(table_ptr->table))
return true;
return false;
}
} // namespace ha_mcs_common

1712
dbcon/mysql/ha_mcs_execplan.cpp
View File

File diff suppressed because it is too large Load Diff

56
dbcon/mysql/ha_mcs_impl.cpp
View File

@@ -832,7 +832,7 @@ uint32_t doUpdateDelete(THD* thd, gp_walk_info& gwi, const std::vector<COND*>& c
{
Message::Args args;
if (isUpdateStatement(thd->lex->sql_command))
if (ha_mcs_common::isUpdateStatement(thd->lex->sql_command))
args.add("Update");
#if 0
else if (thd->rgi_slave && thd->rgi_slave->m_table_map.count() != 0)
@@ -889,7 +889,7 @@ uint32_t doUpdateDelete(THD* thd, gp_walk_info& gwi, const std::vector<COND*>& c
updateCP->isDML(true);
//@Bug 2753. the memory already freed by destructor of UpdateSqlStatement
if (isUpdateStatement(thd->lex->sql_command))
if (ha_mcs_common::isUpdateStatement(thd->lex->sql_command))
{
ColumnAssignment* columnAssignmentPtr = nullptr;
Item_field* item;
@@ -1184,7 +1184,7 @@ uint32_t doUpdateDelete(THD* thd, gp_walk_info& gwi, const std::vector<COND*>& c
}
// Exit early if there is nothing to update
if (colAssignmentListPtr->empty() && isUpdateStatement(thd->lex->sql_command))
if (colAssignmentListPtr->empty() && ha_mcs_common::isUpdateStatement(thd->lex->sql_command))
{
ci->affectedRows = 0;
delete colAssignmentListPtr;
@@ -1198,7 +1198,7 @@ uint32_t doUpdateDelete(THD* thd, gp_walk_info& gwi, const std::vector<COND*>& c
CalpontSystemCatalog::TableName aTableName;
TABLE_LIST* first_table = 0;
if (isUpdateStatement(thd->lex->sql_command))
if (ha_mcs_common::isUpdateStatement(thd->lex->sql_command))
{
aTableName.schema = schemaName;
aTableName.table = tableName;
@@ -1220,7 +1220,7 @@ uint32_t doUpdateDelete(THD* thd, gp_walk_info& gwi, const std::vector<COND*>& c
IDEBUG(cout << "STMT: " << dmlStmt << " and sessionID " << thd->thread_id << endl);
VendorDMLStatement dmlStatement(dmlStmt, sessionID);
if (isUpdateStatement(thd->lex->sql_command))
if (ha_mcs_common::isUpdateStatement(thd->lex->sql_command))
dmlStatement.set_DMLStatementType(DML_UPDATE);
else
dmlStatement.set_DMLStatementType(DML_DELETE);
@@ -1229,7 +1229,7 @@ uint32_t doUpdateDelete(THD* thd, gp_walk_info& gwi, const std::vector<COND*>& c
//@Bug 2753. To make sure the momory is freed.
updateStmt.fColAssignmentListPtr = colAssignmentListPtr;
if (isUpdateStatement(thd->lex->sql_command))
if (ha_mcs_common::isUpdateStatement(thd->lex->sql_command))
{
TableName* qualifiedTablName = new TableName();
qualifiedTablName->fName = tableName;
@@ -1318,7 +1318,7 @@ uint32_t doUpdateDelete(THD* thd, gp_walk_info& gwi, const std::vector<COND*>& c
List<Item> items;
SELECT_LEX select_lex;
if (isUpdateStatement(thd->lex->sql_command))
if (ha_mcs_common::isUpdateStatement(thd->lex->sql_command))
{
items = (thd->lex->first_select_lex()->item_list);
thd->lex->first_select_lex()->item_list = thd->lex->value_list;
@@ -1467,7 +1467,7 @@ uint32_t doUpdateDelete(THD* thd, gp_walk_info& gwi, const std::vector<COND*>& c
returnedCols.push_back((updateCP->columnMap()).begin()->second);
//@Bug 6123. get the correct returned columnlist
if (isDeleteStatement(thd->lex->sql_command))
if (ha_mcs_common::isDeleteStatement(thd->lex->sql_command))
{
returnedCols.clear();
// choose the smallest column to project
@@ -1518,7 +1518,7 @@ uint32_t doUpdateDelete(THD* thd, gp_walk_info& gwi, const std::vector<COND*>& c
updateCP->returnedCols(returnedCols);
if (isUpdateStatement(thd->lex->sql_command))
if (ha_mcs_common::isUpdateStatement(thd->lex->sql_command))
{
const ParseTree* ptsub = updateCP->filters();
@@ -1979,7 +1979,7 @@ int ha_mcs_impl_analyze(THD* thd, TABLE* table)
if (table->s->db.length && strcmp(table->s->db.str, "information_schema") == 0)
return 0;
bool columnStore = (table ? isMCSTable(table) : true);
bool columnStore = (table ? ha_mcs_common::isMCSTable(table) : true);
// Skip non columnstore tables.
if (!columnStore)
return 0;
@@ -2156,7 +2156,8 @@ int ha_mcs_impl_direct_update_delete_rows(bool execute, ha_rows* affected_rows,
gwi.thd = thd;
int rc = 0;
if (thd->slave_thread && !get_replication_slave(thd) && isDMLStatement(thd->lex->sql_command))
if (thd->slave_thread && !get_replication_slave(thd) &&
ha_mcs_common::isDMLStatement(thd->lex->sql_command))
{
if (affected_rows)
*affected_rows = 0;
@@ -2189,7 +2190,7 @@ int ha_mcs::impl_rnd_init(TABLE* table, const std::vector<COND*>& condStack)
gwi.thd = thd;
if (thd->slave_thread && !get_replication_slave(thd) &&
(isDMLStatement(thd->lex->sql_command) || thd->lex->sql_command == SQLCOM_ALTER_TABLE))
(ha_mcs_common::isDMLStatement(thd->lex->sql_command) || thd->lex->sql_command == SQLCOM_ALTER_TABLE))
return 0;
// check whether the system is ready to process statement.
@@ -2241,7 +2242,7 @@ int ha_mcs::impl_rnd_init(TABLE* table, const std::vector<COND*>& condStack)
UPDATE innotab1 SET a=100 WHERE a NOT IN (SELECT a FROM cstab1 WHERE a=1);
*/
if (!isReadOnly() && // make sure the current table is being modified
isUpdateOrDeleteStatement(thd->lex->sql_command))
ha_mcs_common::isUpdateOrDeleteStatement(thd->lex->sql_command))
return doUpdateDelete(thd, gwi, condStack);
uint32_t sessionID = tid2sid(thd->thread_id);
@@ -2578,10 +2579,10 @@ int ha_mcs_impl_rnd_next(uchar* buf, TABLE* table, long timeZone)
THD* thd = current_thd;
if (thd->slave_thread && !get_replication_slave(thd) &&
(isDMLStatement(thd->lex->sql_command) || thd->lex->sql_command == SQLCOM_ALTER_TABLE))
(ha_mcs_common::isDMLStatement(thd->lex->sql_command) || thd->lex->sql_command == SQLCOM_ALTER_TABLE))
return HA_ERR_END_OF_FILE;
if (isMCSTableUpdate(thd) || isMCSTableDelete(thd))
if (ha_mcs_common::isMCSTableUpdate(thd) || ha_mcs_common::isMCSTableDelete(thd))
return HA_ERR_END_OF_FILE;
// @bug 2547
@@ -2661,18 +2662,17 @@ int ha_mcs_impl_rnd_end(TABLE* table, bool is_pushdown_hand)
THD* thd = current_thd;
if (thd->slave_thread && !get_replication_slave(thd) &&
(isDMLStatement(thd->lex->sql_command) || thd->lex->sql_command == SQLCOM_ALTER_TABLE))
return 0;
(ha_mcs_common::isDMLStatement(thd->lex->sql_command) || thd->lex->sql_command == SQLCOM_ALTER_TABLE))
return rc;
cal_connection_info* ci = nullptr;
if (get_fe_conn_info_ptr() != NULL)
ci = reinterpret_cast<cal_connection_info*>(get_fe_conn_info_ptr());
if ((thd->lex)->sql_command == SQLCOM_ALTER_TABLE)
return rc;
if (isMCSTableUpdate(thd) || isMCSTableDelete(thd))
if (ha_mcs_common::isMCSTableUpdate(thd) || ha_mcs_common::isMCSTableDelete(thd))
return rc;
if (!ci)
@@ -3764,7 +3764,7 @@ COND* ha_mcs_impl_cond_push(COND* cond, TABLE* table, std::vector<COND*>& condSt
{
THD* thd = current_thd;
if (isUpdateOrDeleteStatement(thd->lex->sql_command))
if (ha_mcs_common::isUpdateOrDeleteStatement(thd->lex->sql_command))
{
condStack.push_back(cond);
return nullptr;
@@ -3848,7 +3848,7 @@ COND* ha_mcs_impl_cond_push(COND* cond, TABLE* table, std::vector<COND*>& condSt
inline void disableBinlogForDML(THD* thd)
{
if (isDMLStatement(thd->lex->sql_command) && (thd->variables.option_bits & OPTION_BIN_LOG))
if (ha_mcs_common::isDMLStatement(thd->lex->sql_command) && (thd->variables.option_bits & OPTION_BIN_LOG))
{
set_original_option_bits(thd->variables.option_bits, thd);
thd->variables.option_bits &= ~OPTION_BIN_LOG;
@@ -3858,7 +3858,7 @@ inline void disableBinlogForDML(THD* thd)
inline void restoreBinlogForDML(THD* thd)
{
if (isDMLStatement(thd->lex->sql_command))
if (ha_mcs_common::isDMLStatement(thd->lex->sql_command))
{
ulonglong orig_option_bits = get_original_option_bits(thd);
@@ -4027,7 +4027,8 @@ int ha_mcs_impl_pushdown_init(mcs_handler_info* handler_info, TABLE* table, bool
IDEBUG(cout << "pushdown_init for table " << endl);
THD* thd = current_thd;
if (thd->slave_thread && !get_replication_slave(thd) && isDMLStatement(thd->lex->sql_command))
if (thd->slave_thread && !get_replication_slave(thd) &&
ha_mcs_common::isDMLStatement(thd->lex->sql_command))
return 0;
const char* timeZone = thd->variables.time_zone->get_name()->ptr();
@@ -4069,7 +4070,7 @@ int ha_mcs_impl_pushdown_init(mcs_handler_info* handler_info, TABLE* table, bool
// MCOL-4023 We need to test this code path.
// Update and delete code
if (isUpdateOrDeleteStatement(thd->lex->sql_command))
if (ha_mcs_common::isUpdateOrDeleteStatement(thd->lex->sql_command))
return doUpdateDelete(thd, gwi, std::vector<COND*>());
uint32_t sessionID = tid2sid(thd->thread_id);
@@ -4078,7 +4079,7 @@ int ha_mcs_impl_pushdown_init(mcs_handler_info* handler_info, TABLE* table, bool
if (!get_fe_conn_info_ptr())
{
set_fe_conn_info_ptr(reinterpret_cast<void*>(new cal_connection_info(), thd));
set_fe_conn_info_ptr((void*)new cal_connection_info());
thd_set_ha_data(thd, mcs_hton, get_fe_conn_info_ptr());
}
@@ -4493,7 +4494,8 @@ int ha_mcs_impl_select_next(uchar* buf, TABLE* table, long timeZone)
{
THD* thd = current_thd;
if (thd->slave_thread && !get_replication_slave(thd) && isDMLStatement(thd->lex->sql_command))
if (thd->slave_thread && !get_replication_slave(thd) &&
ha_mcs_common::isDMLStatement(thd->lex->sql_command))
return HA_ERR_END_OF_FILE;
int rc = HA_ERR_END_OF_FILE;
@@ -4506,7 +4508,7 @@ int ha_mcs_impl_select_next(uchar* buf, TABLE* table, long timeZone)
cal_connection_info* ci = reinterpret_cast<cal_connection_info*>(get_fe_conn_info_ptr());
if (isUpdateOrDeleteStatement(thd->lex->sql_command))
if (ha_mcs_common::isUpdateOrDeleteStatement(thd->lex->sql_command))
return rc;
// @bug 2547

34
dbcon/mysql/ha_mcs_impl_if.h
View File

@@ -26,7 +26,7 @@
#include <boost/shared_ptr.hpp>
#include <stack>
#include <vector>
#include "basic/string_utils.h"
#include "idb_mysql.h"
#include "ha_mcs_sysvars.h"
@@ -36,6 +36,8 @@ struct st_ha_create_information;
class ha_columnstore_select_handler;
class ha_columnstore_derived_handler;
#include "basic/string_utils.h"
#include "configcpp.h"
#include "idberrorinfo.h"
#include "calpontselectexecutionplan.h"
@@ -117,7 +119,8 @@ struct gp_walk_info
std::vector<execplan::ReturnedColumn*> localCols;
std::stack<execplan::ReturnedColumn*> rcWorkStack;
std::stack<execplan::ParseTree*> ptWorkStack;
boost::shared_ptr<execplan::SimpleColumn> scsp; // while defined as SSCP, it is used as SRCP, nothing specific to SimpleColumn is used in use sites.
boost::shared_ptr<execplan::SimpleColumn> scsp; // while defined as SSCP, it is used as SRCP, nothing
// specific to SimpleColumn is used in use sites.
uint32_t sessionid;
bool fatalParseError;
std::string parseErrorText;
@@ -227,7 +230,6 @@ struct gp_walk_info
{
}
~gp_walk_info();
};
struct SubQueryChainHolder;
@@ -401,11 +403,6 @@ void clearDeleteStacks(gp_walk_info& gwi);
void parse_item(Item* item, std::vector<Item_field*>& field_vec, bool& hasNonSupportItem, uint16& parseInfo,
gp_walk_info* gwip = nullptr);
const std::string bestTableName(const Item_field* ifp);
bool isMCSTable(TABLE* table_ptr);
bool isForeignTableUpdate(THD* thd);
bool isUpdateHasForeignTable(THD* thd);
bool isMCSTableUpdate(THD* thd);
bool isMCSTableDelete(THD* thd);
// execution plan util functions prototypes
execplan::ReturnedColumn* buildReturnedColumn(Item* item, gp_walk_info& gwi, bool& nonSupport,
@@ -452,27 +449,6 @@ bool buildEqualityPredicate(execplan::ReturnedColumn* lhs, execplan::ReturnedCol
boost::shared_ptr<execplan::Operator>& sop, const Item_func::Functype& funcType,
const std::vector<Item*>& itemList, bool isInSubs = false);
inline bool isUpdateStatement(const enum_sql_command& command)
{
return ((command == SQLCOM_UPDATE) || (command == SQLCOM_UPDATE_MULTI));
}
inline bool isDeleteStatement(const enum_sql_command& command)
{
return (command == SQLCOM_DELETE) || (command == SQLCOM_DELETE_MULTI);
}
inline bool isUpdateOrDeleteStatement(const enum_sql_command& command)
{
return isUpdateStatement(command) || isDeleteStatement(command);
}
inline bool isDMLStatement(const enum_sql_command& command)
{
return (command == SQLCOM_INSERT || command == SQLCOM_INSERT_SELECT || command == SQLCOM_TRUNCATE ||
command == SQLCOM_LOAD || isUpdateOrDeleteStatement(command));
}
#ifdef DEBUG_WALK_COND
void debug_walk(const Item* item, void* arg);
#endif

68
dbcon/mysql/ha_mcs_pushdown.cpp
View File

@@ -17,8 +17,40 @@
#include <typeinfo>
#include <string>
// This makes specific MDB classes' attributes public to implement
// MCOL-4740 temporary solution. Search for MCOL-4740
// to get the actual place where it is used.
#define updated_leaves \
updated_leaves; \
\
public:
#include "ha_mcs_pushdown.h"
void update_counters_on_multi_update()
{
if (ha_mcs_common::isMultiUpdateStatement(current_thd->lex->sql_command) &&
!ha_mcs_common::isForeignTableUpdate(current_thd))
{
SELECT_LEX_UNIT* unit = &current_thd->lex->unit;
SELECT_LEX* select_lex = unit->first_select();
auto* multi = (select_lex->join) ? reinterpret_cast<multi_update*>(select_lex->join->result) : nullptr;
if (multi)
{
multi->table_to_update = multi->update_tables ? multi->update_tables->table : 0;
cal_impl_if::cal_connection_info* ci =
reinterpret_cast<cal_impl_if::cal_connection_info*>(get_fe_conn_info_ptr());
if (ci)
{
multi->updated = multi->found = ci->affectedRows;
}
}
}
}
void check_walk(const Item* item, void* arg);
void restore_query_state(ha_columnstore_select_handler* handler)
@@ -391,7 +423,7 @@ derived_handler* create_columnstore_derived_handler(THD* thd, TABLE_LIST* table_
// MCOL-1482 Disable derived_handler if the multi-table update
// statement contains a non-columnstore table.
if (cal_impl_if::isUpdateHasForeignTable(thd))
if (ha_mcs_common::isUpdateHasForeignTable(thd))
{
return handler;
}
@@ -551,18 +583,18 @@ void ha_columnstore_derived_handler::print_error(int, unsigned long)
/*@brief create_columnstore_select_handler_- Creates handler
************************************************************
* DESCRIPTION:
* Creates a select handler if there is no non-equi JOIN, e.g
* t1.c1 > t2.c2 and logical OR in the filter predicates.
* More details in server/sql/select_handler.h
* PARAMETERS:
* thd - THD pointer.
* sel_lex - SELECT_LEX* that describes the query.
* sel_unit - SELECT_LEX_UNIT* that describes the query.
* RETURN:
* select_handler if possible
* NULL in other case
***********************************************************/
* DESCRIPTION:
* Creates a select handler if there is no non-equi JOIN, e.g
* t1.c1 > t2.c2 and logical OR in the filter predicates.
* More details in server/sql/select_handler.h
* PARAMETERS:
* thd - THD pointer.
* sel_lex - SELECT_LEX* that describes the query.
* sel_unit - SELECT_LEX_UNIT* that describes the query.
* RETURN:
* select_handler if possible
* NULL in other case
***********************************************************/
select_handler* create_columnstore_select_handler_(THD* thd, SELECT_LEX* sel_lex, SELECT_LEX_UNIT* sel_unit)
{
mcs_select_handler_mode_t select_handler_mode = get_select_handler_mode(thd);
@@ -578,7 +610,7 @@ select_handler* create_columnstore_select_handler_(THD* thd, SELECT_LEX* sel_lex
// MCOL-1482 Disable select_handler for a multi-table update
// with a non-columnstore table as the target table of the update
// operation.
if (cal_impl_if::isForeignTableUpdate(thd))
if (ha_mcs_common::isForeignTableUpdate(thd))
{
return nullptr;
}
@@ -749,8 +781,7 @@ select_handler* create_columnstore_select_handler_(THD* thd, SELECT_LEX* sel_lex
select_lex != select_lex->master_unit()->fake_select_lex) // (2)
thd->lex->set_limit_rows_examined();
if ((!sel_unit || sel_lex) && !join->tables_list &&
(join->table_count || !select_lex->with_sum_func) &&
if ((!sel_unit || sel_lex) && !join->tables_list && (join->table_count || !select_lex->with_sum_func) &&
!select_lex->have_window_funcs())
{
if (!thd->is_error())
@@ -1010,6 +1041,11 @@ int ha_columnstore_select_handler::end_scan()
scan_ended = true;
// MCOL-4740 multi_update::send_eof(), which outputs the affected
// number of rows to the client, is called after handler::rnd_end().
// So we set multi_update::updated and multi_update::found here.
update_counters_on_multi_update();
int rc = ha_mcs_impl_rnd_end(table, true);
DBUG_RETURN(rc);

1
dbcon/mysql/ha_mcs_pushdown.h
View File

@@ -20,6 +20,7 @@
#include "idb_mysql.h"
#include "ha_mcs.h"
#include "ha_mcs_sysvars.h"
#include "ha_mcs_common.h"
#define NEED_CALPONT_EXTERNS
#include "ha_mcs_impl.h"
#include "ha_mcs_impl_if.h"

14
dbcon/mysql/ha_mcs_sysvars.cpp
View File

@@ -220,7 +220,13 @@ static MYSQL_THDVAR_ULONG(max_allowed_in_values, PLUGIN_VAR_RQCMDARG,
"The maximum length of the entries in the IN query clause.", NULL, NULL, 6000, 1,
~0U, 1);
st_mysql_sys_var* mcs_system_variables[] = {MYSQL_SYSVAR(compression_type),
static my_bool innodb_queries_use_mcs;
static MYSQL_SYSVAR_BOOL(innodb_queries_use_mcs, innodb_queries_use_mcs,
PLUGIN_VAR_OPCMDARG | PLUGIN_VAR_READONLY,
"Direct all InnoDB-only queries into MCS via Select Handler.", NULL, NULL, FALSE);
st_mysql_sys_var* mcs_system_variables[] = {
MYSQL_SYSVAR(compression_type),
MYSQL_SYSVAR(fe_conn_info_ptr),
MYSQL_SYSVAR(original_optimizer_flags),
MYSQL_SYSVAR(original_option_bits),
@@ -260,6 +266,7 @@ st_mysql_sys_var* mcs_system_variables[] = {MYSQL_SYSVAR(compression_type),
MYSQL_SYSVAR(s3_region),
MYSQL_SYSVAR(pron),
MYSQL_SYSVAR(max_allowed_in_values),
MYSQL_SYSVAR(innodb_queries_use_mcs),
NULL};
st_mysql_show_var mcs_status_variables[] = {{"columnstore_version", (char*)&cs_version, SHOW_CHAR},
@@ -655,3 +662,8 @@ void set_max_allowed_in_values(THD* thd, ulong value)
{
THDVAR(thd, max_allowed_in_values) = value;
}
bool get_innodb_queries_uses_mcs()
{
return SYSVAR(innodb_queries_use_mcs);
}

3
dbcon/mysql/ha_view.cpp
View File

@@ -42,6 +42,7 @@ using namespace execplan;
#include "ha_subquery.h"
#include "ha_view.h"
#include "ha_mcs_common.h"
namespace cal_impl_if
{
@@ -131,7 +132,7 @@ void View::transform()
else
{
// check foreign engine tables
bool columnStore = (table_ptr->table ? isMCSTable(table_ptr->table) : true);
bool columnStore = (table_ptr->table ? ha_mcs_common::isMCSTable(table_ptr->table) : true);
// trigger system catalog cache
if (columnStore)

12
ddlproc/CMakeLists.txt
View File

@@ -4,14 +4,6 @@ include_directories(${ENGINE_COMMON_INCLUDES})
set(DDLProc_SRCS ddlproc.cpp ddlprocessor.cpp ../utils/common/crashtrace.cpp)
add_executable(DDLProc ${DDLProc_SRCS})
columnstore_executable(DDLProc ${DDLProc_SRCS})
add_dependencies(DDLProc loggingcpp)
target_link_libraries(DDLProc ${ENGINE_LDFLAGS} ${ENGINE_WRITE_LIBS} ${NETSNMP_LIBRARIES} threadpool)
install(
TARGETS DDLProc
DESTINATION ${ENGINE_BINDIR}
COMPONENT columnstore-engine
)
columnstore_link(DDLProc ${ENGINE_LDFLAGS} ${ENGINE_WRITE_LIBS} ${NETSNMP_LIBRARIES} threadpool loggingcpp)

13
dmlproc/CMakeLists.txt
View File

@@ -6,11 +6,9 @@ set(DMLProc_SRCS dmlproc.cpp dmlprocessor.cpp dmlresultbuffer.cpp batchinsertpro
../utils/common/crashtrace.cpp
)
add_executable(DMLProc ${DMLProc_SRCS})
columnstore_executable(DMLProc ${DMLProc_SRCS})
add_dependencies(DMLProc loggingcpp)
target_link_libraries(
columnstore_link(
DMLProc
${ENGINE_LDFLAGS}
${ENGINE_WRITE_LIBS}
@@ -18,10 +16,5 @@ target_link_libraries(
threadpool
ddlcleanuputil
batchloader
)
install(
TARGETS DMLProc
DESTINATION ${ENGINE_BINDIR}
COMPONENT columnstore-engine
loggingcpp
)

80
mysql-test/columnstore/bugfixes/mcol-4740.result Normal file
View File

@@ -0,0 +1,80 @@
DROP DATABASE IF EXISTS mcol_4740;
CREATE DATABASE mcol_4740;
USE mcol_4740;
CREATE TABLE mcs_1 (a INT, b INT(11), c VARCHAR(100)) engine=columnstore;
CREATE TABLE mcs_2 (a INT, b INT(11), c VARCHAR(100)) engine=columnstore;
INSERT INTO mcs_1 VALUES (33, 99, 1);
INSERT INTO mcs_1 VALUES (33, 99, 2);
INSERT INTO mcs_1 VALUES (33, 99, 3);
INSERT INTO mcs_2 VALUES (33, 11, 1);
INSERT INTO mcs_2 VALUES (33, 11, 2);
INSERT INTO mcs_2 VALUES (33, 11, 3);
SELECT * FROM mcs_1;
a b c
33 99 1
33 99 2
33 99 3
UPDATE mcs_1 A, mcs_2 B SET A.b = B.b WHERE A.c = B.c LIMIT 1;
affected rows: 1
info: Rows matched: 1 Changed: 1 Warnings: 0
SELECT * FROM mcs_1;
a b c
33 11 1
33 99 2
33 99 3
UPDATE mcs_1 A, mcs_2 B SET A.b = A.b + 1, B.b = B.b + 1;
ERROR 42000: The storage engine for the table doesn't support MCS-1012: This version of Columnstore supports update of only one table at a time.
CREATE TABLE `test1` (
`id` text NOT NULL,
`p_date` int(11) DEFAULT NULL,
`con_bfr` text DEFAULT NULL,
`org_bfr` text DEFAULT NULL,
`pat_bfr` text DEFAULT NULL,
`data_type` text DEFAULT NULL,
`order_no` text DEFAULT NULL,
`tra_bfr` text DEFAULT NULL,
`load_bfr` text DEFAULT NULL,
`status` int(11) DEFAULT NULL
) ENGINE=Columnstore DEFAULT CHARSET=utf8;
INSERT INTO test1 values ('index_srgt_id','0','consultation_date_bfr','organization_code_bfr','patient_id_bfr','data_type','order_no','transaction_time_bfr','load_timestamp_bfr','0');
INSERT INTO test1 values ('adt-a55-9000000001','20190610','2.01906E+13','1116508857','PTNT-0001','ADT-01','511','2.01801E+13','2.01801E+13','0');
INSERT INTO test1 values ('adt-a55-9000000002','20190610','2.01906E+13','1116508857','PTNT-0001','ADT-01','511','2.01801E+13','2.01801E+13','0');
INSERT INTO test1 values ('adt-a55-9000000003','20190611','2.01906E+13','1116508857','PTNT-0001','ADT-01','511','2.01801E+13','2.01801E+13','0');
INSERT INTO test1 values ('adt-a55-9000000004','20190612','2.01906E+13','1116508857','PTNT-0001','ADT-01','511','2.01801E+13','2.01801E+13','0');
SELECT status FROM test1;
status
0
0
0
0
0
UPDATE test1 main
INNER JOIN (SELECT id,
con_bfr,
order_no,
ROW_NUMBER() OVER (PARTITION BY IFNULL(p_date,''),
IFNULL(con_bfr,''),
IFNULL(org_bfr,''),
IFNULL(pat_bfr,''),
IFNULL(data_type,'')
ORDER BY IFNULL(tra_bfr,'') DESC,
IFNULL(load_bfr,'') DESC,
id DESC
) AS row_num
FROM test1 tmp
) tmp
ON main.id = tmp.id
AND main.con_bfr = tmp.con_bfr
AND main.order_no = tmp.order_no
AND tmp.row_num = 1
SET main.status = 1;
affected rows: 4
info: Rows matched: 4 Changed: 4 Warnings: 0
SELECT status FROM test1;
status
1
0
1
1
1
DROP DATABASE mcol_4740;

82
mysql-test/columnstore/bugfixes/mcol-4740.test Normal file
View File

@@ -0,0 +1,82 @@
#
# MCOL-4740 UPDATE involving multi-tables returns wrong "Rows matched"
#
--source ../include/have_columnstore.inc
--disable_warnings
DROP DATABASE IF EXISTS mcol_4740;
--enable_warnings
CREATE DATABASE mcol_4740;
USE mcol_4740;
# Test case 1 from MCOL
CREATE TABLE mcs_1 (a INT, b INT(11), c VARCHAR(100)) engine=columnstore;
CREATE TABLE mcs_2 (a INT, b INT(11), c VARCHAR(100)) engine=columnstore;
INSERT INTO mcs_1 VALUES (33, 99, 1);
INSERT INTO mcs_1 VALUES (33, 99, 2);
INSERT INTO mcs_1 VALUES (33, 99, 3);
INSERT INTO mcs_2 VALUES (33, 11, 1);
INSERT INTO mcs_2 VALUES (33, 11, 2);
INSERT INTO mcs_2 VALUES (33, 11, 3);
SELECT * FROM mcs_1;
--enable_info
UPDATE mcs_1 A, mcs_2 B SET A.b = B.b WHERE A.c = B.c LIMIT 1;
--disable_info
SELECT * FROM mcs_1;
--error ER_CHECK_NOT_IMPLEMENTED
UPDATE mcs_1 A, mcs_2 B SET A.b = A.b + 1, B.b = B.b + 1;
# Test case 2 from MCOL
CREATE TABLE `test1` (
`id` text NOT NULL,
`p_date` int(11) DEFAULT NULL,
`con_bfr` text DEFAULT NULL,
`org_bfr` text DEFAULT NULL,
`pat_bfr` text DEFAULT NULL,
`data_type` text DEFAULT NULL,
`order_no` text DEFAULT NULL,
`tra_bfr` text DEFAULT NULL,
`load_bfr` text DEFAULT NULL,
`status` int(11) DEFAULT NULL
) ENGINE=Columnstore DEFAULT CHARSET=utf8;
INSERT INTO test1 values ('index_srgt_id','0','consultation_date_bfr','organization_code_bfr','patient_id_bfr','data_type','order_no','transaction_time_bfr','load_timestamp_bfr','0');
INSERT INTO test1 values ('adt-a55-9000000001','20190610','2.01906E+13','1116508857','PTNT-0001','ADT-01','511','2.01801E+13','2.01801E+13','0');
INSERT INTO test1 values ('adt-a55-9000000002','20190610','2.01906E+13','1116508857','PTNT-0001','ADT-01','511','2.01801E+13','2.01801E+13','0');
INSERT INTO test1 values ('adt-a55-9000000003','20190611','2.01906E+13','1116508857','PTNT-0001','ADT-01','511','2.01801E+13','2.01801E+13','0');
INSERT INTO test1 values ('adt-a55-9000000004','20190612','2.01906E+13','1116508857','PTNT-0001','ADT-01','511','2.01801E+13','2.01801E+13','0');
SELECT status FROM test1;
--enable_info
UPDATE test1 main
INNER JOIN (SELECT id,
con_bfr,
order_no,
ROW_NUMBER() OVER (PARTITION BY IFNULL(p_date,''),
IFNULL(con_bfr,''),
IFNULL(org_bfr,''),
IFNULL(pat_bfr,''),
IFNULL(data_type,'')
ORDER BY IFNULL(tra_bfr,'') DESC,
IFNULL(load_bfr,'') DESC,
id DESC
) AS row_num
FROM test1 tmp
) tmp
ON main.id = tmp.id
AND main.con_bfr = tmp.con_bfr
AND main.order_no = tmp.order_no
AND tmp.row_num = 1
SET main.status = 1;
--disable_info
SELECT status FROM test1;
DROP DATABASE mcol_4740;

3
mysql-test/columnstore/oracle/func_concat_oracle.result
View File

@@ -1,5 +1,6 @@
SET sql_mode=ORACLE;
SET default_storage_engine=ColumnStore;
SET character_set_server = "utf8mb4";
EXPLAIN EXTENDED SELECT 'a'||'b'||'c';
id select_type table type possible_keys key key_len ref rows filtered Extra
1 SIMPLE NULL NULL NULL NULL NULL NULL NULL NULL No tables used
@@ -354,7 +355,7 @@ EXPLAIN EXTENDED SELECT c1 FROM t1 WHERE c1 LIKE '%'||'b' ORDER BY ord;
id select_type table type possible_keys key key_len ref rows filtered Extra
1 PUSHED SELECT NULL NULL NULL NULL NULL NULL NULL NULL NULL
Warnings:
Note 1003 select "test"."t1"."c1" AS "c1" from "test"."t1" where "test"."t1"."c1" like convert(concat('%','b') using utf8mb3) order by "test"."t1"."ord"
Note 1003 select "test"."t1"."c1" AS "c1" from "test"."t1" where "test"."t1"."c1" like convert(concat('%','b') using utf8mb4) order by "test"."t1"."ord"
SELECT c1 FROM t1 WHERE c1 LIKE c2||'%'||'c' ORDER BY ord;
c1
abc

1
mysql-test/columnstore/oracle/func_concat_oracle.test
View File

@@ -4,6 +4,7 @@
SET sql_mode=ORACLE;
SET default_storage_engine=ColumnStore;
SET character_set_server = "utf8mb4";
EXPLAIN EXTENDED SELECT 'a'||'b'||'c';
EXPLAIN EXTENDED SELECT CONCAT('a'||'b'||'c');

6
oam/etc/CMakeLists.txt
View File

@@ -1,5 +1 @@
install(
FILES Columnstore.xml
DESTINATION ${ENGINE_SYSCONFDIR}/columnstore
COMPONENT columnstore-engine
)
columnstore_install_file(Columnstore.xml ${ENGINE_SYSCONFDIR}/columnstore)

70
oam/install_scripts/CMakeLists.txt
View File

@@ -131,49 +131,37 @@ configure_file("${CMAKE_CURRENT_SOURCE_DIR}/mcs-savebrm.py.in" "${CMAKE_CURRENT_
configure_file("${CMAKE_CURRENT_SOURCE_DIR}/mcs-savebrm.py.in" "${CMAKE_CURRENT_SOURCE_DIR}/mcssavebrm.py" @ONLY)
configure_file("${CMAKE_CURRENT_SOURCE_DIR}/columnstoreSyslog.in" "${CMAKE_CURRENT_SOURCE_DIR}/columnstoreSyslog" @ONLY)
install(
PROGRAMS columnstore-post-install
columnstore-pre-uninstall
columnstore_run.sh
post-mysql-install
post-mysqld-install
columnstoreSyslogSetup.sh
mcs-stop-controllernode.sh
mcs-loadbrm.py
mcs-savebrm.py
mariadb-columnstore-start.sh
mariadb-columnstore-stop.sh
loop_process_starter.sh
DESTINATION ${ENGINE_BINDIR}
COMPONENT columnstore-engine
)
columnstore_install_program(columnstore-post-install ${ENGINE_BINDIR})
columnstore_install_program(columnstore-pre-uninstall ${ENGINE_BINDIR})
columnstore_install_program(columnstore_run.sh ${ENGINE_BINDIR})
columnstore_install_program(post-mysql-install ${ENGINE_BINDIR})
columnstore_install_program(post-mysqld-install ${ENGINE_BINDIR})
columnstore_install_program(columnstoreSyslogSetup.sh ${ENGINE_BINDIR})
columnstore_install_program(mcs-stop-controllernode.sh ${ENGINE_BINDIR})
columnstore_install_program(mcs-loadbrm.py ${ENGINE_BINDIR})
columnstore_install_program(mcs-savebrm.py ${ENGINE_BINDIR})
columnstore_install_program(mariadb-columnstore-start.sh ${ENGINE_BINDIR})
columnstore_install_program(mariadb-columnstore-stop.sh ${ENGINE_BINDIR})
columnstore_install_program(loop_process_starter.sh ${ENGINE_BINDIR})
install(
FILES mariadb-columnstore.service
columnstoreAlias
columnstoreSyslog
columnstoreSyslog7
columnstoreSyslog-ng
columnstoreLogRotate
myCnf-include-args.text
myCnf-exclude-args.text
mcs-workernode.service
mcs-controllernode.service
mcs-primproc.service
mcs-writeengineserver.service
mcs-dmlproc.service
mcs-ddlproc.service
mcs-loadbrm.service
mcs-storagemanager.service
DESTINATION ${ENGINE_SUPPORTDIR}
COMPONENT columnstore-engine
)
columnstore_install_file(mariadb-columnstore.service ${ENGINE_SUPPORTDIR})
columnstore_install_file(columnstoreAlias ${ENGINE_SUPPORTDIR})
columnstore_install_file(columnstoreSyslog ${ENGINE_SUPPORTDIR})
columnstore_install_file(columnstoreSyslog7 ${ENGINE_SUPPORTDIR})
columnstore_install_file(columnstoreSyslog-ng ${ENGINE_SUPPORTDIR})
columnstore_install_file(columnstoreLogRotate ${ENGINE_SUPPORTDIR})
columnstore_install_file(myCnf-include-args.text ${ENGINE_SUPPORTDIR})
columnstore_install_file(myCnf-exclude-args.text ${ENGINE_SUPPORTDIR})
columnstore_install_file(mcs-workernode.service ${ENGINE_SUPPORTDIR})
columnstore_install_file(mcs-controllernode.service ${ENGINE_SUPPORTDIR})
columnstore_install_file(mcs-primproc.service ${ENGINE_SUPPORTDIR})
columnstore_install_file(mcs-writeengineserver.service ${ENGINE_SUPPORTDIR})
columnstore_install_file(mcs-dmlproc.service ${ENGINE_SUPPORTDIR})
columnstore_install_file(mcs-ddlproc.service ${ENGINE_SUPPORTDIR})
columnstore_install_file(mcs-loadbrm.service ${ENGINE_SUPPORTDIR})
columnstore_install_file(mcs-storagemanager.service ${ENGINE_SUPPORTDIR})
install(
FILES module
DESTINATION ${ENGINE_DATADIR}/local
COMPONENT columnstore-engine
)
columnstore_install_file(module ${ENGINE_DATADIR}/local)
find_package(Python3 COMPONENTS Interpreter)
if(Python3_FOUND)

11
oam/oamcpp/CMakeLists.txt
View File

@@ -5,15 +5,6 @@ include_directories(${ENGINE_COMMON_INCLUDES})
set(oamcpp_LIB_SRCS liboamcpp.cpp oamcache.cpp)
columnstore_library(oamcpp ${oamcpp_LIB_SRCS})
add_dependencies(oamcpp loggingcpp)
target_link_libraries(oamcpp)
columnstore_link(oamcpp loggingcpp)
target_compile_options(oamcpp PRIVATE -Wno-unused-result)
install(
TARGETS oamcpp
DESTINATION ${ENGINE_LIBDIR}
COMPONENT columnstore-engine
)

10
oamapps/columnstoreDB/CMakeLists.txt
View File

@@ -4,12 +4,6 @@ include_directories(${ENGINE_COMMON_INCLUDES})
set(columnstoreDBWrite_SRCS columnstoreDB.cpp)
add_executable(columnstoreDBWrite ${columnstoreDBWrite_SRCS})
columnstore_executable(columnstoreDBWrite ${columnstoreDBWrite_SRCS})
target_link_libraries(columnstoreDBWrite ${ENGINE_LDFLAGS} ncurses ${ENGINE_EXEC_LIBS})
install(
TARGETS columnstoreDBWrite
DESTINATION ${ENGINE_BINDIR}
COMPONENT columnstore-engine
)
columnstore_link(columnstoreDBWrite ${ENGINE_LDFLAGS} ncurses ${ENGINE_EXEC_LIBS})

23
oamapps/columnstoreSupport/CMakeLists.txt
View File

@@ -4,20 +4,13 @@ include_directories(${ENGINE_COMMON_INCLUDES})
set(columnstoreSupport_SRCS columnstoreSupport.cpp mcsSupportUtil.cpp)
add_executable(columnstoreSupport ${columnstoreSupport_SRCS})
columnstore_executable(columnstoreSupport ${columnstoreSupport_SRCS})
target_compile_options(columnstoreSupport PRIVATE -Wno-unused-result)
columnstore_link(columnstoreSupport ${ENGINE_LDFLAGS} ncurses ${ENGINE_EXEC_LIBS})
target_link_libraries(columnstoreSupport ${ENGINE_LDFLAGS} ncurses ${ENGINE_EXEC_LIBS})
install(
TARGETS columnstoreSupport
DESTINATION ${ENGINE_BINDIR}
COMPONENT columnstore-engine
)
install(
PROGRAMS dbmsReport.sh bulklogReport.sh configReport.sh hardwareReport.sh logReport.sh resourceReport.sh
DESTINATION ${ENGINE_BINDIR}
COMPONENT columnstore-engine
)
columnstore_install_program(dbmsReport.sh, ${ENGINE_BINDIR})
columnstore_install_program(bulklogReport.sh, ${ENGINE_BINDIR})
columnstore_install_program(configReport.sh, ${ENGINE_BINDIR})
columnstore_install_program(hardwareReport.sh, ${ENGINE_BINDIR})
columnstore_install_program(logReport.sh, ${ENGINE_BINDIR})
columnstore_install_program(resourceReport.sh, ${ENGINE_BINDIR})

12
oamapps/postConfigure/CMakeLists.txt
View File

@@ -4,14 +4,6 @@ include_directories(${ENGINE_COMMON_INCLUDES})
set(mycnfUpgrade_SRCS mycnfUpgrade.cpp)
add_executable(mycnfUpgrade ${mycnfUpgrade_SRCS})
columnstore_executable(mycnfUpgrade ${mycnfUpgrade_SRCS})
target_compile_options(mycnfUpgrade PRIVATE -Wno-unused-result)
target_link_libraries(mycnfUpgrade ${ENGINE_LDFLAGS} ${ENGINE_READLINE_LIBRARY} ncurses ${ENGINE_EXEC_LIBS})
install(
TARGETS mycnfUpgrade
DESTINATION ${ENGINE_BINDIR}
COMPONENT columnstore-engine
)
columnstore_link(mycnfUpgrade ${ENGINE_LDFLAGS} ${ENGINE_READLINE_LIBRARY} ncurses ${ENGINE_EXEC_LIBS})

3
oamapps/postConfigure/mycnfUpgrade.cpp
View File

@@ -46,6 +46,7 @@
#include <stdio.h>
#include <string>
#include <unistd.h>
#include <tuple>
#include <vector>
#include <sys/sysinfo.h>
@@ -282,7 +283,7 @@ int main(int argc, char* argv[])
USER = p;
string cmd = "chown " + USER + ":" + USER + " " + mycnfFile;
system(cmd.c_str());
std::ignore = std::system(cmd.c_str());
exit(0);
}

12
primitives/blockcache/CMakeLists.txt
View File

@@ -13,13 +13,5 @@ set(dbbc_STAT_SRCS
stats.cpp
fsutils.cpp
)
# libdbbc_a_CXXFLAGS = $(march_flags) $(AM_CXXFLAGS)
add_library(dbbc STATIC ${dbbc_STAT_SRCS})
add_dependencies(dbbc loggingcpp)
target_link_libraries(dbbc ${NETSNMP_LIBRARIES})
install(TARGETS dbbc DESTINATION ${ENGINE_LIBDIR})
columnstore_static_library(dbbc ${dbbc_STAT_SRCS})
columnstore_link(dbbc ${NETSNMP_LIBRARIES} loggingcpp)

9
primitives/linux-port/CMakeLists.txt
View File

@@ -4,10 +4,5 @@ include_directories(${ENGINE_COMMON_INCLUDES} ../blockcache ../primproc)
set(processor_STAT_SRCS primitiveprocessor.cpp dictionary.cpp column.cpp)
add_library(processor STATIC ${processor_STAT_SRCS})
add_dependencies(processor loggingcpp)
target_link_libraries(processor ${NETSNMP_LIBRARIES})
install(TARGETS processor DESTINATION ${ENGINE_LIBDIR})
columnstore_static_library(processor ${processor_STAT_SRCS})
columnstore_link(processor ${NETSNMP_LIBRARIES} loggingcpp)

12
primitives/primproc/CMakeLists.txt
View File

@@ -25,11 +25,10 @@ set(PrimProc_SRCS
../../utils/common/crashtrace.cpp
)
add_executable(PrimProc ${PrimProc_SRCS})
columnstore_executable(PrimProc ${PrimProc_SRCS})
add_dependencies(PrimProc loggingcpp)
target_include_directories(PrimProc PRIVATE ${Boost_INCLUDE_DIRS})
target_link_libraries(
columnstore_link(
PrimProc
${ENGINE_LDFLAGS}
${NETSNMP_LIBRARIES}
@@ -38,10 +37,5 @@ target_link_libraries(
cacheutils
dbbc
processor
)
install(
TARGETS PrimProc
DESTINATION ${ENGINE_BINDIR}
COMPONENT columnstore-engine
loggingcpp
)

57
storage-manager/CMakeLists.txt
View File

@@ -64,11 +64,11 @@ configure_file(
link_directories(${CMAKE_BINARY_DIR}/lib)
set(CMAKE_INSTALL_RPATH $ORIGIN $ORIGIN/../lib)
add_library(storagemanager SHARED ${storagemanager_SRCS})
columnstore_library(storagemanager SHARED ${storagemanager_SRCS})
add_dependencies(storagemanager marias3 external_boost)
target_compile_definitions(storagemanager PUBLIC BOOST_NO_CXX11_SCOPED_ENUMS)
target_link_libraries(
columnstore_link(
storagemanager
boost_chrono
boost_system
@@ -80,8 +80,8 @@ target_link_libraries(
)
target_include_directories(storagemanager PRIVATE ${Boost_INCLUDE_DIRS})
add_executable(StorageManager src/main.cpp)
target_link_libraries(StorageManager storagemanager)
columnstore_executable(StorageManager src/main.cpp)
columnstore_link(StorageManager storagemanager)
set_property(TARGET StorageManager PROPERTY CXX_STANDARD 20)
set(TMPDIR ${CMAKE_RUNTIME_OUTPUT_DIRECTORY})
@@ -89,11 +89,11 @@ set(CMAKE_RUNTIME_OUTPUT_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR})
add_executable(unit_tests src/unit_tests.cpp)
target_compile_definitions(unit_tests PUBLIC BOOST_NO_CXX11_SCOPED_ENUMS)
target_link_libraries(unit_tests storagemanager)
columnstore_link(unit_tests storagemanager)
add_executable(testS3Connection src/testS3Connection.cpp)
columnstore_executable(testS3Connection src/testS3Connection.cpp)
target_compile_definitions(testS3Connection PUBLIC BOOST_NO_CXX11_SCOPED_ENUMS)
target_link_libraries(testS3Connection storagemanager)
columnstore_link(testS3Connection storagemanager)
set(CMAKE_RUNTIME_OUTPUT_DIRECTORY ${TMPDIR})
@@ -112,42 +112,25 @@ add_custom_command(
# The includes and lib linkages required to link against cloudio ... pretty crazy. When lib dependencies are eventually
# config'd right, change this to only include and link against cloudio.
include_directories(${ENGINE_SRC_DIR}/utils/cloudio ${ENGINE_COMMON_INCLUDES})
add_executable(smcat src/smcat.cpp)
target_link_libraries(smcat storagemanager cloudio ${ENGINE_LDFLAGS} ${ENGINE_EXEC_LIBS})
columnstore_executable(smcat src/smcat.cpp)
columnstore_link(smcat storagemanager cloudio ${ENGINE_LDFLAGS} ${ENGINE_EXEC_LIBS})
add_executable(smput src/smput.cpp)
target_link_libraries(smput storagemanager cloudio ${ENGINE_LDFLAGS} ${ENGINE_EXEC_LIBS})
columnstore_executable(smput src/smput.cpp)
columnstore_link(smput storagemanager cloudio ${ENGINE_LDFLAGS} ${ENGINE_EXEC_LIBS})
add_executable(smls src/smls.cpp)
target_link_libraries(smls storagemanager cloudio ${ENGINE_LDFLAGS} ${ENGINE_EXEC_LIBS})
columnstore_executable(smls src/smls.cpp)
columnstore_link(smls storagemanager cloudio ${ENGINE_LDFLAGS} ${ENGINE_EXEC_LIBS})
add_executable(smrm src/smrm.cpp)
target_link_libraries(smrm storagemanager cloudio ${ENGINE_LDFLAGS} ${ENGINE_EXEC_LIBS})
columnstore_executable(smrm src/smrm.cpp)
columnstore_link(smrm storagemanager cloudio ${ENGINE_LDFLAGS} ${ENGINE_EXEC_LIBS})
add_executable(smps src/smps.cpp)
target_link_libraries(smps storagemanager cloudio ${ENGINE_LDFLAGS} ${ENGINE_EXEC_LIBS})
columnstore_executable(smps src/smps.cpp)
columnstore_link(smps storagemanager cloudio ${ENGINE_LDFLAGS} ${ENGINE_EXEC_LIBS})
add_executable(smkill src/smkill.cpp)
target_link_libraries(smkill storagemanager cloudio ${ENGINE_LDFLAGS} ${ENGINE_EXEC_LIBS})
install(TARGETS storagemanager LIBRARY DESTINATION ${ENGINE_LIBDIR} COMPONENT columnstore-engine)
columnstore_executable(smkill src/smkill.cpp)
columnstore_link(smkill storagemanager cloudio ${ENGINE_LDFLAGS} ${ENGINE_EXEC_LIBS})
install(
TARGETS StorageManager
smcat
smkill
smps
smput
smls
smrm
testS3Connection
RUNTIME DESTINATION ${ENGINE_BINDIR} COMPONENT columnstore-engine
)
install(
FILES storagemanager.cnf
DESTINATION ${ENGINE_SYSCONFDIR}/columnstore
COMPONENT columnstore-engine
)
columnstore_install_file(storagemanager.cnf ${ENGINE_SYSCONFDIR}/columnstore)
install(
FILES storagemanager.cnf

34
tests/CMakeLists.txt
View File

@@ -30,82 +30,80 @@ if(WITH_UNITTESTS)
add_executable(rowgroup_tests rowgroup-tests.cpp)
target_compile_options(rowgroup_tests PRIVATE -Wno-sign-compare)
add_dependencies(rowgroup_tests googletest)
target_link_libraries(rowgroup_tests ${ENGINE_LDFLAGS} ${GTEST_LIBRARIES} ${ENGINE_EXEC_LIBS})
columnstore_link(rowgroup_tests ${ENGINE_LDFLAGS} ${GTEST_LIBRARIES} ${ENGINE_EXEC_LIBS})
gtest_add_tests(TARGET rowgroup_tests TEST_PREFIX columnstore:)
add_executable(rewritetest rewritetest.cpp)
add_dependencies(rewritetest googletest)
target_link_libraries(rewritetest ${ENGINE_LDFLAGS} ${GTEST_LIBRARIES} ${ENGINE_EXEC_LIBS})
columnstore_link(rewritetest ${ENGINE_LDFLAGS} ${GTEST_LIBRARIES} ${ENGINE_EXEC_LIBS})
gtest_add_tests(TARGET rewritetest TEST_PREFIX columnstore:)
add_executable(mcs_decimal_tests mcs_decimal-tests.cpp)
add_dependencies(mcs_decimal_tests googletest)
target_link_libraries(mcs_decimal_tests ${ENGINE_LDFLAGS} ${GTEST_LIBRARIES} ${ENGINE_EXEC_LIBS})
columnstore_link(mcs_decimal_tests ${ENGINE_LDFLAGS} ${GTEST_LIBRARIES} ${ENGINE_EXEC_LIBS})
gtest_add_tests(TARGET mcs_decimal_tests TEST_PREFIX columnstore:)
add_executable(dataconvert_tests dataconvert-tests.cpp)
add_dependencies(dataconvert_tests googletest)
target_link_libraries(dataconvert_tests ${ENGINE_LDFLAGS} ${GTEST_LIBRARIES} ${ENGINE_EXEC_LIBS})
columnstore_link(dataconvert_tests ${ENGINE_LDFLAGS} ${GTEST_LIBRARIES} ${ENGINE_EXEC_LIBS})
gtest_add_tests(TARGET dataconvert_tests TEST_PREFIX columnstore:)
add_executable(rebuild_em_tests rebuild-em-tests.cpp)
add_dependencies(rebuild_em_tests googletest)
target_link_libraries(rebuild_em_tests ${ENGINE_LDFLAGS} ${GTEST_LIBRARIES} ${ENGINE_WRITE_LIBS})
columnstore_link(rebuild_em_tests ${ENGINE_LDFLAGS} ${GTEST_LIBRARIES} ${ENGINE_WRITE_LIBS})
gtest_add_tests(TARGET rebuild_em_tests TEST_PREFIX columnstore:)
add_executable(compression_tests compression-tests.cpp)
add_dependencies(compression_tests googletest)
target_link_libraries(compression_tests ${ENGINE_LDFLAGS} ${GTEST_LIBRARIES} ${ENGINE_WRITE_LIBS})
columnstore_link(compression_tests ${ENGINE_LDFLAGS} ${GTEST_LIBRARIES} ${ENGINE_WRITE_LIBS})
gtest_add_tests(TARGET compression_tests TEST_PREFIX columnstore:)
add_executable(column_scan_filter_tests primitives_column_scan_and_filter.cpp)
target_compile_options(column_scan_filter_tests PRIVATE -Wno-error -Wno-sign-compare)
add_dependencies(column_scan_filter_tests googletest)
target_link_libraries(
column_scan_filter_tests ${ENGINE_LDFLAGS} ${ENGINE_WRITE_LIBS} ${GTEST_LIBRARIES} processor dbbc
)
columnstore_link(column_scan_filter_tests ${ENGINE_LDFLAGS} ${ENGINE_WRITE_LIBS} ${GTEST_LIBRARIES} processor dbbc)
gtest_add_tests(TARGET column_scan_filter_tests TEST_PREFIX columnstore:)
add_executable(simd_processors simd_processors.cpp)
target_compile_options(simd_processors PRIVATE -Wno-error)
add_dependencies(simd_processors googletest)
target_link_libraries(simd_processors ${ENGINE_LDFLAGS} ${ENGINE_WRITE_LIBS} ${GTEST_LIBRARIES} processor dbbc)
columnstore_link(simd_processors ${ENGINE_LDFLAGS} ${ENGINE_WRITE_LIBS} ${GTEST_LIBRARIES} processor dbbc)
gtest_add_tests(TARGET simd_processors TEST_PREFIX columnstore:)
add_executable(fair_threadpool_test fair_threadpool.cpp)
add_dependencies(fair_threadpool_test googletest)
target_link_libraries(fair_threadpool_test ${ENGINE_LDFLAGS} ${ENGINE_WRITE_LIBS} ${GTEST_LIBRARIES} processor dbbc)
columnstore_link(fair_threadpool_test ${ENGINE_LDFLAGS} ${ENGINE_WRITE_LIBS} ${GTEST_LIBRARIES} processor dbbc)
gtest_add_tests(TARGET fair_threadpool_test TEST_PREFIX columnstore:)
add_executable(counting_allocator counting_allocator.cpp)
target_compile_options(counting_allocator PRIVATE -Wno-sign-compare)
add_dependencies(counting_allocator googletest)
target_link_libraries(counting_allocator ${ENGINE_LDFLAGS} ${ENGINE_WRITE_LIBS} ${GTEST_LIBRARIES})
columnstore_link(counting_allocator ${ENGINE_LDFLAGS} ${ENGINE_WRITE_LIBS} ${GTEST_LIBRARIES})
gtest_add_tests(TARGET counting_allocator TEST_PREFIX columnstore:)
add_executable(poolallocator poolallocator.cpp)
target_compile_options(poolallocator PRIVATE -Wno-sign-compare)
add_dependencies(poolallocator googletest)
target_link_libraries(poolallocator ${ENGINE_LDFLAGS} ${ENGINE_WRITE_LIBS} ${GTEST_LIBRARIES})
columnstore_link(poolallocator ${ENGINE_LDFLAGS} ${ENGINE_WRITE_LIBS} ${GTEST_LIBRARIES})
gtest_add_tests(TARGET poolallocator TEST_PREFIX columnstore:)
add_executable(stlpoolallocator stlpoolallocator.cpp)
target_compile_options(stlpoolallocator PRIVATE -Wno-sign-compare)
add_dependencies(stlpoolallocator googletest)
target_link_libraries(stlpoolallocator ${ENGINE_LDFLAGS} ${ENGINE_WRITE_LIBS} ${GTEST_LIBRARIES})
columnstore_link(stlpoolallocator ${ENGINE_LDFLAGS} ${ENGINE_WRITE_LIBS} ${GTEST_LIBRARIES})
gtest_add_tests(TARGET stlpoolallocator TEST_PREFIX columnstore:)
add_executable(comparators_tests comparators-tests.cpp)
target_link_libraries(comparators_tests ${ENGINE_LDFLAGS} ${ENGINE_WRITE_LIBS} ${CPPUNIT_LIBRARIES} cppunit)
columnstore_link(comparators_tests ${ENGINE_LDFLAGS} ${ENGINE_WRITE_LIBS} ${CPPUNIT_LIBRARIES} cppunit)
add_test(NAME columnstore:comparators_tests COMMAND comparators_tests)
add_executable(bytestream bytestream.cpp)
target_link_libraries(bytestream ${ENGINE_LDFLAGS} ${ENGINE_WRITE_LIBS} ${CPPUNIT_LIBRARIES} cppunit)
columnstore_link(bytestream ${ENGINE_LDFLAGS} ${ENGINE_WRITE_LIBS} ${CPPUNIT_LIBRARIES} cppunit)
add_test(NAME columnstore:bytestream COMMAND bytestream)
# standalone EM routines test
add_executable(brm_em_standalone brm-em-standalone.cpp)
target_link_libraries(brm_em_standalone ${ENGINE_LDFLAGS} ${ENGINE_WRITE_LIBS} ${CPPUNIT_LIBRARIES} cppunit)
columnstore_link(brm_em_standalone ${ENGINE_LDFLAGS} ${ENGINE_WRITE_LIBS} ${CPPUNIT_LIBRARIES} cppunit)
add_test(NAME columnstore:brm_em_standalone COMMAND brm_em_standalone)
set_tests_properties(columnstore:brm_em_standalone PROPERTIES DISABLED True)
endif()
@@ -116,7 +114,7 @@ if(WITH_MICROBENCHMARKS AND (NOT CMAKE_BUILD_TYPE STREQUAL "Debug"))
target_include_directories(
primitives_scan_bench PUBLIC ${ENGINE_COMMON_INCLUDES} ${ENGINE_BLOCKCACHE_INCLUDE} ${ENGINE_PRIMPROC_INCLUDE}
)
target_link_libraries(
columnstore_link(
primitives_scan_bench
${ENGINE_LDFLAGS}
${ENGINE_WRITE_LIBS}

258
tools/bincvt/li2bin.cpp
View File

@@ -1,258 +0,0 @@
#include <iostream>
#include <inttypes.h>
#include <string>
#include <cstring>
#include <cstdlib>
//#define NDEBUG
#include <cassert>
using namespace std;
#include <boost/tokenizer.hpp>
#include <boost/static_assert.hpp>
using namespace boost;
// 1|155190|7706|1|17|21168.23|0.04|0.02|N|O|1996-03-13|1996-02-12|1996-03-22|DELIVER IN PERSON|TRUCK|egular
// courts above the|
#pragma pack(1)
struct lineitem_image
{
int32_t l_orderkey;
int32_t l_partkey;
int32_t l_suppkey; // 12
int64_t l_linenumber;
int64_t l_quantity;
int64_t l_extendedprice;
int64_t l_discount;
int64_t l_tax; // 40
char l_returnflag;
char l_linestatus; // 2
int32_t l_shipdate;
int32_t l_commitdate;
int32_t l_receiptdate; // 12
char l_shipinstruct[25];
char l_shipmode[10];
char l_comment[44]; // 79
};
BOOST_STATIC_ASSERT(sizeof(struct lineitem_image) == 145);
// version of lineitem_image that treats some integer columns as unsigned ints.
// Decimal, date, and date/time columns are still treated as integer.
struct unsigned_lineitem_image
{
uint32_t l_orderkey;
uint32_t l_partkey;
uint32_t l_suppkey; // 12
uint64_t l_linenumber;
int64_t l_quantity;
int64_t l_extendedprice;
int64_t l_discount;
int64_t l_tax; // 40
char l_returnflag;
char l_linestatus; // 2
int32_t l_shipdate;
int32_t l_commitdate;
int32_t l_receiptdate; // 12
char l_shipinstruct[25];
char l_shipmode[10];
char l_comment[44]; // 79
};
BOOST_STATIC_ASSERT(sizeof(struct unsigned_lineitem_image) == 145);
struct Date
{
unsigned spare : 6;
unsigned day : 6;
unsigned month : 4;
unsigned year : 16;
};
BOOST_STATIC_ASSERT(sizeof(struct Date) == 4);
union date_image
{
struct Date d;
int32_t i;
};
namespace
{
typedef boost::tokenizer<boost::char_separator<char> > tokenizer;
int64_t idbbigdec2(const string& str)
{
int64_t val = 0;
string::size_type idx = string::npos;
string tmp(str);
idx = tmp.find('.');
if (idx != string::npos)
tmp.erase(idx, 1);
else
tmp.append("00");
val = atoll(tmp.c_str());
return val;
}
int32_t idbdate(const string& str)
{
date_image di;
di.i = 0;
boost::char_separator<char> sep("-");
tokenizer tokens(str, sep);
tokenizer::iterator tok_iter = tokens.begin();
// Note that the spare bits must be set, instead of leaving them
// initialized to 0; to be compatible with internal date format.
if (distance(tok_iter, tokens.end()) == 3)
{
di.d.spare = 0x3E;
di.d.year = atoi(tok_iter->c_str());
++tok_iter;
di.d.month = atoi(tok_iter->c_str());
++tok_iter;
di.d.day = atoi(tok_iter->c_str());
++tok_iter;
}
return di.i;
}
int parseinto(lineitem_image& img, const string& buf)
{
memset(&img, 0, sizeof(img));
boost::char_separator<char> sep("|");
tokenizer tokens(buf, sep);
tokenizer::iterator tok_iter = tokens.begin();
if (distance(tok_iter, tokens.end()) < 16)
return -1;
img.l_orderkey = atoi(tok_iter->c_str());
++tok_iter;
img.l_partkey = atoi(tok_iter->c_str());
++tok_iter;
img.l_suppkey = atoi(tok_iter->c_str());
++tok_iter;
img.l_linenumber = atoll(tok_iter->c_str());
++tok_iter;
img.l_quantity = idbbigdec2(tok_iter->c_str());
++tok_iter;
img.l_extendedprice = idbbigdec2(tok_iter->c_str());
++tok_iter;
img.l_discount = idbbigdec2(tok_iter->c_str());
++tok_iter;
img.l_tax = idbbigdec2(tok_iter->c_str());
++tok_iter;
img.l_returnflag = tok_iter->at(0);
++tok_iter;
img.l_linestatus = tok_iter->at(0);
++tok_iter;
img.l_shipdate = idbdate(tok_iter->c_str());
++tok_iter;
img.l_commitdate = idbdate(tok_iter->c_str());
++tok_iter;
img.l_receiptdate = idbdate(tok_iter->c_str());
++tok_iter;
memcpy(&img.l_shipinstruct[0], tok_iter->c_str(), tok_iter->size());
++tok_iter;
memcpy(&img.l_shipmode[0], tok_iter->c_str(), tok_iter->size());
++tok_iter;
memcpy(&img.l_comment[0], tok_iter->c_str(), tok_iter->size());
++tok_iter;
return 0;
}
int unsigned_parseinto(unsigned_lineitem_image& img, const string& buf)
{
memset(&img, 0, sizeof(img));
boost::char_separator<char> sep("|");
tokenizer tokens(buf, sep);
tokenizer::iterator tok_iter = tokens.begin();
if (distance(tok_iter, tokens.end()) < 16)
return -1;
img.l_orderkey = strtoul(tok_iter->c_str(), 0, 10);
++tok_iter;
img.l_partkey = strtoul(tok_iter->c_str(), 0, 10);
++tok_iter;
img.l_suppkey = strtoul(tok_iter->c_str(), 0, 10);
++tok_iter;
img.l_linenumber = strtoull(tok_iter->c_str(), 0, 10);
++tok_iter;
img.l_quantity = idbbigdec2(tok_iter->c_str());
++tok_iter;
img.l_extendedprice = idbbigdec2(tok_iter->c_str());
++tok_iter;
img.l_discount = idbbigdec2(tok_iter->c_str());
++tok_iter;
img.l_tax = idbbigdec2(tok_iter->c_str());
++tok_iter;
img.l_returnflag = tok_iter->at(0);
++tok_iter;
img.l_linestatus = tok_iter->at(0);
++tok_iter;
img.l_shipdate = idbdate(tok_iter->c_str());
++tok_iter;
img.l_commitdate = idbdate(tok_iter->c_str());
++tok_iter;
img.l_receiptdate = idbdate(tok_iter->c_str());
++tok_iter;
memcpy(&img.l_shipinstruct[0], tok_iter->c_str(), tok_iter->size());
++tok_iter;
memcpy(&img.l_shipmode[0], tok_iter->c_str(), tok_iter->size());
++tok_iter;
memcpy(&img.l_comment[0], tok_iter->c_str(), tok_iter->size());
++tok_iter;
return 0;
}
} // namespace
int main(int argc, char** argv)
{
if ((argc > 1) && (strcmp(argv[1], "-h") == 0))
{
std::cerr << "li2bin [-u]" << std::endl;
std::cerr << " -u Create first 4 fields as unsigned integers" << std::endl;
return 0;
}
string input;
getline(cin, input);
if ((argc > 1) && (strcmp(argv[1], "-u") == 0))
{
unsigned_lineitem_image i;
while (!cin.eof())
{
if (unsigned_parseinto(i, input) == 0)
cout.write(reinterpret_cast<const char*>(&i), sizeof(i));
getline(cin, input);
}
}
else
{
lineitem_image i;
while (!cin.eof())
{
if (parseinto(i, input) == 0)
cout.write(reinterpret_cast<const char*>(&i), sizeof(i));
getline(cin, input);
}
}
return 0;
}

77
tools/cfread/CMakeLists.txt
View File

@@ -1,77 +0,0 @@
include_directories(${KDE4_INCLUDES} ${KDE4_INCLUDE_DIR} ${QT_INCLUDES})
# ########## next target ###############
set(cfread_SRCS cfread.cpp)
kde4_add_executable(cfread ${cfread_SRCS})
target_link_libraries(
cfread
${KDE4_KDECORE_LIBS}
pthread
writeengine
brm
rwlock
messageqcpp
dl
configcpp
xml2
loggingcpp
cacheutils
boost_idb
pthread
rt
)
install(TARGETS cfread ${INSTALL_TARGETS_DEFAULT_ARGS})
# ########## next target ###############
set(mtread_SRCS mtread.cpp)
kde4_add_executable(mtread ${mtread_SRCS})
target_link_libraries(
mtread
${KDE4_KDECORE_LIBS}
pthread
writeengine
brm
rwlock
messageqcpp
dl
configcpp
xml2
loggingcpp
cacheutils
boost_idb
pthread
rt
)
install(TARGETS mtread ${INSTALL_TARGETS_DEFAULT_ARGS})
# ########## install files ###############
# original Makefile.am contents follow:
# $Id: Makefile.am 333 2009-04-03 20:35:04Z rdempsey $ Process this file with automake to produce Makefile.in
#
# AM_CPPFLAGS = $(idb_cppflags) AM_CFLAGS = $(idb_cflags) AM_CXXFLAGS = $(idb_cxxflags) AM_LDFLAGS = $(idb_ldflags)
# bin_PROGRAMS = cfread mtread cfread_SOURCES = cfread.cpp cfread_CPPFLAGS = @idb_common_includes@ $(AM_CPPFLAGS)
# cfread_LDFLAGS = @idb_common_ldflags@ -lpthread -lwriteengine -lbrm -lrwlock -lmessageqcpp -ldl -lconfigcpp -lxml2
# -lloggingcpp -lcacheutils -lboost_idb -lpthread -lrt $(AM_LDFLAGS) mtread_SOURCES = mtread.cpp mtread_CPPFLAGS =
# @idb_common_includes@ $(AM_CPPFLAGS) mtread_LDFLAGS = @idb_common_ldflags@ -lpthread -lwriteengine -lbrm -lrwlock
# -lmessageqcpp -ldl -lconfigcpp -lxml2 -lloggingcpp -lcacheutils -lboost_idb -lpthread -lrt $(AM_LDFLAGS)
#
# test:
#
# coverage:
#
# leakcheck:
#
# docs:
#
# bootstrap: install-data-am
#

183
tools/cfread/cfread.cpp
View File

@@ -1,183 +0,0 @@
//
#include <iostream>
#include <iomanip>
#include <boost/scoped_array.hpp>
#include <errno.h>
#define NDEBUG
#include <cassert>
#include "writeengine.h"
using namespace std;
void timespec_sub(const struct timespec& tv1, const struct timespec& tv2, struct timespec& diff)
{
if (tv2.tv_nsec < tv1.tv_nsec)
{
diff.tv_sec = tv2.tv_sec - tv1.tv_sec - 1;
diff.tv_nsec = tv1.tv_nsec - tv2.tv_nsec;
}
else
{
diff.tv_sec = tv2.tv_sec - tv1.tv_sec;
diff.tv_nsec = tv2.tv_nsec - tv1.tv_nsec;
}
}
// TODO:
// add getopt for CLAs
// add threads for reading a file
// add threads for reading multiple files
// main()
//
int main(int argc, char** argv)
{
uint64_t acc = 0;
uint64_t readBlocks = 0; // read size ib blocks
uint64_t readSize = 0; // read size ib bytes
uint64_t readBufferSz = 0;
const uint64_t blockSize = 8192;
char* alignedbuff = 0;
boost::scoped_array<char> realbuff;
const unsigned pageSize = 4096; // getpagesize();
WriteEngine::FileOp fFileOp;
BRM::OID_t oid;
char fname[256];
struct timespec tm;
struct timespec tm2;
struct timespec tm3;
struct timespec starttm;
struct timespec endtm;
struct timespec tottm;
bool odirect = true;
int fd = 0;
char response = 'Y';
if (argc <= 1)
{
cerr << "usage: testread <oid> <buffer size in blocks>" << endl;
return -1;
}
oid = atoi(argv[1]);
if (oid <= 0)
exit(-1);
if (argc >= 2)
{
readBlocks = atoi(argv[2]);
if (readBlocks <= 0)
readBlocks = 8;
}
if (argc >= 4)
{
odirect = false;
}
readSize = readBlocks * blockSize;
readBufferSz = readSize + pageSize;
realbuff.reset(new char[readBufferSz]);
if (realbuff.get() == 0)
{
cerr << "thr_popper: Can't allocate space for a whole extent in memory" << endl;
return 0;
}
if (fFileOp.getFileName(oid, fname) != WriteEngine::NO_ERROR)
{
fname[0] = 0;
throw std::runtime_error("fileOp.getFileName failed");
}
else
{
cout << "Reading oid: " << oid << " od: " << odirect << " file: " << fname << endl;
}
#if __LP64__
alignedbuff = (char*)((((ptrdiff_t)realbuff.get() >> 12) << 12) + pageSize);
#else
alignedbuff = (char*)(((((ptrdiff_t)realbuff.get() >> 12) << 12) & 0xffffffff) + pageSize);
#endif
idbassert(((ptrdiff_t)alignedbuff - (ptrdiff_t)realbuff.get()) < (ptrdiff_t)pageSize);
idbassert(((ptrdiff_t)alignedbuff % pageSize) == 0);
if (odirect)
fd = open(fname, O_RDONLY | O_DIRECT | O_LARGEFILE | O_NOATIME);
else
fd = open(fname, O_RDONLY | O_LARGEFILE | O_NOATIME);
if (fd < 0)
{
cerr << "Open failed" << endl;
perror("open");
throw runtime_error("Error opening file");
}
while (toupper(response) != 'N')
{
uint64_t i = 1;
uint64_t rCnt = 0;
clock_gettime(CLOCK_REALTIME, &starttm);
while (i != 0)
{
// clock_gettime(CLOCK_REALTIME, &tm);
i = pread(fd, alignedbuff, readSize, acc);
// clock_gettime(CLOCK_REALTIME, &tm2);
idbassert(i == 0 || i == readSize);
idbassert(i % pageSize == 0);
idbassert(acc % pageSize == 0);
if (i < 0 && errno == EINTR)
{
timespec_sub(tm, tm2, tm3);
cout << "* " << i << " " << right << setw(2) << setfill(' ') << tm3.tv_sec << "." << right << setw(9)
<< setfill('0') << tm3.tv_nsec << endl;
continue;
}
else if (i < 0)
{
timespec_sub(tm, tm2, tm3);
cout << "* i: " << i << " sz: " << readSize << " acc: " << acc << right << setw(2) << setfill(' ')
<< tm3.tv_sec << " " << right << tm3.tv_nsec << endl;
perror("pread");
// make loop exit
i = 0;
}
acc += i;
if (i > 0)
rCnt++;
// timespec_sub(tm, tm2, tm3);
// cout
// << i << " "
// << right << setw(2) << setfill(' ') << tm3.tv_sec << " "
// << right << tm3.tv_nsec
// << endl;
} // while(acc...
clock_gettime(CLOCK_REALTIME, &endtm);
timespec_sub(starttm, endtm, tottm);
cout << "Total reads: " << rCnt << " sz: " << acc / (1024 * 1024) << "MB"
<< " tm: " << tottm.tv_sec << "secs " << tottm.tv_nsec << "ns" << endl;
cout << "Repeat the last scan[Y,N]?" << endl;
cin >> response;
acc = 0;
} // while response...
close(fd);
return 0;
} // main

269
tools/cfread/mtread.cpp
View File

@@ -1,269 +0,0 @@
//
#include <iostream>
#include <iomanip>
#include <getopt.h>
#include <errno.h>
#include <boost/thread.hpp>
#define NDEBUG
#include <cassert>
#include "writeengine.h"
using namespace std;
void timespec_sub(const struct timespec& tv1, const struct timespec& tv2, struct timespec& diff)
{
if (tv2.tv_nsec < tv1.tv_nsec)
{
diff.tv_sec = tv2.tv_sec - tv1.tv_sec - 1;
diff.tv_nsec = tv1.tv_nsec - tv2.tv_nsec;
}
else
{
diff.tv_sec = tv2.tv_sec - tv1.tv_sec;
diff.tv_nsec = tv2.tv_nsec - tv1.tv_nsec;
}
}
// TODO:
// add threads for reading a file
// add threads for reading multiple files
struct readThr
{
public:
readThr(const int oid, const int rSize)
{
fblockSize = 8192;
fpageSize = getpagesize();
freadBlocks = rSize;
freadSize = rSize * fblockSize; // read size ib bytes
freadBufferSz = freadSize + fpageSize;
falignedbuff = 0;
foid = oid;
facc = 0;
memset(fname, 0, sizeof(fname));
memset((char*)&ftm, 0, sizeof(ftm));
memset((char*)&ftm2, 0, sizeof(ftm2));
memset((char*)&ftm3, 0, sizeof(ftm3));
memset((char*)&fstarttm, 0, sizeof(fstarttm));
memset((char*)&fendtm, 0, sizeof(fendtm));
memset((char*)&ftottm, 0, sizeof(ftottm));
fodirect = true;
fd = 0;
cout << "o: " << foid << " r: " << freadBlocks << " b: " << freadBufferSz << endl;
}
void operator()()
{
WriteEngine::FileOp fFileOp;
char frealbuff[freadBufferSz];
memset(frealbuff, 0, freadBufferSz);
if (frealbuff == 0)
{
cerr << "thr_popper: Can't allocate space for a whole extent in memory" << endl;
return;
}
if (fFileOp.getFileName(foid, fname) != WriteEngine::NO_ERROR)
{
fname[0] = 0;
throw std::runtime_error("fileOp.getFileName failed");
}
else
{
cout << "Reading oid: " << foid << " od: " << fodirect << " file: " << fname << endl;
}
#if __LP64__
falignedbuff = (char*)((((ptrdiff_t)frealbuff >> 12) << 12) + fpageSize);
#else
falignedbuff = (char*)(((((ptrdiff_t)frealbuff >> 12) << 12) & 0xffffffff) + fpageSize);
#endif
idbassert(((ptrdiff_t)falignedbuff - (ptrdiff_t)frealbuff) < (ptrdiff_t)fpageSize);
idbassert(((ptrdiff_t)falignedbuff % fpageSize) == 0);
if (fodirect)
fd = open(fname, O_RDONLY | O_DIRECT | O_LARGEFILE | O_NOATIME);
else
fd = open(fname, O_RDONLY | O_LARGEFILE | O_NOATIME);
if (fd < 0)
{
cerr << "Open failed" << endl;
perror("open");
throw runtime_error("Error opening file");
}
uint64_t i = 1;
uint64_t rCnt = 0;
clock_gettime(CLOCK_REALTIME, &fstarttm);
while (i > 0)
{
clock_gettime(CLOCK_REALTIME, &ftm);
i = pread(fd, falignedbuff, freadSize, facc);
clock_gettime(CLOCK_REALTIME, &ftm2);
idbassert(i == 0 || i == freadSize);
idbassert(i % fpageSize == 0);
idbassert(facc % fpageSize == 0);
if (i < 0 && errno == EINTR)
{
timespec_sub(ftm, ftm2, ftm3);
cout << "* " << i << " " << right << setw(2) << setfill(' ') << ftm3.tv_sec << "." << right << setw(9)
<< setfill('0') << ftm3.tv_nsec << endl;
continue;
}
else if (i < 0)
{
timespec_sub(ftm, ftm2, ftm3);
cout << "* i: " << i << " sz: " << freadSize << " acc: " << facc << right << setw(2) << setfill(' ')
<< ftm3.tv_sec << " " << right << ftm3.tv_nsec << endl;
perror("pread");
}
facc += i;
if (i > 0)
rCnt++;
/**
timespec_sub(ftm, ftm2, ftm3);
cout
<< rCnt << " " << facc/(1024*1024)
<< right << setw(2) << setfill(' ') << ftm3.tv_sec << "."
<< right << ftm3.tv_nsec << " i: " << i/(1024*1024)
<< endl;
**/
} // while(acc...
clock_gettime(CLOCK_REALTIME, &fendtm);
timespec_sub(fstarttm, fendtm, ftottm);
cout << "Total reads: " << rCnt << " sz: " << facc / (1024 * 1024) << "MB"
<< " tm: " << ftottm.tv_sec << "secs " << ftottm.tv_nsec << "ns" << endl;
facc = 0;
close(fd);
} // operator()
public:
uint64_t facc;
uint64_t freadBlocks; // read size ib blocks
uint64_t freadSize; // read size ib bytes
uint64_t freadBufferSz;
uint64_t fblockSize;
char* falignedbuff;
unsigned fpageSize;
BRM::OID_t foid;
char fname[256];
struct timespec ftm;
struct timespec ftm2;
struct timespec ftm3;
struct timespec fstarttm;
struct timespec fendtm;
struct timespec ftottm;
bool fodirect;
int fd;
}; // struct readThr
//
void usage()
{
cerr << "usage: mtread -o <oid> -s <read size in blocks>" << endl;
}
// main()
//
int main(int argc, char** argv)
{
int ch = 0;
int readBlocks = 0;
BRM::OID_t oid = 0;
std::vector<BRM::OID_t> oidList;
enum CLA_ENUM
{
OID = (int)0,
READSIZE
};
// longopt struct
// struct option {
// const char *name;
// int has_arg;
// int *flag;
// int val;
//};
static struct option long_options[] = {//{const char *name, int has_arg, int *flag, int val},
{"oid", required_argument, NULL, OID},
{"rsize", required_argument, NULL, READSIZE},
{0, 0, 0, 0}};
if (argc <= 1)
{
return -1;
}
// process command line arguments
while ((ch = getopt_long_only(argc, argv, "o:s:", long_options, NULL)) != -1)
{
// pid_t pidId = getpid();
switch (ch)
{
case OID:
case 'o':
oid = atoi(optarg);
oidList.push_back(oid);
cout << "oid: " << optarg << endl;
break;
case READSIZE:
case 's':
readBlocks = atoi(optarg);
cout << "read size: " << optarg << endl;
if (readBlocks <= 0)
readBlocks = 1;
break;
case '?':
default:
cout << "optarg " << optarg << endl;
usage();
break;
} // switch
} // while...
uint32_t idx = 0;
std::vector<boost::thread*> thrList;
while (idx < oidList.size())
{
struct readThr rdr(oidList[idx++], readBlocks);
boost::thread* thr = new boost::thread(rdr);
thrList.push_back(thr);
}
idx = 0;
while (idx < thrList.size())
{
boost::thread* thr = thrList[idx++];
thr->join();
delete thr;
}
thrList.clear();
} // main

10
tools/clearShm/CMakeLists.txt
View File

@@ -4,12 +4,6 @@ include_directories(${ENGINE_COMMON_INCLUDES})
set(clearShm_SRCS main.cpp)
add_executable(clearShm ${clearShm_SRCS})
columnstore_executable(clearShm ${clearShm_SRCS})
target_link_libraries(clearShm ${ENGINE_LDFLAGS} ${ENGINE_EXEC_LIBS})
install(
TARGETS clearShm
DESTINATION ${ENGINE_BINDIR}
COMPONENT columnstore-engine
)
columnstore_link(clearShm ${ENGINE_LDFLAGS} ${ENGINE_EXEC_LIBS})

10
tools/cleartablelock/CMakeLists.txt
View File

@@ -4,12 +4,6 @@ include_directories(${ENGINE_COMMON_INCLUDES})
set(cleartablelock_SRCS cleartablelock.cpp cleartablelockthread.cpp)
add_executable(cleartablelock ${cleartablelock_SRCS})
columnstore_executable(cleartablelock ${cleartablelock_SRCS})
target_link_libraries(cleartablelock ${ENGINE_LDFLAGS} ${NETSNMP_LIBRARIES} ${ENGINE_WRITE_LIBS})
install(
TARGETS cleartablelock
DESTINATION ${ENGINE_BINDIR}
COMPONENT columnstore-engine
)
columnstore_link(cleartablelock ${ENGINE_LDFLAGS} ${NETSNMP_LIBRARIES} ${ENGINE_WRITE_LIBS})

5
tools/configMgt/CMakeLists.txt
View File

@@ -5,7 +5,4 @@ include_directories(${ENGINE_COMMON_INCLUDES})
set(autoConfigure_SRCS autoConfigure.cpp)
add_executable(autoConfigure ${autoConfigure_SRCS})
target_link_libraries(autoConfigure ${ENGINE_LDFLAGS} ${NETSNMP_LIBRARIES} ${ENGINE_EXEC_LIBS})
# install(TARGETS autoConfigure DESTINATION ${ENGINE_BINDIR} COMPONENT columnstore-engine)
columnstore_link(autoConfigure ${ENGINE_LDFLAGS} ${NETSNMP_LIBRARIES} ${ENGINE_EXEC_LIBS})

10
tools/cplogger/CMakeLists.txt
View File

@@ -4,12 +4,6 @@ include_directories(${ENGINE_COMMON_INCLUDES})
set(cplogger_SRCS main.cpp)
add_executable(cplogger ${cplogger_SRCS})
columnstore_executable(cplogger ${cplogger_SRCS})
target_link_libraries(cplogger ${ENGINE_LDFLAGS} ${ENGINE_EXEC_LIBS})
install(
TARGETS cplogger
DESTINATION ${ENGINE_BINDIR}
COMPONENT columnstore-engine
)
columnstore_link(cplogger ${ENGINE_LDFLAGS} ${ENGINE_EXEC_LIBS})

10
tools/dbbuilder/CMakeLists.txt
View File

@@ -4,12 +4,6 @@ include_directories(${ENGINE_COMMON_INCLUDES})
set(dbbuilder_SRCS dbbuilder.cpp systemcatalog.cpp)
add_executable(dbbuilder ${dbbuilder_SRCS})
columnstore_executable(dbbuilder ${dbbuilder_SRCS})
target_link_libraries(dbbuilder ${ENGINE_LDFLAGS} ${NETSNMP_LIBRARIES} ${ENGINE_WRITE_LIBS})
install(
TARGETS dbbuilder
DESTINATION ${ENGINE_BINDIR}
COMPONENT columnstore-engine
)
columnstore_link(dbbuilder ${ENGINE_LDFLAGS} ${NETSNMP_LIBRARIES} ${ENGINE_WRITE_LIBS})

7
tools/dbbuilder/dbbuilder.cpp
View File

@@ -26,6 +26,7 @@
#include <stdexcept>
using namespace std;
#include "basic/string_utils.h"
#include <boost/algorithm/string.hpp>
#include "mcsconfig.h"
@@ -202,7 +203,11 @@ int main(int argc, char* argv[])
(iter->second).second = true;
isUpgrade = true;
}
messageHandler("", std::string("Upgrade flag is ") + std::to_string(isUpgrade) + std::string(" after checking upgrade candidate OID ") + oam.itoa(iter->first) + std::string(" "), false);
messageHandler("",
std::string("Upgrade flag is ") + std::to_string(isUpgrade) +
std::string(" after checking upgrade candidate OID ") + oam.itoa(iter->first) +
std::string(" "),
false);
}
if (!isUpgrade)

14
tools/dbloadxml/CMakeLists.txt
View File

@@ -4,20 +4,14 @@ include_directories(${ENGINE_COMMON_INCLUDES})
set(dbload_STAT_SRCS inputmgr.cpp)
add_library(dbload STATIC ${dbload_STAT_SRCS})
columnstore_static_library(dbload ${dbload_STAT_SRCS})
add_dependencies(dbload loggingcpp)
columnstore_link(dbload loggingcpp)
# ########## next target ###############
set(colxml_SRCS colxml.cpp)
add_executable(colxml ${colxml_SRCS})
columnstore_executable(colxml ${colxml_SRCS})
target_link_libraries(colxml ${ENGINE_LDFLAGS} dbload ${ENGINE_WRITE_LIBS})
install(
TARGETS colxml
DESTINATION ${ENGINE_BINDIR}
COMPONENT columnstore-engine
)
columnstore_link(colxml ${ENGINE_LDFLAGS} dbload ${ENGINE_WRITE_LIBS})

11
tools/ddlcleanup/CMakeLists.txt
View File

@@ -4,12 +4,5 @@ include_directories(${ENGINE_COMMON_INCLUDES})
set(ddlcleanup_SRCS ddlcleanup.cpp)
add_executable(ddlcleanup ${ddlcleanup_SRCS})
target_link_libraries(ddlcleanup ${ENGINE_LDFLAGS} ${NETSNMP_LIBRARIES} ${ENGINE_WRITE_LIBS} ddlcleanuputil)
install(
TARGETS ddlcleanup
DESTINATION ${ENGINE_BINDIR}
COMPONENT columnstore-engine
)
columnstore_executable(ddlcleanup ${ddlcleanup_SRCS})
columnstore_link(ddlcleanup ${ENGINE_LDFLAGS} ${NETSNMP_LIBRARIES} ${ENGINE_WRITE_LIBS} ddlcleanuputil)

43
tools/ddldriver/CMakeLists.txt
View File

@@ -1,43 +0,0 @@
include_directories(${KDE4_INCLUDES} ${KDE4_INCLUDE_DIR} ${QT_INCLUDES})
# ########## next target ###############
set(ddldriver_SRCS ddldriver.cpp)
kde4_add_executable(ddldriver ${ddldriver_SRCS})
target_link_libraries(ddldriver ${KDE4_KDECORE_LIBS})
install(TARGETS ddldriver ${INSTALL_TARGETS_DEFAULT_ARGS})
# ########## install files ###############
# original Makefile.am contents follow:
# Copyright (C) 2014 InfiniDB, Inc.
#
# This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public
# License as published by the Free Software Foundation; version 2 of the License.
#
# This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied
# warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License along with this program; if not, write to the Free
# Software Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
#
# $Id: Makefile.am 333 2009-04-03 20:35:04Z rdempsey $ Process this file with automake to produce Makefile.in
#
# AM_CPPFLAGS = $(idb_cppflags) AM_CFLAGS = $(idb_cflags) AM_CXXFLAGS = $(idb_cxxflags) AM_LDFLAGS = $(idb_ldflags)
# bin_PROGRAMS = ddldriver ddldriver_SOURCES = ddldriver.cpp ddldriver_CPPFLAGS = @idb_common_includes@ $(AM_CPPFLAGS)
# ddldriver_LDFLAGS = @idb_common_ldflags@ @idb_write_libs@ @idb_common_libs@ @netsnmp_libs@ $(AM_LDFLAGS)
#
# test:
#
# coverage:
#
# leakcheck:
#
# docs:
#
# bootstrap: install-data-am
#

133
tools/ddldriver/ddldriver.cpp
View File

@@ -1,133 +0,0 @@
/* Copyright (C) 2014 InfiniDB, Inc.
This program is free software; you can redistribute it and/or
modify it under the terms of the GNU General Public License
as published by the Free Software Foundation; version 2 of
the License.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program; if not, write to the Free Software
Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
MA 02110-1301, USA. */
// $Id: ddldriver.cpp 2101 2013-01-21 14:12:52Z rdempsey $
#include <unistd.h>
#include <cstdio>
#include <iostream>
#include <cctype>
#include <sstream>
using namespace std;
#include "ddlpkg.h"
#include "sqlparser.h"
using namespace ddlpackage;
#include "bytestream.h"
#include "messagequeue.h"
using namespace messageqcpp;
namespace
{
void usage()
{
cout << "usage: ddlriver [-h] schema sql_text" << endl;
}
const string toupper_(const string& in)
{
string::const_iterator iter = in.begin();
string::const_iterator end = in.end();
ostringstream oss;
while (iter != end)
{
oss << static_cast<char>(toupper(*iter));
++iter;
}
return oss.str();
}
} // namespace
int main(int argc, char** argv)
{
int c;
opterr = 0;
while ((c = getopt(argc, argv, "h")) != EOF)
switch (c)
{
case 'h':
case '?':
default:
usage();
return (c == 'h' ? 0 : 1);
break;
}
if (argc - optind < 2)
{
usage();
return 1;
}
string owner(toupper_(argv[optind++]));
SqlParser parser;
parser.setDefaultSchema(owner);
string stmtStr(toupper_(argv[optind++]));
parser.Parse(stmtStr.c_str());
if (!parser.Good())
{
cerr << "Failed to parse statement: " << stmtStr << endl;
return 1;
}
const ParseTree& ptree = parser.GetParseTree();
SqlStatement& stmt = *ptree.fList[0];
stmt.fSessionID = 1;
stmt.fSql = stmtStr;
stmt.fOwner = owner;
ByteStream bytestream;
bytestream << stmt.fSessionID;
stmt.serialize(bytestream);
MessageQueueClient mq("DDLProc");
ByteStream::byte b;
string errorMsg;
try
{
mq.write(bytestream);
bytestream = mq.read();
bytestream >> b;
bytestream >> errorMsg;
}
catch (runtime_error& rex)
{
cerr << "runtime_error in engine: " << rex.what() << endl;
return 1;
}
catch (...)
{
cerr << "uknown error in engine" << endl;
return 1;
}
if (b != 0)
{
cerr << "DDLProc error: " << errorMsg << endl;
return 1;
}
return 0;
}

44
tools/dmldriver/CMakeLists.txt
View File

@@ -1,44 +0,0 @@
include_directories(${KDE4_INCLUDES} ${KDE4_INCLUDE_DIR} ${QT_INCLUDES})
# ########## next target ###############
set(dmldriver_SRCS dmldriver.cpp tpchrf2.cpp dmlif.cpp)
kde4_add_executable(dmldriver ${dmldriver_SRCS})
target_link_libraries(dmldriver ${KDE4_KDECORE_LIBS})
install(TARGETS dmldriver ${INSTALL_TARGETS_DEFAULT_ARGS})
# ########## install files ###############
# original Makefile.am contents follow:
# Copyright (C) 2014 InfiniDB, Inc.
#
# This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public
# License as published by the Free Software Foundation; version 2 of the License.
#
# This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied
# warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License along with this program; if not, write to the Free
# Software Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
#
# $Id: Makefile.am 333 2009-04-03 20:35:04Z rdempsey $ Process this file with automake to produce Makefile.in
#
# AM_CPPFLAGS = $(idb_cppflags) AM_CFLAGS = $(idb_cflags) AM_CXXFLAGS = $(idb_cxxflags) AM_LDFLAGS = $(idb_ldflags)
# bin_PROGRAMS = dmldriver dmldriver_SOURCES = dmldriver.cpp tpchrf2.cpp dmlif.cpp dmldriver_CPPFLAGS =
# @idb_common_includes@ $(AM_CPPFLAGS) dmldriver_LDFLAGS = @idb_common_ldflags@ @idb_write_libs@ @idb_common_libs@
# @netsnmp_libs@ $(AM_LDFLAGS)
#
# test:
#
# coverage:
#
# leakcheck:
#
# docs:
#
# bootstrap: install-data-am
#

205
tools/dmldriver/dmldriver.cpp
View File

@@ -1,205 +0,0 @@
/* Copyright (C) 2014 InfiniDB, Inc.
This program is free software; you can redistribute it and/or
modify it under the terms of the GNU General Public License
as published by the Free Software Foundation; version 2 of
the License.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program; if not, write to the Free Software
Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
MA 02110-1301, USA. */
// $Id: dmldriver.cpp 2101 2013-01-21 14:12:52Z rdempsey $
//#define NDEBUG
#include <cassert>
#include <iostream>
#include <string>
#include <unistd.h>
#include <cctype>
#include <sstream>
#include <fstream>
#include <limits>
using namespace std;
#include <boost/scoped_array.hpp>
#include <boost/tokenizer.hpp>
using namespace boost;
#include "tpchrf2.h"
#include "dmlif.h"
namespace
{
bool vflg;
bool dflg;
void usage()
{
cout << "usage: dmldriver [-vhd] [-c intvl] [-f file] [-s sid] [-t flgs] [-r file] [-p cnt] [sql_text]"
<< endl;
cout << " -c intvl \tcommit every intvl statements" << endl;
cout << " -f file \tread statements from file (max 15KB/stmt)" << endl;
cout << " -s sid \tset sid as session id" << endl;
cout << " -t flgs \tset trace flags" << endl;
cout << " -v \tdisplay affected row count(s)" << endl;
cout << " -d \tdisplay debug info" << endl;
cout << " -r file \tread orderkeys from file for TPC-H RF2" << endl;
cout << " -p cnt \tpack cnt orderkeys into each delete stmt (only w/ -r)" << endl;
cout << " -e schema\tset the schema name (only w/ -r)" << endl;
cout << " -h \tdisplay this help text" << endl;
}
} // namespace
int main(int argc, char** argv)
{
int c;
opterr = 0;
vflg = false;
dflg = false;
bool fflg = false;
string infilename;
int cIntvl = numeric_limits<int>::max();
uint32_t sessionID = time(0) & 0x7fffffff;
bool rflg = false;
int packCnt = 1;
uint32_t tflg = 0;
string schema;
while ((c = getopt(argc, argv, "e:t:s:c:f:r:p:vhd")) != EOF)
switch (c)
{
case 'v': vflg = true; break;
case 'd': dflg = true; break;
case 'f':
fflg = true;
infilename = optarg;
break;
case 'c': cIntvl = static_cast<int>(strtol(optarg, 0, 0)); break;
case 's': sessionID = static_cast<uint32_t>(strtoul(optarg, 0, 0)); break;
case 't': tflg = static_cast<uint32_t>(strtoul(optarg, 0, 0)); break;
case 'r':
rflg = true;
infilename = optarg;
break;
case 'p': packCnt = static_cast<int>(strtol(optarg, 0, 0)); break;
case 'e': schema = optarg; break;
case 'h':
case '?':
default:
usage();
return (c == 'h' ? 0 : 1);
break;
}
if (!fflg && !rflg && ((argc - optind) < 1))
{
usage();
return 1;
}
if (fflg && rflg)
{
cout << "-f and -r are mutually exclusive!" << endl << endl;
usage();
return 1;
}
if (!schema.empty() && !rflg)
{
cout << "-e requires -r!" << endl << endl;
usage();
return 1;
}
string stmtStr;
if (!fflg && !rflg)
stmtStr = argv[optind++];
int rc = 0;
dmlif::DMLIF dmlif(sessionID, tflg, dflg, vflg);
if (fflg)
{
ifstream ifs(infilename.c_str());
if (!ifs.good())
{
cerr << "Error accessing file " << infilename << endl;
return 1;
}
const streamsize ilinelen = 15 * 1024;
scoped_array<char> iline(new char[ilinelen]);
int cnt = 0;
for (;;)
{
ifs.getline(iline.get(), ilinelen);
if (ifs.eof())
break;
rc = dmlif.sendOne(iline.get());
if (rc != 0)
break;
cnt++;
if ((cnt % cIntvl) == 0)
dmlif.sendOne("COMMIT;");
}
}
else if (rflg)
{
ifstream ifs(infilename.c_str());
if (!ifs.good())
{
cerr << "Error accessing file " << infilename << endl;
return 1;
}
if (schema.empty())
schema = "tpch";
tpch::RF2 rf2(schema, sessionID, tflg, cIntvl, packCnt, dflg, vflg);
rc = rf2.run(ifs);
}
else
{
rc = dmlif.sendOne(stmtStr);
}
if (rc == 0)
dmlif.sendOne("COMMIT;");
return 0;
}

384
tools/dmldriver/dmlif.cpp
View File

@@ -1,384 +0,0 @@
/* Copyright (C) 2014 InfiniDB, Inc.
This program is free software; you can redistribute it and/or
modify it under the terms of the GNU General Public License
as published by the Free Software Foundation; version 2 of
the License.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program; if not, write to the Free Software
Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
MA 02110-1301, USA. */
// $Id: dmlif.cpp 2101 2013-01-21 14:12:52Z rdempsey $
//#define NDEBUG
#include <cassert>
#include <string>
#include <sstream>
using namespace std;
#include <boost/tokenizer.hpp>
using namespace boost;
#include "vendordmlstatement.h"
#include "calpontdmlpackage.h"
#include "calpontdmlfactory.h"
using namespace dmlpackage;
#include "bytestream.h"
#include "messagequeue.h"
using namespace messageqcpp;
#include "simplecolumn.h"
#include "calpontselectexecutionplan.h"
#include "sessionmanager.h"
#include "simplefilter.h"
#include "constantcolumn.h"
#include "constantfilter.h"
using namespace execplan;
#include "brmtypes.h"
#include "dmlif.h"
using namespace dmlif;
namespace dmlif
{
DMLIF::DMLIF(uint32_t sessionid, uint32_t tflg, bool dflg, bool vflg)
: fSessionID(sessionid), fTflg(tflg), fDflg(dflg), fVflg(vflg), fOPt(0), fLPt(0)
{
fMqp.reset(new MessageQueueClient("DMLProc"));
}
DMLIF::~DMLIF()
{
}
int DMLIF::sendOne(const string& stmt)
{
int rc;
string tStmt(stmt);
if (*tStmt.rbegin() != ';')
tStmt += ";";
VendorDMLStatement dmlStmt(tStmt, fSessionID);
CalpontDMLPackage* pDMLPackage = CalpontDMLFactory::makeCalpontDMLPackage(dmlStmt);
if (pDMLPackage == 0)
{
cerr << "Failed to parse statement: " << tStmt << endl;
return -1;
}
string queryString = pDMLPackage->get_QueryString();
if (fDflg)
cout << "qs: >" << queryString << '<' << endl;
string dmlStatement = pDMLPackage->get_DMLStatement();
if (fDflg)
cout << "DML: " << dmlStatement << endl;
bool isDML = true;
if (dmlStatement == "COMMIT" || dmlStatement == "ROLLBACK")
{
isDML = false;
}
if (isDML)
{
char_separator<char> sep(" ");
tokenizer<char_separator<char> > tok(queryString, sep);
tokenizer<char_separator<char> >::iterator iter = tok.begin();
idbassert(iter != tok.end());
string where = *iter;
++iter;
idbassert(iter != tok.end());
string col1 = *iter;
++iter;
idbassert(iter != tok.end());
string op = *iter;
++iter;
idbassert(iter != tok.end());
string col2 = *iter;
++iter;
idbassert(iter == tok.end());
if (fDflg)
cout << "SQL: " << pDMLPackage->get_SQLStatement() << endl;
if (fDflg)
cout << "hf: " << pDMLPackage->HasFilter() << endl;
DMLTable* tp = pDMLPackage->get_Table();
if (fDflg)
cout << "sn: " << tp->get_SchemaName() << " tn: " << tp->get_TableName() << endl;
if (fDflg)
cout << "row count: " << tp->get_RowList().size() << endl;
SRCP srcp(new SimpleColumn(tp->get_SchemaName(), tp->get_TableName(), col1, fSessionID));
CalpontSelectExecutionPlan::ColumnMap cm;
cm.insert(make_pair(col1, srcp));
pDMLPackage->get_ExecutionPlan()->columnMap(cm);
CalpontSelectExecutionPlan::ReturnedColumnList rcl;
rcl.push_back(srcp);
pDMLPackage->get_ExecutionPlan()->returnedCols(rcl);
pDMLPackage->get_ExecutionPlan()->sessionID(fSessionID);
pDMLPackage->get_ExecutionPlan()->traceFlags(fTflg);
SessionManager sm;
BRM::TxnID txnid = sm.getTxnID(fSessionID);
if (!txnid.valid)
txnid = sm.newTxnID(fSessionID);
pDMLPackage->get_ExecutionPlan()->txnID(txnid.id);
pDMLPackage->get_ExecutionPlan()->verID(sm.verID());
ParseTree* pt = new ParseTree();
ReturnedColumn* rc1 = srcp->clone();
ReturnedColumn* rc2 = new ConstantColumn(col2, ConstantColumn::NUM);
SOP sop(new Operator(op));
SimpleFilter* sf = new SimpleFilter(sop, rc1, rc2);
pt->data(sf);
pDMLPackage->get_ExecutionPlan()->filters(pt);
if (fDflg)
cout << "ep: " << *pDMLPackage->get_ExecutionPlan() << endl;
}
ByteStream bytestream;
pDMLPackage->write(bytestream);
delete pDMLPackage;
ByteStream::octbyte rows;
rc = DMLSend(bytestream, rows);
if (isDML && fVflg)
cout << rows << " rows affected" << endl;
return rc;
}
int DMLIF::DMLSend(ByteStream& bytestream, ByteStream::octbyte& rows)
{
ByteStream::byte b;
string errorMsg;
try
{
fMqp->connect();
fMqp->write(bytestream);
bytestream = fMqp->read();
fMqp->shutdown();
if (fDflg)
cout << "read " << bytestream.length() << " bytes from DMLProc" << endl;
bytestream >> b;
if (fDflg)
cout << "b = " << (int)b << endl;
bytestream >> rows;
if (fDflg)
cout << "rows = " << rows << endl;
bytestream >> errorMsg;
if (fDflg)
cout << "errorMsg = " << errorMsg << endl;
}
catch (runtime_error& rex)
{
cerr << "runtime_error in engine: " << rex.what() << endl;
return -1;
}
catch (...)
{
cerr << "uknown error in engine" << endl;
return -1;
}
if (b != 0)
{
cerr << "DMLProc error: " << errorMsg << endl;
return -1;
}
return 0;
}
void DMLIF::rf2Start(const string& sn)
{
fSchema = sn;
fOFilterStr = "";
fLFilterStr = "";
fOPt = 0;
fLPt = 0;
}
void DMLIF::rf2Add(int64_t okey)
{
ostringstream oss;
oss << okey;
string okeyStr(oss.str());
if (fOFilterStr.empty())
{
fOFilterStr = "o_orderkey=" + okeyStr;
ReturnedColumn* rc1 = new SimpleColumn(fSchema, "orders", "o_orderkey", fSessionID);
ReturnedColumn* rc2 = new ConstantColumn(okeyStr, ConstantColumn::NUM);
SOP sop(new Operator("="));
ConstantFilter* cf = new ConstantFilter(sop, rc1, rc2);
sop.reset(new Operator("or"));
cf->op(sop);
fOPt = new ParseTree(cf);
}
else
{
fOFilterStr += " or o_orderkey=" + okeyStr;
ReturnedColumn* rc1 = new SimpleColumn(fSchema, "orders", "o_orderkey", fSessionID);
ReturnedColumn* rc2 = new ConstantColumn(okeyStr, ConstantColumn::NUM);
SOP sop(new Operator("="));
ConstantFilter* cf = dynamic_cast<ConstantFilter*>(fOPt->data());
cf->pushFilter(new SimpleFilter(sop, rc1, rc2));
}
if (fLFilterStr.empty())
{
fLFilterStr = "l_orderkey=" + okeyStr;
ReturnedColumn* rc1 = new SimpleColumn(fSchema, "lineitem", "l_orderkey", fSessionID);
ReturnedColumn* rc2 = new ConstantColumn(okeyStr, ConstantColumn::NUM);
SOP sop(new Operator("="));
ConstantFilter* cf = new ConstantFilter(sop, rc1, rc2);
sop.reset(new Operator("or"));
cf->op(sop);
fLPt = new ParseTree(cf);
}
else
{
fLFilterStr += " or l_orderkey=" + okeyStr;
ReturnedColumn* rc1 = new SimpleColumn(fSchema, "lineitem", "l_orderkey", fSessionID);
ReturnedColumn* rc2 = new ConstantColumn(okeyStr, ConstantColumn::NUM);
SOP sop(new Operator("="));
ConstantFilter* cf = dynamic_cast<ConstantFilter*>(fLPt->data());
cf->pushFilter(new SimpleFilter(sop, rc1, rc2));
}
}
int DMLIF::rf2Send()
{
if (fOFilterStr.empty())
return -1;
int rc = 0;
string dmlstr;
dmlstr = "delete from " + fSchema + ".orders where " + fOFilterStr + ';';
if (fDflg)
cout << dmlstr << endl;
VendorDMLStatement dmlStmt(dmlstr, fSessionID);
CalpontDMLPackage* pDMLPackage = CalpontDMLFactory::makeCalpontDMLPackage(dmlStmt);
if (pDMLPackage == 0)
{
cerr << "Failed to parse statement: " << dmlstr << endl;
return -1;
}
SRCP srcp(new SimpleColumn(fSchema, "orders", "o_orderkey", fSessionID));
CalpontSelectExecutionPlan::ColumnMap cm;
cm.insert(make_pair("o_orderkey", srcp));
pDMLPackage->get_ExecutionPlan()->columnMap(cm);
CalpontSelectExecutionPlan::ReturnedColumnList rcl;
rcl.push_back(srcp);
pDMLPackage->get_ExecutionPlan()->returnedCols(rcl);
pDMLPackage->get_ExecutionPlan()->sessionID(fSessionID);
pDMLPackage->get_ExecutionPlan()->traceFlags(fTflg);
SessionManager sm;
BRM::TxnID txnid = sm.getTxnID(fSessionID);
if (!txnid.valid)
txnid = sm.newTxnID(fSessionID);
pDMLPackage->get_ExecutionPlan()->txnID(txnid.id);
pDMLPackage->get_ExecutionPlan()->verID(sm.verID());
pDMLPackage->get_ExecutionPlan()->filters(fOPt);
if (fDflg)
cout << "ep: " << *pDMLPackage->get_ExecutionPlan() << endl;
ByteStream bytestream;
pDMLPackage->write(bytestream);
delete pDMLPackage;
pDMLPackage = 0;
ByteStream::octbyte rows = 0;
rc = DMLSend(bytestream, rows);
if (fVflg)
cout << rows << " rows affected" << endl;
dmlstr = "delete from " + fSchema + ".lineitem where " + fLFilterStr + ';';
if (fDflg)
cout << dmlstr << endl;
VendorDMLStatement dmlStmt1(dmlstr, fSessionID);
pDMLPackage = CalpontDMLFactory::makeCalpontDMLPackage(dmlStmt1);
if (pDMLPackage == 0)
{
cerr << "Failed to parse statement: " << dmlstr << endl;
return -1;
}
srcp.reset(new SimpleColumn(fSchema, "lineitem", "l_orderkey", fSessionID));
cm.clear();
cm.insert(make_pair("l_orderkey", srcp));
pDMLPackage->get_ExecutionPlan()->columnMap(cm);
rcl.clear();
rcl.push_back(srcp);
pDMLPackage->get_ExecutionPlan()->returnedCols(rcl);
pDMLPackage->get_ExecutionPlan()->sessionID(fSessionID);
pDMLPackage->get_ExecutionPlan()->traceFlags(fTflg);
txnid = sm.getTxnID(fSessionID);
if (!txnid.valid)
txnid = sm.newTxnID(fSessionID);
pDMLPackage->get_ExecutionPlan()->txnID(txnid.id);
pDMLPackage->get_ExecutionPlan()->verID(sm.verID());
pDMLPackage->get_ExecutionPlan()->filters(fLPt);
if (fDflg)
cout << "ep: " << *pDMLPackage->get_ExecutionPlan() << endl;
bytestream.reset();
pDMLPackage->write(bytestream);
delete pDMLPackage;
pDMLPackage = 0;
rows = 0;
rc = DMLSend(bytestream, rows);
if (fVflg)
cout << rows << " rows affected" << endl;
return 0;
}
} // namespace dmlif

65
tools/dmldriver/dmlif.h
View File

@@ -1,65 +0,0 @@
/* Copyright (C) 2014 InfiniDB, Inc.
This program is free software; you can redistribute it and/or
modify it under the terms of the GNU General Public License
as published by the Free Software Foundation; version 2 of
the License.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program; if not, write to the Free Software
Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
MA 02110-1301, USA. */
// $Id: dmlif.h 2101 2013-01-21 14:12:52Z rdempsey $
#pragma once
#include <string>
#include <cstddef>
#include <boost/scoped_ptr.hpp>
#include "messagequeue.h"
#include "expressionparser.h"
namespace dmlif
{
class DMLIF
{
public:
DMLIF(uint32_t sessionid, uint32_t tflg = 0, bool dflg = false, bool vflg = false);
~DMLIF();
int sendOne(const std::string& stmt);
void rf2Start(const std::string& sn);
void rf2Add(int64_t okey);
int rf2Send();
protected:
int DMLSend(messageqcpp::ByteStream& bytestream, messageqcpp::ByteStream::octbyte& rows);
private:
// DMLIF(const DMLIF& rhs);
// DMLIF& operator=(const DMLIF& rhs);
uint32_t fSessionID;
uint32_t fTflg;
bool fDflg;
bool fVflg;
boost::scoped_ptr<messageqcpp::MessageQueueClient> fMqp;
std::string fSchema;
std::string fOFilterStr;
std::string fLFilterStr;
execplan::ParseTree* fOPt;
execplan::ParseTree* fLPt;
};
} // namespace dmlif

86
tools/dmldriver/tpchrf2.cpp
View File

@@ -1,86 +0,0 @@
/* Copyright (C) 2014 InfiniDB, Inc.
This program is free software; you can redistribute it and/or
modify it under the terms of the GNU General Public License
as published by the Free Software Foundation; version 2 of
the License.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program; if not, write to the Free Software
Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
MA 02110-1301, USA. */
// $Id: tpchrf2.cpp 2101 2013-01-21 14:12:52Z rdempsey $
#include <istream>
using namespace std;
#include <boost/scoped_array.hpp>
#include <boost/tokenizer.hpp>
using namespace boost;
#include "tpchrf2.h"
#include "dmlif.h"
namespace tpch
{
RF2::RF2(const string& sn, uint32_t sid, uint32_t tflg, int c, int p, bool d, bool v)
: fSchema(sn), fSessionID(sid), fTflg(tflg), fIntvl(c), fPack(p), fDflg(d), fVflg(v)
{
}
RF2::~RF2()
{
}
int RF2::run(istream& in)
{
const streamsize ilinelen = 1024;
scoped_array<char> iline(new char[ilinelen]);
int cnt = 0;
dmlif::DMLIF dmlif(fSessionID, fTflg, fDflg, fVflg);
for (;;)
{
dmlif.rf2Start(fSchema);
for (int i = 0; i < fPack; i++)
{
in.getline(iline.get(), ilinelen);
if (in.eof())
break;
typedef char_separator<char> cs;
typedef tokenizer<cs> tk;
cs sep("|");
tk tok(string(iline.get()), sep);
tk::iterator iter = tok.begin();
idbassert(iter != tok.end());
string keystr = *iter;
++iter;
// idbassert(iter == tok.end());
int64_t okey = strtol(keystr.c_str(), 0, 0);
dmlif.rf2Add(okey);
}
dmlif.rf2Send();
cnt++;
if ((cnt % fIntvl) == 0)
dmlif.sendOne("COMMIT;");
if (in.eof())
break;
}
dmlif.sendOne("COMMIT;");
return 0;
}
} // namespace tpch

52
tools/dmldriver/tpchrf2.h
View File

@@ -1,52 +0,0 @@
/* Copyright (C) 2014 InfiniDB, Inc.
This program is free software; you can redistribute it and/or
modify it under the terms of the GNU General Public License
as published by the Free Software Foundation; version 2 of
the License.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program; if not, write to the Free Software
Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
MA 02110-1301, USA. */
// $Id: tpchrf2.h 2101 2013-01-21 14:12:52Z rdempsey $
#pragma once
#include <string>
#include <istream>
#include <cstddef>
#include <limits>
#include <stdint.h>
namespace tpch
{
class RF2
{
public:
RF2(const std::string& sn, uint32_t sid, uint32_t tflg = 0, int c = std::numeric_limits<int>::max(),
int p = 1, bool d = false, bool v = false);
~RF2();
int run(std::istream& in);
private:
// RF2(const RF2& rhs);
// RF2& operator=(const RF2& rhs);
std::string fSchema;
uint32_t fSessionID;
uint32_t fTflg;
int fIntvl;
int fPack;
bool fDflg;
bool fVflg;
};
} // namespace tpch

11
tools/editem/CMakeLists.txt
View File

@@ -4,12 +4,5 @@ include_directories(${ENGINE_COMMON_INCLUDES})
set(editem_SRCS editem.cpp)
add_executable(editem ${editem_SRCS})
target_link_libraries(editem ${ENGINE_LDFLAGS} ${NETSNMP_LIBRARIES} ${ENGINE_EXEC_LIBS})
install(
TARGETS editem
DESTINATION ${ENGINE_BINDIR}
COMPONENT columnstore-engine
)
columnstore_executable(editem ${editem_SRCS})
columnstore_link(editem ${ENGINE_LDFLAGS} ${NETSNMP_LIBRARIES} ${ENGINE_EXEC_LIBS})

173
tools/fixdate/fixdate.cpp
View File

@@ -1,173 +0,0 @@
/* Copyright (C) 2014 InfiniDB, Inc.
This program is free software; you can redistribute it and/or
modify it under the terms of the GNU General Public License
as published by the Free Software Foundation; version 2 of
the License.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program; if not, write to the Free Software
Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
MA 02110-1301, USA. */
/*
* $Id: fixdate.cpp 2101 2013-01-21 14:12:52Z rdempsey $
*
*/
#include <iostream>
#include <fstream>
#include <cassert>
#include <vector>
#include <algorithm>
#include <iterator>
using namespace std;
#include <unistd.h>
#include <boost/format.hpp>
#include <boost/scoped_array.hpp>
using namespace boost;
#include "bytestream.h"
using namespace messageqcpp;
#include "dmlpackageprocessor.h"
using namespace dmlpackageprocessor;
namespace
{
const streamsize blkSz = 8192;
void usage()
{
}
u_int64_t fixDate(ByteStream& bs, ostream& out)
{
ByteStream fixed;
ByteStream::quadbyte o;
u_int64_t cnt = 0;
#if 0
DMLPackageProcessor::Date minDate;
DMLPackageProcessor::Date maxDate;
minDate.year = 1992;
minDate.month = 1;
minDate.day = 2;
maxDate.year = 1998;
maxDate.month = 12;
maxDate.day = 25;
ByteStream::quadbyte mxd;
ByteStream::quadbyte mnd;
mxd = *(reinterpret_cast<ByteStream::quadbyte*>(&maxDate));
mnd = *(reinterpret_cast<ByteStream::quadbyte*>(&minDate));
#endif
DMLPackageProcessor::Date fixDate;
fixDate.spare = 0;
ByteStream::quadbyte f;
while (bs.length() > 0)
{
bs >> o;
if (o >= 0xfffffffe)
{
fixed << o;
continue;
}
f = o & 0xffff;
fixDate.year = f;
o >>= 16;
f = o & 0xf;
fixDate.month = f;
o >>= 4;
f = o & 0x3f;
fixDate.day = f;
// o >>= 6;
o = *(reinterpret_cast<ByteStream::quadbyte*>(&fixDate));
fixed << o;
#if 0
cout << DMLPackageProcessor::dateToString(o) << endl;
idbassert(o >= mnd && o <= mxd);
cnt++;
#endif
}
out << fixed;
return cnt;
}
} // namespace
int main(int argc, char* argv[])
{
int c;
opterr = 0;
while ((c = getopt(argc, argv, "h")) != EOF)
switch (c)
{
case 'h':
usage();
return 0;
break;
default:
usage();
return 1;
break;
}
if ((argc - optind) < 1)
{
usage();
return 1;
}
ByteStream bs;
ifstream ifs(argv[optind + 0]);
ByteStream::byte inbuf[blkSz];
streampos fLen;
u_int64_t blkNo = 0;
ifs.seekg(0, ios_base::end);
fLen = ifs.tellg();
ifs.seekg(0, ios_base::beg);
idbassert((fLen % blkSz) == 0);
u_int64_t numBlks = fLen / blkSz;
cout << numBlks << " blocks to fix..." << endl;
ofstream ofs("fixdate.cdf");
cout << "pct done: " << setw(3);
for (;;)
{
ifs.read(reinterpret_cast<char*>(inbuf), blkSz);
if (ifs.eof())
break;
bs.load(inbuf, blkSz);
fixDate(bs, ofs);
cout << "\b\b\b" << setw(3) << (u_int64_t)(blkNo * 100 / numBlks);
// cout << setw(3) << (u_int64_t)(blkNo * 100 / numBlks) << endl;
blkNo++;
}
cout << "\b\b\b" << setw(3) << 100 << endl;
// cout << setw(3) << 100 << endl;
return 0;
}

10
tools/getConfig/CMakeLists.txt
View File

@@ -4,12 +4,6 @@ include_directories(${ENGINE_COMMON_INCLUDES})
set(getConfig_SRCS main.cpp)
add_executable(mcsGetConfig ${getConfig_SRCS})
columnstore_executable(mcsGetConfig ${getConfig_SRCS})
target_link_libraries(mcsGetConfig ${ENGINE_LDFLAGS} ${ENGINE_EXEC_LIBS})
install(
TARGETS mcsGetConfig
DESTINATION ${ENGINE_BINDIR}
COMPONENT columnstore-engine
)
columnstore_link(mcsGetConfig ${ENGINE_LDFLAGS} ${ENGINE_EXEC_LIBS})

265
tools/idb_comp/main.cpp
View File

@@ -1,265 +0,0 @@
#include <unistd.h>
#include <sys/types.h>
#include <sys/stat.h>
#include <fcntl.h>
#include <cerrno>
//#define NDEBUG
#include <cassert>
#include <cstring>
#include <lzo/lzoconf.h>
#include <lzo/lzo1x.h>
#include <iostream>
#include <iomanip>
#include <sstream>
#include <inttypes.h>
#include <cstdlib>
using namespace std;
#include <boost/scoped_array.hpp>
namespace
{
size_t exp_buf_size;
unsigned vflg;
const string lzo_rctos(int r)
{
switch (r)
{
case LZO_E_INPUT_NOT_CONSUMED: return "LZO_E_INPUT_NOT_CONSUMED";
default: break;
}
return "Unknown Error!";
}
void usage()
{
cout << "usage: idb_comp [-e size][-h] file(s)" << endl
<< "\t-e size size (in KB) of expansion buffer" << endl
<< "\t-h display this help" << endl;
}
int doit(const string& infile)
{
int rc = 0;
int ifd = open(infile.c_str(), O_RDONLY | O_LARGEFILE | O_DIRECT);
if (ifd < 0)
{
cerr << infile << ": open: " << strerror(errno) << endl;
return 1;
}
struct stat istatbuf;
fstat(ifd, &istatbuf);
string outname(infile);
string::size_type ptr;
ptr = outname.find_last_of('.');
if (ptr != string::npos)
outname.erase(ptr);
ptr = outname.find_last_of('/');
if (ptr != string::npos)
outname.erase(0, ptr + 1);
outname = "./" + outname + ".cmp";
int ofd = open(outname.c_str(), O_WRONLY | O_CREAT | O_TRUNC | O_LARGEFILE | O_DIRECT, 0644);
if (ofd < 0)
{
cerr << outname << ": open: " << strerror(errno) << endl;
close(ifd);
return 1;
}
lzo_init();
ssize_t nread = -1;
ssize_t nwritten = -1;
lzo_bytep ibuf;
lzo_bytep tibuf;
lzo_bytep cbuf;
lzo_bytep tcbuf;
lzo_bytep wkmem;
lzo_uint32 ibuf_len = 0;
lzo_uint cbuf_len = 0;
ibuf_len = 512 * 1024 * 8;
tibuf = new lzo_byte[ibuf_len + 4095];
if ((ptrdiff_t)tibuf & 0xfffULL)
ibuf = (lzo_bytep)((ptrdiff_t)tibuf & 0xfffffffffffff000ULL) + 4096;
else
ibuf = tibuf;
cbuf_len = 512 * 1024 * 8;
tcbuf = new lzo_byte[cbuf_len + 4095 + exp_buf_size * 1024];
if ((ptrdiff_t)tcbuf & 0xfff)
cbuf = (lzo_bytep)((ptrdiff_t)tcbuf & 0xfffffffffffff000ULL) + 4096;
else
cbuf = tcbuf;
wkmem = new lzo_byte[LZO1X_1_15_MEM_COMPRESS];
int r = LZO_E_OK;
const int TOTAL_HDR_LEN = 4096 * 2;
char* hdrbuf = new char[TOTAL_HDR_LEN + 4095];
memset(hdrbuf, 0, TOTAL_HDR_LEN + 4095);
char* hdrbufp = 0;
if ((ptrdiff_t)hdrbuf & 0xfff)
hdrbufp = (char*)((ptrdiff_t)hdrbuf & 0xfffffffffffff000ULL) + 4096;
else
hdrbufp = hdrbuf;
struct compHdr
{
uint64_t ptrs[512];
};
idbassert(sizeof(compHdr) <= 4096);
compHdr* hdrptr1 = (compHdr*)hdrbufp;
compHdr* hdrptr = hdrptr1 + 1; // advance to 2nd hdr to store compression ptrs
lseek(ofd, TOTAL_HDR_LEN, SEEK_SET);
nread = read(ifd, ibuf, ibuf_len);
int idx = 0;
off_t cmpoff = TOTAL_HDR_LEN;
while (nread > 0)
{
cbuf_len = 512 * 1024 * 8;
memset(cbuf, 0, cbuf_len);
r = lzo1x_1_15_compress(ibuf, nread, cbuf, &cbuf_len, wkmem);
if (r != LZO_E_OK)
{
cerr << "compression failed!: " << r << endl;
rc = 1;
goto out;
}
if (cbuf_len > (unsigned)nread)
{
cerr << "WARNING: expansion detected! (output grew by " << (cbuf_len - nread) << " bytes)" << endl;
idbassert((cbuf_len - nread) <= exp_buf_size * 1024);
}
if (cbuf_len & 0xfff)
cbuf_len = (cbuf_len & 0xfffffffffffff000ULL) + 4096;
// cbuf_len = 512 * 1024 * 8;
nwritten = write(ofd, cbuf, cbuf_len);
if (nwritten < 0 || (unsigned)nwritten != cbuf_len)
{
cerr << outname << ": write: " << strerror(errno) << " (" << nwritten << ')' << endl;
rc = 1;
goto out;
}
if (vflg > 0)
{
lzo_bytep tbuf;
lzo_uint tbuflen = 4 * 1024 * 1024 + 4;
boost::scoped_array<lzo_byte> tbuf_sa(new lzo_byte[tbuflen]);
tbuf = tbuf_sa.get();
cout << "idx: " << idx << " off: " << cmpoff << " size: " << cbuf_len;
r = lzo1x_decompress(cbuf, cbuf_len, tbuf, &tbuflen, 0);
cout << " r: " << lzo_rctos(r) << " size: " << tbuflen << endl;
}
hdrptr->ptrs[idx] = cmpoff;
idx++;
cmpoff += cbuf_len;
nread = read(ifd, ibuf, ibuf_len);
}
if (nread < 0)
{
cerr << infile << ": read: " << strerror(errno) << endl;
rc = 1;
goto out;
}
hdrptr->ptrs[idx] = cmpoff;
idbassert(idx <= 64);
// Fill in meta-data information in first header
hdrptr1->ptrs[0] = 0xfdc119a384d0778eULL;
hdrptr1->ptrs[1] = 1;
hdrptr1->ptrs[2] = 1;
nwritten = pwrite(ofd, hdrbufp, TOTAL_HDR_LEN, 0);
idbassert(nwritten == TOTAL_HDR_LEN);
out:
delete[] wkmem;
delete[] tcbuf;
delete[] tibuf;
fsync(ofd);
struct stat ostatbuf;
fstat(ofd, &ostatbuf);
idbassert(ostatbuf.st_size == (signed)hdrptr->ptrs[idx]);
delete[] hdrbuf;
cout << infile << ": Input Size: " << istatbuf.st_size << " Output size: " << ostatbuf.st_size
<< " Compression: " << (100LL - (ostatbuf.st_size * 100LL / istatbuf.st_size)) << '%' << endl;
close(ofd);
close(ifd);
return rc;
}
} // namespace
int main(int argc, char** argv)
{
opterr = 0;
int c;
exp_buf_size = 128;
vflg = 0;
while ((c = getopt(argc, argv, "e:vh")) != -1)
switch (c)
{
case 'e': exp_buf_size = atoi(optarg); break;
case 'v': vflg++; break;
case 'h':
default:
usage();
return (c == 'h' ? 0 : 1);
break;
}
if ((argc - optind) < 1)
{
usage();
return 1;
}
int rc = 0;
for (int i = optind; i < argc; i++)
if (doit(argv[i]))
rc = 1;
return rc;
}

11
tools/idbmeminfo/CMakeLists.txt
View File

@@ -4,12 +4,5 @@ include_directories(${ENGINE_COMMON_INCLUDES})
set(idbmeminfo_SRCS idbmeminfo.cpp)
add_executable(idbmeminfo ${idbmeminfo_SRCS})
target_link_libraries(idbmeminfo ${ENGINE_LDFLAGS})
install(
TARGETS idbmeminfo
DESTINATION ${ENGINE_BINDIR}
COMPONENT columnstore-engine
)
columnstore_executable(idbmeminfo ${idbmeminfo_SRCS})
columnstore_link(idbmeminfo ${ENGINE_LDFLAGS})

19
tools/passwd/CMakeLists.txt
View File

@@ -4,21 +4,10 @@ include_directories(${ENGINE_COMMON_INCLUDES} ${ENGINE_UTILS_COMMON_INCLUDE})
set(cspasswd_SRCS cspasswd.cpp secrets.cpp)
set(cskeys_SRCS cskeys.cpp secrets.cpp)
add_executable(cspasswd ${cspasswd_SRCS})
add_executable(cskeys ${cskeys_SRCS})
columnstore_executable(cspasswd ${cspasswd_SRCS})
columnstore_executable(cskeys ${cskeys_SRCS})
target_include_directories(cspasswd BEFORE PUBLIC ${OPENSSL_INCLUDE_DIR})
target_include_directories(cskeys BEFORE PUBLIC ${OPENSSL_INCLUDE_DIR})
target_link_libraries(cspasswd ${ENGINE_LDFLAGS} ${ENGINE_EXEC_LIBS} ${SSL_LIBRARIES})
target_link_libraries(cskeys ${ENGINE_LDFLAGS} ${ENGINE_EXEC_LIBS} ${SSL_LIBRARIES})
install(
TARGETS cspasswd
DESTINATION ${ENGINE_BINDIR}
COMPONENT columnstore-engine
)
install(
TARGETS cskeys
DESTINATION ${ENGINE_BINDIR}
COMPONENT columnstore-engine
)
columnstore_link(cspasswd ${ENGINE_LDFLAGS} ${ENGINE_EXEC_LIBS} ${SSL_LIBRARIES})
columnstore_link(cskeys ${ENGINE_LDFLAGS} ${ENGINE_EXEC_LIBS} ${SSL_LIBRARIES})

2357
tools/pingproc/pingproc.cpp
View File

File diff suppressed because it is too large Load Diff

55
tools/qfe/CMakeLists.txt
View File

@@ -1,55 +0,0 @@
include_directories(${KDE4_INCLUDES} ${KDE4_INCLUDE_DIR} ${QT_INCLUDES})
# ########## next target ###############
set(QFE_SRCS
cseputils.cpp
ddlstmts.cpp
parsequery.cpp
returnedrows.cpp
sendcsep.cpp
server.cpp
socketio.cpp
qfelexer.lpp
qfeparser.ypp
)
kde4_add_executable(QFE ${QFE_SRCS})
target_link_libraries(QFE ${KDE4_KDECORE_LIBS})
install(TARGETS QFE ${INSTALL_TARGETS_DEFAULT_ARGS})
# ########## install files ###############
# original Makefile.am contents follow:
# Copyright (C) 2014 InfiniDB, Inc.
#
# This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public
# License as published by the Free Software Foundation; version 2 of the License.
#
# This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied
# warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License along with this program; if not, write to the Free
# Software Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
#
# $Id$ Process this file with automake to produce Makefile.in
#
# AM_CPPFLAGS = $(idb_cppflags) AM_CFLAGS = $(idb_cflags) AM_CXXFLAGS = $(idb_cxxflags) AM_LDFLAGS = $(idb_ldflags)
# AM_YFLAGS = -d -p qfe AM_LFLAGS = -i -Pqfe -olex.yy.c bin_PROGRAMS = QFE QFE_SOURCES = \ cseputils.cpp \ ddlstmts.cpp
# \ parsequery.cpp \ returnedrows.cpp \ sendcsep.cpp \ server.cpp \ socketio.cpp \ qfelexer.lpp \ qfeparser.ypp
# QFE_CPPFLAGS = @idb_common_includes@ $(AM_CPPFLAGS) QFE_LDFLAGS = @idb_common_ldflags@ @idb_write_libs@
# @idb_exec_libs@ $(AM_LDFLAGS) BUILT_SOURCES = qfeparser.cpp qfelexer.cpp qfeparser.h
#
# test:
#
# coverage:
#
# leakcheck:
#
# docs:
#
# bootstrap: install-data-am
#

148
tools/qfe/cli.cpp
View File

@@ -1,148 +0,0 @@
#include <poll.h>
#include <sys/socket.h>
#include <netinet/in.h>
#include <arpa/inet.h>
#include <netdb.h>
#include <iostream>
#include <string>
#include <sstream>
#include <iomanip>
#include <sys/types.h>
#include <sys/stat.h>
#include <fcntl.h>
#include <unistd.h>
#include <cerrno>
#include <cstring>
#include <cstdlib>
using namespace std;
#include "socktype.h"
#include "socketio.h"
using namespace qfe;
namespace
{
void usage()
{
cout << "usage: cli [-h] [-s serverip] [-p port] [-c schema] <query>" << endl;
cout << '\t' << "-s serverip connect to serverip instead of 127.0.0.1" << endl;
cout << '\t' << "-p port connect on port instead of 9198" << endl;
cout << '\t' << "-c schema use schema as default instead of tpch1" << endl;
cout << '\t' << "-h display this help" << endl;
}
} // namespace
int main(int argc, char** argv)
{
opterr = 0;
int c;
string serverip("127.0.0.1");
string schema("tpch1");
short PortNo = 9198;
while ((c = getopt(argc, argv, "s:c:p:h")) != -1)
switch (c)
{
case 's': serverip = optarg; break;
case 'c': schema = optarg; break;
case 'p':
PortNo = atoi(optarg);
if (PortNo == 0)
{
usage();
return 1;
}
break;
case 'h':
case '?':
default:
usage();
return 0;
break;
}
if (argc - optind < 1)
{
usage();
return 1;
}
string query(argv[optind + 0]);
SockType fd = -1;
fd = ::socket(PF_INET, SOCK_STREAM, IPPROTO_TCP);
if (fd < 0)
{
cerr << "socket create error: " << strerror(errno) << endl;
return 1;
}
int rc = 0;
struct sockaddr_in serv_addr;
struct addrinfo hints;
struct addrinfo* res = 0;
memset(&hints, 0, sizeof(struct addrinfo));
hints.ai_family = PF_INET;
hints.ai_socktype = SOCK_STREAM;
hints.ai_protocol = IPPROTO_TCP;
rc = getaddrinfo(serverip.c_str(), 0, &hints, &res);
if (rc != 0)
{
cerr << "Error resolving '" << serverip << "': " << gai_strerror(rc) << endl;
return 1;
}
sockaddr_in* sain = 0;
sain = reinterpret_cast<sockaddr_in*>(res->ai_addr);
memset(&serv_addr, 0, sizeof(serv_addr));
serv_addr.sin_family = PF_INET;
serv_addr.sin_addr = sain->sin_addr;
serv_addr.sin_port = htons(PortNo);
freeaddrinfo(res);
rc = ::connect(fd, (sockaddr*)&serv_addr, sizeof(serv_addr));
if (rc < 0)
{
cerr << "socket connect error: " << strerror(errno) << endl;
return 1;
}
socketio::writeString(fd, schema);
socketio::writeString(fd, query);
uint32_t flag = 0;
string row;
row = socketio::readString(fd);
if (row != "OK")
{
cerr << "query failed: " << row << endl;
goto bailout;
}
row = socketio::readString(fd);
while (!row.empty())
{
cout << row << endl;
row = socketio::readString(fd);
}
flag = 0;
SockWriteFcn(fd, &flag, 4);
bailout:
::shutdown(fd, SHUT_RDWR);
::close(fd);
fd = -1;
return 0;
}

141
tools/qfe/cseputils.cpp
View File

@@ -1,141 +0,0 @@
#include <unistd.h>
#include <string>
using namespace std;
#include "constantcolumn.h"
#include "simplefilter.h"
#include "calpontsystemcatalog.h"
#include "parsetree.h"
#include "simplecolumn.h"
#include "calpontselectexecutionplan.h"
using namespace execplan;
#include "cseputils.h"
namespace qfe
{
extern string DefaultSchema;
namespace utils
{
ConstantColumn* createConstCol(const string& valstr)
{
ConstantColumn* cc = new ConstantColumn(valstr);
cc->alias(valstr);
return cc;
}
template <typename T>
ConstantColumn* createConstCol(const string& valstr, T val)
{
ConstantColumn* cc = new ConstantColumn(valstr, val);
cc->alias(valstr);
return cc;
}
SimpleFilter* createSimpleFilter(boost::shared_ptr<CalpontSystemCatalog>& csc,
const CalpontSystemCatalog::TableColName& tcn, const string& opstr,
ConstantColumn* cc)
{
SimpleFilter* lsf = new SimpleFilter();
Operator* op = new Operator();
op->data(opstr);
CalpontSystemCatalog::ColType ccct;
ccct = op->resultType();
ccct.colDataType = cc->resultType().colDataType;
op->operationType(ccct);
SOP sop(op);
lsf->op(sop);
CalpontSystemCatalog::OID oid = csc->lookupOID(tcn);
CalpontSystemCatalog::ColType ct = csc->colType(oid);
SimpleColumn* sc = new SimpleColumn();
sc->schemaName(tcn.schema);
sc->tableName(tcn.table);
sc->tableAlias(tcn.table);
sc->columnName(tcn.column);
sc->oid(oid);
sc->resultType(ct);
sc->alias(tcn.toString());
lsf->lhs(sc);
lsf->rhs(cc);
return lsf;
}
void appendSimpleFilter(ParseTree*& ptree, SimpleFilter* filter)
{
if (ptree->data() == 0)
{
// degenerate case, this filter goes at this node
ptree->data(filter);
}
else if (ptree->right() == 0 && ptree->left() == 0)
{
// this will be the case when there is a single node in the tree
// that contains a filter. Here we want to make the root node an
// 'and' operator, push the existing down to the lhs and make a
// new node for the new filter
ParseTree* newLhs = new ParseTree(ptree->data());
ParseTree* newRhs = new ParseTree(filter);
Operator* op = new Operator();
op->data("and");
ptree->data(op);
ptree->left(newLhs);
ptree->right(newRhs);
}
else
{
// this will be the case once we have a tree with an 'and' at the
// root node, a filter in the lhs, and an arbitrary height tree
// with the same properties on the rhs. Because all operators
// are guaranteed to be and for now we simply insert a new rhs
// node and "push down" the existing tree
Operator* op = new Operator();
op->data("and");
ParseTree* newRhs = new ParseTree(op);
newRhs->left(new ParseTree(filter));
newRhs->right(ptree->right());
ptree->right(newRhs);
}
}
void updateParseTree(boost::shared_ptr<execplan::CalpontSystemCatalog>& csc,
execplan::CalpontSelectExecutionPlan*& csep, execplan::SimpleColumn* sc,
const std::string& relop, pair<int, string> cval)
{
execplan::ConstantColumn* cc = 0;
if (cval.first == 0)
cc = createConstCol(cval.second, static_cast<int64_t>(atoll(cval.second.c_str())));
else
cc = createConstCol(cval.second);
if (sc->schemaName() == "infinidb_unknown" && !DefaultSchema.empty())
sc->schemaName(DefaultSchema);
execplan::SimpleFilter* sf = 0;
sf = createSimpleFilter(csc, execplan::make_tcn(sc->schemaName(), sc->tableName(), sc->columnName()), relop,
cc);
execplan::ParseTree* ptp = 0;
ptp = csep->filters();
if (ptp == 0)
ptp = new execplan::ParseTree();
appendSimpleFilter(ptp, sf);
csep->filters(ptp);
}
// template instantiations
template ConstantColumn* createConstCol<int64_t>(const string& valstr, int64_t val);
} // namespace utils
} // namespace qfe

34
tools/qfe/cseputils.h
View File

@@ -1,34 +0,0 @@
#pragma once
#include <string>
namespace execplan
{
class ConstantColumn;
class SimpleFilter;
class ParseTree;
class SimpleColumn;
} // namespace execplan
#include "calpontsystemcatalog.h"
namespace qfe
{
namespace utils
{
execplan::ConstantColumn* createConstCol(const std::string& valstr);
template <typename T>
execplan::ConstantColumn* createConstCol(const std::string& valstr, T val);
execplan::SimpleFilter* createSimpleFilter(boost::shared_ptr<execplan::CalpontSystemCatalog>& csc,
const execplan::CalpontSystemCatalog::TableColName& tcn,
const std::string& opstr, execplan::ConstantColumn* cc);
void appendSimpleFilter(execplan::ParseTree*& ptree, execplan::SimpleFilter* filter);
void updateParseTree(boost::shared_ptr<execplan::CalpontSystemCatalog>&,
execplan::CalpontSelectExecutionPlan*&, execplan::SimpleColumn*, const std::string&,
pair<int, string>);
} // namespace utils
} // namespace qfe

80
tools/qfe/ddlstmts.cpp
View File

@@ -1,80 +0,0 @@
#include <unistd.h>
#include <string>
#include <stdexcept>
using namespace std;
#include <boost/scoped_ptr.hpp>
using namespace boost;
#include "ddlpkg.h"
#include "sqlparser.h"
using namespace ddlpackage;
#include "bytestream.h"
#include "messagequeue.h"
using namespace messageqcpp;
#include "ddlstmts.h"
namespace qfe
{
extern string DefaultSchema;
}
namespace
{
using namespace qfe;
int processDDLStmt(const string& stmt, uint32_t sid)
{
MessageQueueClient* mq = 0;
SqlParser parser;
parser.setDefaultSchema(DefaultSchema);
parser.Parse(stmt.c_str());
if (parser.Good())
{
const ddlpackage::ParseTree& ptree = parser.GetParseTree();
SqlStatement& ddlstmt = *ptree.fList[0];
ddlstmt.fSessionID = sid;
ddlstmt.fSql = stmt;
ddlstmt.fOwner = DefaultSchema;
ByteStream bytestream;
bytestream << ddlstmt.fSessionID;
ddlstmt.serialize(bytestream);
mq = new MessageQueueClient("DDLProc");
scoped_ptr<MessageQueueClient> smq(mq);
ByteStream::byte b = 0;
mq->write(bytestream);
bytestream = mq->read();
bytestream >> b;
string emsg;
bytestream >> emsg;
if (b != 0)
throw runtime_error(emsg);
}
else
{
throw runtime_error("syntax error");
}
return 0;
}
} // namespace
namespace qfe
{
void processCreateStmt(const string& stmt, uint32_t sid)
{
processDDLStmt(stmt, sid);
}
void processDropStmt(const string& stmt, uint32_t sid)
{
processDDLStmt(stmt, sid);
}
} // namespace qfe

10
tools/qfe/ddlstmts.h
View File

@@ -1,10 +0,0 @@
#pragma once
#include <unistd.h>
#include <string>
namespace qfe
{
void processCreateStmt(const std::string&, uint32_t);
void processDropStmt(const std::string&, uint32_t);
} // namespace qfe

97
tools/qfe/parsequery.cpp
View File

@@ -1,97 +0,0 @@
#include <unistd.h>
#include <string>
#include <memory>
using namespace std;
#include <boost/thread/mutex.hpp>
#include <boost/scoped_ptr.hpp>
using namespace boost;
#include "cseputils.h"
#include "parsequery.h"
#include "calpontselectexecutionplan.h"
#include "simplecolumn.h"
#include "simplefilter.h"
#include "constantcolumn.h"
#include "calpontsystemcatalog.h"
#include "sessionmanager.h"
using namespace execplan;
#include "brmtypes.h"
using namespace BRM;
extern int qfeparse();
extern int qfedebug;
struct yy_buffer_state;
extern yy_buffer_state* qfe_scan_string(const char*);
extern void qfe_delete_buffer(yy_buffer_state*);
extern CalpontSelectExecutionPlan* ParserCSEP;
extern boost::shared_ptr<CalpontSystemCatalog> ParserCSC;
namespace
{
mutex ParserMutex;
} // namespace
namespace qfe
{
extern string DefaultSchema;
CalpontSelectExecutionPlan* parseQuery(const string& query, const uint32_t sid)
{
// We're going to make parsing the query single-threaded for now. This makes it a lot
// easier to interface with the parser and doesn;t materially affect overall query
// performance (I think)
mutex::scoped_lock lk(ParserMutex);
boost::shared_ptr<CalpontSystemCatalog> csc = CalpontSystemCatalog::makeCalpontSystemCatalog(sid);
CalpontSelectExecutionPlan* csep = 0;
csep = new CalpontSelectExecutionPlan();
// we use an auto_ptr here with some trepidation. We only want auto delete on an execption.
// If the parseing and plan build succeed, we want the ptr to stay around. boost::scoped_ptr<>
// doesn't have an API to release ownership, so we use auto_ptr...
auto_ptr<CalpontSelectExecutionPlan> scsep(csep);
yy_buffer_state* ybs = 0;
ybs = qfe_scan_string(query.c_str());
if (ybs != 0)
{
ParserCSEP = csep;
ParserCSC = csc;
if (qfeparse() != 0)
throw runtime_error("syntax error");
qfe_delete_buffer(ybs);
}
else
throw runtime_error("Internal parser memory error");
csep->data(query);
SessionManager sm;
TxnID txnID;
txnID = sm.getTxnID(sid);
if (!txnID.valid)
{
txnID.id = 0;
txnID.valid = true;
}
QueryContext verID;
verID = sm.verID();
csep->txnID(txnID.id);
csep->verID(verID);
csep->sessionID(sid);
// cout << *csep << endl;
scsep.release();
return csep;
}
} // namespace qfe

12
tools/qfe/parsequery.h
View File

@@ -1,12 +0,0 @@
#pragma once
#include <unistd.h>
#include <string>
#include "calpontselectexecutionplan.h"
namespace qfe
{
execplan::CalpontSelectExecutionPlan* parseQuery(const std::string&, uint32_t);
}

51
tools/qfe/qfe-cli.py
View File

@@ -1,51 +0,0 @@
#!/usr/bin/python
'''
Created on May 1, 2013
@author: rtw
'''
import struct
import binascii
import socket
import sys
QFE_HOST = 'localhost'
#QFE_HOST = 'winsvr2008r2.calpont.com'
QFE_SOCK = 9198
def do_query(query):
client_socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
client_socket.connect((QFE_HOST, QFE_SOCK))
defschem = "tpch1";
lenstruct = struct.pack("I", len(defschem))
client_socket.send(lenstruct)
client_socket.send(defschem)
lenstruct = struct.pack("I", len(query))
client_socket.send(lenstruct)
client_socket.send(query)
rowlen = 1
while rowlen:
rowlen = client_socket.recv(4)
if len(rowlen) < 4:
break
rowval = struct.unpack("I",rowlen)[0]
if rowval == 0:
lenstruct = struct.pack("I", 0);
client_socket.send(lenstruct)
break
row = client_socket.recv(rowval)
print row
client_socket.close()
if __name__ == '__main__':
if len(sys.argv) < 2:
query = raw_input ( "QFE-cli > " )
else:
query = sys.argv[1]
sys.exit(do_query(query))

65
tools/qfe/qfelexer.lpp
View File

@@ -1,65 +0,0 @@
%{
#include <utility>
#include <string>
#include "returnedcolumn.h"
#ifndef _MSC_VER
#include "qfeparser.h"
#else
#include "bison-win.h"
#endif
extern int qfeerror(const char *s);
extern std::string* newstr(const char* cp);
%}
%option noyywrap
%option yylineno
%option nounput
%x CCONST
digit [0-9]
int_const [+-]?{digit}+
letter [a-zA-Z_]
objname {letter}({letter}|{digit})*
%%
"SELECT" { return QFEP_SELECT; }
"FROM" { return QFEP_FROM; }
"WHERE" { return QFEP_WHERE; }
"GROUP BY" { return GROUPBY; }
"AND" { qfelval.cp = newstr(qfetext); return LOGICOP; }
"OR" { qfelval.cp = newstr(qfetext); return LOGICOP; }
"LIMIT" { return LIMIT; }
"ORDER BY" { return ORDERBY; }
"ASC" { return ASC; }
"DESC" { return DESC; }
"AS" { return AS; }
"SUM" { qfelval.cp = newstr(qfetext); return FUNC; }
"MIN" { qfelval.cp = newstr(qfetext); return FUNC; }
"MAX" { qfelval.cp = newstr(qfetext); return FUNC; }
"AVG" { qfelval.cp = newstr(qfetext); return FUNC; }
{objname} { qfelval.cp = newstr(qfetext); return OBJNAME; }
"'" { BEGIN(CCONST); }
<CCONST>[^']* { qfelval.cp = newstr(qfetext); return CHAR_CONST; }
<CCONST>"'" { BEGIN(INITIAL); }
{int_const} { qfelval.cp = newstr(qfetext); return INT_CONST; }
">" { qfelval.cp = newstr(qfetext); return RELOP; }
">=" { qfelval.cp = newstr(qfetext); return RELOP; }
"<" { qfelval.cp = newstr(qfetext); return RELOP; }
"<=" { qfelval.cp = newstr(qfetext); return RELOP; }
"<>" { qfelval.cp = newstr(qfetext); return RELOP; }
"=" { qfelval.cp = newstr(qfetext); return RELOP; }
[ \t]* {}
[\n] { qfelineno++; }
. { return qfetext[0]; }

306
tools/qfe/qfeparser.ypp
View File

@@ -1,306 +0,0 @@
%{
#include <iostream>
#include <string>
#include <utility>
using namespace std;
namespace qfe
{
extern string DefaultSchema;
}
int qfeerror(const char *s);
int qfelex(void);
#include "calpontselectexecutionplan.h"
#include "simplecolumn.h"
#include "calpontsystemcatalog.h"
#include "aggregatecolumn.h"
execplan::CalpontSelectExecutionPlan* ParserCSEP;
execplan::CalpontSystemCatalog* ParserCSC;
namespace execplan
{
class ReturnedColumn;
}
#include "cseputils.h"
%}
%union
{
execplan::ReturnedColumn* rcp;
std::string* cp;
std::pair<int, std::string>* cvp;
}
//Some of these defines cause conflicts in calpontselectexecutionplan.h, so we prepend QFEP_
%token CHAR_CONST RELOP LOGICOP QFEP_SELECT QFEP_FROM QFEP_WHERE GROUPBY OBJNAME INT_CONST LIMIT ORDERBY ASC
%token DESC AS FUNC
%type <cp> select_column_list table_name filter_predicate_list obj_name_aux opt_alias opt_as
%type <cp> opt_limit_clause
%type <cp> OBJNAME CHAR_CONST INT_CONST RELOP LOGICOP FUNC AS
%type <rcp> aliased_column_name select_column_name obj_name func_col groupby_item
%type <cvp> constant
%destructor { delete $$; } OBJNAME CHAR_CONST INT_CONST RELOP LOGICOP FUNC AS
%destructor { delete $$; } aliased_column_name select_column_name obj_name func_col groupby_item
%destructor { delete $$; } constant
%%
statement: QFEP_SELECT select_column_list
QFEP_FROM table_spec
opt_where_clause
opt_groupby_clause
opt_orderby_clause
opt_limit_clause ';'
;
select_column_list: aliased_column_name
{
execplan::SRCP rcp($1);
ParserCSEP->returnedCols().push_back(rcp);
$$ = 0;
}
| select_column_list ',' aliased_column_name
{
execplan::SRCP rcp($3);
ParserCSEP->returnedCols().push_back(rcp);
$$ = 0;
}
;
select_column_name: obj_name
{
execplan::SimpleColumn* sc = dynamic_cast<execplan::SimpleColumn*>($1);
if (sc->schemaName() == "infinidb_unknown" && !qfe::DefaultSchema.empty())
sc->schemaName(qfe::DefaultSchema);
sc->setOID();
sc->tableAlias(sc->tableName());
sc->alias(execplan::make_tcn(sc->schemaName(), sc->tableName(), sc->columnName()).toString());
$$ = $1;
}
| func_col
;
func_col: FUNC '(' select_column_name ')'
{
execplan::AggregateColumn* ag = new execplan::AggregateColumn(execplan::AggregateColumn::agname2num(*($1)), $3);
delete $1;
$$ = ag;
}
;
aliased_column_name: select_column_name opt_alias
;
opt_alias: /* empty */
{ $$ = 0; }
| opt_as OBJNAME
{
delete $2;
$$ = $1;
}
| opt_as CHAR_CONST
{
delete $2;
$$ = $1;
}
;
opt_as: /* empty */
{ $$ = 0; }
| AS
;
table_spec: aliased_table_name
;
/* this is too loose for a table name, but it's easier to parse */
table_name: obj_name
{
execplan::SimpleColumn* sc=0;
sc = dynamic_cast<execplan::SimpleColumn*>($1);
execplan::CalpontSystemCatalog::TableAliasName tan;
tan.schema = sc->tableName();
if (tan.schema == "infinidb_unknown" && !qfe::DefaultSchema.empty())
tan.schema = qfe::DefaultSchema;
if (qfe::DefaultSchema.empty())
qfe::DefaultSchema = tan.schema;
tan.table = sc->columnName();
tan.alias = tan.table;
execplan::CalpontSelectExecutionPlan::TableList tl;
tl.push_back(tan);
ParserCSEP->tableList(tl);
delete sc;
$$ = 0;
}
;
aliased_table_name: table_name opt_alias
;
opt_where_clause: /* empty */
| QFEP_WHERE filter_predicate_list
;
filter_predicate_list: obj_name RELOP constant
{
pair<int, string> cval = *($3);
delete $3;
execplan::SimpleColumn* sc=0;
sc = dynamic_cast<execplan::SimpleColumn*>($1);
qfe::utils::updateParseTree(ParserCSC, ParserCSEP,
sc, *($2), cval);
delete sc;
delete $2;
$$ = 0;
}
| filter_predicate_list LOGICOP obj_name RELOP constant
{
//string logicop = *($2);
delete $2;
pair<int, string> cval = *($5);
delete $5;
execplan::SimpleColumn* sc=0;
sc = dynamic_cast<execplan::SimpleColumn*>($3);
qfe::utils::updateParseTree(ParserCSC, ParserCSEP,
sc, *($4), cval);
delete sc;
delete $4;
$$ = 0;
}
;
obj_name: obj_name_aux
{
//This is possibly a table name, but we shove it into a SimpleColumn. We'll
// fix this in the table_spec production
execplan::SimpleColumn* sc = new execplan::SimpleColumn("infinidb_unknown", "infinidb_unknown", *($1));
sc->tableAlias("infinidb_unknown");
sc->alias(execplan::make_tcn("infinidb_unknown", "infinidb_unknown", *($1)).toString());
//cerr << "inside parser: " << *sc << endl;
delete $1;
$$ = sc;
}
| obj_name_aux '.' obj_name_aux
{
//This is possibly a table name, but we shove it into a SimpleColumn. We'll
// fix this in the table_spec production
execplan::SimpleColumn* sc = new execplan::SimpleColumn("infinidb_unknown", *($1), *($3));
sc->tableAlias(*($1));
sc->alias(execplan::make_tcn("infinidb_unknown", *($1), *($3)).toString());
//cerr << "inside parser: " << *sc << endl;
delete $1;
delete $3;
$$ = sc;
}
| obj_name_aux '.' obj_name_aux '.' obj_name_aux
{
if (qfe::DefaultSchema.empty())
qfe::DefaultSchema = *($1);
execplan::SimpleColumn* sc = new execplan::SimpleColumn(*($1), *($3), *($5));
sc->tableAlias(*($3));
sc->alias(execplan::make_tcn(*($1), *($3), *($5)).toString());
//cerr << "inside parser: " << *sc << endl;
delete $1;
delete $3;
delete $5;
$$ = sc;
}
;
obj_name_aux: OBJNAME
| '`' OBJNAME '`'
{
$$ = $2;
}
;
constant: CHAR_CONST
{
pair<int, string>* p = new pair<int, string>(1, *($1));
delete $1;
$$ = p;
}
| INT_CONST
{
pair<int, string>* p = new pair<int, string>(0, *($1));
delete $1;
$$ = p;
}
;
opt_groupby_clause: /* empty */
| GROUPBY groupby_items_list
{ YYERROR; }
;
groupby_items_list: groupby_item
| groupby_items_list ',' groupby_item
{ delete $3; }
;
groupby_item: obj_name
| INT_CONST
{
delete $1;
$$ = 0;
}
;
opt_orderby_clause: /* empty */
| ORDERBY groupby_items_list opt_direction
{ YYERROR; }
;
opt_direction: /* empty */
| ASC
| DESC
;
opt_limit_clause: /* empty */
{ $$ = 0; }
| LIMIT INT_CONST
{
delete $2;
YYERROR;
$$ = 0;
}
| LIMIT INT_CONST ',' INT_CONST
{
delete $2;
delete $4;
YYERROR;
$$ = 0;
}
;
%%
int qfeerror(const string& s)
{
extern int qfelineno; // defined and maintained in lex.c
extern char *qfetext; // defined and maintained in lex.c
cerr << "ERROR: " << s << " at symbol \"" << qfetext;
cerr << "\" on line " << qfelineno << endl;
return -1;
}
int qfeerror(const char *s)
{
return qfeerror(string(s));
}
string* newstr(const char* cp)
{
string* strp;
strp = new string(cp);
return strp;
}

95
tools/qfe/returnedrows.cpp
View File

@@ -1,95 +0,0 @@
#include <unistd.h>
#include <iostream>
#include <string>
#include <cerrno>
using namespace std;
#include <boost/scoped_ptr.hpp>
#include <boost/scoped_array.hpp>
using namespace boost;
#include "mcsconfig.h"
#include "socktype.h"
#include "exceptclasses.h"
#include "socketio.h"
#include "bytestream.h"
#include "messagequeue.h"
using namespace messageqcpp;
#include "rowgroup.h"
using namespace rowgroup;
namespace qfe
{
void processReturnedRows(MessageQueueClient* mqc, SockType fd)
{
scoped_ptr<MessageQueueClient> cleaner(mqc);
SBS sbs;
sbs = mqc->read();
// cerr << "got a bs of " << sbs->length() << " bytes" << endl;
RowGroup rg;
rg.deserialize(*sbs);
// cerr << "got a base rowgroup with rows of " << rg.getRowSize() << " bytes" << endl;
// cerr << rg.toString() << endl;
ByteStream bs;
ByteStream::quadbyte tableOID = 100;
bs.reset();
bs << tableOID;
mqc->write(bs);
sbs = mqc->read();
// cerr << "got a bs of " << sbs->length() << " bytes" << endl;
RGData rgd;
rgd.deserialize(*sbs, true);
rg.setData(&rgd);
// cerr << "got a rowgroup with: " << rg.getRowCount() << " rows" << endl;
socketio::writeString(fd, "OK");
Row r;
while (rg.getRowCount() > 0)
{
rg.initRow(&r);
rg.getRow(0, &r);
string csv;
bs.reset();
for (unsigned i = 0; i < rg.getRowCount(); i++)
{
csv = r.toCSV();
bs << csv;
r.nextRow();
}
// cerr << "writing " << bs.length() << " bytes back to client" << endl;
SockWriteFcn(fd, bs.buf(), bs.length());
bs.reset();
bs << tableOID;
mqc->write(bs);
sbs = mqc->read();
// cerr << "got a bs of " << sbs->length() << " bytes" << endl;
rgd.deserialize(*sbs, true);
rg.setData(&rgd);
// cerr << "got a rowgroup with: " << rg.getRowCount() << " rows" << endl;
}
tableOID = 0;
bs.reset();
bs << tableOID;
mqc->write(bs);
// sync with the client on end-of-results
SockWriteFcn(fd, &tableOID, 4);
SockReadFcn(fd, &tableOID, 4);
}
} // namespace qfe

10
tools/qfe/returnedrows.h
View File

@@ -1,10 +0,0 @@
#pragma once
#include "socktype.h"
#include "messagequeue.h"
namespace qfe
{
void processReturnedRows(messageqcpp::MessageQueueClient*, SockType);
}

53
tools/qfe/sendcsep.cpp
View File

@@ -1,53 +0,0 @@
#include <unistd.h>
#include <stdexcept>
#include <memory>
using namespace std;
#include <boost/scoped_ptr.hpp>
using namespace boost;
#include "calpontselectexecutionplan.h"
using namespace execplan;
#include "bytestream.h"
#include "messagequeue.h"
using namespace messageqcpp;
namespace qfe
{
MessageQueueClient* sendCSEP(CalpontSelectExecutionPlan* csep)
{
scoped_ptr<CalpontSelectExecutionPlan> cleaner(csep);
ByteStream bs;
MessageQueueClient* mqc = 0;
mqc = new MessageQueueClient("ExeMgr1");
auto_ptr<MessageQueueClient> smqc(mqc);
bs.reset();
ByteStream::quadbyte wantTuples = 4;
bs << wantTuples;
mqc->write(bs);
bs.reset();
csep->serialize(bs);
mqc->write(bs);
SBS sbs;
sbs = mqc->read();
*sbs >> wantTuples;
// cerr << "got flag: " << wantTuples << endl;
string msg;
sbs = mqc->read();
*sbs >> msg;
// cerr << "got msg: " << msg << endl;
if (wantTuples != 0)
throw runtime_error(msg);
smqc.release();
return mqc;
}
} // namespace qfe

12
tools/qfe/sendcsep.h
View File

@@ -1,12 +0,0 @@
#pragma once
#include "messagequeue.h"
#include "calpontselectexecutionplan.h"
namespace qfe
{
// Takes ownership of the alloc'd ptr
// Returns an alloc'd mqc
messageqcpp::MessageQueueClient* sendCSEP(execplan::CalpontSelectExecutionPlan*);
} // namespace qfe

429
tools/qfe/server.cpp
View File

@@ -1,429 +0,0 @@
/* Copyright (C) 2014 InfiniDB, Inc.
This program is free software; you can redistribute it and/or
modify it under the terms of the GNU General Public License
as published by the Free Software Foundation; version 2 of
the License.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program; if not, write to the Free Software
Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
MA 02110-1301, USA. */
/*
Protocol definition:
*/
#include <poll.h>
#include <sys/socket.h>
#include <netinet/in.h>
#include <arpa/inet.h>
#include <iostream>
#include <string>
#include <stdexcept>
#include <sstream>
#include <iomanip>
#include <sys/types.h>
#include <sys/stat.h>
#include <fcntl.h>
#include <unistd.h>
#include <signal.h>
#include <cerrno>
using namespace std;
#include <boost/thread.hpp>
#include <boost/scoped_ptr.hpp>
#include <boost/scoped_array.hpp>
#include <boost/tokenizer.hpp>
#include <boost/algorithm/string.hpp>
using namespace boost;
#include "mcsconfig.h"
#include "socktype.h"
#include "parsequery.h"
#include "sendcsep.h"
#include "returnedrows.h"
#include "socketio.h"
#include "ddlstmts.h"
#include "exceptclasses.h" //brings in idbassert_s macro
#include "messagequeue.h"
using namespace messageqcpp;
#include "atomicops.h"
namespace execplan
{
class CalpontSelectExecutionPlan;
}
#define SINGLE_THREADED
namespace qfe
{
string DefaultSchema;
}
namespace
{
using namespace qfe;
enum StmtType
{
UNKNOWN,
QUERY,
CREATE,
DROP,
SHOW,
};
volatile uint32_t SystemSID;
void log(const string& s)
{
cerr << s << endl;
}
struct QueryMessage
{
QueryMessage() : isValid(false)
{
}
~QueryMessage()
{
}
string toString() const;
bool isValid;
string queryText;
string defaultSchema;
};
string QueryMessage::toString() const
{
ostringstream oss;
oss << "valid: " << boolalpha << isValid << ", "
<< "queryText: " << queryText << ", "
<< "defaultSchema: " << defaultSchema;
return oss.str();
}
ostream& operator<<(ostream& os, const QueryMessage& rhs)
{
os << rhs.toString();
return os;
}
class ThreadFunc
{
public:
ThreadFunc(SockType fd) : fFd(fd)
{
}
~ThreadFunc()
{
}
void run();
void operator()()
{
run();
}
private:
ThreadFunc(const ThreadFunc& rhs);
ThreadFunc& operator=(const ThreadFunc& rhs);
SockType fFd;
};
bool serverInit()
{
setsid();
// Handle certain signals (we want these to return EINTR so we can throw)
// SIGPIPE
// I don't think we'll get any of these from init (except possibly HUP, but that's an indication
// of bad things anyway)
// SIGHUP?
// SIGUSR1?
// SIGUSR2?
// SIGPOLL?
struct sigaction sa;
memset(&sa, 0, sizeof(struct sigaction));
sa.sa_handler = SIG_IGN;
sigaction(SIGPIPE, &sa, 0);
sigaction(SIGHUP, &sa, 0);
sigaction(SIGUSR1, &sa, 0);
sigaction(SIGUSR2, &sa, 0);
#ifndef __FreeBSD__
sigaction(SIGPOLL, &sa, 0);
#endif
#if 0
int fd;
close(2);
fd = open("/tmp/qfe.err", O_CREAT | O_TRUNC | O_WRONLY, 0644);
if (fd >= 0 && fd != 2)
{
dup2(fd, 2);
close(fd);
}
#endif
return true;
}
SockType initListenSock(short portNo)
{
SockType listenSock = -1;
listenSock = socket(PF_INET, SOCK_STREAM, IPPROTO_TCP);
idbassert_s(listenSock >= 0, string("socket create error: ") + strerror(errno));
// if (listenSock < 0) throw runtime_error(string("socket create error: ") + strerror(errno));
int optval = 1;
setsockopt(listenSock, SOL_SOCKET, SO_REUSEADDR, reinterpret_cast<const char*>(&optval), sizeof(optval));
int rc = 0;
struct sockaddr_in serv_addr;
memset(&serv_addr, 0, sizeof(serv_addr));
serv_addr.sin_family = AF_INET;
serv_addr.sin_addr.s_addr = INADDR_ANY;
serv_addr.sin_port = htons(portNo);
const int MaxTries = 5 * 60 / 10;
int tries = 0;
again:
rc = ::bind(listenSock, (sockaddr*)&serv_addr, sizeof(serv_addr));
if (rc < 0)
{
if (errno == EADDRINUSE)
{
// cerr << "Addr in use..." << endl;
if (++tries >= MaxTries)
{
log("Waited too long for socket to bind...giving up");
// cerr << "Waited too long for socket to bind...giving up" << endl;
exit(1);
}
sleep(10);
goto again;
}
idbassert_s(0, string("socket bind error: ") + strerror(errno));
// throw runtime_error(string("socket bind error: ") + strerror(errno));
}
rc = listen(listenSock, 16);
idbassert_s(rc >= 0, string("socket listen error") + strerror(errno));
// if (rc < 0) throw runtime_error(string("socket listen error") + strerror(errno));
return listenSock;
}
QueryMessage getNextMsg(SockType fd)
{
QueryMessage msg;
try
{
msg.defaultSchema = socketio::readString(fd);
msg.queryText = socketio::readString(fd);
msg.isValid = true;
}
catch (runtime_error& rex)
{
cerr << "re reading ctl msg: " << rex.what() << endl;
msg.queryText = "";
}
catch (...)
{
cerr << "ex reading ctl msg" << endl;
msg.queryText = "";
}
return msg;
}
StmtType guessStatementType(const string& stmt)
{
typedef boost::tokenizer<boost::char_separator<char> > tokenizer;
char_separator<char> sep;
tokenizer tokens(stmt, sep);
tokenizer::iterator tok_iter = tokens.begin();
string first_word;
first_word = *tok_iter;
algorithm::to_lower(first_word);
if (first_word == "select")
return QUERY;
if (first_word == "create")
return CREATE;
if (first_word == "drop")
return DROP;
if (first_word == "show")
return SHOW;
return UNKNOWN;
}
struct ScopedCleaner
{
ScopedCleaner(SockType fd = -1) : fFd(fd)
{
}
~ScopedCleaner()
{
if (fFd >= 0)
shutdown(fFd, SHUT_RDWR);
close(fFd);
}
SockType fFd;
};
void ThreadFunc::run()
{
QueryMessage m;
execplan::CalpontSelectExecutionPlan* csep = 0;
MessageQueueClient* msgqcl;
ScopedCleaner cleaner(fFd);
uint32_t sid = 1;
sid = atomicops::atomicInc(&SystemSID);
try
{
m = getNextMsg(fFd);
if (m.isValid)
{
DefaultSchema = m.defaultSchema;
StmtType st = guessStatementType(m.queryText);
switch (st)
{
case QUERY:
csep = parseQuery(m.queryText, sid);
// sendCSEP takes ownership of the ptr from parseQuery
msgqcl = sendCSEP(csep);
// processReturnedRows takes ownership of the ptr from sendCSEP
processReturnedRows(msgqcl, fFd);
break;
case CREATE: processCreateStmt(m.queryText, sid); break;
case DROP: processDropStmt(m.queryText, sid); break;
case SHOW:
{
ostringstream oss;
oss << "select calpontsys.systable.tablename from calpontsys.systable where "
"calpontsys.systable.schema='"
<< m.defaultSchema << "';";
csep = parseQuery(oss.str(), sid);
msgqcl = sendCSEP(csep);
processReturnedRows(msgqcl, fFd);
break;
}
default: throw runtime_error("couldn't guess the statement type"); break;
}
}
}
catch (std::exception& ex)
{
socketio::writeString(fFd, ex.what());
throw; // in a multi-threaded server this will simply cause this thread to exit
}
catch (...)
{
socketio::writeString(fFd, "internal query processing error");
throw;
}
}
} // namespace
int main(int argc, char** argv)
{
int c;
SockType listenSock;
short portNo;
portNo = 0;
char* p = getenv("IDB_QFE_PORT");
if (p && *p)
portNo = atoi(p);
if (portNo <= 0)
portNo = 9198;
listenSock = -1;
opterr = 0;
while ((c = getopt(argc, argv, "p:")) != -1)
switch (c)
{
case 'p': portNo = atoi(optarg); break;
case '?':
default: break;
}
if (!serverInit())
{
log("Could not initialize the QFE Server!");
cerr << "Could not initialize the QFE Server!" << endl;
return 1;
}
listenSock = initListenSock(portNo);
SystemSID = 0;
for (;;)
{
int querySock = -1;
querySock = accept(listenSock, 0, 0);
idbassert_s(querySock >= 0, string("socket accept error: ") + strerror(errno));
// ThreadFunc now owns querySock and is responsible for cleaning it up
ThreadFunc tf(querySock);
#ifdef SINGLE_THREADED
try
{
tf.run();
}
catch (std::exception& ex)
{
cerr << "ThreadFunc run threw an exception: " << ex.what() << endl;
}
catch (...)
{
cerr << "ThreadFunc run threw an exception" << endl;
}
#else
thread t(tf);
#endif
}
return 0;
}

133
tools/qfe/socketio.cpp
View File

@@ -1,133 +0,0 @@
#include <unistd.h>
#include <poll.h>
#include <sys/socket.h>
#include <netinet/in.h>
#include <arpa/inet.h>
#include <cerrno>
#include <sstream>
using namespace std;
#include <boost/scoped_array.hpp>
using namespace boost;
#include "mcsconfig.h"
#include "exceptclasses.h"
#include "socketio.h"
namespace qfe
{
namespace socketio
{
void readn(int fd, void* buf, const size_t wanted)
{
size_t needed = wanted;
size_t sofar = 0;
char* p = static_cast<char*>(buf);
ssize_t rrc = -1;
pollfd fds[1];
int en = 0;
fds[0].fd = fd;
fds[0].events = POLLIN;
while (wanted > sofar)
{
fds[0].revents = 0;
poll(fds, 1, -1);
errno = 0;
rrc = read(fd, (p + sofar), needed);
en = errno;
if (rrc < 0)
{
if (en == EAGAIN || en == EINTR || en == 512)
continue;
ostringstream oss;
oss << "qfe: readn: read() returned " << rrc << " (" << strerror(en) << ")";
idbassert_s(0, oss.str());
}
needed -= rrc;
sofar += rrc;
}
}
size_t writen(int fd, const void* data, const size_t nbytes)
{
size_t nleft;
ssize_t nwritten;
const char* bufp = static_cast<const char*>(data);
nleft = nbytes;
while (nleft > 0)
{
// the O_NONBLOCK flag is not set, this is a blocking I/O.
if ((nwritten = ::write(fd, bufp, nleft)) < 0)
{
if (errno == EINTR)
nwritten = 0;
else
{
// save the error no first
int e = errno;
string errorMsg = "qfe: writen: write() error: ";
scoped_array<char> buf(new char[80]);
#if STRERROR_R_CHAR_P
const char* p;
if ((p = strerror_r(e, buf.get(), 80)) != 0)
errorMsg += p;
#else
int p;
if ((p = strerror_r(e, buf.get(), 80)) == 0)
errorMsg += buf.get();
#endif
idbassert_s(0, errorMsg);
}
}
nleft -= nwritten;
bufp += nwritten;
}
return nbytes;
}
uint32_t readNumber32(SockType fd)
{
uint32_t np;
SockReadFcn(fd, &np, 4);
return np;
}
string readString(SockType fd)
{
string s;
uint32_t len = readNumber32(fd);
if (len > 0)
{
scoped_array<char> buf(new char[len + 1]);
SockReadFcn(fd, buf.get(), len);
buf[len] = 0;
s = buf.get();
}
return s;
}
void writeString(SockType fd, const string& data)
{
uint32_t len = data.length();
SockWriteFcn(fd, &len, 4);
SockWriteFcn(fd, data.c_str(), len);
}
} // namespace socketio
} // namespace qfe

20
tools/qfe/socketio.h
View File

@@ -1,20 +0,0 @@
#pragma once
#include <stdint.h>
#include <unistd.h>
#include <string>
#include "socktype.h"
namespace qfe
{
namespace socketio
{
void readn(int fd, void* buf, const size_t wanted);
size_t writen(int fd, const void* data, const size_t nbytes);
uint32_t readNumber32(SockType);
std::string readString(SockType);
void writeString(SockType, const std::string&);
} // namespace socketio
} // namespace qfe

6
tools/qfe/socktype.h
View File

@@ -1,6 +0,0 @@
#pragma once
typedef int SockType;
#define SockReadFcn qfe::socketio::readn
#define SockWriteFcn qfe::socketio::writen

9
tools/rebuildEM/CMakeLists.txt
View File

@@ -1,10 +1,5 @@
include_directories(${ENGINE_COMMON_INCLUDES})
set(rebuildEM_SRCS main.cpp rebuildEM.cpp)
add_executable(mcsRebuildEM ${rebuildEM_SRCS})
target_link_libraries(mcsRebuildEM ${ENGINE_LDFLAGS} ${ENGINE_WRITE_LIBS} boost_system boost_filesystem)
install(
TARGETS mcsRebuildEM
DESTINATION ${ENGINE_BINDIR}
COMPONENT columnstore-engine
)
columnstore_executable(mcsRebuildEM ${rebuildEM_SRCS})
columnstore_link(mcsRebuildEM ${ENGINE_LDFLAGS} ${ENGINE_WRITE_LIBS} boost_system boost_filesystem)

10
tools/rgprint/CMakeLists.txt
View File

@@ -4,12 +4,6 @@ include_directories(${ENGINE_COMMON_INCLUDES})
set(rgprint_SRCS rgprint.cpp)
add_executable(rgprint ${rgprint_SRCS})
columnstore_executable(rgprint ${rgprint_SRCS})
target_link_libraries(rgprint ${ENGINE_LDFLAGS} ${NETSNMP_LIBRARIES} ${ENGINE_WRITE_LIBS})
install(
TARGETS rgprint
DESTINATION ${ENGINE_BINDIR}
COMPONENT columnstore-engine
)
columnstore_link(rgprint ${ENGINE_LDFLAGS} ${NETSNMP_LIBRARIES} ${ENGINE_WRITE_LIBS})

32
tools/sendPlan/CMakeLists.txt
View File

@@ -1,32 +0,0 @@
include_directories(${KDE4_INCLUDES} ${KDE4_INCLUDE_DIR} ${QT_INCLUDES})
# ########## next target ###############
set(sendPlan_SRCS sendplan.cpp)
kde4_add_executable(sendPlan ${sendPlan_SRCS})
target_link_libraries(sendPlan ${KDE4_KDECORE_LIBS})
install(TARGETS sendPlan ${INSTALL_TARGETS_DEFAULT_ARGS})
# ########## install files ###############
# original Makefile.am contents follow:
# $Id: Makefile.am 333 2009-04-03 20:35:04Z rdempsey $ Process this file with automake to produce Makefile.in
#
# AM_CPPFLAGS = $(idb_cppflags) AM_CFLAGS = $(idb_cflags) AM_CXXFLAGS = $(idb_cxxflags) AM_LDFLAGS = $(idb_ldflags)
# bin_PROGRAMS = sendPlan sendPlan_SOURCES = sendplan.cpp sendPlan_CPPFLAGS = @idb_common_includes@ $(AM_CPPFLAGS)
# sendPlan_LDFLAGS = @idb_common_ldflags@ @idb_exec_libs@ $(AM_LDFLAGS)
#
# test:
#
# coverage:
#
# leakcheck:
#
# docs:
#
# bootstrap: install-data-am
#

392
tools/sendPlan/sendplan.cpp
View File

@@ -1,392 +0,0 @@
// $Id: sendplan.cpp 1739 2012-03-22 12:57:59Z pleblanc $
#include <iostream>
#include <fstream>
#include <sstream>
#include <unistd.h>
#include <sys/time.h>
#include <iomanip>
using namespace std;
#include "bytestream.h"
using namespace messageqcpp;
#include "calpontselectexecutionplan.h"
#include "sessionmanager.h"
using namespace execplan;
#include "calpontsystemcatalog.h"
using namespace execplan;
#include "tableband.h"
#include "joblist.h"
#include "joblistfactory.h"
using namespace joblist;
#include "configcpp.h"
#include "errorcodes.h"
#include "rowgroup.h"
using namespace rowgroup;
typedef CalpontSystemCatalog::OID OID;
namespace
{
bool vflg;
void usage()
{
cout << "usage: sendPlan [-v|h|d|B] [-t lvl] [-s sid] plan_file ..." << endl;
cout << "-v verbose output" << endl;
cout << "-t lvl set trace level to lvl" << endl;
cout << "-s sid set session id to sid" << endl;
cout << "-d display the query, but don't run it" << endl;
cout << "-B Bob's personal preferred output format" << endl;
cout << "-h display this help" << endl;
}
double tm_diff(const struct timeval* st, const struct timeval* et)
{
double sd = (double)st->tv_sec + (double)st->tv_usec / 1000000.0;
double ed = (double)et->tv_sec + (double)et->tv_usec / 1000000.0;
return (ed - sd);
}
//...Extract the stats string to be printed and the runtime start time from
//...the full stats string we receive from ExeMgr.
void parseStatsString(const string& fullStatsString, string& printStatsString,
struct timeval& queryRunStartTime)
{
string::size_type delimPos = fullStatsString.find('|');
printStatsString = fullStatsString.substr(0, delimPos);
istringstream startTimeString(fullStatsString.substr(delimPos + 1));
startTimeString >> queryRunStartTime.tv_sec >> queryRunStartTime.tv_usec;
}
} // namespace
int main(int argc, char** argv)
{
vflg = false;
uint32_t tlvl = 0;
bool dflg = false;
int c;
int32_t sid = -1;
bool Bflg = false;
opterr = 0;
while ((c = getopt(argc, argv, "vt:ds:Bh")) != EOF)
switch (c)
{
case 't': tlvl = static_cast<uint32_t>(strtoul(optarg, 0, 0)); break;
case 'v': vflg = true; break;
case 'd': dflg = true; break;
case 's': sid = static_cast<int32_t>(strtol(optarg, 0, 0)); break;
case 'B': Bflg = true; break;
case 'h':
case '?':
default:
usage();
return (c == 'h' ? 0 : 1);
break;
}
if (dflg)
vflg = true;
if ((argc - optind) < 1)
{
usage();
return 1;
}
ifstream inputf;
ByteStream bs;
ByteStream dbs;
ByteStream eoq;
ByteStream tbs;
ByteStream statsStream;
ByteStream::quadbyte q = 0;
eoq << q;
uint32_t sessionid;
time_t t;
SJLP jl;
DeliveredTableMap tm;
DeliveredTableMap::iterator iter;
DeliveredTableMap::iterator end;
CalpontSelectExecutionPlan csep;
struct timeval start_time;
struct timeval end_time;
MessageQueueClient* mqc = 0;
if (!dflg)
mqc = new MessageQueueClient("ExeMgr1");
if (sid == -1)
{
time(&t);
sessionid = static_cast<uint32_t>(t);
}
else
{
sessionid = static_cast<uint32_t>(sid);
}
sessionid &= 0x7fffffff;
logging::ErrorCodes errorCodes;
for (; optind < argc; optind++)
{
inputf.open(argv[optind]);
if (!inputf.good())
{
cerr << "error opening plan stream " << argv[optind] << endl;
return 1;
}
bs.reset();
inputf >> bs;
inputf.close();
csep.unserialize(bs);
csep.sessionID(sessionid);
SessionManager sm;
csep.verID(sm.verID());
csep.traceFlags(0);
ResourceManager* rm = ResourceManager::instance();
jl = JobListFactory::makeJobList(&csep, rm);
csep.traceFlags(tlvl);
if (vflg)
{
if (dflg)
cout << endl << "Query:" << endl;
else
{
cout << endl << "Session: " << sessionid << ", Sending Query";
if (Bflg)
cout << " (" << argv[optind] << ')';
cout << ':' << endl;
}
if (!Bflg)
cout << csep.data() << endl << endl;
}
if (dflg)
continue;
try
{
dbs.reset();
csep.serialize(dbs);
gettimeofday(&start_time, 0);
// try tuples first, but expect the worst...
bool expectTuples = false;
ByteStream tbs;
ByteStream::quadbyte tqb = 4;
tbs << tqb;
mqc->write(tbs);
// send the CSEP
mqc->write(dbs);
// read the response to the tuple request
tbs = mqc->read();
idbassert(tbs.length() == 4);
tbs >> tqb;
if (tqb == 4)
expectTuples = true;
if (!expectTuples)
cout << "Using TableBand I/F" << endl;
else
cout << "Using tuple I/F" << endl;
tm = jl->deliveredTables();
iter = tm.begin();
end = tm.end();
OID toid;
uint64_t rowTot;
bool reported = false;
bool needRGCtor = true;
while (iter != end)
{
toid = iter->first;
q = static_cast<ByteStream::quadbyte>(toid);
tbs.reset();
tbs << q;
mqc->write(tbs);
ByteStream tbbs;
TableBand tb;
RowGroup rg;
rowTot = 0;
uint16_t status = 0;
TableBand::VBA::size_type rc;
ofstream out;
for (;;)
{
tbbs = mqc->read();
#if 0
cout << tbbs.length() << endl;
out.open("bs1.dat");
idbassert(out.good());
out << tbbs;
out.close();
tbbs = mqc->read();
cout << tbbs.length() << endl;
out.open("bs2.dat");
idbassert(out.good());
out << tbbs;
out.close();
tbbs = mqc->read();
cout << tbbs.length() << endl;
out.open("bs3.dat");
idbassert(out.good());
out << tbbs;
out.close();
#endif
if (tbbs.length())
{
if (!expectTuples)
tb.unserialize(tbbs);
else
{
if (needRGCtor)
{
rg.deserialize(tbbs);
needRGCtor = false;
tbbs = mqc->read();
}
rg.setData((uint8_t*)tbbs.buf());
}
}
else
{
//@bug 1346
if (!status)
status = logging::makeJobListErr;
break;
}
if (!expectTuples)
{
rc = tb.getRowCount();
status = tb.getStatus();
}
else
{
rc = rg.getRowCount();
status = rg.getStatus();
if (rc == 0)
status = 0;
}
if (rc == 0)
break;
rowTot += rc;
}
BatchPrimitive* step = dynamic_cast<BatchPrimitive*>(iter->second.get());
if (vflg && step)
{
cout << "For table " << step->tableName();
if (!Bflg)
cout << " " << toid;
cout << ": read " << rowTot << " rows" << endl;
}
if (status && !reported)
{
cout << "### Query failed: " << errorCodes.errorString(status) << " Check crit.log\n";
reported = true;
}
if (!step && !reported)
{
cout << "### Query failed: Did not return project BatchPrimitive. Check crit.log\n";
reported = true;
}
++iter;
}
if (vflg)
{
gettimeofday(&end_time, 0);
cout << "Query time: " << fixed << setprecision(1) << tm_diff(&start_time, &end_time) << " secs"
<< endl;
//...Ask for query stats through special table id of 3
const OID TABLE_ID_TO_GET_QUERY_STATS = 3;
if (!Bflg)
cout << "Retrieving stats..." << endl;
toid = TABLE_ID_TO_GET_QUERY_STATS;
q = static_cast<ByteStream::quadbyte>(toid);
statsStream.reset();
statsStream << q;
mqc->write(statsStream);
ByteStream bs_statsString;
bs_statsString = mqc->read();
string statsString;
bs_statsString >> statsString;
string printStatsString;
struct timeval startRunTime;
parseStatsString(statsString, printStatsString, startRunTime);
cout << printStatsString << "; QuerySetupTime-" << tm_diff(&start_time, &startRunTime) << "secs"
<< endl;
}
//...Close this query/session
mqc->write(eoq);
jl.reset();
}
catch (const exception& ex)
{
cout << "### SendPlan caught an exception: " << ex.what() << endl;
}
}
// jl.reset();
CalpontSystemCatalog::removeCalpontSystemCatalog(sessionid);
config::Config::deleteInstanceMap();
delete mqc;
return 0;
}

Some files were not shown because too many files have changed in this diff Show More