First attempt to automate my benchmark

benchmarks/bench.sh

@@ -0,0 +1,121 @@
+#!/usr/bin/env bash
+
+set -Eeuo pipefail
+
+trap cleanup SIGINT SIGTERM ERR EXIT
+
+usage() {
+  cat <<EOF
+Usage: $(basename "${BASH_SOURCE[0]}") branch lua_script [-h] [-d data.tbl] [-s 1000000]
+
+A script to run benchmark(for now) to compare the pefromanceo of min/max
+calculation on develop and provided branch. Runs the provided .lua script using sysbench.
+
+Available options:
+
+-h, --help      Print this help and exit
+-d, --data      Data for table that will be given to cpimport; if no name provided it will be generated.
+-s, --size      Size of the dataset to generate
+EOF
+  exit
+}
+
+SCRIPT_LOCATION=$(cd "$(dirname "${BASH_SOURCE[0]}")" &>/dev/null && pwd -P)
+MDB_SOURCE_PATH=$(realpath $SCRIPT_LOCATION/../../..)
+DATA="data.tbl"
+
+cleanup() {
+  trap - SIGINT SIGTERM ERR EXIT
+  if [ -f $DATA ]
+     then
+         sudo rm $DATA
+  fi
+}
+
+die() {
+  local msg=$1
+  local code=${2-1} # default exit status 1
+  echo "$msg"
+  exit "$code"
+}
+
+parse_params() {
+
+  args=("$@")
+  # check required params and arguments
+  [[ ${#args[@]} -eq 0 ]] && die "Missing script arguments"
+
+  BRANCH="$1"
+
+  if [ $BRANCH == "--help" ] || [ $BRANCH == "-h" ]
+     then
+         usage
+  fi
+
+  [[ ${#args[@]} < 2 ]] && die "Missing script arguments"
+
+  SCRIPT="$2"
+  RANGE=1000000
+
+  while :; do
+    case "${1-}" in
+    -d | --data) DATA="${2-}"
+                 shift
+                 ;;
+    -s | --size) RANGE="${2-}"
+                 shift
+                 ;;
+    -?*) die "Unknown option: $1" ;;
+    *) break ;;
+    esac
+    shift
+  done
+
+  return 0
+}
+
+parse_params "$@"
+cd $MDB_SOURCE_PATH/columnstore/columnstore/benchmarks
+seq 1 $RANGE > "$DATA"
+
+git checkout $BRANCH
+sudo $MDB_SOURCE_PATH/columnstore/columnstore/build/bootstrap_mcs.sh -t RelWithDebInfo
+echo "Build done; benchmarking $BRANCH now"
+#Prepare should only create the table, we will fill it with cpimport
+sysbench $SCRIPT \
+        --mysql-socket=/run/mysqld/mysqld.sock \
+        --db-driver=mysql \
+        --mysql-db=test \
+        prepare
+
+sudo cpimport test t1 "$DATA"
+
+sysbench $SCRIPT \
+        --mysql-socket=/run/mysqld/mysqld.sock \
+        --db-driver=mysql \
+        --mysql-db=test \
+        --time=30 run | tail -n +12 > "${BRANCH}_bench.txt"
+
+git checkout develop
+sudo $MDB_SOURCE_PATH/columnstore/columnstore/build/bootstrap_mcs.sh -t RelWithDebInfo
+echo "Build done; benchmarking develop now"
+sysbench $SCRIPT \
+        --mysql-socket=/run/mysqld/mysqld.sock \
+        --db-driver=mysql \
+        --mysql-db=test \
+        prepare
+
+sudo cpimport test t1 "$DATA"
+
+sysbench $SCRIPT \
+        --mysql-socket=/run/mysqld/mysqld.sock \
+        --db-driver=mysql \
+        --mysql-db=test \
+        --time=30 run | tail -n +12 > develop_bench.txt
+
+sysbench $SCRIPT --mysql-socket=/run/mysqld/mysqld.sock \
+        --db-driver=mysql \
+        --mysql-db=test \
+        cleanup
+
+python3 parse_bench.py "$BRANCH" "${BRANCH}_bench.txt" "develop_bench.txt"

benchmarks/bench_report.lua

@@ -0,0 +1,49 @@
+function sysbench.report_json(stat)
+   if not gobj then
+      io.write('[\n')
+      -- hack to print the closing bracket when the Lua state of the reporting
+      -- thread is closed
+      gobj = newproxy(true)
+      getmetatable(gobj).__gc = function () io.write('\n]\n') end
+   else
+      io.write(',\n')
+   end
+
+   local seconds = stat.time_interval
+   io.write(([[
+  {
+    "time": %4.0f,
+    "threads": %u,
+    "transactions": %u,
+    "queries": {
+       "total": %u,
+       "reads": %u,
+       "writes": %u,
+       "other": %u
+    },
+    "tps": %4.2f,
+    "qps": {
+      "total": %4.2f,
+      "reads": %4.2f,
+      "writes": %4.2f,
+      "other": %4.2f
+    },
+    "latency": %4.2f,
+    "errors": %4.2f,
+    "reconnects": %4.2f
+  }]]):format(
+            stat.time_total,
+            stat.threads_running,
+            stat.events,
+            stat.reads + stat.writes + stat.other,
+            stat.reads, stat.writes, stat.other,
+            stat.events / seconds,
+            (stat.reads + stat.writes + stat.other) / seconds,
+            stat.reads / seconds,
+            stat.writes / seconds,
+            stat.other / seconds,
+            stat.latency_pct * 1000,
+            stat.errors / seconds,
+            stat.reconnects / seconds
+   ))
+end

benchmarks/parse_bench.py

@@ -0,0 +1,58 @@
+#!/usr/bin/env python3
+
+import json
+import sys
+
+branch_name = sys.argv[1]
+data_branch = []
+data_develop = []
+time_spent = 30
+
+with open(sys.argv[2],'r') as jf1:
+    data_branch = json.load(jf1)
+
+with open(sys.argv[3],'r') as jf2:
+    data_develop = json.load(jf2)
+
+time = '''Time spent: {time}'''.format(time = time_spent)
+
+comparison = '''Queries made:
+for {name}
+  -- {reads1} reads ({reads1ps} per second)
+  -- {writes1} writes ({writes1ps} per second)
+  -- {other1} other ({other1ps} per second)
+  -- {total1} in total ({total1ps} per second)
+for develop
+  -- {reads2} reads ({reads2ps} per second)
+  -- {writes2} writes ({writes2ps} per second)
+  -- {other2} other ({other2ps} per second)
+  -- {total2} in total ({total2ps} per second)
+'''.format(name = branch_name,
+           reads1 = data_branch[0]["queries"]["reads"],
+           writes1 = data_branch[0]["queries"]["writes"],
+           other1 = data_branch[0]["queries"]["other"],
+           total1 = data_branch[0]["queries"]["total"],
+           reads2 = data_develop[0]["queries"]["reads"],
+           writes2 = data_develop[0]["queries"]["writes"],
+           other2 = data_develop[0]["queries"]["other"],
+           total2 = data_develop[0]["queries"]["total"],
+           reads1ps = data_branch[0]["qps"]["reads"],
+           writes1ps = data_branch[0]["qps"]["writes"],
+           other1ps = data_branch[0]["qps"]["other"],
+           total1ps = data_branch[0]["qps"]["total"],
+           reads2ps = data_develop[0]["qps"]["reads"],
+           writes2ps = data_develop[0]["qps"]["writes"],
+           other2ps = data_develop[0]["qps"]["other"],
+           total2ps = data_develop[0]["qps"]["total"],)
+
+relation = ""
+if data_branch[0]["qps"]["total"] >= data_develop[0]["qps"]["total"]:
+    relation = '''The speed increase (total / total) is {inc}
+               '''.format(inc = data_branch[0]["qps"]["total"]/data_develop[0]["qps"]["total"])
+else:
+    relation = '''The speed decrease (total / total) is {inc}
+               '''.format(inc = data_develop[0]["qps"]["total"]/data_branch[0]["qps"]["total"])
+
+print(time)
+print(comparison)
+print(relation)

benchmarks/select_bench.lua

@@ -0,0 +1,29 @@
+require("bench_report")
+
+function prepare ()
+  local i
+  print("creating table test.t1 ...")
+  db_query("create table t1 (c1 int)engine=columnstore")
+end
+
+function cleanup()
+  db_query("drop table t1")
+end
+
+function help()
+  print("sysbench Lua demo; no special command line options available")
+end
+
+function thread_init(thread_id)
+end
+
+function thread_done(thread_id)
+  db_disconnect()
+end
+
+function event(thread_id)
+  db_query("select c1 from t1 where c1 = 5 or c1 = 10")
+end
+
+sysbench.hooks.report_intermediate = sysbench.report_json
+sysbench.hooks.report_cumulative = sysbench.report_json