From f5ca4077d805c3dab1f119f1192c082851205db2 Mon Sep 17 00:00:00 2001 From: Dishon Merkhai Date: Mon, 13 Mar 2023 15:04:54 -0400 Subject: [PATCH 1/6] MDEV-30839: Add new options to mini-benchmark and fixes Add new options to mini-benchmark.sh so it is more useful, such as `--log` to avoid output from `lscpu` and other commands being lost. Also fix following errors: - Fix if condition for the cycle count regression verdict. The condition checked if the cycle count was greater than 850 billion, but `$RESULT` is only 3 digits and represents the number of cycles in billions. - Fix flamegraph width. The original width caused the flamegraph to be cut-off on most screen sizes in the browser. - Fix ShellCheck warnings and suggestions. - Fix condition to check if perf has permission to run on the system. All new code of the whole pull request, including one or several files that are either new files or modified ones, are contributed under the BSD-new license. I am contributing on behalf of my employer Amazon Web Services, Inc. --- support-files/mini-benchmark.sh | 91 +++++++++++++++++++++++++-------- 1 file changed, 71 insertions(+), 20 deletions(-) diff --git a/support-files/mini-benchmark.sh b/support-files/mini-benchmark.sh index 18de6dbec51..9b7cb6dc698 100755 --- a/support-files/mini-benchmark.sh +++ b/support-files/mini-benchmark.sh @@ -10,6 +10,18 @@ display_help() { echo "regressions." echo echo "optional arguments:" + echo " --name STRING identifier for the benchmark, added to the " + echo " folder name and (if --log is set) the log file " + echo " --threads \"STRING\" quoted string of space-separated integers " + echo " representing the threads to run." + echo " example: --threads \"1 32 64 128\"" + echo " default: \"1 2 4 8 16\"" + echo " --duration INTEGER duration of each thread run in seconds" + echo " default: 60" + echo " --workload STRING sysbench workload to execute" + echo " default: oltp_read_write" + echo " --log logs the mini-benchmark stdout/stderr into the" + echo " benchmark folder." echo " --perf measure CPU cycles and instruction count in for " echo " sysbench runs" echo " --perf-flamegraph record performance counters in perf.data.* and" @@ -17,6 +29,12 @@ display_help() { echo " -h, --help display this help and exit" } +# Default parameters +BENCHMARK_NAME='mini-benchmark' +THREADS='1 2 4 8 16' +DURATION=60 +WORKLOAD='oltp_read_write' + while : do case "$1" in @@ -28,6 +46,31 @@ do display_version exit 0 ;; + --name) + shift + BENCHMARK_NAME+='-' + BENCHMARK_NAME+=$1 + shift + ;; + --threads) + shift + THREADS=$1 + shift + ;; + --duration) + shift + DURATION=$1 + shift + ;; + --workload) + shift + WORKLOAD=$1 + shift + ;; + --log) + LOG=true + shift + ;; --perf) PERF=true shift @@ -47,6 +90,13 @@ do esac done +# Save results of this run in a subdirectory so that they are not overwritten by +# the next run +TIMESTAMP="$(date -Iseconds)" +mkdir "$BENCHMARK_NAME-$TIMESTAMP" +cd "$BENCHMARK_NAME-$TIMESTAMP" || exit 1 + +( # Check that the dependencies of this script are available if [ ! -e /usr/bin/pgrep ] then @@ -62,6 +112,7 @@ fi # If there are multiple processes, assume the last one is the actual server and # any potential other ones were just part of the service wrapper chain +# shellcheck disable=SC2005 MARIADB_SERVER_PID="$(echo "$(pgrep -f mariadbd || pgrep -f mysqld)" | tail -n 1)" if [ -z "$MARIADB_SERVER_PID" ] @@ -102,12 +153,13 @@ then echo "Ensure the MariaDB Server debug symbols are installed" for x in $(ldd /usr/sbin/mariadbd | grep -oE " /.* ") do - rpm -q --whatprovides --qf '%{name}' $x | cut -d : -f 1 + rpm -q --whatprovides --qf '%{name}' "$x" | cut -d : -f 1 done | sort -u > mariadbd-dependencies.txt # shellcheck disable=SC2046 debuginfo-install -y mariadb-server $(cat mariadbd-dependencies.txt) - - if [ ! $(perf record echo "testing perf" > /dev/null 2>&1) ] + + perf record echo "testing perf" > /dev/null 2>&1 + if [ $? -ne 0 ] then echo "perf does not have permission to run on this system. Skipping." PERF="" @@ -120,7 +172,8 @@ elif [ -e /usr/bin/perf ] then # If flamegraphs were not requested, log normal perf counters if possible - if [ ! $(perf stat echo "testing perf" > /dev/null 2>&1) ] + perf stat echo "testing perf" > /dev/null 2>&1 + if [ $? -ne 0 ] then echo "perf does not have permission to run on this system. Skipping." PERF="" @@ -156,28 +209,23 @@ mariadb -e " CREATE USER IF NOT EXISTS sbtest@localhost; GRANT ALL PRIVILEGES ON sbtest.* TO sbtest@localhost" -sysbench oltp_read_write prepare --tables=20 --table-size=100000 | tee sysbench-prepare.log +sysbench "$WORKLOAD" prepare --tables=20 --table-size=100000 | tee sysbench-prepare.log sync && sleep 1 # Ensure writes were propagated to disk -# Save results of this run in a subdirectory so that they are not overwritten by -# the next run -TIMESTAMP="$(date -Iseconds)" -mkdir "mini-benchmark-$TIMESTAMP" -cd "mini-benchmark-$TIMESTAMP" || exit 1 - # Run benchmark with increasing thread counts. The MariaDB Server will be using # around 300 MB of RAM and mostly reading and writing in RAM, so I/O usage is # also low. The benchmark will most likely be CPU bound to due to the load # profile, and also guaranteed to be CPU bound because of being limited to a # single CPU with 'tasksel'. -for t in 1 2 4 8 16 +for t in $THREADS do # Prepend command with perf if defined - # Output stderr to stdout as perf outpus everything in stderr - $PERF $TASKSET_SYSBENCH sysbench oltp_read_write run --threads=$t --time=60 --report-interval=10 2>&1 | tee sysbench-run-$t.log + # Output stderr to stdout as perf outputs everything in stderr + # shellcheck disable=SC2086 + $PERF $TASKSET_SYSBENCH sysbench "$WORKLOAD" run --threads=$t --time=$DURATION --report-interval=10 2>&1 | tee sysbench-run-$t.log done -sysbench oltp_read_write cleanup --tables=20 | tee sysbench-cleanup.log +sysbench "$WORKLOAD" cleanup --tables=20 | tee sysbench-cleanup.log # Store results from 4 thread run in a Gitlab-CI compatible metrics file grep -oE '[a-z]+:[ ]+[0-9.]+' sysbench-run-4.log | sed -r 's/\s+/ /g' | tail -n 15 > metrics.txt @@ -197,10 +245,10 @@ then # Final verdict based on cpu cycle count RESULT="$(grep -h -e cycles sysbench-run-*.log | sort -k 1 | awk '{s+=$1}END{print s}')" - if [ "$RESULT" -gt 850000000000 ] + if [ "$RESULT" -gt 850 ] then echo # Newline improves readability - echo "Benchmark exceeded 8.5 billion cpu cycles, performance most likely regressed!" + echo "Benchmark exceeded 850 billion cpu cycles, performance most likely regressed!" exit 1 fi fi @@ -216,12 +264,12 @@ if [ "$PERF_RECORD" == true ] then for f in perf.data.* do - perf script -i $f | stackcollapse-perf.pl | flamegraph.pl --width 3000 > $f.svg + perf script -i "$f" | stackcollapse-perf.pl | flamegraph.pl --width 1800 > "$f".svg done - echo "Flamegraphs stored in folder mini-benchmark-$TIMESTAMP/" + echo "Flamegraphs stored in folder $BENCHMARK_NAME-$TIMESTAMP/" fi -# Fallback if CPU cycle count not availalbe: final verdict based on peak QPS +# Fallback if CPU cycle count not available: final verdict based on peak QPS RESULT="$(sort -k 9 -h sysbench-run-*.log | tail -n 1 | grep -oE "qps: [0-9]+" | grep -oE "[0-9]+")" case $RESULT in ''|*[!0-9]*) @@ -240,3 +288,6 @@ case $RESULT in fi ;; esac +# Record the output into the log file, if requested +) 2>&1 | ($LOG && tee "$BENCHMARK_NAME"-"$TIMESTAMP".log) + From ea9a6a1494b80bfc1a2e99509ffbb4dfed8b49ab Mon Sep 17 00:00:00 2001 From: Daniel Black Date: Fri, 2 Feb 2024 11:38:00 +1100 Subject: [PATCH 2/6] MDEV-33095 MariaDB-backup - no OS_DATA_FILE_NO_O_DIRECT on some platforms Postfix for a6290a5bc5f3cba096854595c354d19d9267743d, in 10.11 where OS_DATA_FILE_NO_O_DIRECT gets used. Same #ifdef conditions as other uses of OS_DATA_FILE_NO_O_DIRECT. Noticed on aarch64-macos builder. --- extra/mariabackup/xtrabackup.cc | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/extra/mariabackup/xtrabackup.cc b/extra/mariabackup/xtrabackup.cc index 680404e6986..3b828dec8a4 100644 --- a/extra/mariabackup/xtrabackup.cc +++ b/extra/mariabackup/xtrabackup.cc @@ -2428,7 +2428,12 @@ static bool innodb_init() os_file_delete_if_exists_func(ib_logfile0.c_str(), nullptr); os_file_t file= os_file_create_func(ib_logfile0.c_str(), OS_FILE_CREATE, OS_FILE_NORMAL, - OS_DATA_FILE_NO_O_DIRECT, false, &ret); +#if defined _WIN32 || defined HAVE_FCNTL_DIRECT + OS_DATA_FILE_NO_O_DIRECT, +#else + OS_DATA_FILE, +#endif + false, &ret); if (!ret) { invalid_log: From 93189df44eeb431110bf9ca759bafa856ae90c82 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marko=20M=C3=A4kel=C3=A4?= Date: Fri, 2 Feb 2024 10:48:10 +0200 Subject: [PATCH 3/6] MDEV-33361 Excessive delays in SET GLOBAL innodb_log_file_size innodb_log_file_size_update(): Wait for buf_pool.done_flush_list (that a flush batch has completed), not buf_pool.do_flush_list (that the buf_flush_page_cleaner was woken up). The condition variable buf_pool.done_flush_list is broadcast by the buf_flush_page_cleaner() at the end of each batch, which is when the log checkpoint can advance and where any log resizing may be completed. The purpose of the condition variable buf_pool.do_flush_list is to wake up the buf_flush_page_cleaner() thread because there is work to do. If no thread is signaling that condition variable, this loop could unnecessarily wait for up to 5 seconds too long for the log resizing to be completed. By consuming signals it could also prevent the buf_flush_page_cleaner() thread from waking up. Tested by: Matthias Leich --- storage/innobase/handler/ha_innodb.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/storage/innobase/handler/ha_innodb.cc b/storage/innobase/handler/ha_innodb.cc index 407834f2008..06af558a2e7 100644 --- a/storage/innobase/handler/ha_innodb.cc +++ b/storage/innobase/handler/ha_innodb.cc @@ -18452,7 +18452,7 @@ static void innodb_log_file_size_update(THD *thd, st_mysql_sys_var*, const bool in_progress(buf_pool.get_oldest_modification(LSN_MAX) < log_sys.resize_in_progress()); if (in_progress) - my_cond_timedwait(&buf_pool.do_flush_list, + my_cond_timedwait(&buf_pool.done_flush_list, &buf_pool.flush_list_mutex.m_mutex, &abstime); mysql_mutex_unlock(&buf_pool.flush_list_mutex); if (!log_sys.resize_in_progress()) From c79f19a5edd284d34addf175f8fa4f95e68c1f99 Mon Sep 17 00:00:00 2001 From: Vladislav Vaintroub Date: Fri, 2 Feb 2024 17:32:32 +0100 Subject: [PATCH 4/6] MDEV-33374 main.mysql_connector_net fails on new Windows 11 Additionally pass `-ExecutionPolicy Bypass` to powershell.exe when executing script --- mysql-test/main/mysql_connector_net.test | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mysql-test/main/mysql_connector_net.test b/mysql-test/main/mysql_connector_net.test index c1dce65adc8..ad048c20b3d 100644 --- a/mysql-test/main/mysql_connector_net.test +++ b/mysql-test/main/mysql_connector_net.test @@ -4,7 +4,7 @@ let $sys_errno=0; # Error 100 is returned by the powershell script # if MySql.Data is not installed --error 0,100 ---exec powershell -NoLogo -NoProfile -File main\mysql_connector_net.ps1 +--exec powershell -ExecutionPolicy Bypass -NoLogo -NoProfile -File main\mysql_connector_net.ps1 if ($sys_errno != 0) { --skip Connector/NET is not installed From 7f3839ab8cda2a5596b5cf02c4975076b22005f0 Mon Sep 17 00:00:00 2001 From: Daniel Bartholomew Date: Tue, 6 Feb 2024 08:25:30 -0500 Subject: [PATCH 5/6] bump the VERSION --- VERSION | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/VERSION b/VERSION index 7dd56698644..96bdb06ebf4 100644 --- a/VERSION +++ b/VERSION @@ -1,4 +1,4 @@ MYSQL_VERSION_MAJOR=10 MYSQL_VERSION_MINOR=11 -MYSQL_VERSION_PATCH=7 +MYSQL_VERSION_PATCH=8 SERVER_MATURITY=stable From c4c167778edcbd14f673fba161c91185c2899399 Mon Sep 17 00:00:00 2001 From: Alexander Barkov Date: Wed, 7 Feb 2024 09:05:28 +0400 Subject: [PATCH 6/6] MDEV-33392 Server crashes when using RANDOM_BYTES function and GROUP BY clause on a column with a negative value Item_func_random_bytes did not set its NULL-ability flag. --- mysql-test/main/func_str.result | 34 +++++++++++++++++++++++++++++++++ mysql-test/main/func_str.test | 22 +++++++++++++++++++++ sql/item_strfunc.cc | 1 + 3 files changed, 57 insertions(+) diff --git a/mysql-test/main/func_str.result b/mysql-test/main/func_str.result index daf97dd8023..7ca082962ad 100644 --- a/mysql-test/main/func_str.result +++ b/mysql-test/main/func_str.result @@ -5460,3 +5460,37 @@ Warning 1292 Truncated incorrect BINARY(2) value: '...random bytes...' # # End of 10.10 tests # +# +# Start of 10.11 tests +# +# +# MDEV-33392 Server crashes when using RANDOM_BYTES function and GROUP BY clause on a column with a negative value +# +SET sql_mode=''; +CREATE TABLE t1 (a VARCHAR(255)) ENGINE=MyISAM; +INSERT INTO t1 VALUES (9494),(9495),(9496),(9497),(9498),(9499),(9500),(9501),(9502),(9503); +SELECT RANDOM_BYTES (-1) f1,a f2 FROM t1 GROUP BY f1,f2; +f1 f2 +NULL 9494 +NULL 9495 +NULL 9496 +NULL 9497 +NULL 9498 +NULL 9499 +NULL 9500 +NULL 9501 +NULL 9502 +NULL 9503 +CREATE TABLE t2 AS SELECT RANDOM_BYTES (-1) f1,a f2 FROM t1 GROUP BY f1,f2; +SHOW CREATE TABLE t2; +Table Create Table +t2 CREATE TABLE `t2` ( + `f1` binary(0) DEFAULT NULL, + `f2` varchar(255) DEFAULT NULL +) ENGINE=MyISAM DEFAULT CHARSET=latin1 COLLATE=latin1_swedish_ci +DROP TABLE t2; +DROP TABLE t1; +SET sql_mode=DEFAULT; +# +# End of 10.11 tests +# diff --git a/mysql-test/main/func_str.test b/mysql-test/main/func_str.test index bbd84fde829..1670858e130 100644 --- a/mysql-test/main/func_str.test +++ b/mysql-test/main/func_str.test @@ -2435,3 +2435,25 @@ select "a" in ("abc", (convert(random_bytes(8) ,binary(2)))); --echo # --echo # End of 10.10 tests --echo # + +--echo # +--echo # Start of 10.11 tests +--echo # + +--echo # +--echo # MDEV-33392 Server crashes when using RANDOM_BYTES function and GROUP BY clause on a column with a negative value +--echo # + +SET sql_mode=''; +CREATE TABLE t1 (a VARCHAR(255)) ENGINE=MyISAM; +INSERT INTO t1 VALUES (9494),(9495),(9496),(9497),(9498),(9499),(9500),(9501),(9502),(9503); +SELECT RANDOM_BYTES (-1) f1,a f2 FROM t1 GROUP BY f1,f2; +CREATE TABLE t2 AS SELECT RANDOM_BYTES (-1) f1,a f2 FROM t1 GROUP BY f1,f2; +SHOW CREATE TABLE t2; +DROP TABLE t2; +DROP TABLE t1; +SET sql_mode=DEFAULT; + +--echo # +--echo # End of 10.11 tests +--echo # diff --git a/sql/item_strfunc.cc b/sql/item_strfunc.cc index ad7d3a96518..49d7ff5752d 100644 --- a/sql/item_strfunc.cc +++ b/sql/item_strfunc.cc @@ -1509,6 +1509,7 @@ String *Item_func_sformat::val_str(String *res) bool Item_func_random_bytes::fix_length_and_dec(THD *thd) { + set_maybe_null(); used_tables_cache|= RAND_TABLE_BIT; if (args[0]->can_eval_in_optimize()) {