MDEV-36848: identify tests with various MSAN suitability

With MSAN the following test behavious where observed:
* funcs_1.myisam_views-big - normal big test for non-debug
* innodb_gis.rtree_purge - normal big test with MSAN
* main.alter_table_lock - very quick - unclear why disabled
* main.cte_recursive - slow on Debug only
* main.join_cache_notasan - special MSAN handing for returning OOM added
* main.sum_distinct-big - 90 seconds on non-debug - still big however
* maria.max_length - normal big test with MSAN
* perfschema.statement_digest_long_query - overflows stack on debug

Timingsi (on old memory constrained hardware):

non-debug:

funcs_1.myisam_views-big                 w2 [ pass ]  78564
innodb_gis.rtree_purge '16k'             w2 [ pass ]   5784
innodb_gis.rtree_purge '32k'             w2 [ pass ]   5242
innodb_gis.rtree_purge '4k'              w1 [ pass ]   8303
innodb_gis.rtree_purge '64k'             w1 [ pass ]   6348
innodb_gis.rtree_purge '8k'              w2 [ pass ]   5870
main.alter_table_lock                    w1 [ pass ]     41
main.cte_recursive                       w1 [ pass ]  15485
main.join_cache_notasan                  w1 [ pass ]     39
main.sum_distinct-big                    w2 [ pass ]  96256
maria.max_length                         w1 [ pass ]  92990
perfschema.statement_digest_long_query   w2 [ pass ]      8

debug:

funcs_1.myisam_views-big                 w1 [ skipped ]  Can't be run WITH_MSAN and CMAKE_BUILD_TYPE=Debug
innodb_gis.rtree_purge '16k'             w2 [ pass ]  109788
innodb_gis.rtree_purge '32k'             w2 [ pass ]  62361
innodb_gis.rtree_purge '4k'              w1 [ pass ]  89423
innodb_gis.rtree_purge '64k'             w1 [ pass ]  72082
innodb_gis.rtree_purge '8k'              w1 [ pass ]  98452
main.alter_table_lock                    w2 [ pass ]     38
main.cte_recursive                       w2 [ pass ]  180047
main.join_cache_notasan                  w1 [ pass ]    166
main.sum_distinct-big                    w1 [ skipped ]  Can't be run WITH_MSAN and CMAKE_BUILD_TYPE=Debug
maria.max_length                         w1 [ skipped ]  Can't be run WITH_MSAN and CMAKE_BUILD_TYPE=Debug
perfschema.statement_digest_long_query   w1 [ skipped ]  Can't be run WITH_MSAN and CMAKE_BUILD_TYPE=Debug

mysql-test/include/no_msan_without_big.inc

@@ -0,0 +1,8 @@
+# Slow with MSAN, but if mtr --big-test specified, then it should complete
+if (!$BIG_TEST)
+{
+  if (`select count(*) from information_schema.system_variables where variable_name='have_sanitizer' and global_value like "MSAN%"`)
+  {
+    --skip Can't be run WITH_MSAN unless using --big-test
+  }
+}

mysql-test/include/not_msan.inc

@@ -1,4 +1,5 @@
-# This file should only be used with tests that are too big or slow for MSAN.
+# This file should only be used with tests that are too big or slow for MSAN (even with --big-test).
+# Use no_msan_without_big instead unless this really won't complete in a test timeout period.
 
 if (`select count(*) from information_schema.system_variables where variable_name='have_sanitizer' and global_value like "MSAN%"`)
 {

mysql-test/include/not_msan_with_debug.inc

@@ -0,0 +1,9 @@
+# This file should only be used with tests that are too big or slow for MSAN with Debug.
+
+if (`select count(*) from information_schema.system_variables where variable_name='have_sanitizer' and global_value like "MSAN%"`)
+{
+  if (`select version() like '%debug%'`)
+  {
+    --skip Can't be run WITH_MSAN and CMAKE_BUILD_TYPE=Debug
+  }
+}

mysql-test/main/alter_table_lock.test

@@ -1,5 +1,3 @@
---source include/not_msan.inc
-
 --echo #
 --echo # MDEV-23836: Assertion `! is_set() || m_can_overwrite_status' in
 --echo # Diagnostics_area::set_error_status (interrupted ALTER TABLE under LOCK)

mysql-test/main/cte_recursive.test

@@ -1,6 +1,6 @@
 --source include/default_optimizer_switch.inc
 # This is too slow on MSAN
---source include/not_msan.inc
+--source include/no_msan_without_big.inc
 --source include/not_valgrind.inc
 --source include/have_innodb.inc
 

mysql-test/main/join_cache_notasan.cnf

@@ -0,0 +1,5 @@
+!include include/default_my.cnf
+
+[ENV]
+MSAN_OPTIONS=allocator_may_return_null=1:abort_on_error=1
+

mysql-test/main/join_cache_notasan.result

@@ -1,3 +1,4 @@
+call mtr.add_suppression("MemorySanitizer failed to allocate");
 #
 # MDEV-28217 Incorrect Join Execution When Controlling Join Buffer Size
 #

mysql-test/main/join_cache_notasan.test

@@ -4,11 +4,13 @@
 --source include/have_64bit.inc
 # Disable asan it asan builds crashes when trying to allocate too much memory
 --source include/not_asan.inc
---source include/not_msan.inc
 # Valgrind is useful here, but very slow as lots of memory is allocated
 --source include/no_valgrind_without_big.inc
 --source include/have_innodb.inc
 
+# MSAN runs, but ignore its notice. ER_OUTOFMEMORY is expected by tests
+call mtr.add_suppression("MemorySanitizer failed to allocate");
+
 --echo #
 --echo # MDEV-28217 Incorrect Join Execution When Controlling Join Buffer Size
 --echo #

mysql-test/main/sum_distinct-big.test

@@ -5,7 +5,7 @@
 --source include/big_test.inc
 # Test will take more than one hour with valgrind
 --source include/not_valgrind.inc
---source include/not_msan.inc
+--source include/not_msan_with_debug.inc
 --source include/have_innodb.inc
 --source include/have_sequence.inc
 

mysql-test/suite/funcs_1/t/myisam_views-big.test

@@ -4,8 +4,7 @@
 # because of a pair of slow Solaris Sparc machines in pb2,
 # this test is marked as big:
 --source include/big_test.inc
-# This test often times out with MSAN
---source include/not_msan.inc
+--source include/not_msan_with_debug.inc
 
 # MyISAM tables should be used
 #

mysql-test/suite/innodb_gis/t/rtree_purge.test

@@ -4,8 +4,7 @@
 --source include/innodb_page_size.inc
 --source include/have_sequence.inc
 --source include/not_valgrind.inc
-# This test often times out with MSAN
---source include/not_msan.inc
+--source include/no_msan_without_big.inc
 
 create table t (
   b point not null,d point not null, spatial key (d),spatial key (b)

mysql-test/suite/maria/max_length.test

@@ -6,7 +6,7 @@
 --source include/big_test.inc
 # This test is too slow for valgrind
 --source include/not_valgrind.inc
---source include/not_msan.inc
+--source include/not_msan_with_debug.inc
 
 drop table if exists t1,t2;
 

mysql-test/suite/perfschema/t/statement_digest_long_query.test

@@ -6,6 +6,7 @@
 --source include/have_perfschema.inc
 # Test requires: sp-protocol/ps-protocol/view-protocol/cursor-protocol disabled
 --source include/no_protocol.inc
+--source include/not_msan_with_debug.inc
 # Thread stack overrun on solaris
 let $have_solaris = `select convert(@@version_compile_os using latin1) LIKE ("solaris%")`;
 if ($have_solaris)